]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu/gmc7: disable legacy vga features in gmc init
[mirror_ubuntu-hirsute-kernel.git] / drivers / gpu / drm / amd / amdgpu / gmc_v8_0.c
CommitLineData
aaa36a97
AD		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
248a1d6f 24#include <drm/drmP.h>
aaa36a97
AD		 25#include "amdgpu.h"
26#include "gmc_v8_0.h"
27#include "amdgpu_ucode.h"
28
29#include "gmc/gmc_8_1_d.h"
30#include "gmc/gmc_8_1_sh_mask.h"
31
32#include "bif/bif_5_0_d.h"
33#include "bif/bif_5_0_sh_mask.h"
34
35#include "oss/oss_3_0_d.h"
36#include "oss/oss_3_0_sh_mask.h"
37
38#include "vid.h"
39#include "vi.h"
40
1ce65f52
HW		 41#include "amdgpu_atombios.h"
42
81c59f54 43
aaa36a97
AD		 44static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
45static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
34e3205e 46static int gmc_v8_0_wait_for_idle(void *handle);
aaa36a97 47
c65444fe 48MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
2cc0c0b5
FC		 49MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");
50MODULE_FIRMWARE("amdgpu/polaris10_mc.bin");
c4642a47 51MODULE_FIRMWARE("amdgpu/polaris12_mc.bin");
aaa36a97
AD		 52
53static const u32 golden_settings_tonga_a11[] =
54{
55 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
56 mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
57 mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
58 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
59 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
60 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
61 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62};
63
64static const u32 tonga_mgcg_cgcg_init[] =
65{
66 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
67};
68
127a2628
DZ		 69static const u32 golden_settings_fiji_a10[] =
70{
71 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
72 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
73 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
74 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
75};
76
77static const u32 fiji_mgcg_cgcg_init[] =
78{
79 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
80};
81
2cc0c0b5 82static const u32 golden_settings_polaris11_a11[] =
c9778572
FC		 83{
84 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
85 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
86 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
87 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
88};
89
2cc0c0b5 90static const u32 golden_settings_polaris10_a11[] =
c9778572
FC		 91{
92 mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
93 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
94 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
95 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
96 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
97};
98
aaa36a97
AD		 99static const u32 cz_mgcg_cgcg_init[] =
100{
101 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
102};
103
aade2f04
SL		 104static const u32 stoney_mgcg_cgcg_init[] =
105{
0711257e 106 mmATC_MISC_CG, 0xffffffff, 0x000c0200,
aade2f04
SL		 107 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
108};
109
6d51c813
HR		 110static const u32 golden_settings_stoney_common[] =
111{
112 mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004,
113 mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000
114};
aade2f04 115
aaa36a97
AD		 116static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
117{
118 switch (adev->asic_type) {
127a2628
DZ		 119 case CHIP_FIJI:
120 amdgpu_program_register_sequence(adev,
121 fiji_mgcg_cgcg_init,
122 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
123 amdgpu_program_register_sequence(adev,
124 golden_settings_fiji_a10,
125 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
126 break;
aaa36a97
AD		 127 case CHIP_TONGA:
128 amdgpu_program_register_sequence(adev,
129 tonga_mgcg_cgcg_init,
130 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
131 amdgpu_program_register_sequence(adev,
132 golden_settings_tonga_a11,
133 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
134 break;
2cc0c0b5 135 case CHIP_POLARIS11:
c4642a47 136 case CHIP_POLARIS12:
c9778572 137 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 138 golden_settings_polaris11_a11,
139 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
c9778572 140 break;
2cc0c0b5 141 case CHIP_POLARIS10:
c9778572 142 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 143 golden_settings_polaris10_a11,
144 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
c9778572 145 break;
aaa36a97
AD		 146 case CHIP_CARRIZO:
147 amdgpu_program_register_sequence(adev,
148 cz_mgcg_cgcg_init,
149 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
150 break;
aade2f04
SL		 151 case CHIP_STONEY:
152 amdgpu_program_register_sequence(adev,
153 stoney_mgcg_cgcg_init,
154 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
6d51c813
HR		 155 amdgpu_program_register_sequence(adev,
156 golden_settings_stoney_common,
157 (const u32)ARRAY_SIZE(golden_settings_stoney_common));
aade2f04 158 break;
aaa36a97
AD		 159 default:
160 break;
161 }
162}
163
e4f6b39e 164static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)
aaa36a97
AD		 165{
166 u32 blackout;
167
34e3205e 168 gmc_v8_0_wait_for_idle(adev);
aaa36a97
AD		 169
170 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
171 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
172 /* Block CPU access */
173 WREG32(mmBIF_FB_EN, 0);
174 /* blackout the MC */
175 blackout = REG_SET_FIELD(blackout,
176 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);
177 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
178 }
179 /* wait for the MC to settle */
180 udelay(100);
181}
182
e4f6b39e 183static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)
aaa36a97
AD		 184{
185 u32 tmp;
186
187 /* unblackout the MC */
188 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
189 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
190 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
191 /* allow CPU access */
192 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
193 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
194 WREG32(mmBIF_FB_EN, tmp);
aaa36a97
AD		 195}
196
197/**
198 * gmc_v8_0_init_microcode - load ucode images from disk
199 *
200 * @adev: amdgpu_device pointer
201 *
202 * Use the firmware interface to load the ucode images into
203 * the driver (not loaded into hw).
204 * Returns 0 on success, error on failure.
205 */
206static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
207{
208 const char *chip_name;
209 char fw_name[30];
210 int err;
211
212 DRM_DEBUG("\n");
213
214 switch (adev->asic_type) {
aaa36a97
AD		 215 case CHIP_TONGA:
216 chip_name = "tonga";
217 break;
2cc0c0b5
FC		 218 case CHIP_POLARIS11:
219 chip_name = "polaris11";
c9778572 220 break;
2cc0c0b5
FC		 221 case CHIP_POLARIS10:
222 chip_name = "polaris10";
c9778572 223 break;
c4642a47
JZ		 224 case CHIP_POLARIS12:
225 chip_name = "polaris12";
226 break;
127a2628 227 case CHIP_FIJI:
aaa36a97 228 case CHIP_CARRIZO:
aade2f04 229 case CHIP_STONEY:
aaa36a97
AD		 230 return 0;
231 default: BUG();
232 }
233
c65444fe 234 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
aaa36a97
AD		 235 err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
236 if (err)
237 goto out;
238 err = amdgpu_ucode_validate(adev->mc.fw);
239
240out:
241 if (err) {
7ca85295 242 pr_err("mc: Failed to load firmware \"%s\"\n", fw_name);
aaa36a97
AD		 243 release_firmware(adev->mc.fw);
244 adev->mc.fw = NULL;
245 }
246 return err;
247}
248
249/**
0d52c6a1 250 * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw
aaa36a97
AD		 251 *
252 * @adev: amdgpu_device pointer
253 *
254 * Load the GDDR MC ucode into the hw (CIK).
255 * Returns 0 on success, error on failure.
256 */
0d52c6a1 257static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev)
aaa36a97
AD		 258{
259 const struct mc_firmware_header_v1_0 *hdr;
260 const __le32 *fw_data = NULL;
261 const __le32 *io_mc_regs = NULL;
887656f0 262 u32 running;
aaa36a97
AD		 263 int i, ucode_size, regs_size;
264
c12d2871
AD		 265 /* Skip MC ucode loading on SR-IOV capable boards.
266 * vbios does this for us in asic_init in that case.
4e99a44e
ML		 267 * Skip MC ucode loading on VF, because hypervisor will do that
268 * for this adaptor.
c12d2871 269 */
4e99a44e 270 if (amdgpu_sriov_bios(adev))
c12d2871
AD		 271 return 0;
272
0d52c6a1
RZ		 273 if (!adev->mc.fw)
274 return -EINVAL;
275
aaa36a97
AD		 276 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
277 amdgpu_ucode_print_mc_hdr(&hdr->header);
278
279 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
280 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
281 io_mc_regs = (const __le32 *)
282 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
283 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
284 fw_data = (const __le32 *)
285 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
286
287 running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
288
289 if (running == 0) {
aaa36a97
AD		 290 /* reset the engine and set to writable */
291 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
292 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
293
294 /* load mc io regs */
295 for (i = 0; i < regs_size; i++) {
296 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
297 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
298 }
299 /* load the MC ucode */
300 for (i = 0; i < ucode_size; i++)
301 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
302
303 /* put the engine back into the active state */
304 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
305 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
306 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
307
308 /* wait for training to complete */
309 for (i = 0; i < adev->usec_timeout; i++) {
310 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
311 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
312 break;
313 udelay(1);
314 }
315 for (i = 0; i < adev->usec_timeout; i++) {
316 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
317 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
318 break;
319 udelay(1);
320 }
aaa36a97
AD		 321 }
322
323 return 0;
324}
325
0d52c6a1
RZ		 326static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev)
327{
328 const struct mc_firmware_header_v1_0 *hdr;
329 const __le32 *fw_data = NULL;
330 const __le32 *io_mc_regs = NULL;
331 u32 data, vbios_version;
332 int i, ucode_size, regs_size;
333
334 /* Skip MC ucode loading on SR-IOV capable boards.
335 * vbios does this for us in asic_init in that case.
336 * Skip MC ucode loading on VF, because hypervisor will do that
337 * for this adaptor.
338 */
339 if (amdgpu_sriov_bios(adev))
340 return 0;
341
342 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
343 data = RREG32(mmMC_SEQ_IO_DEBUG_DATA);
344 vbios_version = data & 0xf;
345
346 if (vbios_version == 0)
347 return 0;
348
349 if (!adev->mc.fw)
350 return -EINVAL;
351
352 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
353 amdgpu_ucode_print_mc_hdr(&hdr->header);
354
355 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
356 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
357 io_mc_regs = (const __le32 *)
358 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
359 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
360 fw_data = (const __le32 *)
361 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
362
363 data = RREG32(mmMC_SEQ_MISC0);
364 data &= ~(0x40);
365 WREG32(mmMC_SEQ_MISC0, data);
366
367 /* load mc io regs */
368 for (i = 0; i < regs_size; i++) {
369 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
370 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
371 }
372
373 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
374 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
375
376 /* load the MC ucode */
377 for (i = 0; i < ucode_size; i++)
378 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
379
380 /* put the engine back into the active state */
381 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
382 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
383 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
384
385 /* wait for training to complete */
386 for (i = 0; i < adev->usec_timeout; i++) {
387 data = RREG32(mmMC_SEQ_MISC0);
388 if (data & 0x80)
389 break;
390 udelay(1);
391 }
392
393 return 0;
394}
395
aaa36a97
AD		 396static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
397 struct amdgpu_mc *mc)
398{
e72b9912
ED		 399 u64 base = 0;
400
401 if (!amdgpu_sriov_vf(adev))
402 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
9081c4cf
AD		 403 base <<= 24;
404
aaa36a97
AD		 405 if (mc->mc_vram_size > 0xFFC0000000ULL) {
406 /* leave room for at least 1024M GTT */
407 dev_warn(adev->dev, "limiting VRAM\n");
408 mc->real_vram_size = 0xFFC0000000ULL;
409 mc->mc_vram_size = 0xFFC0000000ULL;
410 }
9081c4cf 411 amdgpu_vram_location(adev, &adev->mc, base);
6f02a696 412 amdgpu_gart_location(adev, mc);
aaa36a97
AD		 413}
414
415/**
416 * gmc_v8_0_mc_program - program the GPU memory controller
417 *
418 * @adev: amdgpu_device pointer
419 *
420 * Set the location of vram, gart, and AGP in the GPU's
421 * physical address space (CIK).
422 */
423static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
424{
aaa36a97
AD		 425 u32 tmp;
426 int i, j;
427
428 /* Initialize HDP */
429 for (i = 0, j = 0; i < 32; i++, j += 0x6) {
430 WREG32((0xb05 + j), 0x00000000);
431 WREG32((0xb06 + j), 0x00000000);
432 WREG32((0xb07 + j), 0x00000000);
433 WREG32((0xb08 + j), 0x00000000);
434 WREG32((0xb09 + j), 0x00000000);
435 }
436 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
437
34e3205e 438 if (gmc_v8_0_wait_for_idle((void *)adev)) {
aaa36a97
AD		 439 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
440 }
441 /* Update configuration */
442 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
443 adev->mc.vram_start >> 12);
444 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
445 adev->mc.vram_end >> 12);
446 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
447 adev->vram_scratch.gpu_addr >> 12);
e72b9912
ED		 448
449 if (amdgpu_sriov_vf(adev)) {
450 tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
451 tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
452 WREG32(mmMC_VM_FB_LOCATION, tmp);
453 /* XXX double check these! */
454 WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
455 WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
456 WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
457 }
458
aaa36a97
AD		 459 WREG32(mmMC_VM_AGP_BASE, 0);
460 WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
461 WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
34e3205e 462 if (gmc_v8_0_wait_for_idle((void *)adev)) {
aaa36a97
AD		 463 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
464 }
aaa36a97
AD		 465
466 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
467
468 tmp = RREG32(mmHDP_MISC_CNTL);
13459bd0 469 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
aaa36a97
AD		 470 WREG32(mmHDP_MISC_CNTL, tmp);
471
472 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
473 WREG32(mmHDP_HOST_PATH_CNTL, tmp);
474}
475
476/**
477 * gmc_v8_0_mc_init - initialize the memory controller driver params
478 *
479 * @adev: amdgpu_device pointer
480 *
481 * Look up the amount of vram, vram width, and decide how to place
482 * vram and gart within the GPU's physical address space (CIK).
483 * Returns 0 for success.
484 */
485static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
486{
1ce65f52
HW		 487 adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
488 if (!adev->mc.vram_width) {
489 u32 tmp;
490 int chansize, numchan;
491
492 /* Get VRAM informations */
493 tmp = RREG32(mmMC_ARB_RAMCFG);
494 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
495 chansize = 64;
496 } else {
497 chansize = 32;
498 }
499 tmp = RREG32(mmMC_SHARED_CHMAP);
500 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
501 case 0:
502 default:
503 numchan = 1;
504 break;
505 case 1:
506 numchan = 2;
507 break;
508 case 2:
509 numchan = 4;
510 break;
511 case 3:
512 numchan = 8;
513 break;
514 case 4:
515 numchan = 3;
516 break;
517 case 5:
518 numchan = 6;
519 break;
520 case 6:
521 numchan = 10;
522 break;
523 case 7:
524 numchan = 12;
525 break;
526 case 8:
527 numchan = 16;
528 break;
529 }
530 adev->mc.vram_width = numchan * chansize;
aaa36a97 531 }
aaa36a97
AD		 532 /* Could aper size report 0 ? */
533 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
534 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
535 /* size in MB on si */
536 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
537 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
999446a7
CK		 538
539#ifdef CONFIG_X86_64
540 if (adev->flags & AMD_IS_APU) {
541 adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
542 adev->mc.aper_size = adev->mc.real_vram_size;
543 }
544#endif
aaa36a97 545
a1493cd5 546 /* In case the PCI BAR is larger than the actual amount of vram */
999446a7 547 adev->mc.visible_vram_size = adev->mc.aper_size;
a1493cd5
AD		 548 if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
549 adev->mc.visible_vram_size = adev->mc.real_vram_size;
550
011d4bbe 551 amdgpu_gart_set_defaults(adev);
aaa36a97
AD		 552 gmc_v8_0_vram_gtt_location(adev, &adev->mc);
553
554 return 0;
555}
556
557/*
558 * GART
559 * VMID 0 is the physical GPU addresses as used by the kernel.
560 * VMIDs 1-15 are used for userspace clients and are handled
561 * by the amdgpu vm/hsa code.
562 */
563
564/**
565 * gmc_v8_0_gart_flush_gpu_tlb - gart tlb flush callback
566 *
567 * @adev: amdgpu_device pointer
568 * @vmid: vm instance to flush
569 *
570 * Flush the TLB for the requested page table (CIK).
571 */
572static void gmc_v8_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
573 uint32_t vmid)
574{
575 /* flush hdp cache */
576 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
577
578 /* bits 0-15 are the VM contexts0-15 */
579 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
580}
581
582/**
583 * gmc_v8_0_gart_set_pte_pde - update the page tables using MMIO
584 *
585 * @adev: amdgpu_device pointer
586 * @cpu_pt_addr: cpu address of the page table
587 * @gpu_page_idx: entry in the page table to update
588 * @addr: dst addr to write into pte/pde
589 * @flags: access flags
590 *
591 * Update the page tables using the CPU.
592 */
593static int gmc_v8_0_gart_set_pte_pde(struct amdgpu_device *adev,
594 void *cpu_pt_addr,
595 uint32_t gpu_page_idx,
596 uint64_t addr,
6b777607 597 uint64_t flags)
aaa36a97
AD		 598{
599 void __iomem *ptr = (void *)cpu_pt_addr;
600 uint64_t value;
601
602 /*
603 * PTE format on VI:
604 * 63:40 reserved
605 * 39:12 4k physical page base address
606 * 11:7 fragment
607 * 6 write
608 * 5 read
609 * 4 exe
610 * 3 reserved
611 * 2 snooped
612 * 1 system
613 * 0 valid
614 *
615 * PDE format on VI:
616 * 63:59 block fragment size
617 * 58:40 reserved
618 * 39:1 physical base address of PTE
619 * bits 5:1 must be 0.
620 * 0 valid
621 */
622 value = addr & 0x000000FFFFFFF000ULL;
623 value |= flags;
624 writeq(value, ptr + (gpu_page_idx * 8));
625
626 return 0;
627}
628
5463545b
AX		 629static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
630 uint32_t flags)
631{
632 uint64_t pte_flag = 0;
633
634 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
635 pte_flag |= AMDGPU_PTE_EXECUTABLE;
636 if (flags & AMDGPU_VM_PAGE_READABLE)
637 pte_flag |= AMDGPU_PTE_READABLE;
638 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
639 pte_flag |= AMDGPU_PTE_WRITEABLE;
640 if (flags & AMDGPU_VM_PAGE_PRT)
641 pte_flag |= AMDGPU_PTE_PRT;
642
643 return pte_flag;
644}
645
b1166325
CK		 646static uint64_t gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
647{
648 BUG_ON(addr & 0xFFFFFF0000000FFFULL);
649 return addr;
650}
651
d9c13156
CK		 652/**
653 * gmc_v8_0_set_fault_enable_default - update VM fault handling
654 *
655 * @adev: amdgpu_device pointer
656 * @value: true redirects VM faults to the default page
657 */
658static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev,
659 bool value)
660{
661 u32 tmp;
662
663 tmp = RREG32(mmVM_CONTEXT1_CNTL);
664 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
665 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
666 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
667 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
668 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
669 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
670 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
671 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
672 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
673 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
674 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
675 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
676 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
677 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
678 WREG32(mmVM_CONTEXT1_CNTL, tmp);
679}
680
603adfe8
CK		 681/**
682 * gmc_v8_0_set_prt - set PRT VM fault
683 *
684 * @adev: amdgpu_device pointer
685 * @enable: enable/disable VM fault handling for PRT
686*/
687static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
688{
689 u32 tmp;
690
691 if (enable && !adev->mc.prt_warning) {
692 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
693 adev->mc.prt_warning = true;
694 }
695
696 tmp = RREG32(mmVM_PRT_CNTL);
697 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
698 CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
699 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
700 CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
701 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
702 TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
703 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
704 TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
705 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
706 L2_CACHE_STORE_INVALID_ENTRIES, enable);
707 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
708 L1_TLB_STORE_INVALID_ENTRIES, enable);
709 tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
710 MASK_PDE0_FAULT, enable);
711 WREG32(mmVM_PRT_CNTL, tmp);
712
713 if (enable) {
714 uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
715 uint32_t high = adev->vm_manager.max_pfn;
716
717 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
718 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
719 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
720 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
721 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
722 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
723 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
724 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
725 } else {
726 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
727 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
728 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
729 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
730 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
731 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
732 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
733 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
734 }
735}
736
aaa36a97
AD		 737/**
738 * gmc_v8_0_gart_enable - gart enable
739 *
740 * @adev: amdgpu_device pointer
741 *
742 * This sets up the TLBs, programs the page tables for VMID0,
743 * sets up the hw for VMIDs 1-15 which are allocated on
744 * demand, and sets up the global locations for the LDS, GDS,
745 * and GPUVM for FSA64 clients (CIK).
746 * Returns 0 for success, errors for failure.
747 */
748static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
749{
750 int r, i;
751 u32 tmp;
752
753 if (adev->gart.robj == NULL) {
754 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
755 return -EINVAL;
756 }
757 r = amdgpu_gart_table_vram_pin(adev);
758 if (r)
759 return r;
760 /* Setup TLB control */
761 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
762 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
763 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
764 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
765 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
766 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
767 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
768 /* Setup L2 cache */
769 tmp = RREG32(mmVM_L2_CNTL);
770 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
771 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
772 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
773 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
774 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
775 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
a80b3047 776 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
aaa36a97
AD		 777 WREG32(mmVM_L2_CNTL, tmp);
778 tmp = RREG32(mmVM_L2_CNTL2);
779 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
780 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
781 WREG32(mmVM_L2_CNTL2, tmp);
782 tmp = RREG32(mmVM_L2_CNTL3);
783 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
784 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
785 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
786 WREG32(mmVM_L2_CNTL3, tmp);
787 /* XXX: set to enable PTE/PDE in system memory */
788 tmp = RREG32(mmVM_L2_CNTL4);
789 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0);
790 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0);
791 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0);
792 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0);
793 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0);
794 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0);
795 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0);
796 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0);
797 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0);
798 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0);
799 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0);
800 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);
801 WREG32(mmVM_L2_CNTL4, tmp);
802 /* setup context0 */
6f02a696
CK		 803 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
804 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
aaa36a97
AD		 805 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
806 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
807 (u32)(adev->dummy_page.addr >> 12));
808 WREG32(mmVM_CONTEXT0_CNTL2, 0);
809 tmp = RREG32(mmVM_CONTEXT0_CNTL);
810 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
811 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
812 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
813 WREG32(mmVM_CONTEXT0_CNTL, tmp);
814
815 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0);
816 WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0);
817 WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0);
818
819 /* empty context1-15 */
820 /* FIXME start with 4G, once using 2 level pt switch to full
821 * vm size space
822 */
823 /* set vm size, must be a multiple of 4 */
824 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
25a595e4 825 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
aaa36a97
AD		 826 for (i = 1; i < 16; i++) {
827 if (i < 8)
828 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
829 adev->gart.table_addr >> 12);
830 else
831 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
832 adev->gart.table_addr >> 12);
833 }
834
835 /* enable context1-15 */
836 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
837 (u32)(adev->dummy_page.addr >> 12));
838 WREG32(mmVM_CONTEXT1_CNTL2, 4);
839 tmp = RREG32(mmVM_CONTEXT1_CNTL);
840 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
841 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
aaa36a97 842 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97 843 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97 844 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97 845 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97 846 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97 847 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
aaa36a97
AD		 848 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
849 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
36b32a68 850 adev->vm_manager.block_size - 9);
aaa36a97 851 WREG32(mmVM_CONTEXT1_CNTL, tmp);
d9c13156
CK		 852 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
853 gmc_v8_0_set_fault_enable_default(adev, false);
854 else
855 gmc_v8_0_set_fault_enable_default(adev, true);
aaa36a97
AD		 856
857 gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
858 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
6f02a696 859 (unsigned)(adev->mc.gart_size >> 20),
aaa36a97
AD		 860 (unsigned long long)adev->gart.table_addr);
861 adev->gart.ready = true;
862 return 0;
863}
864
865static int gmc_v8_0_gart_init(struct amdgpu_device *adev)
866{
867 int r;
868
869 if (adev->gart.robj) {
870 WARN(1, "R600 PCIE GART already initialized\n");
871 return 0;
872 }
873 /* Initialize common gart structure */
874 r = amdgpu_gart_init(adev);
875 if (r)
876 return r;
877 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
4b98e0c4 878 adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE;
aaa36a97
AD		 879 return amdgpu_gart_table_vram_alloc(adev);
880}
881
882/**
883 * gmc_v8_0_gart_disable - gart disable
884 *
885 * @adev: amdgpu_device pointer
886 *
887 * This disables all VM page table (CIK).
888 */
889static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
890{
891 u32 tmp;
892
893 /* Disable all tables */
894 WREG32(mmVM_CONTEXT0_CNTL, 0);
895 WREG32(mmVM_CONTEXT1_CNTL, 0);
896 /* Setup TLB control */
897 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
898 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
899 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
900 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
901 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
902 /* Setup L2 cache */
903 tmp = RREG32(mmVM_L2_CNTL);
904 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
905 WREG32(mmVM_L2_CNTL, tmp);
906 WREG32(mmVM_L2_CNTL2, 0);
907 amdgpu_gart_table_vram_unpin(adev);
908}
909
910/**
911 * gmc_v8_0_gart_fini - vm fini callback
912 *
913 * @adev: amdgpu_device pointer
914 *
915 * Tears down the driver GART/VM setup (CIK).
916 */
917static void gmc_v8_0_gart_fini(struct amdgpu_device *adev)
918{
919 amdgpu_gart_table_vram_free(adev);
920 amdgpu_gart_fini(adev);
921}
922
aaa36a97
AD		 923/**
924 * gmc_v8_0_vm_decode_fault - print human readable fault info
925 *
926 * @adev: amdgpu_device pointer
927 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
928 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
929 *
930 * Print human readable fault information (CIK).
931 */
932static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
933 u32 status, u32 addr, u32 mc_client)
934{
935 u32 mc_id;
936 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
937 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
938 PROTECTIONS);
939 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
940 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
941
942 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
943 MEMORY_CLIENT_ID);
944
b01dd025 945 dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
aaa36a97
AD		 946 protections, vmid, addr,
947 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
948 MEMORY_CLIENT_RW) ?
949 "write" : "read", block, mc_client, mc_id);
950}
951
81c59f54
KW		 952static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type)
953{
954 switch (mc_seq_vram_type) {
955 case MC_SEQ_MISC0__MT__GDDR1:
956 return AMDGPU_VRAM_TYPE_GDDR1;
957 case MC_SEQ_MISC0__MT__DDR2:
958 return AMDGPU_VRAM_TYPE_DDR2;
959 case MC_SEQ_MISC0__MT__GDDR3:
960 return AMDGPU_VRAM_TYPE_GDDR3;
961 case MC_SEQ_MISC0__MT__GDDR4:
962 return AMDGPU_VRAM_TYPE_GDDR4;
963 case MC_SEQ_MISC0__MT__GDDR5:
964 return AMDGPU_VRAM_TYPE_GDDR5;
965 case MC_SEQ_MISC0__MT__HBM:
966 return AMDGPU_VRAM_TYPE_HBM;
967 case MC_SEQ_MISC0__MT__DDR3:
968 return AMDGPU_VRAM_TYPE_DDR3;
969 default:
970 return AMDGPU_VRAM_TYPE_UNKNOWN;
971 }
972}
973
5fc3aeeb 974static int gmc_v8_0_early_init(void *handle)
aaa36a97 975{
5fc3aeeb 976 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
977
aaa36a97
AD		 978 gmc_v8_0_set_gart_funcs(adev);
979 gmc_v8_0_set_irq_funcs(adev);
980
8fe73328
JZ		 981 adev->mc.shared_aperture_start = 0x2000000000000000ULL;
982 adev->mc.shared_aperture_end =
983 adev->mc.shared_aperture_start + (4ULL << 30) - 1;
984 adev->mc.private_aperture_start =
985 adev->mc.shared_aperture_end + 1;
986 adev->mc.private_aperture_end =
987 adev->mc.private_aperture_start + (4ULL << 30) - 1;
988
aaa36a97
AD		 989 return 0;
990}
991
140b519f
CK		 992static int gmc_v8_0_late_init(void *handle)
993{
994 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
995
afc45421
FC		 996 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
997 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
998 else
999 return 0;
140b519f
CK		 1000}
1001
b634de4f
AD		 1002#define mmMC_SEQ_MISC0_FIJI 0xA71
1003
5fc3aeeb 1004static int gmc_v8_0_sw_init(void *handle)
aaa36a97
AD		 1005{
1006 int r;
1007 int dma_bits;
5fc3aeeb 1008 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97 1009
d1518a1d
AD		 1010 if (adev->flags & AMD_IS_APU) {
1011 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
1012 } else {
b634de4f
AD		 1013 u32 tmp;
1014
1015 if (adev->asic_type == CHIP_FIJI)
1016 tmp = RREG32(mmMC_SEQ_MISC0_FIJI);
1017 else
1018 tmp = RREG32(mmMC_SEQ_MISC0);
d1518a1d
AD		 1019 tmp &= MC_SEQ_MISC0__MT__MASK;
1020 adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp);
1021 }
1022
d766e6a3 1023 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
aaa36a97
AD		 1024 if (r)
1025 return r;
1026
d766e6a3 1027 r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
aaa36a97
AD		 1028 if (r)
1029 return r;
1030
1031 /* Adjust VM size here.
1032 * Currently set to 4GB ((1 << 20) 4k pages).
1033 * Max GPUVM size for cayman and SI is 40 bits.
1034 */
bab4fee7 1035 amdgpu_vm_adjust_size(adev, 64);
36b32a68
ZJ		 1036 adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
1037
aaa36a97
AD		 1038 /* Set the internal MC address mask
1039 * This is the max address of the GPU's
1040 * internal address space.
1041 */
1042 adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
1043
916910ad
HR		 1044 adev->mc.stolen_size = 256 * 1024;
1045
aaa36a97
AD		 1046 /* set DMA mask + need_dma32 flags.
1047 * PCIE - can handle 40-bits.
1048 * IGP - can handle 40-bits
1049 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1050 */
1051 adev->need_dma32 = false;
1052 dma_bits = adev->need_dma32 ? 32 : 40;
1053 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1054 if (r) {
1055 adev->need_dma32 = true;
1056 dma_bits = 32;
7ca85295 1057 pr_warn("amdgpu: No suitable DMA available\n");
aaa36a97
AD		 1058 }
1059 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1060 if (r) {
1061 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
7ca85295 1062 pr_warn("amdgpu: No coherent DMA available\n");
aaa36a97
AD		 1063 }
1064
1065 r = gmc_v8_0_init_microcode(adev);
1066 if (r) {
1067 DRM_ERROR("Failed to load mc firmware!\n");
1068 return r;
1069 }
1070
1071 r = gmc_v8_0_mc_init(adev);
1072 if (r)
1073 return r;
1074
1075 /* Memory manager */
1076 r = amdgpu_bo_init(adev);
1077 if (r)
1078 return r;
1079
1080 r = gmc_v8_0_gart_init(adev);
1081 if (r)
1082 return r;
1083
05ec3eda
CK		 1084 /*
1085 * number of VMs
1086 * VMID 0 is reserved for System
1087 * amdgpu graphics/compute will use VMIDs 1-7
1088 * amdkfd will use VMIDs 8-15
1089 */
1090 adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1091 adev->vm_manager.num_level = 1;
1092 amdgpu_vm_manager_init(adev);
1093
1094 /* base offset of vram pages */
1095 if (adev->flags & AMD_IS_APU) {
1096 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
1097
1098 tmp <<= 22;
1099 adev->vm_manager.vram_base_offset = tmp;
1100 } else {
1101 adev->vm_manager.vram_base_offset = 0;
aaa36a97
AD		 1102 }
1103
05ec3eda 1104 return 0;
aaa36a97
AD		 1105}
1106
5fc3aeeb 1107static int gmc_v8_0_sw_fini(void *handle)
aaa36a97 1108{
5fc3aeeb 1109 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97 1110
05ec3eda 1111 amdgpu_vm_manager_fini(adev);
aaa36a97 1112 gmc_v8_0_gart_fini(adev);
418aa0c2 1113 amdgpu_gem_force_release(adev);
aaa36a97
AD		 1114 amdgpu_bo_fini(adev);
1115
1116 return 0;
1117}
1118
5fc3aeeb 1119static int gmc_v8_0_hw_init(void *handle)
aaa36a97
AD		 1120{
1121 int r;
5fc3aeeb 1122 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 1123
1124 gmc_v8_0_init_golden_registers(adev);
1125
1126 gmc_v8_0_mc_program(adev);
1127
8878d854 1128 if (adev->asic_type == CHIP_TONGA) {
0d52c6a1
RZ		 1129 r = gmc_v8_0_tonga_mc_load_microcode(adev);
1130 if (r) {
1131 DRM_ERROR("Failed to load MC firmware!\n");
1132 return r;
1133 }
1134 } else if (adev->asic_type == CHIP_POLARIS11 ||
1135 adev->asic_type == CHIP_POLARIS10 ||
1136 adev->asic_type == CHIP_POLARIS12) {
1137 r = gmc_v8_0_polaris_mc_load_microcode(adev);
aaa36a97
AD		 1138 if (r) {
1139 DRM_ERROR("Failed to load MC firmware!\n");
1140 return r;
1141 }
1142 }
1143
1144 r = gmc_v8_0_gart_enable(adev);
1145 if (r)
1146 return r;
1147
1148 return r;
1149}
1150
5fc3aeeb 1151static int gmc_v8_0_hw_fini(void *handle)
aaa36a97 1152{
5fc3aeeb 1153 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1154
140b519f 1155 amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
aaa36a97
AD		 1156 gmc_v8_0_gart_disable(adev);
1157
1158 return 0;
1159}
1160
5fc3aeeb 1161static int gmc_v8_0_suspend(void *handle)
aaa36a97 1162{
5fc3aeeb 1163 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97 1164
aaa36a97
AD		 1165 gmc_v8_0_hw_fini(adev);
1166
1167 return 0;
1168}
1169
5fc3aeeb 1170static int gmc_v8_0_resume(void *handle)
aaa36a97
AD		 1171{
1172 int r;
5fc3aeeb 1173 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 1174
1175 r = gmc_v8_0_hw_init(adev);
1176 if (r)
1177 return r;
1178
b3c85a0f 1179 amdgpu_vm_reset_all_ids(adev);
aaa36a97 1180
b3c85a0f 1181 return 0;
aaa36a97
AD		 1182}
1183
5fc3aeeb 1184static bool gmc_v8_0_is_idle(void *handle)
aaa36a97 1185{
5fc3aeeb 1186 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 1187 u32 tmp = RREG32(mmSRBM_STATUS);
1188
1189 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1190 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1191 return false;
1192
1193 return true;
1194}
1195
5fc3aeeb 1196static int gmc_v8_0_wait_for_idle(void *handle)
aaa36a97
AD		 1197{
1198 unsigned i;
1199 u32 tmp;
5fc3aeeb 1200 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 1201
1202 for (i = 0; i < adev->usec_timeout; i++) {
1203 /* read MC_STATUS */
1204 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1205 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1206 SRBM_STATUS__MCC_BUSY_MASK |
1207 SRBM_STATUS__MCD_BUSY_MASK |
1208 SRBM_STATUS__VMC_BUSY_MASK |
1209 SRBM_STATUS__VMC1_BUSY_MASK);
1210 if (!tmp)
1211 return 0;
1212 udelay(1);
1213 }
1214 return -ETIMEDOUT;
1215
1216}
1217
da146d3b 1218static bool gmc_v8_0_check_soft_reset(void *handle)
aaa36a97 1219{
aaa36a97 1220 u32 srbm_soft_reset = 0;
5fc3aeeb 1221 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 1222 u32 tmp = RREG32(mmSRBM_STATUS);
1223
1224 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1225 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1226 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1227
1228 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1229 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
2f7d10b3 1230 if (!(adev->flags & AMD_IS_APU))
aaa36a97
AD		 1231 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1232 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1233 }
aaa36a97 1234 if (srbm_soft_reset) {
50b0197a 1235 adev->mc.srbm_soft_reset = srbm_soft_reset;
da146d3b 1236 return true;
50b0197a 1237 } else {
50b0197a 1238 adev->mc.srbm_soft_reset = 0;
da146d3b 1239 return false;
50b0197a 1240 }
50b0197a 1241}
aaa36a97 1242
50b0197a
CZ		 1243static int gmc_v8_0_pre_soft_reset(void *handle)
1244{
1245 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1246
da146d3b 1247 if (!adev->mc.srbm_soft_reset)
50b0197a
CZ		 1248 return 0;
1249
e4f6b39e 1250 gmc_v8_0_mc_stop(adev);
50b0197a
CZ		 1251 if (gmc_v8_0_wait_for_idle(adev)) {
1252 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1253 }
1254
1255 return 0;
1256}
aaa36a97 1257
50b0197a
CZ		 1258static int gmc_v8_0_soft_reset(void *handle)
1259{
1260 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1261 u32 srbm_soft_reset;
1262
da146d3b 1263 if (!adev->mc.srbm_soft_reset)
50b0197a
CZ		 1264 return 0;
1265 srbm_soft_reset = adev->mc.srbm_soft_reset;
1266
1267 if (srbm_soft_reset) {
1268 u32 tmp;
aaa36a97
AD		 1269
1270 tmp = RREG32(mmSRBM_SOFT_RESET);
1271 tmp |= srbm_soft_reset;
1272 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1273 WREG32(mmSRBM_SOFT_RESET, tmp);
1274 tmp = RREG32(mmSRBM_SOFT_RESET);
1275
1276 udelay(50);
1277
1278 tmp &= ~srbm_soft_reset;
1279 WREG32(mmSRBM_SOFT_RESET, tmp);
1280 tmp = RREG32(mmSRBM_SOFT_RESET);
1281
1282 /* Wait a little for things to settle down */
1283 udelay(50);
aaa36a97
AD		 1284 }
1285
1286 return 0;
1287}
1288
50b0197a
CZ		 1289static int gmc_v8_0_post_soft_reset(void *handle)
1290{
1291 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1292
da146d3b 1293 if (!adev->mc.srbm_soft_reset)
50b0197a
CZ		 1294 return 0;
1295
e4f6b39e 1296 gmc_v8_0_mc_resume(adev);
50b0197a
CZ		 1297 return 0;
1298}
1299
aaa36a97
AD		 1300static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1301 struct amdgpu_irq_src *src,
1302 unsigned type,
1303 enum amdgpu_interrupt_state state)
1304{
1305 u32 tmp;
1306 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1307 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1308 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1309 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1310 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1311 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1312 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1313
1314 switch (state) {
1315 case AMDGPU_IRQ_STATE_DISABLE:
1316 /* system context */
1317 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1318 tmp &= ~bits;
1319 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1320 /* VMs */
1321 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1322 tmp &= ~bits;
1323 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1324 break;
1325 case AMDGPU_IRQ_STATE_ENABLE:
1326 /* system context */
1327 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1328 tmp |= bits;
1329 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1330 /* VMs */
1331 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1332 tmp |= bits;
1333 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1334 break;
1335 default:
1336 break;
1337 }
1338
1339 return 0;
1340}
1341
1342static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
1343 struct amdgpu_irq_src *source,
1344 struct amdgpu_iv_entry *entry)
1345{
1346 u32 addr, status, mc_client;
1347
edcafc02
PD		 1348 if (amdgpu_sriov_vf(adev)) {
1349 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
7ccf5aa8 1350 entry->src_id, entry->src_data[0]);
edcafc02
PD		 1351 dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n");
1352 return 0;
1353 }
1354
aaa36a97
AD		 1355 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1356 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1357 mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
ce0c6bcd
CK		 1358 /* reset addr and status */
1359 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1360
1361 if (!addr && !status)
1362 return 0;
1363
d9c13156
CK		 1364 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1365 gmc_v8_0_set_fault_enable_default(adev, false);
1366
01615881
EC		 1367 if (printk_ratelimit()) {
1368 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
7ccf5aa8 1369 entry->src_id, entry->src_data[0]);
01615881
EC		 1370 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1371 addr);
1372 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1373 status);
1374 gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
1375 }
aaa36a97
AD		 1376
1377 return 0;
1378}
1379
a0d69786 1380static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev,
3fde56b8 1381 bool enable)
a0d69786
EH		 1382{
1383 uint32_t data;
1384
3fde56b8 1385 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
a0d69786
EH		 1386 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1387 data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK;
1388 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1389
1390 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1391 data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK;
1392 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1393
1394 data = RREG32(mmMC_HUB_MISC_VM_CG);
1395 data |= MC_HUB_MISC_VM_CG__ENABLE_MASK;
1396 WREG32(mmMC_HUB_MISC_VM_CG, data);
1397
1398 data = RREG32(mmMC_XPB_CLK_GAT);
1399 data |= MC_XPB_CLK_GAT__ENABLE_MASK;
1400 WREG32(mmMC_XPB_CLK_GAT, data);
1401
1402 data = RREG32(mmATC_MISC_CG);
1403 data |= ATC_MISC_CG__ENABLE_MASK;
1404 WREG32(mmATC_MISC_CG, data);
1405
1406 data = RREG32(mmMC_CITF_MISC_WR_CG);
1407 data |= MC_CITF_MISC_WR_CG__ENABLE_MASK;
1408 WREG32(mmMC_CITF_MISC_WR_CG, data);
1409
1410 data = RREG32(mmMC_CITF_MISC_RD_CG);
1411 data |= MC_CITF_MISC_RD_CG__ENABLE_MASK;
1412 WREG32(mmMC_CITF_MISC_RD_CG, data);
1413
1414 data = RREG32(mmMC_CITF_MISC_VM_CG);
1415 data |= MC_CITF_MISC_VM_CG__ENABLE_MASK;
1416 WREG32(mmMC_CITF_MISC_VM_CG, data);
1417
1418 data = RREG32(mmVM_L2_CG);
1419 data |= VM_L2_CG__ENABLE_MASK;
1420 WREG32(mmVM_L2_CG, data);
1421 } else {
1422 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1423 data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK;
1424 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1425
1426 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1427 data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK;
1428 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1429
1430 data = RREG32(mmMC_HUB_MISC_VM_CG);
1431 data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK;
1432 WREG32(mmMC_HUB_MISC_VM_CG, data);
1433
1434 data = RREG32(mmMC_XPB_CLK_GAT);
1435 data &= ~MC_XPB_CLK_GAT__ENABLE_MASK;
1436 WREG32(mmMC_XPB_CLK_GAT, data);
1437
1438 data = RREG32(mmATC_MISC_CG);
1439 data &= ~ATC_MISC_CG__ENABLE_MASK;
1440 WREG32(mmATC_MISC_CG, data);
1441
1442 data = RREG32(mmMC_CITF_MISC_WR_CG);
1443 data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK;
1444 WREG32(mmMC_CITF_MISC_WR_CG, data);
1445
1446 data = RREG32(mmMC_CITF_MISC_RD_CG);
1447 data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK;
1448 WREG32(mmMC_CITF_MISC_RD_CG, data);
1449
1450 data = RREG32(mmMC_CITF_MISC_VM_CG);
1451 data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK;
1452 WREG32(mmMC_CITF_MISC_VM_CG, data);
1453
1454 data = RREG32(mmVM_L2_CG);
1455 data &= ~VM_L2_CG__ENABLE_MASK;
1456 WREG32(mmVM_L2_CG, data);
1457 }
1458}
1459
1460static void fiji_update_mc_light_sleep(struct amdgpu_device *adev,
3fde56b8 1461 bool enable)
a0d69786
EH		 1462{
1463 uint32_t data;
1464
3fde56b8 1465 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) {
a0d69786
EH		 1466 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1467 data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
1468 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1469
1470 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1471 data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
1472 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1473
1474 data = RREG32(mmMC_HUB_MISC_VM_CG);
1475 data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1476 WREG32(mmMC_HUB_MISC_VM_CG, data);
1477
1478 data = RREG32(mmMC_XPB_CLK_GAT);
1479 data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
1480 WREG32(mmMC_XPB_CLK_GAT, data);
1481
1482 data = RREG32(mmATC_MISC_CG);
1483 data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK;
1484 WREG32(mmATC_MISC_CG, data);
1485
1486 data = RREG32(mmMC_CITF_MISC_WR_CG);
1487 data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
1488 WREG32(mmMC_CITF_MISC_WR_CG, data);
1489
1490 data = RREG32(mmMC_CITF_MISC_RD_CG);
1491 data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
1492 WREG32(mmMC_CITF_MISC_RD_CG, data);
1493
1494 data = RREG32(mmMC_CITF_MISC_VM_CG);
1495 data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1496 WREG32(mmMC_CITF_MISC_VM_CG, data);
1497
1498 data = RREG32(mmVM_L2_CG);
1499 data |= VM_L2_CG__MEM_LS_ENABLE_MASK;
1500 WREG32(mmVM_L2_CG, data);
1501 } else {
1502 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1503 data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK;
1504 WREG32(mmMC_HUB_MISC_HUB_CG, data);
1505
1506 data = RREG32(mmMC_HUB_MISC_SIP_CG);
1507 data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK;
1508 WREG32(mmMC_HUB_MISC_SIP_CG, data);
1509
1510 data = RREG32(mmMC_HUB_MISC_VM_CG);
1511 data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1512 WREG32(mmMC_HUB_MISC_VM_CG, data);
1513
1514 data = RREG32(mmMC_XPB_CLK_GAT);
1515 data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK;
1516 WREG32(mmMC_XPB_CLK_GAT, data);
1517
1518 data = RREG32(mmATC_MISC_CG);
1519 data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK;
1520 WREG32(mmATC_MISC_CG, data);
1521
1522 data = RREG32(mmMC_CITF_MISC_WR_CG);
1523 data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK;
1524 WREG32(mmMC_CITF_MISC_WR_CG, data);
1525
1526 data = RREG32(mmMC_CITF_MISC_RD_CG);
1527 data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK;
1528 WREG32(mmMC_CITF_MISC_RD_CG, data);
1529
1530 data = RREG32(mmMC_CITF_MISC_VM_CG);
1531 data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK;
1532 WREG32(mmMC_CITF_MISC_VM_CG, data);
1533
1534 data = RREG32(mmVM_L2_CG);
1535 data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK;
1536 WREG32(mmVM_L2_CG, data);
1537 }
1538}
1539
5fc3aeeb 1540static int gmc_v8_0_set_clockgating_state(void *handle,
1541 enum amd_clockgating_state state)
aaa36a97 1542{
a0d69786
EH		 1543 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1544
ce137c04
ML		 1545 if (amdgpu_sriov_vf(adev))
1546 return 0;
1547
a0d69786
EH		 1548 switch (adev->asic_type) {
1549 case CHIP_FIJI:
1550 fiji_update_mc_medium_grain_clock_gating(adev,
7e913664 1551 state == AMD_CG_STATE_GATE);
a0d69786 1552 fiji_update_mc_light_sleep(adev,
7e913664 1553 state == AMD_CG_STATE_GATE);
a0d69786
EH		 1554 break;
1555 default:
1556 break;
1557 }
aaa36a97
AD		 1558 return 0;
1559}
1560
5fc3aeeb 1561static int gmc_v8_0_set_powergating_state(void *handle,
1562 enum amd_powergating_state state)
aaa36a97
AD		 1563{
1564 return 0;
1565}
1566
8bcab092
HR		 1567static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags)
1568{
1569 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1570 int data;
1571
ce137c04
ML		 1572 if (amdgpu_sriov_vf(adev))
1573 *flags = 0;
1574
8bcab092
HR		 1575 /* AMD_CG_SUPPORT_MC_MGCG */
1576 data = RREG32(mmMC_HUB_MISC_HUB_CG);
1577 if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK)
1578 *flags |= AMD_CG_SUPPORT_MC_MGCG;
1579
1580 /* AMD_CG_SUPPORT_MC_LS */
1581 if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK)
1582 *flags |= AMD_CG_SUPPORT_MC_LS;
1583}
1584
a1255107 1585static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
88a907d6 1586 .name = "gmc_v8_0",
aaa36a97 1587 .early_init = gmc_v8_0_early_init,
140b519f 1588 .late_init = gmc_v8_0_late_init,
aaa36a97
AD		 1589 .sw_init = gmc_v8_0_sw_init,
1590 .sw_fini = gmc_v8_0_sw_fini,
1591 .hw_init = gmc_v8_0_hw_init,
1592 .hw_fini = gmc_v8_0_hw_fini,
1593 .suspend = gmc_v8_0_suspend,
1594 .resume = gmc_v8_0_resume,
1595 .is_idle = gmc_v8_0_is_idle,
1596 .wait_for_idle = gmc_v8_0_wait_for_idle,
50b0197a
CZ		 1597 .check_soft_reset = gmc_v8_0_check_soft_reset,
1598 .pre_soft_reset = gmc_v8_0_pre_soft_reset,
aaa36a97 1599 .soft_reset = gmc_v8_0_soft_reset,
50b0197a 1600 .post_soft_reset = gmc_v8_0_post_soft_reset,
aaa36a97
AD		 1601 .set_clockgating_state = gmc_v8_0_set_clockgating_state,
1602 .set_powergating_state = gmc_v8_0_set_powergating_state,
8bcab092 1603 .get_clockgating_state = gmc_v8_0_get_clockgating_state,
aaa36a97
AD		 1604};
1605
1606static const struct amdgpu_gart_funcs gmc_v8_0_gart_funcs = {
1607 .flush_gpu_tlb = gmc_v8_0_gart_flush_gpu_tlb,
1608 .set_pte_pde = gmc_v8_0_gart_set_pte_pde,
603adfe8 1609 .set_prt = gmc_v8_0_set_prt,
b1166325
CK		 1610 .get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
1611 .get_vm_pde = gmc_v8_0_get_vm_pde
aaa36a97
AD		 1612};
1613
1614static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {
1615 .set = gmc_v8_0_vm_fault_interrupt_state,
1616 .process = gmc_v8_0_process_interrupt,
1617};
1618
1619static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev)
1620{
1621 if (adev->gart.gart_funcs == NULL)
1622 adev->gart.gart_funcs = &gmc_v8_0_gart_funcs;
1623}
1624
1625static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)
1626{
1627 adev->mc.vm_fault.num_types = 1;
1628 adev->mc.vm_fault.funcs = &gmc_v8_0_irq_funcs;
1629}
a1255107
AD		 1630
1631const struct amdgpu_ip_block_version gmc_v8_0_ip_block =
1632{
1633 .type = AMD_IP_BLOCK_TYPE_GMC,
1634 .major = 8,
1635 .minor = 0,
1636 .rev = 0,
1637 .funcs = &gmc_v8_0_ip_funcs,
1638};
1639
1640const struct amdgpu_ip_block_version gmc_v8_1_ip_block =
1641{
1642 .type = AMD_IP_BLOCK_TYPE_GMC,
1643 .major = 8,
1644 .minor = 1,
1645 .rev = 0,
1646 .funcs = &gmc_v8_0_ip_funcs,
1647};
1648
1649const struct amdgpu_ip_block_version gmc_v8_5_ip_block =
1650{
1651 .type = AMD_IP_BLOCK_TYPE_GMC,
1652 .major = 8,
1653 .minor = 5,
1654 .rev = 0,
1655 .funcs = &gmc_v8_0_ip_funcs,
1656};
Ubuntu Hirsute Linux kernel mirror