]>
Commit | Line | Data |
---|---|---|
aaa36a97 AD |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
47b757fb | 23 | |
aaa36a97 | 24 | #include <linux/firmware.h> |
47b757fb SR |
25 | #include <linux/module.h> |
26 | #include <linux/pci.h> | |
27 | ||
fd5fd480 | 28 | #include <drm/drm_cache.h> |
aaa36a97 AD |
29 | #include "amdgpu.h" |
30 | #include "gmc_v8_0.h" | |
31 | #include "amdgpu_ucode.h" | |
b97dfa27 | 32 | #include "amdgpu_amdkfd.h" |
2cddc50e | 33 | #include "amdgpu_gem.h" |
aaa36a97 AD |
34 | |
35 | #include "gmc/gmc_8_1_d.h" | |
36 | #include "gmc/gmc_8_1_sh_mask.h" | |
37 | ||
38 | #include "bif/bif_5_0_d.h" | |
39 | #include "bif/bif_5_0_sh_mask.h" | |
40 | ||
41 | #include "oss/oss_3_0_d.h" | |
42 | #include "oss/oss_3_0_sh_mask.h" | |
43 | ||
2e2bfd90 AD |
44 | #include "dce/dce_10_0_d.h" |
45 | #include "dce/dce_10_0_sh_mask.h" | |
46 | ||
aaa36a97 AD |
47 | #include "vid.h" |
48 | #include "vi.h" | |
49 | ||
1ce65f52 HW |
50 | #include "amdgpu_atombios.h" |
51 | ||
091aec0b AG |
52 | #include "ivsrcid/ivsrcid_vislands30.h" |
53 | ||
132f34e4 | 54 | static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev); |
aaa36a97 | 55 | static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev); |
34e3205e | 56 | static int gmc_v8_0_wait_for_idle(void *handle); |
aaa36a97 | 57 | |
c65444fe | 58 | MODULE_FIRMWARE("amdgpu/tonga_mc.bin"); |
2cc0c0b5 FC |
59 | MODULE_FIRMWARE("amdgpu/polaris11_mc.bin"); |
60 | MODULE_FIRMWARE("amdgpu/polaris10_mc.bin"); | |
c4642a47 | 61 | MODULE_FIRMWARE("amdgpu/polaris12_mc.bin"); |
a81a7c9c AD |
62 | MODULE_FIRMWARE("amdgpu/polaris11_k_mc.bin"); |
63 | MODULE_FIRMWARE("amdgpu/polaris10_k_mc.bin"); | |
d7fd6765 | 64 | MODULE_FIRMWARE("amdgpu/polaris12_k_mc.bin"); |
aaa36a97 AD |
65 | |
66 | static const u32 golden_settings_tonga_a11[] = | |
67 | { | |
68 | mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000, | |
69 | mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028, | |
70 | mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991, | |
71 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
72 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
73 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
74 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
75 | }; | |
76 | ||
77 | static const u32 tonga_mgcg_cgcg_init[] = | |
78 | { | |
79 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 | |
80 | }; | |
81 | ||
127a2628 DZ |
82 | static const u32 golden_settings_fiji_a10[] = |
83 | { | |
84 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
85 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
86 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
87 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
88 | }; | |
89 | ||
90 | static const u32 fiji_mgcg_cgcg_init[] = | |
91 | { | |
92 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 | |
93 | }; | |
94 | ||
2cc0c0b5 | 95 | static const u32 golden_settings_polaris11_a11[] = |
c9778572 FC |
96 | { |
97 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
98 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
99 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
100 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff | |
101 | }; | |
102 | ||
2cc0c0b5 | 103 | static const u32 golden_settings_polaris10_a11[] = |
c9778572 FC |
104 | { |
105 | mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000, | |
106 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
107 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
108 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, | |
109 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff | |
110 | }; | |
111 | ||
aaa36a97 AD |
112 | static const u32 cz_mgcg_cgcg_init[] = |
113 | { | |
114 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 | |
115 | }; | |
116 | ||
aade2f04 SL |
117 | static const u32 stoney_mgcg_cgcg_init[] = |
118 | { | |
0711257e | 119 | mmATC_MISC_CG, 0xffffffff, 0x000c0200, |
aade2f04 SL |
120 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 |
121 | }; | |
122 | ||
6d51c813 HR |
123 | static const u32 golden_settings_stoney_common[] = |
124 | { | |
125 | mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004, | |
126 | mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000 | |
127 | }; | |
aade2f04 | 128 | |
aaa36a97 AD |
129 | static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) |
130 | { | |
131 | switch (adev->asic_type) { | |
127a2628 | 132 | case CHIP_FIJI: |
9c3f2b54 AD |
133 | amdgpu_device_program_register_sequence(adev, |
134 | fiji_mgcg_cgcg_init, | |
135 | ARRAY_SIZE(fiji_mgcg_cgcg_init)); | |
136 | amdgpu_device_program_register_sequence(adev, | |
137 | golden_settings_fiji_a10, | |
138 | ARRAY_SIZE(golden_settings_fiji_a10)); | |
127a2628 | 139 | break; |
aaa36a97 | 140 | case CHIP_TONGA: |
9c3f2b54 AD |
141 | amdgpu_device_program_register_sequence(adev, |
142 | tonga_mgcg_cgcg_init, | |
143 | ARRAY_SIZE(tonga_mgcg_cgcg_init)); | |
144 | amdgpu_device_program_register_sequence(adev, | |
145 | golden_settings_tonga_a11, | |
146 | ARRAY_SIZE(golden_settings_tonga_a11)); | |
aaa36a97 | 147 | break; |
2cc0c0b5 | 148 | case CHIP_POLARIS11: |
c4642a47 | 149 | case CHIP_POLARIS12: |
13b75aac | 150 | case CHIP_VEGAM: |
9c3f2b54 AD |
151 | amdgpu_device_program_register_sequence(adev, |
152 | golden_settings_polaris11_a11, | |
153 | ARRAY_SIZE(golden_settings_polaris11_a11)); | |
c9778572 | 154 | break; |
2cc0c0b5 | 155 | case CHIP_POLARIS10: |
9c3f2b54 AD |
156 | amdgpu_device_program_register_sequence(adev, |
157 | golden_settings_polaris10_a11, | |
158 | ARRAY_SIZE(golden_settings_polaris10_a11)); | |
c9778572 | 159 | break; |
aaa36a97 | 160 | case CHIP_CARRIZO: |
9c3f2b54 AD |
161 | amdgpu_device_program_register_sequence(adev, |
162 | cz_mgcg_cgcg_init, | |
163 | ARRAY_SIZE(cz_mgcg_cgcg_init)); | |
aaa36a97 | 164 | break; |
aade2f04 | 165 | case CHIP_STONEY: |
9c3f2b54 AD |
166 | amdgpu_device_program_register_sequence(adev, |
167 | stoney_mgcg_cgcg_init, | |
168 | ARRAY_SIZE(stoney_mgcg_cgcg_init)); | |
169 | amdgpu_device_program_register_sequence(adev, | |
170 | golden_settings_stoney_common, | |
171 | ARRAY_SIZE(golden_settings_stoney_common)); | |
aade2f04 | 172 | break; |
aaa36a97 AD |
173 | default: |
174 | break; | |
175 | } | |
176 | } | |
177 | ||
e4f6b39e | 178 | static void gmc_v8_0_mc_stop(struct amdgpu_device *adev) |
aaa36a97 AD |
179 | { |
180 | u32 blackout; | |
181 | ||
34e3205e | 182 | gmc_v8_0_wait_for_idle(adev); |
aaa36a97 AD |
183 | |
184 | blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL); | |
185 | if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) { | |
186 | /* Block CPU access */ | |
187 | WREG32(mmBIF_FB_EN, 0); | |
188 | /* blackout the MC */ | |
189 | blackout = REG_SET_FIELD(blackout, | |
190 | MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1); | |
191 | WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout); | |
192 | } | |
193 | /* wait for the MC to settle */ | |
194 | udelay(100); | |
195 | } | |
196 | ||
e4f6b39e | 197 | static void gmc_v8_0_mc_resume(struct amdgpu_device *adev) |
aaa36a97 AD |
198 | { |
199 | u32 tmp; | |
200 | ||
201 | /* unblackout the MC */ | |
202 | tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL); | |
203 | tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0); | |
204 | WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp); | |
205 | /* allow CPU access */ | |
206 | tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1); | |
207 | tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1); | |
208 | WREG32(mmBIF_FB_EN, tmp); | |
aaa36a97 AD |
209 | } |
210 | ||
211 | /** | |
212 | * gmc_v8_0_init_microcode - load ucode images from disk | |
213 | * | |
214 | * @adev: amdgpu_device pointer | |
215 | * | |
216 | * Use the firmware interface to load the ucode images into | |
217 | * the driver (not loaded into hw). | |
218 | * Returns 0 on success, error on failure. | |
219 | */ | |
220 | static int gmc_v8_0_init_microcode(struct amdgpu_device *adev) | |
221 | { | |
222 | const char *chip_name; | |
223 | char fw_name[30]; | |
224 | int err; | |
225 | ||
226 | DRM_DEBUG("\n"); | |
227 | ||
228 | switch (adev->asic_type) { | |
aaa36a97 AD |
229 | case CHIP_TONGA: |
230 | chip_name = "tonga"; | |
231 | break; | |
2cc0c0b5 | 232 | case CHIP_POLARIS11: |
a81a7c9c AD |
233 | if (((adev->pdev->device == 0x67ef) && |
234 | ((adev->pdev->revision == 0xe0) || | |
235 | (adev->pdev->revision == 0xe5))) || | |
236 | ((adev->pdev->device == 0x67ff) && | |
237 | ((adev->pdev->revision == 0xcf) || | |
238 | (adev->pdev->revision == 0xef) || | |
239 | (adev->pdev->revision == 0xff)))) | |
240 | chip_name = "polaris11_k"; | |
241 | else if ((adev->pdev->device == 0x67ef) && | |
242 | (adev->pdev->revision == 0xe2)) | |
243 | chip_name = "polaris11_k"; | |
244 | else | |
245 | chip_name = "polaris11"; | |
c9778572 | 246 | break; |
2cc0c0b5 | 247 | case CHIP_POLARIS10: |
a81a7c9c AD |
248 | if ((adev->pdev->device == 0x67df) && |
249 | ((adev->pdev->revision == 0xe1) || | |
250 | (adev->pdev->revision == 0xf7))) | |
251 | chip_name = "polaris10_k"; | |
252 | else | |
253 | chip_name = "polaris10"; | |
c9778572 | 254 | break; |
c4642a47 | 255 | case CHIP_POLARIS12: |
d7fd6765 JZ |
256 | if (((adev->pdev->device == 0x6987) && |
257 | ((adev->pdev->revision == 0xc0) || | |
258 | (adev->pdev->revision == 0xc3))) || | |
259 | ((adev->pdev->device == 0x6981) && | |
260 | ((adev->pdev->revision == 0x00) || | |
261 | (adev->pdev->revision == 0x01) || | |
a81a7c9c | 262 | (adev->pdev->revision == 0x10)))) |
d7fd6765 | 263 | chip_name = "polaris12_k"; |
a81a7c9c AD |
264 | else |
265 | chip_name = "polaris12"; | |
c4642a47 | 266 | break; |
127a2628 | 267 | case CHIP_FIJI: |
aaa36a97 | 268 | case CHIP_CARRIZO: |
aade2f04 | 269 | case CHIP_STONEY: |
589ecd75 | 270 | case CHIP_VEGAM: |
aaa36a97 AD |
271 | return 0; |
272 | default: BUG(); | |
273 | } | |
274 | ||
c65444fe | 275 | snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name); |
770d13b1 | 276 | err = request_firmware(&adev->gmc.fw, fw_name, adev->dev); |
aaa36a97 AD |
277 | if (err) |
278 | goto out; | |
770d13b1 | 279 | err = amdgpu_ucode_validate(adev->gmc.fw); |
aaa36a97 AD |
280 | |
281 | out: | |
282 | if (err) { | |
7ca85295 | 283 | pr_err("mc: Failed to load firmware \"%s\"\n", fw_name); |
770d13b1 CK |
284 | release_firmware(adev->gmc.fw); |
285 | adev->gmc.fw = NULL; | |
aaa36a97 AD |
286 | } |
287 | return err; | |
288 | } | |
289 | ||
290 | /** | |
0d52c6a1 | 291 | * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw |
aaa36a97 AD |
292 | * |
293 | * @adev: amdgpu_device pointer | |
294 | * | |
5980bcaa | 295 | * Load the GDDR MC ucode into the hw (VI). |
aaa36a97 AD |
296 | * Returns 0 on success, error on failure. |
297 | */ | |
0d52c6a1 | 298 | static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev) |
aaa36a97 AD |
299 | { |
300 | const struct mc_firmware_header_v1_0 *hdr; | |
301 | const __le32 *fw_data = NULL; | |
302 | const __le32 *io_mc_regs = NULL; | |
887656f0 | 303 | u32 running; |
aaa36a97 AD |
304 | int i, ucode_size, regs_size; |
305 | ||
c12d2871 AD |
306 | /* Skip MC ucode loading on SR-IOV capable boards. |
307 | * vbios does this for us in asic_init in that case. | |
4e99a44e ML |
308 | * Skip MC ucode loading on VF, because hypervisor will do that |
309 | * for this adaptor. | |
c12d2871 | 310 | */ |
4e99a44e | 311 | if (amdgpu_sriov_bios(adev)) |
c12d2871 AD |
312 | return 0; |
313 | ||
770d13b1 | 314 | if (!adev->gmc.fw) |
0d52c6a1 RZ |
315 | return -EINVAL; |
316 | ||
770d13b1 | 317 | hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data; |
aaa36a97 AD |
318 | amdgpu_ucode_print_mc_hdr(&hdr->header); |
319 | ||
770d13b1 | 320 | adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version); |
aaa36a97 AD |
321 | regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); |
322 | io_mc_regs = (const __le32 *) | |
770d13b1 | 323 | (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); |
aaa36a97 AD |
324 | ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; |
325 | fw_data = (const __le32 *) | |
770d13b1 | 326 | (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); |
aaa36a97 AD |
327 | |
328 | running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN); | |
329 | ||
330 | if (running == 0) { | |
aaa36a97 AD |
331 | /* reset the engine and set to writable */ |
332 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); | |
333 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010); | |
334 | ||
335 | /* load mc io regs */ | |
336 | for (i = 0; i < regs_size; i++) { | |
337 | WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++)); | |
338 | WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++)); | |
339 | } | |
340 | /* load the MC ucode */ | |
341 | for (i = 0; i < ucode_size; i++) | |
342 | WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++)); | |
343 | ||
344 | /* put the engine back into the active state */ | |
345 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); | |
346 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004); | |
347 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001); | |
348 | ||
349 | /* wait for training to complete */ | |
350 | for (i = 0; i < adev->usec_timeout; i++) { | |
351 | if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL), | |
352 | MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0)) | |
353 | break; | |
354 | udelay(1); | |
355 | } | |
356 | for (i = 0; i < adev->usec_timeout; i++) { | |
357 | if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL), | |
358 | MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1)) | |
359 | break; | |
360 | udelay(1); | |
361 | } | |
aaa36a97 AD |
362 | } |
363 | ||
364 | return 0; | |
365 | } | |
366 | ||
0d52c6a1 RZ |
367 | static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev) |
368 | { | |
369 | const struct mc_firmware_header_v1_0 *hdr; | |
370 | const __le32 *fw_data = NULL; | |
371 | const __le32 *io_mc_regs = NULL; | |
b52b6738 | 372 | u32 data; |
0d52c6a1 RZ |
373 | int i, ucode_size, regs_size; |
374 | ||
375 | /* Skip MC ucode loading on SR-IOV capable boards. | |
376 | * vbios does this for us in asic_init in that case. | |
377 | * Skip MC ucode loading on VF, because hypervisor will do that | |
378 | * for this adaptor. | |
379 | */ | |
380 | if (amdgpu_sriov_bios(adev)) | |
0d52c6a1 RZ |
381 | return 0; |
382 | ||
770d13b1 | 383 | if (!adev->gmc.fw) |
0d52c6a1 RZ |
384 | return -EINVAL; |
385 | ||
770d13b1 | 386 | hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data; |
0d52c6a1 RZ |
387 | amdgpu_ucode_print_mc_hdr(&hdr->header); |
388 | ||
770d13b1 | 389 | adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version); |
0d52c6a1 RZ |
390 | regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); |
391 | io_mc_regs = (const __le32 *) | |
770d13b1 | 392 | (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); |
0d52c6a1 RZ |
393 | ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; |
394 | fw_data = (const __le32 *) | |
770d13b1 | 395 | (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); |
0d52c6a1 RZ |
396 | |
397 | data = RREG32(mmMC_SEQ_MISC0); | |
398 | data &= ~(0x40); | |
399 | WREG32(mmMC_SEQ_MISC0, data); | |
400 | ||
401 | /* load mc io regs */ | |
402 | for (i = 0; i < regs_size; i++) { | |
403 | WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++)); | |
404 | WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++)); | |
405 | } | |
406 | ||
407 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); | |
408 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010); | |
409 | ||
410 | /* load the MC ucode */ | |
411 | for (i = 0; i < ucode_size; i++) | |
412 | WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++)); | |
413 | ||
414 | /* put the engine back into the active state */ | |
415 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); | |
416 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004); | |
417 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001); | |
418 | ||
419 | /* wait for training to complete */ | |
420 | for (i = 0; i < adev->usec_timeout; i++) { | |
421 | data = RREG32(mmMC_SEQ_MISC0); | |
422 | if (data & 0x80) | |
423 | break; | |
424 | udelay(1); | |
425 | } | |
426 | ||
427 | return 0; | |
428 | } | |
429 | ||
aaa36a97 | 430 | static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev, |
770d13b1 | 431 | struct amdgpu_gmc *mc) |
aaa36a97 | 432 | { |
e72b9912 ED |
433 | u64 base = 0; |
434 | ||
435 | if (!amdgpu_sriov_vf(adev)) | |
436 | base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; | |
9081c4cf AD |
437 | base <<= 24; |
438 | ||
83afe835 | 439 | amdgpu_gmc_vram_location(adev, mc, base); |
961c75cf | 440 | amdgpu_gmc_gart_location(adev, mc); |
aaa36a97 AD |
441 | } |
442 | ||
443 | /** | |
444 | * gmc_v8_0_mc_program - program the GPU memory controller | |
445 | * | |
446 | * @adev: amdgpu_device pointer | |
447 | * | |
448 | * Set the location of vram, gart, and AGP in the GPU's | |
5980bcaa | 449 | * physical address space (VI). |
aaa36a97 AD |
450 | */ |
451 | static void gmc_v8_0_mc_program(struct amdgpu_device *adev) | |
452 | { | |
aaa36a97 AD |
453 | u32 tmp; |
454 | int i, j; | |
455 | ||
456 | /* Initialize HDP */ | |
457 | for (i = 0, j = 0; i < 32; i++, j += 0x6) { | |
458 | WREG32((0xb05 + j), 0x00000000); | |
459 | WREG32((0xb06 + j), 0x00000000); | |
460 | WREG32((0xb07 + j), 0x00000000); | |
461 | WREG32((0xb08 + j), 0x00000000); | |
462 | WREG32((0xb09 + j), 0x00000000); | |
463 | } | |
464 | WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0); | |
465 | ||
34e3205e | 466 | if (gmc_v8_0_wait_for_idle((void *)adev)) { |
aaa36a97 AD |
467 | dev_warn(adev->dev, "Wait for MC idle timedout !\n"); |
468 | } | |
2e2bfd90 AD |
469 | if (adev->mode_info.num_crtc) { |
470 | /* Lockout access through VGA aperture*/ | |
471 | tmp = RREG32(mmVGA_HDP_CONTROL); | |
472 | tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); | |
473 | WREG32(mmVGA_HDP_CONTROL, tmp); | |
474 | ||
475 | /* disable VGA render */ | |
476 | tmp = RREG32(mmVGA_RENDER_CONTROL); | |
477 | tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); | |
478 | WREG32(mmVGA_RENDER_CONTROL, tmp); | |
479 | } | |
aaa36a97 AD |
480 | /* Update configuration */ |
481 | WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, | |
770d13b1 | 482 | adev->gmc.vram_start >> 12); |
aaa36a97 | 483 | WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, |
770d13b1 | 484 | adev->gmc.vram_end >> 12); |
aaa36a97 AD |
485 | WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, |
486 | adev->vram_scratch.gpu_addr >> 12); | |
e72b9912 ED |
487 | |
488 | if (amdgpu_sriov_vf(adev)) { | |
770d13b1 CK |
489 | tmp = ((adev->gmc.vram_end >> 24) & 0xFFFF) << 16; |
490 | tmp |= ((adev->gmc.vram_start >> 24) & 0xFFFF); | |
e72b9912 ED |
491 | WREG32(mmMC_VM_FB_LOCATION, tmp); |
492 | /* XXX double check these! */ | |
770d13b1 | 493 | WREG32(mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); |
e72b9912 ED |
494 | WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); |
495 | WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF); | |
496 | } | |
497 | ||
aaa36a97 AD |
498 | WREG32(mmMC_VM_AGP_BASE, 0); |
499 | WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF); | |
500 | WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF); | |
34e3205e | 501 | if (gmc_v8_0_wait_for_idle((void *)adev)) { |
aaa36a97 AD |
502 | dev_warn(adev->dev, "Wait for MC idle timedout !\n"); |
503 | } | |
aaa36a97 AD |
504 | |
505 | WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); | |
506 | ||
507 | tmp = RREG32(mmHDP_MISC_CNTL); | |
13459bd0 | 508 | tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0); |
aaa36a97 AD |
509 | WREG32(mmHDP_MISC_CNTL, tmp); |
510 | ||
511 | tmp = RREG32(mmHDP_HOST_PATH_CNTL); | |
512 | WREG32(mmHDP_HOST_PATH_CNTL, tmp); | |
513 | } | |
514 | ||
515 | /** | |
516 | * gmc_v8_0_mc_init - initialize the memory controller driver params | |
517 | * | |
518 | * @adev: amdgpu_device pointer | |
519 | * | |
520 | * Look up the amount of vram, vram width, and decide how to place | |
5980bcaa | 521 | * vram and gart within the GPU's physical address space (VI). |
aaa36a97 AD |
522 | * Returns 0 for success. |
523 | */ | |
524 | static int gmc_v8_0_mc_init(struct amdgpu_device *adev) | |
525 | { | |
d6895ad3 CK |
526 | int r; |
527 | ||
770d13b1 CK |
528 | adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev); |
529 | if (!adev->gmc.vram_width) { | |
1ce65f52 HW |
530 | u32 tmp; |
531 | int chansize, numchan; | |
532 | ||
533 | /* Get VRAM informations */ | |
534 | tmp = RREG32(mmMC_ARB_RAMCFG); | |
535 | if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) { | |
536 | chansize = 64; | |
537 | } else { | |
538 | chansize = 32; | |
539 | } | |
540 | tmp = RREG32(mmMC_SHARED_CHMAP); | |
541 | switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) { | |
542 | case 0: | |
543 | default: | |
544 | numchan = 1; | |
545 | break; | |
546 | case 1: | |
547 | numchan = 2; | |
548 | break; | |
549 | case 2: | |
550 | numchan = 4; | |
551 | break; | |
552 | case 3: | |
553 | numchan = 8; | |
554 | break; | |
555 | case 4: | |
556 | numchan = 3; | |
557 | break; | |
558 | case 5: | |
559 | numchan = 6; | |
560 | break; | |
561 | case 6: | |
562 | numchan = 10; | |
563 | break; | |
564 | case 7: | |
565 | numchan = 12; | |
566 | break; | |
567 | case 8: | |
568 | numchan = 16; | |
569 | break; | |
570 | } | |
770d13b1 | 571 | adev->gmc.vram_width = numchan * chansize; |
aaa36a97 | 572 | } |
aaa36a97 | 573 | /* size in MB on si */ |
770d13b1 CK |
574 | adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; |
575 | adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL; | |
999446a7 | 576 | |
d6895ad3 CK |
577 | if (!(adev->flags & AMD_IS_APU)) { |
578 | r = amdgpu_device_resize_fb_bar(adev); | |
579 | if (r) | |
580 | return r; | |
581 | } | |
770d13b1 CK |
582 | adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); |
583 | adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); | |
d6895ad3 | 584 | |
999446a7 CK |
585 | #ifdef CONFIG_X86_64 |
586 | if (adev->flags & AMD_IS_APU) { | |
770d13b1 CK |
587 | adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; |
588 | adev->gmc.aper_size = adev->gmc.real_vram_size; | |
999446a7 CK |
589 | } |
590 | #endif | |
aaa36a97 | 591 | |
a1493cd5 | 592 | /* In case the PCI BAR is larger than the actual amount of vram */ |
770d13b1 CK |
593 | adev->gmc.visible_vram_size = adev->gmc.aper_size; |
594 | if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) | |
595 | adev->gmc.visible_vram_size = adev->gmc.real_vram_size; | |
a1493cd5 | 596 | |
c3db7b5a AD |
597 | /* set the gart size */ |
598 | if (amdgpu_gart_size == -1) { | |
599 | switch (adev->asic_type) { | |
c3db7b5a | 600 | case CHIP_POLARIS10: /* all engines support GPUVM */ |
f43c72ba | 601 | case CHIP_POLARIS11: /* all engines support GPUVM */ |
c3db7b5a | 602 | case CHIP_POLARIS12: /* all engines support GPUVM */ |
f43c72ba | 603 | case CHIP_VEGAM: /* all engines support GPUVM */ |
c3db7b5a | 604 | default: |
770d13b1 | 605 | adev->gmc.gart_size = 256ULL << 20; |
c3db7b5a AD |
606 | break; |
607 | case CHIP_TONGA: /* UVD, VCE do not support GPUVM */ | |
608 | case CHIP_FIJI: /* UVD, VCE do not support GPUVM */ | |
609 | case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */ | |
610 | case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */ | |
770d13b1 | 611 | adev->gmc.gart_size = 1024ULL << 20; |
c3db7b5a AD |
612 | break; |
613 | } | |
614 | } else { | |
770d13b1 | 615 | adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; |
c3db7b5a AD |
616 | } |
617 | ||
770d13b1 | 618 | gmc_v8_0_vram_gtt_location(adev, &adev->gmc); |
aaa36a97 AD |
619 | |
620 | return 0; | |
621 | } | |
622 | ||
623 | /* | |
624 | * GART | |
625 | * VMID 0 is the physical GPU addresses as used by the kernel. | |
626 | * VMIDs 1-15 are used for userspace clients and are handled | |
627 | * by the amdgpu vm/hsa code. | |
628 | */ | |
629 | ||
630 | /** | |
132f34e4 | 631 | * gmc_v8_0_flush_gpu_tlb - gart tlb flush callback |
aaa36a97 AD |
632 | * |
633 | * @adev: amdgpu_device pointer | |
634 | * @vmid: vm instance to flush | |
635 | * | |
5980bcaa | 636 | * Flush the TLB for the requested page table (VI). |
aaa36a97 | 637 | */ |
132f34e4 | 638 | static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev, |
2a79d868 | 639 | uint32_t vmid, uint32_t flush_type) |
aaa36a97 | 640 | { |
aaa36a97 AD |
641 | /* bits 0-15 are the VM contexts0-15 */ |
642 | WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); | |
643 | } | |
644 | ||
5518625d | 645 | static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, |
c633c00b | 646 | unsigned vmid, uint64_t pd_addr) |
5518625d CK |
647 | { |
648 | uint32_t reg; | |
649 | ||
650 | if (vmid < 8) | |
651 | reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid; | |
652 | else | |
653 | reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8; | |
654 | amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12); | |
655 | ||
656 | /* bits 0-15 are the VM contexts0-15 */ | |
657 | amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid); | |
658 | ||
659 | return pd_addr; | |
660 | } | |
661 | ||
c633c00b CK |
662 | static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, |
663 | unsigned pasid) | |
664 | { | |
665 | amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid); | |
666 | } | |
667 | ||
6490bd76 YZ |
668 | /* |
669 | * PTE format on VI: | |
670 | * 63:40 reserved | |
671 | * 39:12 4k physical page base address | |
672 | * 11:7 fragment | |
673 | * 6 write | |
674 | * 5 read | |
675 | * 4 exe | |
676 | * 3 reserved | |
677 | * 2 snooped | |
678 | * 1 system | |
679 | * 0 valid | |
aaa36a97 | 680 | * |
6490bd76 YZ |
681 | * PDE format on VI: |
682 | * 63:59 block fragment size | |
683 | * 58:40 reserved | |
684 | * 39:1 physical base address of PTE | |
685 | * bits 5:1 must be 0. | |
686 | * 0 valid | |
aaa36a97 | 687 | */ |
aaa36a97 | 688 | |
5463545b AX |
689 | static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev, |
690 | uint32_t flags) | |
691 | { | |
692 | uint64_t pte_flag = 0; | |
693 | ||
694 | if (flags & AMDGPU_VM_PAGE_EXECUTABLE) | |
695 | pte_flag |= AMDGPU_PTE_EXECUTABLE; | |
696 | if (flags & AMDGPU_VM_PAGE_READABLE) | |
697 | pte_flag |= AMDGPU_PTE_READABLE; | |
698 | if (flags & AMDGPU_VM_PAGE_WRITEABLE) | |
699 | pte_flag |= AMDGPU_PTE_WRITEABLE; | |
700 | if (flags & AMDGPU_VM_PAGE_PRT) | |
701 | pte_flag |= AMDGPU_PTE_PRT; | |
702 | ||
703 | return pte_flag; | |
704 | } | |
705 | ||
3de676d8 CK |
706 | static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level, |
707 | uint64_t *addr, uint64_t *flags) | |
b1166325 | 708 | { |
3de676d8 | 709 | BUG_ON(*addr & 0xFFFFFF0000000FFFULL); |
b1166325 CK |
710 | } |
711 | ||
d9c13156 CK |
712 | /** |
713 | * gmc_v8_0_set_fault_enable_default - update VM fault handling | |
714 | * | |
715 | * @adev: amdgpu_device pointer | |
716 | * @value: true redirects VM faults to the default page | |
717 | */ | |
718 | static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev, | |
719 | bool value) | |
720 | { | |
721 | u32 tmp; | |
722 | ||
723 | tmp = RREG32(mmVM_CONTEXT1_CNTL); | |
724 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
725 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
726 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
727 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
728 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
729 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
730 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
731 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
732 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
733 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
734 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
735 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
736 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
737 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
738 | WREG32(mmVM_CONTEXT1_CNTL, tmp); | |
739 | } | |
740 | ||
603adfe8 CK |
741 | /** |
742 | * gmc_v8_0_set_prt - set PRT VM fault | |
743 | * | |
744 | * @adev: amdgpu_device pointer | |
745 | * @enable: enable/disable VM fault handling for PRT | |
746 | */ | |
747 | static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable) | |
748 | { | |
749 | u32 tmp; | |
750 | ||
770d13b1 | 751 | if (enable && !adev->gmc.prt_warning) { |
603adfe8 | 752 | dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n"); |
770d13b1 | 753 | adev->gmc.prt_warning = true; |
603adfe8 CK |
754 | } |
755 | ||
756 | tmp = RREG32(mmVM_PRT_CNTL); | |
757 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
758 | CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable); | |
759 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
760 | CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable); | |
761 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
762 | TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable); | |
763 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
764 | TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable); | |
765 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
766 | L2_CACHE_STORE_INVALID_ENTRIES, enable); | |
767 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
768 | L1_TLB_STORE_INVALID_ENTRIES, enable); | |
769 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, | |
770 | MASK_PDE0_FAULT, enable); | |
771 | WREG32(mmVM_PRT_CNTL, tmp); | |
772 | ||
773 | if (enable) { | |
774 | uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT; | |
a3e9a15a CK |
775 | uint32_t high = adev->vm_manager.max_pfn - |
776 | (AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT); | |
603adfe8 CK |
777 | |
778 | WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low); | |
779 | WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low); | |
780 | WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low); | |
781 | WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low); | |
782 | WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high); | |
783 | WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high); | |
784 | WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high); | |
785 | WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high); | |
786 | } else { | |
787 | WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff); | |
788 | WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff); | |
789 | WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff); | |
790 | WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff); | |
791 | WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0); | |
792 | WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0); | |
793 | WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0); | |
794 | WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0); | |
795 | } | |
796 | } | |
797 | ||
aaa36a97 AD |
798 | /** |
799 | * gmc_v8_0_gart_enable - gart enable | |
800 | * | |
801 | * @adev: amdgpu_device pointer | |
802 | * | |
803 | * This sets up the TLBs, programs the page tables for VMID0, | |
804 | * sets up the hw for VMIDs 1-15 which are allocated on | |
805 | * demand, and sets up the global locations for the LDS, GDS, | |
5980bcaa | 806 | * and GPUVM for FSA64 clients (VI). |
aaa36a97 AD |
807 | * Returns 0 for success, errors for failure. |
808 | */ | |
809 | static int gmc_v8_0_gart_enable(struct amdgpu_device *adev) | |
810 | { | |
bdb1922a | 811 | uint64_t table_addr; |
ce1b1b66 | 812 | int r, i; |
e618d306 | 813 | u32 tmp, field; |
aaa36a97 | 814 | |
1123b989 | 815 | if (adev->gart.bo == NULL) { |
aaa36a97 AD |
816 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); |
817 | return -EINVAL; | |
818 | } | |
ce1b1b66 ML |
819 | r = amdgpu_gart_table_vram_pin(adev); |
820 | if (r) | |
821 | return r; | |
bdb1922a MD |
822 | |
823 | table_addr = amdgpu_bo_gpu_offset(adev->gart.bo); | |
824 | ||
aaa36a97 AD |
825 | /* Setup TLB control */ |
826 | tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL); | |
827 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); | |
828 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1); | |
829 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); | |
830 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1); | |
831 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); | |
832 | WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp); | |
833 | /* Setup L2 cache */ | |
834 | tmp = RREG32(mmVM_L2_CNTL); | |
835 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); | |
836 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1); | |
837 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1); | |
838 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1); | |
839 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7); | |
840 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); | |
a80b3047 | 841 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); |
aaa36a97 AD |
842 | WREG32(mmVM_L2_CNTL, tmp); |
843 | tmp = RREG32(mmVM_L2_CNTL2); | |
844 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); | |
845 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); | |
846 | WREG32(mmVM_L2_CNTL2, tmp); | |
e618d306 RH |
847 | |
848 | field = adev->vm_manager.fragment_size; | |
aaa36a97 AD |
849 | tmp = RREG32(mmVM_L2_CNTL3); |
850 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1); | |
e618d306 RH |
851 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field); |
852 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field); | |
aaa36a97 AD |
853 | WREG32(mmVM_L2_CNTL3, tmp); |
854 | /* XXX: set to enable PTE/PDE in system memory */ | |
855 | tmp = RREG32(mmVM_L2_CNTL4); | |
856 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0); | |
857 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0); | |
858 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0); | |
859 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0); | |
860 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0); | |
861 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0); | |
862 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0); | |
863 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0); | |
864 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0); | |
865 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0); | |
866 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0); | |
867 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0); | |
868 | WREG32(mmVM_L2_CNTL4, tmp); | |
869 | /* setup context0 */ | |
770d13b1 CK |
870 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12); |
871 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12); | |
4e830fb1 | 872 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12); |
aaa36a97 | 873 | WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, |
92e71b06 | 874 | (u32)(adev->dummy_page_addr >> 12)); |
aaa36a97 AD |
875 | WREG32(mmVM_CONTEXT0_CNTL2, 0); |
876 | tmp = RREG32(mmVM_CONTEXT0_CNTL); | |
877 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); | |
878 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); | |
879 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); | |
880 | WREG32(mmVM_CONTEXT0_CNTL, tmp); | |
881 | ||
882 | WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0); | |
883 | WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0); | |
884 | WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0); | |
885 | ||
886 | /* empty context1-15 */ | |
887 | /* FIXME start with 4G, once using 2 level pt switch to full | |
888 | * vm size space | |
889 | */ | |
890 | /* set vm size, must be a multiple of 4 */ | |
891 | WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); | |
25a595e4 | 892 | WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1); |
aaa36a97 AD |
893 | for (i = 1; i < 16; i++) { |
894 | if (i < 8) | |
895 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i, | |
4e830fb1 | 896 | table_addr >> 12); |
aaa36a97 AD |
897 | else |
898 | WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8, | |
4e830fb1 | 899 | table_addr >> 12); |
aaa36a97 AD |
900 | } |
901 | ||
902 | /* enable context1-15 */ | |
903 | WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, | |
92e71b06 | 904 | (u32)(adev->dummy_page_addr >> 12)); |
aaa36a97 AD |
905 | WREG32(mmVM_CONTEXT1_CNTL2, 4); |
906 | tmp = RREG32(mmVM_CONTEXT1_CNTL); | |
907 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); | |
908 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1); | |
aaa36a97 | 909 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 | 910 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 | 911 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 | 912 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 | 913 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 | 914 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
aaa36a97 AD |
915 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
916 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, | |
36b32a68 | 917 | adev->vm_manager.block_size - 9); |
aaa36a97 | 918 | WREG32(mmVM_CONTEXT1_CNTL, tmp); |
d9c13156 CK |
919 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) |
920 | gmc_v8_0_set_fault_enable_default(adev, false); | |
921 | else | |
922 | gmc_v8_0_set_fault_enable_default(adev, true); | |
aaa36a97 | 923 | |
2a79d868 | 924 | gmc_v8_0_flush_gpu_tlb(adev, 0, 0); |
aaa36a97 | 925 | DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", |
770d13b1 | 926 | (unsigned)(adev->gmc.gart_size >> 20), |
4e830fb1 | 927 | (unsigned long long)table_addr); |
aaa36a97 AD |
928 | adev->gart.ready = true; |
929 | return 0; | |
930 | } | |
931 | ||
932 | static int gmc_v8_0_gart_init(struct amdgpu_device *adev) | |
933 | { | |
934 | int r; | |
935 | ||
1123b989 | 936 | if (adev->gart.bo) { |
aaa36a97 AD |
937 | WARN(1, "R600 PCIE GART already initialized\n"); |
938 | return 0; | |
939 | } | |
940 | /* Initialize common gart structure */ | |
941 | r = amdgpu_gart_init(adev); | |
942 | if (r) | |
943 | return r; | |
944 | adev->gart.table_size = adev->gart.num_gpu_pages * 8; | |
4b98e0c4 | 945 | adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE; |
aaa36a97 AD |
946 | return amdgpu_gart_table_vram_alloc(adev); |
947 | } | |
948 | ||
949 | /** | |
950 | * gmc_v8_0_gart_disable - gart disable | |
951 | * | |
952 | * @adev: amdgpu_device pointer | |
953 | * | |
5980bcaa | 954 | * This disables all VM page table (VI). |
aaa36a97 AD |
955 | */ |
956 | static void gmc_v8_0_gart_disable(struct amdgpu_device *adev) | |
957 | { | |
958 | u32 tmp; | |
959 | ||
960 | /* Disable all tables */ | |
961 | WREG32(mmVM_CONTEXT0_CNTL, 0); | |
962 | WREG32(mmVM_CONTEXT1_CNTL, 0); | |
963 | /* Setup TLB control */ | |
964 | tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL); | |
965 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); | |
966 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0); | |
967 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0); | |
968 | WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp); | |
969 | /* Setup L2 cache */ | |
970 | tmp = RREG32(mmVM_L2_CNTL); | |
971 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0); | |
972 | WREG32(mmVM_L2_CNTL, tmp); | |
973 | WREG32(mmVM_L2_CNTL2, 0); | |
ce1b1b66 | 974 | amdgpu_gart_table_vram_unpin(adev); |
aaa36a97 AD |
975 | } |
976 | ||
aaa36a97 AD |
977 | /** |
978 | * gmc_v8_0_vm_decode_fault - print human readable fault info | |
979 | * | |
980 | * @adev: amdgpu_device pointer | |
981 | * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value | |
982 | * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value | |
983 | * | |
5980bcaa | 984 | * Print human readable fault information (VI). |
aaa36a97 | 985 | */ |
904a3374 CK |
986 | static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev, u32 status, |
987 | u32 addr, u32 mc_client, unsigned pasid) | |
aaa36a97 | 988 | { |
aaa36a97 AD |
989 | u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); |
990 | u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, | |
991 | PROTECTIONS); | |
992 | char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, | |
993 | (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; | |
904a3374 | 994 | u32 mc_id; |
aaa36a97 AD |
995 | |
996 | mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, | |
997 | MEMORY_CLIENT_ID); | |
998 | ||
904a3374 CK |
999 | dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n", |
1000 | protections, vmid, pasid, addr, | |
aaa36a97 AD |
1001 | REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1002 | MEMORY_CLIENT_RW) ? | |
1003 | "write" : "read", block, mc_client, mc_id); | |
1004 | } | |
1005 | ||
81c59f54 KW |
1006 | static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type) |
1007 | { | |
1008 | switch (mc_seq_vram_type) { | |
1009 | case MC_SEQ_MISC0__MT__GDDR1: | |
1010 | return AMDGPU_VRAM_TYPE_GDDR1; | |
1011 | case MC_SEQ_MISC0__MT__DDR2: | |
1012 | return AMDGPU_VRAM_TYPE_DDR2; | |
1013 | case MC_SEQ_MISC0__MT__GDDR3: | |
1014 | return AMDGPU_VRAM_TYPE_GDDR3; | |
1015 | case MC_SEQ_MISC0__MT__GDDR4: | |
1016 | return AMDGPU_VRAM_TYPE_GDDR4; | |
1017 | case MC_SEQ_MISC0__MT__GDDR5: | |
1018 | return AMDGPU_VRAM_TYPE_GDDR5; | |
1019 | case MC_SEQ_MISC0__MT__HBM: | |
1020 | return AMDGPU_VRAM_TYPE_HBM; | |
1021 | case MC_SEQ_MISC0__MT__DDR3: | |
1022 | return AMDGPU_VRAM_TYPE_DDR3; | |
1023 | default: | |
1024 | return AMDGPU_VRAM_TYPE_UNKNOWN; | |
1025 | } | |
1026 | } | |
1027 | ||
5fc3aeeb | 1028 | static int gmc_v8_0_early_init(void *handle) |
aaa36a97 | 1029 | { |
5fc3aeeb | 1030 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1031 | ||
132f34e4 | 1032 | gmc_v8_0_set_gmc_funcs(adev); |
aaa36a97 AD |
1033 | gmc_v8_0_set_irq_funcs(adev); |
1034 | ||
770d13b1 CK |
1035 | adev->gmc.shared_aperture_start = 0x2000000000000000ULL; |
1036 | adev->gmc.shared_aperture_end = | |
1037 | adev->gmc.shared_aperture_start + (4ULL << 30) - 1; | |
1038 | adev->gmc.private_aperture_start = | |
1039 | adev->gmc.shared_aperture_end + 1; | |
1040 | adev->gmc.private_aperture_end = | |
1041 | adev->gmc.private_aperture_start + (4ULL << 30) - 1; | |
8fe73328 | 1042 | |
aaa36a97 AD |
1043 | return 0; |
1044 | } | |
1045 | ||
140b519f CK |
1046 | static int gmc_v8_0_late_init(void *handle) |
1047 | { | |
1048 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1049 | ||
6f752ec2 AG |
1050 | amdgpu_bo_late_init(adev); |
1051 | ||
afc45421 | 1052 | if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS) |
770d13b1 | 1053 | return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); |
afc45421 FC |
1054 | else |
1055 | return 0; | |
140b519f CK |
1056 | } |
1057 | ||
ebdef28e AD |
1058 | static unsigned gmc_v8_0_get_vbios_fb_size(struct amdgpu_device *adev) |
1059 | { | |
1060 | u32 d1vga_control = RREG32(mmD1VGA_CONTROL); | |
1061 | unsigned size; | |
1062 | ||
1063 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { | |
1064 | size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ | |
1065 | } else { | |
1066 | u32 viewport = RREG32(mmVIEWPORT_SIZE); | |
1067 | size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * | |
1068 | REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * | |
1069 | 4); | |
1070 | } | |
1071 | /* return 0 if the pre-OS buffer uses up most of vram */ | |
1072 | if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) | |
1073 | return 0; | |
1074 | return size; | |
1075 | } | |
1076 | ||
b634de4f AD |
1077 | #define mmMC_SEQ_MISC0_FIJI 0xA71 |
1078 | ||
5fc3aeeb | 1079 | static int gmc_v8_0_sw_init(void *handle) |
aaa36a97 AD |
1080 | { |
1081 | int r; | |
5fc3aeeb | 1082 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 | 1083 | |
d1518a1d | 1084 | if (adev->flags & AMD_IS_APU) { |
770d13b1 | 1085 | adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; |
d1518a1d | 1086 | } else { |
b634de4f AD |
1087 | u32 tmp; |
1088 | ||
f43c72ba LL |
1089 | if ((adev->asic_type == CHIP_FIJI) || |
1090 | (adev->asic_type == CHIP_VEGAM)) | |
b634de4f AD |
1091 | tmp = RREG32(mmMC_SEQ_MISC0_FIJI); |
1092 | else | |
1093 | tmp = RREG32(mmMC_SEQ_MISC0); | |
d1518a1d | 1094 | tmp &= MC_SEQ_MISC0__MT__MASK; |
770d13b1 | 1095 | adev->gmc.vram_type = gmc_v8_0_convert_vram_type(tmp); |
d1518a1d AD |
1096 | } |
1097 | ||
1ffdeca6 | 1098 | r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault); |
aaa36a97 AD |
1099 | if (r) |
1100 | return r; | |
1101 | ||
1ffdeca6 | 1102 | r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault); |
aaa36a97 AD |
1103 | if (r) |
1104 | return r; | |
1105 | ||
1106 | /* Adjust VM size here. | |
1107 | * Currently set to 4GB ((1 << 20) 4k pages). | |
1108 | * Max GPUVM size for cayman and SI is 40 bits. | |
1109 | */ | |
f3368128 | 1110 | amdgpu_vm_adjust_size(adev, 64, 9, 1, 40); |
36b32a68 | 1111 | |
aaa36a97 AD |
1112 | /* Set the internal MC address mask |
1113 | * This is the max address of the GPU's | |
1114 | * internal address space. | |
1115 | */ | |
770d13b1 | 1116 | adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ |
aaa36a97 | 1117 | |
244511f3 | 1118 | r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40)); |
aaa36a97 | 1119 | if (r) { |
7ca85295 | 1120 | pr_warn("amdgpu: No suitable DMA available\n"); |
244511f3 | 1121 | return r; |
aaa36a97 | 1122 | } |
244511f3 | 1123 | adev->need_swiotlb = drm_need_swiotlb(40); |
aaa36a97 AD |
1124 | |
1125 | r = gmc_v8_0_init_microcode(adev); | |
1126 | if (r) { | |
1127 | DRM_ERROR("Failed to load mc firmware!\n"); | |
1128 | return r; | |
1129 | } | |
1130 | ||
1131 | r = gmc_v8_0_mc_init(adev); | |
1132 | if (r) | |
1133 | return r; | |
1134 | ||
ebdef28e AD |
1135 | adev->gmc.stolen_size = gmc_v8_0_get_vbios_fb_size(adev); |
1136 | ||
aaa36a97 AD |
1137 | /* Memory manager */ |
1138 | r = amdgpu_bo_init(adev); | |
1139 | if (r) | |
1140 | return r; | |
1141 | ||
1142 | r = gmc_v8_0_gart_init(adev); | |
1143 | if (r) | |
1144 | return r; | |
1145 | ||
05ec3eda CK |
1146 | /* |
1147 | * number of VMs | |
1148 | * VMID 0 is reserved for System | |
1149 | * amdgpu graphics/compute will use VMIDs 1-7 | |
1150 | * amdkfd will use VMIDs 8-15 | |
1151 | */ | |
1152 | adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS; | |
05ec3eda CK |
1153 | amdgpu_vm_manager_init(adev); |
1154 | ||
1155 | /* base offset of vram pages */ | |
1156 | if (adev->flags & AMD_IS_APU) { | |
1157 | u64 tmp = RREG32(mmMC_VM_FB_OFFSET); | |
1158 | ||
1159 | tmp <<= 22; | |
1160 | adev->vm_manager.vram_base_offset = tmp; | |
1161 | } else { | |
1162 | adev->vm_manager.vram_base_offset = 0; | |
aaa36a97 AD |
1163 | } |
1164 | ||
b97dfa27 | 1165 | adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info), |
1166 | GFP_KERNEL); | |
1167 | if (!adev->gmc.vm_fault_info) | |
1168 | return -ENOMEM; | |
1169 | atomic_set(&adev->gmc.vm_fault_info_updated, 0); | |
1170 | ||
05ec3eda | 1171 | return 0; |
aaa36a97 AD |
1172 | } |
1173 | ||
5fc3aeeb | 1174 | static int gmc_v8_0_sw_fini(void *handle) |
aaa36a97 | 1175 | { |
5fc3aeeb | 1176 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 | 1177 | |
f59548c8 | 1178 | amdgpu_gem_force_release(adev); |
05ec3eda | 1179 | amdgpu_vm_manager_fini(adev); |
b97dfa27 | 1180 | kfree(adev->gmc.vm_fault_info); |
a3d9103e | 1181 | amdgpu_gart_table_vram_free(adev); |
aaa36a97 | 1182 | amdgpu_bo_fini(adev); |
a3d9103e | 1183 | amdgpu_gart_fini(adev); |
770d13b1 CK |
1184 | release_firmware(adev->gmc.fw); |
1185 | adev->gmc.fw = NULL; | |
aaa36a97 AD |
1186 | |
1187 | return 0; | |
1188 | } | |
1189 | ||
5fc3aeeb | 1190 | static int gmc_v8_0_hw_init(void *handle) |
aaa36a97 AD |
1191 | { |
1192 | int r; | |
5fc3aeeb | 1193 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
1194 | |
1195 | gmc_v8_0_init_golden_registers(adev); | |
1196 | ||
1197 | gmc_v8_0_mc_program(adev); | |
1198 | ||
8878d854 | 1199 | if (adev->asic_type == CHIP_TONGA) { |
0d52c6a1 RZ |
1200 | r = gmc_v8_0_tonga_mc_load_microcode(adev); |
1201 | if (r) { | |
1202 | DRM_ERROR("Failed to load MC firmware!\n"); | |
1203 | return r; | |
1204 | } | |
1205 | } else if (adev->asic_type == CHIP_POLARIS11 || | |
1206 | adev->asic_type == CHIP_POLARIS10 || | |
1207 | adev->asic_type == CHIP_POLARIS12) { | |
1208 | r = gmc_v8_0_polaris_mc_load_microcode(adev); | |
aaa36a97 AD |
1209 | if (r) { |
1210 | DRM_ERROR("Failed to load MC firmware!\n"); | |
1211 | return r; | |
1212 | } | |
1213 | } | |
1214 | ||
1215 | r = gmc_v8_0_gart_enable(adev); | |
1216 | if (r) | |
1217 | return r; | |
1218 | ||
1219 | return r; | |
1220 | } | |
1221 | ||
5fc3aeeb | 1222 | static int gmc_v8_0_hw_fini(void *handle) |
aaa36a97 | 1223 | { |
5fc3aeeb | 1224 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1225 | ||
770d13b1 | 1226 | amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); |
aaa36a97 AD |
1227 | gmc_v8_0_gart_disable(adev); |
1228 | ||
1229 | return 0; | |
1230 | } | |
1231 | ||
5fc3aeeb | 1232 | static int gmc_v8_0_suspend(void *handle) |
aaa36a97 | 1233 | { |
5fc3aeeb | 1234 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 | 1235 | |
aaa36a97 AD |
1236 | gmc_v8_0_hw_fini(adev); |
1237 | ||
1238 | return 0; | |
1239 | } | |
1240 | ||
5fc3aeeb | 1241 | static int gmc_v8_0_resume(void *handle) |
aaa36a97 AD |
1242 | { |
1243 | int r; | |
5fc3aeeb | 1244 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
1245 | |
1246 | r = gmc_v8_0_hw_init(adev); | |
1247 | if (r) | |
1248 | return r; | |
1249 | ||
620f774f | 1250 | amdgpu_vmid_reset_all(adev); |
aaa36a97 | 1251 | |
b3c85a0f | 1252 | return 0; |
aaa36a97 AD |
1253 | } |
1254 | ||
5fc3aeeb | 1255 | static bool gmc_v8_0_is_idle(void *handle) |
aaa36a97 | 1256 | { |
5fc3aeeb | 1257 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
1258 | u32 tmp = RREG32(mmSRBM_STATUS); |
1259 | ||
1260 | if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | | |
1261 | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK)) | |
1262 | return false; | |
1263 | ||
1264 | return true; | |
1265 | } | |
1266 | ||
5fc3aeeb | 1267 | static int gmc_v8_0_wait_for_idle(void *handle) |
aaa36a97 AD |
1268 | { |
1269 | unsigned i; | |
1270 | u32 tmp; | |
5fc3aeeb | 1271 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
1272 | |
1273 | for (i = 0; i < adev->usec_timeout; i++) { | |
1274 | /* read MC_STATUS */ | |
1275 | tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK | | |
1276 | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | | |
1277 | SRBM_STATUS__MCC_BUSY_MASK | | |
1278 | SRBM_STATUS__MCD_BUSY_MASK | | |
1279 | SRBM_STATUS__VMC_BUSY_MASK | | |
1280 | SRBM_STATUS__VMC1_BUSY_MASK); | |
1281 | if (!tmp) | |
1282 | return 0; | |
1283 | udelay(1); | |
1284 | } | |
1285 | return -ETIMEDOUT; | |
1286 | ||
1287 | } | |
1288 | ||
da146d3b | 1289 | static bool gmc_v8_0_check_soft_reset(void *handle) |
aaa36a97 | 1290 | { |
aaa36a97 | 1291 | u32 srbm_soft_reset = 0; |
5fc3aeeb | 1292 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
aaa36a97 AD |
1293 | u32 tmp = RREG32(mmSRBM_STATUS); |
1294 | ||
1295 | if (tmp & SRBM_STATUS__VMC_BUSY_MASK) | |
1296 | srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
1297 | SRBM_SOFT_RESET, SOFT_RESET_VMC, 1); | |
1298 | ||
1299 | if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | | |
1300 | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) { | |
2f7d10b3 | 1301 | if (!(adev->flags & AMD_IS_APU)) |
aaa36a97 AD |
1302 | srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, |
1303 | SRBM_SOFT_RESET, SOFT_RESET_MC, 1); | |
1304 | } | |
aaa36a97 | 1305 | if (srbm_soft_reset) { |
770d13b1 | 1306 | adev->gmc.srbm_soft_reset = srbm_soft_reset; |
da146d3b | 1307 | return true; |
50b0197a | 1308 | } else { |
770d13b1 | 1309 | adev->gmc.srbm_soft_reset = 0; |
da146d3b | 1310 | return false; |
50b0197a | 1311 | } |
50b0197a | 1312 | } |
aaa36a97 | 1313 | |
50b0197a CZ |
1314 | static int gmc_v8_0_pre_soft_reset(void *handle) |
1315 | { | |
1316 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1317 | ||
770d13b1 | 1318 | if (!adev->gmc.srbm_soft_reset) |
50b0197a CZ |
1319 | return 0; |
1320 | ||
e4f6b39e | 1321 | gmc_v8_0_mc_stop(adev); |
50b0197a CZ |
1322 | if (gmc_v8_0_wait_for_idle(adev)) { |
1323 | dev_warn(adev->dev, "Wait for GMC idle timed out !\n"); | |
1324 | } | |
1325 | ||
1326 | return 0; | |
1327 | } | |
aaa36a97 | 1328 | |
50b0197a CZ |
1329 | static int gmc_v8_0_soft_reset(void *handle) |
1330 | { | |
1331 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1332 | u32 srbm_soft_reset; | |
1333 | ||
770d13b1 | 1334 | if (!adev->gmc.srbm_soft_reset) |
50b0197a | 1335 | return 0; |
770d13b1 | 1336 | srbm_soft_reset = adev->gmc.srbm_soft_reset; |
50b0197a CZ |
1337 | |
1338 | if (srbm_soft_reset) { | |
1339 | u32 tmp; | |
aaa36a97 AD |
1340 | |
1341 | tmp = RREG32(mmSRBM_SOFT_RESET); | |
1342 | tmp |= srbm_soft_reset; | |
1343 | dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
1344 | WREG32(mmSRBM_SOFT_RESET, tmp); | |
1345 | tmp = RREG32(mmSRBM_SOFT_RESET); | |
1346 | ||
1347 | udelay(50); | |
1348 | ||
1349 | tmp &= ~srbm_soft_reset; | |
1350 | WREG32(mmSRBM_SOFT_RESET, tmp); | |
1351 | tmp = RREG32(mmSRBM_SOFT_RESET); | |
1352 | ||
1353 | /* Wait a little for things to settle down */ | |
1354 | udelay(50); | |
aaa36a97 AD |
1355 | } |
1356 | ||
1357 | return 0; | |
1358 | } | |
1359 | ||
50b0197a CZ |
1360 | static int gmc_v8_0_post_soft_reset(void *handle) |
1361 | { | |
1362 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1363 | ||
770d13b1 | 1364 | if (!adev->gmc.srbm_soft_reset) |
50b0197a CZ |
1365 | return 0; |
1366 | ||
e4f6b39e | 1367 | gmc_v8_0_mc_resume(adev); |
50b0197a CZ |
1368 | return 0; |
1369 | } | |
1370 | ||
aaa36a97 AD |
1371 | static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev, |
1372 | struct amdgpu_irq_src *src, | |
1373 | unsigned type, | |
1374 | enum amdgpu_interrupt_state state) | |
1375 | { | |
1376 | u32 tmp; | |
1377 | u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1378 | VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1379 | VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1380 | VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1381 | VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1382 | VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | | |
1383 | VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK); | |
1384 | ||
1385 | switch (state) { | |
1386 | case AMDGPU_IRQ_STATE_DISABLE: | |
1387 | /* system context */ | |
1388 | tmp = RREG32(mmVM_CONTEXT0_CNTL); | |
1389 | tmp &= ~bits; | |
1390 | WREG32(mmVM_CONTEXT0_CNTL, tmp); | |
1391 | /* VMs */ | |
1392 | tmp = RREG32(mmVM_CONTEXT1_CNTL); | |
1393 | tmp &= ~bits; | |
1394 | WREG32(mmVM_CONTEXT1_CNTL, tmp); | |
1395 | break; | |
1396 | case AMDGPU_IRQ_STATE_ENABLE: | |
1397 | /* system context */ | |
1398 | tmp = RREG32(mmVM_CONTEXT0_CNTL); | |
1399 | tmp |= bits; | |
1400 | WREG32(mmVM_CONTEXT0_CNTL, tmp); | |
1401 | /* VMs */ | |
1402 | tmp = RREG32(mmVM_CONTEXT1_CNTL); | |
1403 | tmp |= bits; | |
1404 | WREG32(mmVM_CONTEXT1_CNTL, tmp); | |
1405 | break; | |
1406 | default: | |
1407 | break; | |
1408 | } | |
1409 | ||
1410 | return 0; | |
1411 | } | |
1412 | ||
1413 | static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, | |
1414 | struct amdgpu_irq_src *source, | |
1415 | struct amdgpu_iv_entry *entry) | |
1416 | { | |
b97dfa27 | 1417 | u32 addr, status, mc_client, vmid; |
aaa36a97 | 1418 | |
edcafc02 PD |
1419 | if (amdgpu_sriov_vf(adev)) { |
1420 | dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n", | |
7ccf5aa8 | 1421 | entry->src_id, entry->src_data[0]); |
edcafc02 PD |
1422 | dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n"); |
1423 | return 0; | |
1424 | } | |
1425 | ||
aaa36a97 AD |
1426 | addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR); |
1427 | status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS); | |
1428 | mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); | |
ce0c6bcd CK |
1429 | /* reset addr and status */ |
1430 | WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1); | |
1431 | ||
1432 | if (!addr && !status) | |
1433 | return 0; | |
1434 | ||
d9c13156 CK |
1435 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST) |
1436 | gmc_v8_0_set_fault_enable_default(adev, false); | |
1437 | ||
01615881 | 1438 | if (printk_ratelimit()) { |
05794eff | 1439 | struct amdgpu_task_info task_info; |
efaa9646 | 1440 | |
05794eff | 1441 | memset(&task_info, 0, sizeof(struct amdgpu_task_info)); |
efaa9646 AG |
1442 | amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); |
1443 | ||
1444 | dev_err(adev->dev, "GPU fault detected: %d 0x%08x for process %s pid %d thread %s pid %d\n", | |
1445 | entry->src_id, entry->src_data[0], task_info.process_name, | |
1446 | task_info.tgid, task_info.task_name, task_info.pid); | |
01615881 EC |
1447 | dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", |
1448 | addr); | |
1449 | dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", | |
1450 | status); | |
904a3374 CK |
1451 | gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client, |
1452 | entry->pasid); | |
01615881 | 1453 | } |
aaa36a97 | 1454 | |
b97dfa27 | 1455 | vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1456 | VMID); | |
1457 | if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid) | |
1458 | && !atomic_read(&adev->gmc.vm_fault_info_updated)) { | |
1459 | struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info; | |
1460 | u32 protections = REG_GET_FIELD(status, | |
1461 | VM_CONTEXT1_PROTECTION_FAULT_STATUS, | |
1462 | PROTECTIONS); | |
1463 | ||
1464 | info->vmid = vmid; | |
1465 | info->mc_id = REG_GET_FIELD(status, | |
1466 | VM_CONTEXT1_PROTECTION_FAULT_STATUS, | |
1467 | MEMORY_CLIENT_ID); | |
1468 | info->status = status; | |
1469 | info->page_addr = addr; | |
1470 | info->prot_valid = protections & 0x7 ? true : false; | |
1471 | info->prot_read = protections & 0x8 ? true : false; | |
1472 | info->prot_write = protections & 0x10 ? true : false; | |
1473 | info->prot_exec = protections & 0x20 ? true : false; | |
1474 | mb(); | |
1475 | atomic_set(&adev->gmc.vm_fault_info_updated, 1); | |
1476 | } | |
1477 | ||
aaa36a97 AD |
1478 | return 0; |
1479 | } | |
1480 | ||
a0d69786 | 1481 | static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev, |
3fde56b8 | 1482 | bool enable) |
a0d69786 EH |
1483 | { |
1484 | uint32_t data; | |
1485 | ||
3fde56b8 | 1486 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) { |
a0d69786 EH |
1487 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1488 | data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK; | |
1489 | WREG32(mmMC_HUB_MISC_HUB_CG, data); | |
1490 | ||
1491 | data = RREG32(mmMC_HUB_MISC_SIP_CG); | |
1492 | data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK; | |
1493 | WREG32(mmMC_HUB_MISC_SIP_CG, data); | |
1494 | ||
1495 | data = RREG32(mmMC_HUB_MISC_VM_CG); | |
1496 | data |= MC_HUB_MISC_VM_CG__ENABLE_MASK; | |
1497 | WREG32(mmMC_HUB_MISC_VM_CG, data); | |
1498 | ||
1499 | data = RREG32(mmMC_XPB_CLK_GAT); | |
1500 | data |= MC_XPB_CLK_GAT__ENABLE_MASK; | |
1501 | WREG32(mmMC_XPB_CLK_GAT, data); | |
1502 | ||
1503 | data = RREG32(mmATC_MISC_CG); | |
1504 | data |= ATC_MISC_CG__ENABLE_MASK; | |
1505 | WREG32(mmATC_MISC_CG, data); | |
1506 | ||
1507 | data = RREG32(mmMC_CITF_MISC_WR_CG); | |
1508 | data |= MC_CITF_MISC_WR_CG__ENABLE_MASK; | |
1509 | WREG32(mmMC_CITF_MISC_WR_CG, data); | |
1510 | ||
1511 | data = RREG32(mmMC_CITF_MISC_RD_CG); | |
1512 | data |= MC_CITF_MISC_RD_CG__ENABLE_MASK; | |
1513 | WREG32(mmMC_CITF_MISC_RD_CG, data); | |
1514 | ||
1515 | data = RREG32(mmMC_CITF_MISC_VM_CG); | |
1516 | data |= MC_CITF_MISC_VM_CG__ENABLE_MASK; | |
1517 | WREG32(mmMC_CITF_MISC_VM_CG, data); | |
1518 | ||
1519 | data = RREG32(mmVM_L2_CG); | |
1520 | data |= VM_L2_CG__ENABLE_MASK; | |
1521 | WREG32(mmVM_L2_CG, data); | |
1522 | } else { | |
1523 | data = RREG32(mmMC_HUB_MISC_HUB_CG); | |
1524 | data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK; | |
1525 | WREG32(mmMC_HUB_MISC_HUB_CG, data); | |
1526 | ||
1527 | data = RREG32(mmMC_HUB_MISC_SIP_CG); | |
1528 | data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK; | |
1529 | WREG32(mmMC_HUB_MISC_SIP_CG, data); | |
1530 | ||
1531 | data = RREG32(mmMC_HUB_MISC_VM_CG); | |
1532 | data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK; | |
1533 | WREG32(mmMC_HUB_MISC_VM_CG, data); | |
1534 | ||
1535 | data = RREG32(mmMC_XPB_CLK_GAT); | |
1536 | data &= ~MC_XPB_CLK_GAT__ENABLE_MASK; | |
1537 | WREG32(mmMC_XPB_CLK_GAT, data); | |
1538 | ||
1539 | data = RREG32(mmATC_MISC_CG); | |
1540 | data &= ~ATC_MISC_CG__ENABLE_MASK; | |
1541 | WREG32(mmATC_MISC_CG, data); | |
1542 | ||
1543 | data = RREG32(mmMC_CITF_MISC_WR_CG); | |
1544 | data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK; | |
1545 | WREG32(mmMC_CITF_MISC_WR_CG, data); | |
1546 | ||
1547 | data = RREG32(mmMC_CITF_MISC_RD_CG); | |
1548 | data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK; | |
1549 | WREG32(mmMC_CITF_MISC_RD_CG, data); | |
1550 | ||
1551 | data = RREG32(mmMC_CITF_MISC_VM_CG); | |
1552 | data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK; | |
1553 | WREG32(mmMC_CITF_MISC_VM_CG, data); | |
1554 | ||
1555 | data = RREG32(mmVM_L2_CG); | |
1556 | data &= ~VM_L2_CG__ENABLE_MASK; | |
1557 | WREG32(mmVM_L2_CG, data); | |
1558 | } | |
1559 | } | |
1560 | ||
1561 | static void fiji_update_mc_light_sleep(struct amdgpu_device *adev, | |
3fde56b8 | 1562 | bool enable) |
a0d69786 EH |
1563 | { |
1564 | uint32_t data; | |
1565 | ||
3fde56b8 | 1566 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) { |
a0d69786 EH |
1567 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1568 | data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK; | |
1569 | WREG32(mmMC_HUB_MISC_HUB_CG, data); | |
1570 | ||
1571 | data = RREG32(mmMC_HUB_MISC_SIP_CG); | |
1572 | data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK; | |
1573 | WREG32(mmMC_HUB_MISC_SIP_CG, data); | |
1574 | ||
1575 | data = RREG32(mmMC_HUB_MISC_VM_CG); | |
1576 | data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK; | |
1577 | WREG32(mmMC_HUB_MISC_VM_CG, data); | |
1578 | ||
1579 | data = RREG32(mmMC_XPB_CLK_GAT); | |
1580 | data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK; | |
1581 | WREG32(mmMC_XPB_CLK_GAT, data); | |
1582 | ||
1583 | data = RREG32(mmATC_MISC_CG); | |
1584 | data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK; | |
1585 | WREG32(mmATC_MISC_CG, data); | |
1586 | ||
1587 | data = RREG32(mmMC_CITF_MISC_WR_CG); | |
1588 | data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK; | |
1589 | WREG32(mmMC_CITF_MISC_WR_CG, data); | |
1590 | ||
1591 | data = RREG32(mmMC_CITF_MISC_RD_CG); | |
1592 | data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK; | |
1593 | WREG32(mmMC_CITF_MISC_RD_CG, data); | |
1594 | ||
1595 | data = RREG32(mmMC_CITF_MISC_VM_CG); | |
1596 | data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK; | |
1597 | WREG32(mmMC_CITF_MISC_VM_CG, data); | |
1598 | ||
1599 | data = RREG32(mmVM_L2_CG); | |
1600 | data |= VM_L2_CG__MEM_LS_ENABLE_MASK; | |
1601 | WREG32(mmVM_L2_CG, data); | |
1602 | } else { | |
1603 | data = RREG32(mmMC_HUB_MISC_HUB_CG); | |
1604 | data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK; | |
1605 | WREG32(mmMC_HUB_MISC_HUB_CG, data); | |
1606 | ||
1607 | data = RREG32(mmMC_HUB_MISC_SIP_CG); | |
1608 | data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK; | |
1609 | WREG32(mmMC_HUB_MISC_SIP_CG, data); | |
1610 | ||
1611 | data = RREG32(mmMC_HUB_MISC_VM_CG); | |
1612 | data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK; | |
1613 | WREG32(mmMC_HUB_MISC_VM_CG, data); | |
1614 | ||
1615 | data = RREG32(mmMC_XPB_CLK_GAT); | |
1616 | data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK; | |
1617 | WREG32(mmMC_XPB_CLK_GAT, data); | |
1618 | ||
1619 | data = RREG32(mmATC_MISC_CG); | |
1620 | data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK; | |
1621 | WREG32(mmATC_MISC_CG, data); | |
1622 | ||
1623 | data = RREG32(mmMC_CITF_MISC_WR_CG); | |
1624 | data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK; | |
1625 | WREG32(mmMC_CITF_MISC_WR_CG, data); | |
1626 | ||
1627 | data = RREG32(mmMC_CITF_MISC_RD_CG); | |
1628 | data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK; | |
1629 | WREG32(mmMC_CITF_MISC_RD_CG, data); | |
1630 | ||
1631 | data = RREG32(mmMC_CITF_MISC_VM_CG); | |
1632 | data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK; | |
1633 | WREG32(mmMC_CITF_MISC_VM_CG, data); | |
1634 | ||
1635 | data = RREG32(mmVM_L2_CG); | |
1636 | data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK; | |
1637 | WREG32(mmVM_L2_CG, data); | |
1638 | } | |
1639 | } | |
1640 | ||
5fc3aeeb | 1641 | static int gmc_v8_0_set_clockgating_state(void *handle, |
1642 | enum amd_clockgating_state state) | |
aaa36a97 | 1643 | { |
a0d69786 EH |
1644 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1645 | ||
ce137c04 ML |
1646 | if (amdgpu_sriov_vf(adev)) |
1647 | return 0; | |
1648 | ||
a0d69786 EH |
1649 | switch (adev->asic_type) { |
1650 | case CHIP_FIJI: | |
1651 | fiji_update_mc_medium_grain_clock_gating(adev, | |
7e913664 | 1652 | state == AMD_CG_STATE_GATE); |
a0d69786 | 1653 | fiji_update_mc_light_sleep(adev, |
7e913664 | 1654 | state == AMD_CG_STATE_GATE); |
a0d69786 EH |
1655 | break; |
1656 | default: | |
1657 | break; | |
1658 | } | |
aaa36a97 AD |
1659 | return 0; |
1660 | } | |
1661 | ||
5fc3aeeb | 1662 | static int gmc_v8_0_set_powergating_state(void *handle, |
1663 | enum amd_powergating_state state) | |
aaa36a97 AD |
1664 | { |
1665 | return 0; | |
1666 | } | |
1667 | ||
8bcab092 HR |
1668 | static void gmc_v8_0_get_clockgating_state(void *handle, u32 *flags) |
1669 | { | |
1670 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1671 | int data; | |
1672 | ||
ce137c04 ML |
1673 | if (amdgpu_sriov_vf(adev)) |
1674 | *flags = 0; | |
1675 | ||
8bcab092 HR |
1676 | /* AMD_CG_SUPPORT_MC_MGCG */ |
1677 | data = RREG32(mmMC_HUB_MISC_HUB_CG); | |
1678 | if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK) | |
1679 | *flags |= AMD_CG_SUPPORT_MC_MGCG; | |
1680 | ||
1681 | /* AMD_CG_SUPPORT_MC_LS */ | |
1682 | if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK) | |
1683 | *flags |= AMD_CG_SUPPORT_MC_LS; | |
1684 | } | |
1685 | ||
a1255107 | 1686 | static const struct amd_ip_funcs gmc_v8_0_ip_funcs = { |
88a907d6 | 1687 | .name = "gmc_v8_0", |
aaa36a97 | 1688 | .early_init = gmc_v8_0_early_init, |
140b519f | 1689 | .late_init = gmc_v8_0_late_init, |
aaa36a97 AD |
1690 | .sw_init = gmc_v8_0_sw_init, |
1691 | .sw_fini = gmc_v8_0_sw_fini, | |
1692 | .hw_init = gmc_v8_0_hw_init, | |
1693 | .hw_fini = gmc_v8_0_hw_fini, | |
1694 | .suspend = gmc_v8_0_suspend, | |
1695 | .resume = gmc_v8_0_resume, | |
1696 | .is_idle = gmc_v8_0_is_idle, | |
1697 | .wait_for_idle = gmc_v8_0_wait_for_idle, | |
50b0197a CZ |
1698 | .check_soft_reset = gmc_v8_0_check_soft_reset, |
1699 | .pre_soft_reset = gmc_v8_0_pre_soft_reset, | |
aaa36a97 | 1700 | .soft_reset = gmc_v8_0_soft_reset, |
50b0197a | 1701 | .post_soft_reset = gmc_v8_0_post_soft_reset, |
aaa36a97 AD |
1702 | .set_clockgating_state = gmc_v8_0_set_clockgating_state, |
1703 | .set_powergating_state = gmc_v8_0_set_powergating_state, | |
8bcab092 | 1704 | .get_clockgating_state = gmc_v8_0_get_clockgating_state, |
aaa36a97 AD |
1705 | }; |
1706 | ||
132f34e4 CK |
1707 | static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = { |
1708 | .flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb, | |
5518625d | 1709 | .emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb, |
c633c00b | 1710 | .emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping, |
603adfe8 | 1711 | .set_prt = gmc_v8_0_set_prt, |
b1166325 CK |
1712 | .get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags, |
1713 | .get_vm_pde = gmc_v8_0_get_vm_pde | |
aaa36a97 AD |
1714 | }; |
1715 | ||
1716 | static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = { | |
1717 | .set = gmc_v8_0_vm_fault_interrupt_state, | |
1718 | .process = gmc_v8_0_process_interrupt, | |
1719 | }; | |
1720 | ||
132f34e4 | 1721 | static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev) |
aaa36a97 | 1722 | { |
f54b30d7 | 1723 | adev->gmc.gmc_funcs = &gmc_v8_0_gmc_funcs; |
aaa36a97 AD |
1724 | } |
1725 | ||
1726 | static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev) | |
1727 | { | |
770d13b1 CK |
1728 | adev->gmc.vm_fault.num_types = 1; |
1729 | adev->gmc.vm_fault.funcs = &gmc_v8_0_irq_funcs; | |
aaa36a97 | 1730 | } |
a1255107 AD |
1731 | |
1732 | const struct amdgpu_ip_block_version gmc_v8_0_ip_block = | |
1733 | { | |
1734 | .type = AMD_IP_BLOCK_TYPE_GMC, | |
1735 | .major = 8, | |
1736 | .minor = 0, | |
1737 | .rev = 0, | |
1738 | .funcs = &gmc_v8_0_ip_funcs, | |
1739 | }; | |
1740 | ||
1741 | const struct amdgpu_ip_block_version gmc_v8_1_ip_block = | |
1742 | { | |
1743 | .type = AMD_IP_BLOCK_TYPE_GMC, | |
1744 | .major = 8, | |
1745 | .minor = 1, | |
1746 | .rev = 0, | |
1747 | .funcs = &gmc_v8_0_ip_funcs, | |
1748 | }; | |
1749 | ||
1750 | const struct amdgpu_ip_block_version gmc_v8_5_ip_block = | |
1751 | { | |
1752 | .type = AMD_IP_BLOCK_TYPE_GMC, | |
1753 | .major = 8, | |
1754 | .minor = 5, | |
1755 | .rev = 0, | |
1756 | .funcs = &gmc_v8_0_ip_funcs, | |
1757 | }; |