]>
Commit | Line | Data |
---|---|---|
e60f8db5 AX |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | #include "amdgpu.h" | |
24 | #include "mmhub_v1_0.h" | |
25 | ||
65417d9f FX |
26 | #include "mmhub/mmhub_1_0_offset.h" |
27 | #include "mmhub/mmhub_1_0_sh_mask.h" | |
28 | #include "mmhub/mmhub_1_0_default.h" | |
6ce68225 FX |
29 | #include "athub/athub_1_0_offset.h" |
30 | #include "athub/athub_1_0_sh_mask.h" | |
fb960bd2 | 31 | #include "vega10_enum.h" |
e60f8db5 AX |
32 | |
33 | #include "soc15_common.h" | |
34 | ||
2547a7aa HR |
35 | #define mmDAGB0_CNTL_MISC2_RV 0x008f |
36 | #define mmDAGB0_CNTL_MISC2_RV_BASE_IDX 0 | |
37 | ||
e60f8db5 AX |
38 | u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev) |
39 | { | |
2a419183 | 40 | u64 base = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE); |
6fdd68b1 | 41 | u64 top = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP); |
e60f8db5 AX |
42 | |
43 | base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; | |
44 | base <<= 24; | |
45 | ||
6fdd68b1 AD |
46 | top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; |
47 | top <<= 24; | |
48 | ||
49 | adev->gmc.fb_start = base; | |
50 | adev->gmc.fb_end = top; | |
51 | ||
e60f8db5 AX |
52 | return base; |
53 | } | |
54 | ||
a51dca4f HR |
55 | static void mmhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev) |
56 | { | |
11c3a249 | 57 | uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo); |
a51dca4f | 58 | |
2a419183 HR |
59 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, |
60 | lower_32_bits(value)); | |
a51dca4f | 61 | |
2a419183 HR |
62 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, |
63 | upper_32_bits(value)); | |
a51dca4f HR |
64 | } |
65 | ||
9bbad6fd HR |
66 | static void mmhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev) |
67 | { | |
68 | mmhub_v1_0_init_gart_pt_regs(adev); | |
69 | ||
2a419183 | 70 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, |
770d13b1 | 71 | (u32)(adev->gmc.gart_start >> 12)); |
2a419183 | 72 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, |
770d13b1 | 73 | (u32)(adev->gmc.gart_start >> 44)); |
2a419183 HR |
74 | |
75 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, | |
770d13b1 | 76 | (u32)(adev->gmc.gart_end >> 12)); |
2a419183 | 77 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, |
770d13b1 | 78 | (u32)(adev->gmc.gart_end >> 44)); |
9bbad6fd HR |
79 | } |
80 | ||
fc4b884b | 81 | static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev) |
e60f8db5 | 82 | { |
fc4b884b HR |
83 | uint64_t value; |
84 | uint32_t tmp; | |
e60f8db5 | 85 | |
c3e1b43c | 86 | /* Program the AGP BAR */ |
2a419183 | 87 | WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BASE, 0); |
c3e1b43c CK |
88 | WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); |
89 | WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); | |
a51dca4f | 90 | |
fc4b884b | 91 | /* Program the system aperture low logical page number. */ |
2a419183 | 92 | WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, |
c3e1b43c | 93 | min(adev->gmc.vram_start, adev->gmc.agp_start) >> 18); |
76006776 HR |
94 | |
95 | if (adev->asic_type == CHIP_RAVEN && adev->rev_id >= 0x8) | |
96 | /* | |
97 | * Raven2 has a HW issue that it is unable to use the vram which | |
98 | * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the | |
99 | * workaround that increase system aperture high address (add 1) | |
100 | * to get rid of the VM fault and hardware hang. | |
101 | */ | |
102 | WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, | |
103 | (max(adev->gmc.vram_end, adev->gmc.agp_end) >> 18) + 0x1); | |
104 | else | |
105 | WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, | |
106 | max(adev->gmc.vram_end, adev->gmc.agp_end) >> 18); | |
fc4b884b HR |
107 | |
108 | /* Set default page address. */ | |
770d13b1 | 109 | value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start + |
e60f8db5 | 110 | adev->vm_manager.vram_base_offset; |
2a419183 HR |
111 | WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, |
112 | (u32)(value >> 12)); | |
113 | WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, | |
114 | (u32)(value >> 44)); | |
fc4b884b HR |
115 | |
116 | /* Program "protection fault". */ | |
2a419183 | 117 | WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, |
92e71b06 | 118 | (u32)(adev->dummy_page_addr >> 12)); |
2a419183 | 119 | WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, |
92e71b06 | 120 | (u32)((u64)adev->dummy_page_addr >> 44)); |
2a419183 HR |
121 | |
122 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2); | |
fc4b884b HR |
123 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, |
124 | ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); | |
2a419183 | 125 | WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2, tmp); |
fc4b884b HR |
126 | } |
127 | ||
34269839 HR |
128 | static void mmhub_v1_0_init_tlb_regs(struct amdgpu_device *adev) |
129 | { | |
130 | uint32_t tmp; | |
131 | ||
132 | /* Setup TLB control */ | |
2a419183 | 133 | tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL); |
34269839 HR |
134 | |
135 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); | |
136 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); | |
137 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, | |
138 | ENABLE_ADVANCED_DRIVER_MODEL, 1); | |
139 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, | |
140 | SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); | |
141 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0); | |
142 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, | |
143 | MTYPE, MTYPE_UC);/* XXX for emulation. */ | |
144 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); | |
145 | ||
2a419183 | 146 | WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp); |
34269839 HR |
147 | } |
148 | ||
41f6f311 HR |
149 | static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev) |
150 | { | |
a3ce3645 | 151 | uint32_t tmp; |
41f6f311 HR |
152 | |
153 | /* Setup L2 cache */ | |
2a419183 | 154 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL); |
41f6f311 | 155 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); |
6be7adb3 | 156 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1); |
41f6f311 HR |
157 | /* XXX for emulation, Refer to closed source code.*/ |
158 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE, | |
159 | 0); | |
160 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1); | |
161 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); | |
162 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); | |
2a419183 | 163 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp); |
41f6f311 | 164 | |
2a419183 | 165 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2); |
41f6f311 HR |
166 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); |
167 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); | |
2a419183 | 168 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp); |
41f6f311 | 169 | |
770d13b1 | 170 | if (adev->gmc.translate_further) { |
6a42fd6f CK |
171 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); |
172 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, | |
173 | L2_CACHE_BIGK_FRAGMENT_SIZE, 9); | |
174 | } else { | |
175 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); | |
176 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, | |
177 | L2_CACHE_BIGK_FRAGMENT_SIZE, 6); | |
178 | } | |
41f6f311 HR |
179 | |
180 | tmp = mmVM_L2_CNTL4_DEFAULT; | |
181 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); | |
182 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); | |
2a419183 | 183 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL4, tmp); |
41f6f311 HR |
184 | } |
185 | ||
02c4704b HR |
186 | static void mmhub_v1_0_enable_system_domain(struct amdgpu_device *adev) |
187 | { | |
188 | uint32_t tmp; | |
189 | ||
2a419183 | 190 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL); |
02c4704b HR |
191 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); |
192 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); | |
2a419183 | 193 | WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL, tmp); |
02c4704b HR |
194 | } |
195 | ||
d5c87390 HR |
196 | static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev) |
197 | { | |
2a419183 HR |
198 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, |
199 | 0XFFFFFFFF); | |
200 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, | |
201 | 0x0000000F); | |
202 | ||
203 | WREG32_SOC15(MMHUB, 0, | |
204 | mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0); | |
205 | WREG32_SOC15(MMHUB, 0, | |
206 | mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0); | |
207 | ||
208 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, | |
209 | 0); | |
210 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, | |
211 | 0); | |
d5c87390 HR |
212 | } |
213 | ||
3dff4cc4 | 214 | static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) |
fc4b884b | 215 | { |
6a42fd6f | 216 | unsigned num_level, block_size; |
3dff4cc4 | 217 | uint32_t tmp; |
6a42fd6f CK |
218 | int i; |
219 | ||
220 | num_level = adev->vm_manager.num_level; | |
221 | block_size = adev->vm_manager.block_size; | |
770d13b1 | 222 | if (adev->gmc.translate_further) |
6a42fd6f CK |
223 | num_level -= 1; |
224 | else | |
225 | block_size -= 9; | |
e60f8db5 AX |
226 | |
227 | for (i = 0; i <= 14; i++) { | |
deca8322 | 228 | tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i); |
6a42fd6f CK |
229 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); |
230 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, | |
231 | num_level); | |
e60f8db5 | 232 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 233 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 234 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f CK |
235 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, |
236 | 1); | |
e60f8db5 | 237 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 238 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 239 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 240 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 241 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 242 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 243 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 244 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 245 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f | 246 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
e60f8db5 | 247 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
6a42fd6f CK |
248 | PAGE_TABLE_BLOCK_SIZE, |
249 | block_size); | |
9f57f7b4 JC |
250 | /* Send no-retry XNACK on fault to suppress VM fault storm. */ |
251 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, | |
252 | RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); | |
deca8322 TSD |
253 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp); |
254 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0); | |
255 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0); | |
256 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2, | |
22770e5a | 257 | lower_32_bits(adev->vm_manager.max_pfn - 1)); |
deca8322 | 258 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2, |
22770e5a | 259 | upper_32_bits(adev->vm_manager.max_pfn - 1)); |
e60f8db5 | 260 | } |
3dff4cc4 HR |
261 | } |
262 | ||
1e4eccda HR |
263 | static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev) |
264 | { | |
265 | unsigned i; | |
266 | ||
267 | for (i = 0; i < 18; ++i) { | |
deca8322 TSD |
268 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, |
269 | 2 * i, 0xffffffff); | |
270 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, | |
271 | 2 * i, 0x1f); | |
1e4eccda HR |
272 | } |
273 | } | |
274 | ||
a95890b4 HZ |
275 | void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev, |
276 | bool enable) | |
277 | { | |
a95890b4 HZ |
278 | if (amdgpu_sriov_vf(adev)) |
279 | return; | |
280 | ||
f8386b35 | 281 | if (enable && adev->pg_flags & AMD_PG_SUPPORT_MMHUB) { |
b92c6287 RZ |
282 | if (adev->powerplay.pp_funcs->set_powergating_by_smu) |
283 | amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GMC, true); | |
7e452ce9 | 284 | |
a95890b4 HZ |
285 | } |
286 | } | |
287 | ||
3dff4cc4 HR |
288 | int mmhub_v1_0_gart_enable(struct amdgpu_device *adev) |
289 | { | |
290 | if (amdgpu_sriov_vf(adev)) { | |
291 | /* | |
292 | * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are | |
293 | * VF copy registers so vbios post doesn't program them, for | |
294 | * SRIOV driver need to program them | |
295 | */ | |
2a419183 | 296 | WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE, |
770d13b1 | 297 | adev->gmc.vram_start >> 24); |
2a419183 | 298 | WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP, |
770d13b1 | 299 | adev->gmc.vram_end >> 24); |
3dff4cc4 HR |
300 | } |
301 | ||
302 | /* GART Enable. */ | |
303 | mmhub_v1_0_init_gart_aperture_regs(adev); | |
304 | mmhub_v1_0_init_system_aperture_regs(adev); | |
305 | mmhub_v1_0_init_tlb_regs(adev); | |
306 | mmhub_v1_0_init_cache_regs(adev); | |
307 | ||
308 | mmhub_v1_0_enable_system_domain(adev); | |
309 | mmhub_v1_0_disable_identity_aperture(adev); | |
310 | mmhub_v1_0_setup_vmid_config(adev); | |
1e4eccda | 311 | mmhub_v1_0_program_invalidation(adev); |
e60f8db5 AX |
312 | |
313 | return 0; | |
314 | } | |
315 | ||
316 | void mmhub_v1_0_gart_disable(struct amdgpu_device *adev) | |
317 | { | |
318 | u32 tmp; | |
319 | u32 i; | |
320 | ||
321 | /* Disable all tables */ | |
322 | for (i = 0; i < 16; i++) | |
deca8322 | 323 | WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL, i, 0); |
e60f8db5 AX |
324 | |
325 | /* Setup TLB control */ | |
2a419183 | 326 | tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL); |
e60f8db5 AX |
327 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); |
328 | tmp = REG_SET_FIELD(tmp, | |
329 | MC_VM_MX_L1_TLB_CNTL, | |
330 | ENABLE_ADVANCED_DRIVER_MODEL, | |
331 | 0); | |
2a419183 | 332 | WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp); |
e60f8db5 AX |
333 | |
334 | /* Setup L2 cache */ | |
2a419183 | 335 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL); |
e60f8db5 | 336 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0); |
2a419183 HR |
337 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp); |
338 | WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0); | |
e60f8db5 AX |
339 | } |
340 | ||
341 | /** | |
342 | * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling | |
343 | * | |
344 | * @adev: amdgpu_device pointer | |
345 | * @value: true redirects VM faults to the default page | |
346 | */ | |
347 | void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) | |
348 | { | |
349 | u32 tmp; | |
2a419183 | 350 | tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL); |
e60f8db5 AX |
351 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
352 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
353 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
354 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
355 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
356 | PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
357 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
358 | PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
359 | tmp = REG_SET_FIELD(tmp, | |
360 | VM_L2_PROTECTION_FAULT_CNTL, | |
361 | TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, | |
362 | value); | |
363 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
364 | NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
365 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
366 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
367 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
368 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
369 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
370 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
371 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
372 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
373 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
374 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); | |
4bd9a67e ML |
375 | if (!value) { |
376 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
377 | CRASH_ON_NO_RETRY_FAULT, 1); | |
378 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, | |
379 | CRASH_ON_RETRY_FAULT, 1); | |
380 | } | |
381 | ||
2a419183 | 382 | WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp); |
e60f8db5 AX |
383 | } |
384 | ||
77f6c763 | 385 | void mmhub_v1_0_init(struct amdgpu_device *adev) |
e60f8db5 | 386 | { |
e60f8db5 AX |
387 | struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB]; |
388 | ||
389 | hub->ctx0_ptb_addr_lo32 = | |
390 | SOC15_REG_OFFSET(MMHUB, 0, | |
391 | mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); | |
392 | hub->ctx0_ptb_addr_hi32 = | |
393 | SOC15_REG_OFFSET(MMHUB, 0, | |
394 | mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); | |
395 | hub->vm_inv_eng0_req = | |
396 | SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_REQ); | |
397 | hub->vm_inv_eng0_ack = | |
398 | SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ACK); | |
399 | hub->vm_context0_cntl = | |
400 | SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL); | |
401 | hub->vm_l2_pro_fault_status = | |
402 | SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_STATUS); | |
403 | hub->vm_l2_pro_fault_cntl = | |
404 | SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL); | |
405 | ||
77f6c763 HR |
406 | } |
407 | ||
e60f8db5 AX |
408 | static void mmhub_v1_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, |
409 | bool enable) | |
410 | { | |
2547a7aa | 411 | uint32_t def, data, def1, data1, def2 = 0, data2 = 0; |
e60f8db5 | 412 | |
2a419183 | 413 | def = data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG); |
2547a7aa | 414 | |
741deade | 415 | if (adev->asic_type != CHIP_RAVEN) { |
2a419183 HR |
416 | def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2); |
417 | def2 = data2 = RREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2); | |
2547a7aa | 418 | } else |
2a419183 | 419 | def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV); |
e60f8db5 AX |
420 | |
421 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) { | |
422 | data |= ATC_L2_MISC_CG__ENABLE_MASK; | |
423 | ||
424 | data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | | |
425 | DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | | |
426 | DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | | |
427 | DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | | |
428 | DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | | |
429 | DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); | |
430 | ||
741deade | 431 | if (adev->asic_type != CHIP_RAVEN) |
2547a7aa HR |
432 | data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | |
433 | DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK | | |
434 | DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | | |
435 | DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK | | |
436 | DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | | |
437 | DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); | |
e60f8db5 AX |
438 | } else { |
439 | data &= ~ATC_L2_MISC_CG__ENABLE_MASK; | |
440 | ||
441 | data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | | |
442 | DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | | |
443 | DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | | |
444 | DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | | |
445 | DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | | |
446 | DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); | |
447 | ||
741deade | 448 | if (adev->asic_type != CHIP_RAVEN) |
2547a7aa HR |
449 | data2 |= (DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | |
450 | DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK | | |
451 | DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | | |
452 | DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK | | |
453 | DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | | |
454 | DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); | |
e60f8db5 AX |
455 | } |
456 | ||
457 | if (def != data) | |
2a419183 | 458 | WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data); |
e60f8db5 | 459 | |
2547a7aa | 460 | if (def1 != data1) { |
741deade | 461 | if (adev->asic_type != CHIP_RAVEN) |
2a419183 | 462 | WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1); |
2547a7aa | 463 | else |
2a419183 | 464 | WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV, data1); |
2547a7aa | 465 | } |
e60f8db5 | 466 | |
741deade | 467 | if (adev->asic_type != CHIP_RAVEN && def2 != data2) |
2a419183 | 468 | WREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2, data2); |
e60f8db5 AX |
469 | } |
470 | ||
471 | static void athub_update_medium_grain_clock_gating(struct amdgpu_device *adev, | |
472 | bool enable) | |
473 | { | |
474 | uint32_t def, data; | |
475 | ||
2a419183 | 476 | def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); |
e60f8db5 AX |
477 | |
478 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) | |
479 | data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK; | |
480 | else | |
481 | data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK; | |
482 | ||
483 | if (def != data) | |
2a419183 | 484 | WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data); |
e60f8db5 AX |
485 | } |
486 | ||
487 | static void mmhub_v1_0_update_medium_grain_light_sleep(struct amdgpu_device *adev, | |
488 | bool enable) | |
489 | { | |
490 | uint32_t def, data; | |
491 | ||
2a419183 | 492 | def = data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG); |
e60f8db5 AX |
493 | |
494 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) | |
495 | data |= ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; | |
496 | else | |
497 | data &= ~ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; | |
498 | ||
499 | if (def != data) | |
2a419183 | 500 | WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data); |
e60f8db5 AX |
501 | } |
502 | ||
503 | static void athub_update_medium_grain_light_sleep(struct amdgpu_device *adev, | |
504 | bool enable) | |
505 | { | |
506 | uint32_t def, data; | |
507 | ||
2a419183 | 508 | def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); |
e60f8db5 AX |
509 | |
510 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) && | |
511 | (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) | |
512 | data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK; | |
513 | else | |
514 | data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK; | |
515 | ||
516 | if(def != data) | |
2a419183 | 517 | WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data); |
e60f8db5 AX |
518 | } |
519 | ||
d5583d4f HR |
520 | int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev, |
521 | enum amd_clockgating_state state) | |
e60f8db5 | 522 | { |
98c24b24 XY |
523 | if (amdgpu_sriov_vf(adev)) |
524 | return 0; | |
525 | ||
e60f8db5 AX |
526 | switch (adev->asic_type) { |
527 | case CHIP_VEGA10: | |
f8d27677 | 528 | case CHIP_VEGA12: |
c2d7fd2b | 529 | case CHIP_VEGA20: |
2547a7aa | 530 | case CHIP_RAVEN: |
e60f8db5 AX |
531 | mmhub_v1_0_update_medium_grain_clock_gating(adev, |
532 | state == AMD_CG_STATE_GATE ? true : false); | |
533 | athub_update_medium_grain_clock_gating(adev, | |
534 | state == AMD_CG_STATE_GATE ? true : false); | |
535 | mmhub_v1_0_update_medium_grain_light_sleep(adev, | |
536 | state == AMD_CG_STATE_GATE ? true : false); | |
537 | athub_update_medium_grain_light_sleep(adev, | |
538 | state == AMD_CG_STATE_GATE ? true : false); | |
539 | break; | |
540 | default: | |
541 | break; | |
542 | } | |
543 | ||
544 | return 0; | |
545 | } | |
546 | ||
13052be5 | 547 | void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags) |
e2a4cd69 | 548 | { |
e2a4cd69 HR |
549 | int data; |
550 | ||
551 | if (amdgpu_sriov_vf(adev)) | |
552 | *flags = 0; | |
553 | ||
554 | /* AMD_CG_SUPPORT_MC_MGCG */ | |
2a419183 | 555 | data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL); |
e2a4cd69 HR |
556 | if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK) |
557 | *flags |= AMD_CG_SUPPORT_MC_MGCG; | |
558 | ||
559 | /* AMD_CG_SUPPORT_MC_LS */ | |
2a419183 | 560 | data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG); |
e2a4cd69 HR |
561 | if (data & ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK) |
562 | *flags |= AMD_CG_SUPPORT_MC_LS; | |
563 | } |