]>
Commit | Line | Data |
---|---|---|
130e0371 OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include "amdgpu_amdkfd.h" | |
2f7d10b3 | 24 | #include "amd_shared.h" |
130e0371 OG |
25 | #include <drm/drmP.h> |
26 | #include "amdgpu.h" | |
2db0cdbe | 27 | #include "amdgpu_gfx.h" |
130e0371 OG |
28 | #include <linux/module.h> |
29 | ||
130e0371 | 30 | const struct kgd2kfd_calls *kgd2kfd; |
8eabaf54 | 31 | bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**); |
130e0371 | 32 | |
efb1c658 | 33 | int amdgpu_amdkfd_init(void) |
130e0371 | 34 | { |
efb1c658 OG |
35 | int ret; |
36 | ||
130e0371 | 37 | #if defined(CONFIG_HSA_AMD_MODULE) |
8eabaf54 | 38 | int (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**); |
130e0371 OG |
39 | |
40 | kgd2kfd_init_p = symbol_request(kgd2kfd_init); | |
41 | ||
42 | if (kgd2kfd_init_p == NULL) | |
efb1c658 OG |
43 | return -ENOENT; |
44 | ||
45 | ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd); | |
46 | if (ret) { | |
47 | symbol_put(kgd2kfd_init); | |
48 | kgd2kfd = NULL; | |
49 | } | |
50 | ||
51 | #elif defined(CONFIG_HSA_AMD) | |
52 | ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd); | |
53 | if (ret) | |
54 | kgd2kfd = NULL; | |
55 | ||
56 | #else | |
57 | ret = -ENOENT; | |
130e0371 | 58 | #endif |
efb1c658 OG |
59 | |
60 | return ret; | |
130e0371 OG |
61 | } |
62 | ||
5c33f214 FK |
63 | void amdgpu_amdkfd_fini(void) |
64 | { | |
65 | if (kgd2kfd) { | |
66 | kgd2kfd->exit(); | |
67 | symbol_put(kgd2kfd_init); | |
68 | } | |
69 | } | |
70 | ||
71 | void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) | |
130e0371 | 72 | { |
5c33f214 FK |
73 | const struct kfd2kgd_calls *kfd2kgd; |
74 | ||
75 | if (!kgd2kfd) | |
76 | return; | |
77 | ||
dc102c43 | 78 | switch (adev->asic_type) { |
41548ef7 | 79 | #ifdef CONFIG_DRM_AMDGPU_CIK |
130e0371 | 80 | case CHIP_KAVERI: |
30d13424 | 81 | case CHIP_HAWAII: |
32c22e99 OG |
82 | kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions(); |
83 | break; | |
41548ef7 | 84 | #endif |
ff758a12 | 85 | case CHIP_CARRIZO: |
30d13424 FK |
86 | case CHIP_TONGA: |
87 | case CHIP_FIJI: | |
88 | case CHIP_POLARIS10: | |
89 | case CHIP_POLARIS11: | |
ff758a12 BG |
90 | kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); |
91 | break; | |
130e0371 | 92 | default: |
9953b72f | 93 | dev_dbg(adev->dev, "kfd not supported on this ASIC\n"); |
5c33f214 | 94 | return; |
130e0371 OG |
95 | } |
96 | ||
5c33f214 FK |
97 | adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev, |
98 | adev->pdev, kfd2kgd); | |
130e0371 OG |
99 | } |
100 | ||
22cb0164 AD |
101 | /** |
102 | * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to | |
103 | * setup amdkfd | |
104 | * | |
105 | * @adev: amdgpu_device pointer | |
106 | * @aperture_base: output returning doorbell aperture base physical address | |
107 | * @aperture_size: output returning doorbell aperture size in bytes | |
108 | * @start_offset: output returning # of doorbell bytes reserved for amdgpu. | |
109 | * | |
110 | * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up, | |
111 | * takes doorbells required for its own rings and reports the setup to amdkfd. | |
112 | * amdgpu reserved doorbells are at the start of the doorbell aperture. | |
113 | */ | |
114 | static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, | |
115 | phys_addr_t *aperture_base, | |
116 | size_t *aperture_size, | |
117 | size_t *start_offset) | |
118 | { | |
119 | /* | |
120 | * The first num_doorbells are used by amdgpu. | |
121 | * amdkfd takes whatever's left in the aperture. | |
122 | */ | |
123 | if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) { | |
124 | *aperture_base = adev->doorbell.base; | |
125 | *aperture_size = adev->doorbell.size; | |
126 | *start_offset = adev->doorbell.num_doorbells * sizeof(u32); | |
127 | } else { | |
128 | *aperture_base = 0; | |
129 | *aperture_size = 0; | |
130 | *start_offset = 0; | |
131 | } | |
132 | } | |
133 | ||
dc102c43 | 134 | void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) |
130e0371 | 135 | { |
d0b63bb3 AR |
136 | int i; |
137 | int last_valid_bit; | |
dc102c43 | 138 | if (adev->kfd) { |
130e0371 OG |
139 | struct kgd2kfd_shared_resources gpu_resources = { |
140 | .compute_vmid_bitmap = 0xFF00, | |
d0b63bb3 AR |
141 | .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec, |
142 | .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe | |
130e0371 OG |
143 | }; |
144 | ||
d0b63bb3 AR |
145 | /* this is going to have a few of the MSBs set that we need to |
146 | * clear */ | |
147 | bitmap_complement(gpu_resources.queue_bitmap, | |
148 | adev->gfx.mec.queue_bitmap, | |
149 | KGD_MAX_QUEUES); | |
150 | ||
7b2124a5 AR |
151 | /* remove the KIQ bit as well */ |
152 | if (adev->gfx.kiq.ring.ready) | |
2db0cdbe AD |
153 | clear_bit(amdgpu_gfx_queue_to_bit(adev, |
154 | adev->gfx.kiq.ring.me - 1, | |
155 | adev->gfx.kiq.ring.pipe, | |
156 | adev->gfx.kiq.ring.queue), | |
7b2124a5 AR |
157 | gpu_resources.queue_bitmap); |
158 | ||
d0b63bb3 AR |
159 | /* According to linux/bitmap.h we shouldn't use bitmap_clear if |
160 | * nbits is not compile time constant */ | |
3447d220 | 161 | last_valid_bit = 1 /* only first MEC can have compute queues */ |
d0b63bb3 AR |
162 | * adev->gfx.mec.num_pipe_per_mec |
163 | * adev->gfx.mec.num_queue_per_pipe; | |
164 | for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i) | |
165 | clear_bit(i, gpu_resources.queue_bitmap); | |
166 | ||
dc102c43 | 167 | amdgpu_doorbell_get_kfd_info(adev, |
130e0371 OG |
168 | &gpu_resources.doorbell_physical_address, |
169 | &gpu_resources.doorbell_aperture_size, | |
170 | &gpu_resources.doorbell_start_offset); | |
171 | ||
dc102c43 | 172 | kgd2kfd->device_init(adev->kfd, &gpu_resources); |
130e0371 OG |
173 | } |
174 | } | |
175 | ||
dc102c43 | 176 | void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev) |
130e0371 | 177 | { |
dc102c43 AR |
178 | if (adev->kfd) { |
179 | kgd2kfd->device_exit(adev->kfd); | |
180 | adev->kfd = NULL; | |
130e0371 OG |
181 | } |
182 | } | |
183 | ||
dc102c43 | 184 | void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, |
130e0371 OG |
185 | const void *ih_ring_entry) |
186 | { | |
dc102c43 AR |
187 | if (adev->kfd) |
188 | kgd2kfd->interrupt(adev->kfd, ih_ring_entry); | |
130e0371 OG |
189 | } |
190 | ||
dc102c43 | 191 | void amdgpu_amdkfd_suspend(struct amdgpu_device *adev) |
130e0371 | 192 | { |
dc102c43 AR |
193 | if (adev->kfd) |
194 | kgd2kfd->suspend(adev->kfd); | |
130e0371 OG |
195 | } |
196 | ||
dc102c43 | 197 | int amdgpu_amdkfd_resume(struct amdgpu_device *adev) |
130e0371 OG |
198 | { |
199 | int r = 0; | |
200 | ||
dc102c43 AR |
201 | if (adev->kfd) |
202 | r = kgd2kfd->resume(adev->kfd); | |
130e0371 OG |
203 | |
204 | return r; | |
205 | } | |
206 | ||
130e0371 OG |
207 | int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, |
208 | void **mem_obj, uint64_t *gpu_addr, | |
209 | void **cpu_ptr) | |
210 | { | |
dc102c43 | 211 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
473fee47 | 212 | struct amdgpu_bo *bo = NULL; |
130e0371 | 213 | int r; |
473fee47 YZ |
214 | uint64_t gpu_addr_tmp = 0; |
215 | void *cpu_ptr_tmp = NULL; | |
130e0371 | 216 | |
dc102c43 | 217 | r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT, |
473fee47 | 218 | AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &bo); |
130e0371 | 219 | if (r) { |
dc102c43 | 220 | dev_err(adev->dev, |
130e0371 OG |
221 | "failed to allocate BO for amdkfd (%d)\n", r); |
222 | return r; | |
223 | } | |
224 | ||
225 | /* map the buffer */ | |
473fee47 | 226 | r = amdgpu_bo_reserve(bo, true); |
130e0371 | 227 | if (r) { |
dc102c43 | 228 | dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); |
130e0371 OG |
229 | goto allocate_mem_reserve_bo_failed; |
230 | } | |
231 | ||
473fee47 YZ |
232 | r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, |
233 | &gpu_addr_tmp); | |
130e0371 | 234 | if (r) { |
dc102c43 | 235 | dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); |
130e0371 OG |
236 | goto allocate_mem_pin_bo_failed; |
237 | } | |
130e0371 | 238 | |
473fee47 | 239 | r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp); |
130e0371 | 240 | if (r) { |
dc102c43 | 241 | dev_err(adev->dev, |
130e0371 OG |
242 | "(%d) failed to map bo to kernel for amdkfd\n", r); |
243 | goto allocate_mem_kmap_bo_failed; | |
244 | } | |
130e0371 | 245 | |
473fee47 YZ |
246 | *mem_obj = bo; |
247 | *gpu_addr = gpu_addr_tmp; | |
248 | *cpu_ptr = cpu_ptr_tmp; | |
249 | ||
250 | amdgpu_bo_unreserve(bo); | |
130e0371 OG |
251 | |
252 | return 0; | |
253 | ||
254 | allocate_mem_kmap_bo_failed: | |
473fee47 | 255 | amdgpu_bo_unpin(bo); |
130e0371 | 256 | allocate_mem_pin_bo_failed: |
473fee47 | 257 | amdgpu_bo_unreserve(bo); |
130e0371 | 258 | allocate_mem_reserve_bo_failed: |
473fee47 | 259 | amdgpu_bo_unref(&bo); |
130e0371 OG |
260 | |
261 | return r; | |
262 | } | |
263 | ||
264 | void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) | |
265 | { | |
473fee47 YZ |
266 | struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj; |
267 | ||
268 | amdgpu_bo_reserve(bo, true); | |
269 | amdgpu_bo_kunmap(bo); | |
270 | amdgpu_bo_unpin(bo); | |
271 | amdgpu_bo_unreserve(bo); | |
272 | amdgpu_bo_unref(&(bo)); | |
130e0371 OG |
273 | } |
274 | ||
30f1c042 HK |
275 | void get_local_mem_info(struct kgd_dev *kgd, |
276 | struct kfd_local_mem_info *mem_info) | |
277 | { | |
278 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; | |
279 | uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask : | |
280 | ~((1ULL << 32) - 1); | |
770d13b1 | 281 | resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size; |
30f1c042 HK |
282 | |
283 | memset(mem_info, 0, sizeof(*mem_info)); | |
770d13b1 CK |
284 | if (!(adev->gmc.aper_base & address_mask || aper_limit & address_mask)) { |
285 | mem_info->local_mem_size_public = adev->gmc.visible_vram_size; | |
286 | mem_info->local_mem_size_private = adev->gmc.real_vram_size - | |
287 | adev->gmc.visible_vram_size; | |
30f1c042 HK |
288 | } else { |
289 | mem_info->local_mem_size_public = 0; | |
770d13b1 | 290 | mem_info->local_mem_size_private = adev->gmc.real_vram_size; |
30f1c042 | 291 | } |
770d13b1 | 292 | mem_info->vram_width = adev->gmc.vram_width; |
30f1c042 | 293 | |
fb8baefc | 294 | pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n", |
770d13b1 | 295 | &adev->gmc.aper_base, &aper_limit, |
30f1c042 HK |
296 | mem_info->local_mem_size_public, |
297 | mem_info->local_mem_size_private); | |
298 | ||
4a2ba394 SL |
299 | if (amdgpu_emu_mode == 1) { |
300 | mem_info->mem_clk_max = 100; | |
301 | return; | |
302 | } | |
303 | ||
30f1c042 HK |
304 | if (amdgpu_sriov_vf(adev)) |
305 | mem_info->mem_clk_max = adev->clock.default_mclk / 100; | |
306 | else | |
307 | mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100; | |
308 | } | |
309 | ||
130e0371 OG |
310 | uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) |
311 | { | |
dc102c43 | 312 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
130e0371 | 313 | |
dc102c43 AR |
314 | if (adev->gfx.funcs->get_gpu_clock_counter) |
315 | return adev->gfx.funcs->get_gpu_clock_counter(adev); | |
130e0371 OG |
316 | return 0; |
317 | } | |
318 | ||
319 | uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) | |
320 | { | |
dc102c43 | 321 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; |
130e0371 | 322 | |
a9efcc19 | 323 | /* the sclk is in quantas of 10kHz */ |
4a2ba394 SL |
324 | if (amdgpu_emu_mode == 1) |
325 | return 100; | |
326 | ||
a9efcc19 FK |
327 | if (amdgpu_sriov_vf(adev)) |
328 | return adev->clock.default_sclk / 100; | |
329 | ||
330 | return amdgpu_dpm_get_sclk(adev, false) / 100; | |
130e0371 | 331 | } |
ebdebf42 FC |
332 | |
333 | void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info) | |
334 | { | |
335 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; | |
336 | struct amdgpu_cu_info acu_info = adev->gfx.cu_info; | |
337 | ||
338 | memset(cu_info, 0, sizeof(*cu_info)); | |
339 | if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap)) | |
340 | return; | |
341 | ||
342 | cu_info->cu_active_number = acu_info.number; | |
343 | cu_info->cu_ao_mask = acu_info.ao_cu_mask; | |
344 | memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0], | |
345 | sizeof(acu_info.bitmap)); | |
346 | cu_info->num_shader_engines = adev->gfx.config.max_shader_engines; | |
347 | cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; | |
348 | cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh; | |
349 | cu_info->simd_per_cu = acu_info.simd_per_cu; | |
350 | cu_info->max_waves_per_simd = acu_info.max_waves_per_simd; | |
351 | cu_info->wave_front_size = acu_info.wave_front_size; | |
352 | cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu; | |
353 | cu_info->lds_size = acu_info.lds_size; | |
354 | } | |
9f0a0b41 KR |
355 | |
356 | uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd) | |
357 | { | |
358 | struct amdgpu_device *adev = (struct amdgpu_device *)kgd; | |
359 | ||
360 | return amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); | |
361 | } |