]>
Commit | Line | Data |
---|---|---|
6a7f76e7 CK |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Christian König | |
23 | */ | |
24 | ||
25 | #include <drm/drmP.h> | |
26 | #include "amdgpu.h" | |
27 | ||
28 | struct amdgpu_vram_mgr { | |
29 | struct drm_mm mm; | |
30 | spinlock_t lock; | |
3c848bb3 CK |
31 | atomic64_t usage; |
32 | atomic64_t vis_usage; | |
6a7f76e7 CK |
33 | }; |
34 | ||
35 | /** | |
36 | * amdgpu_vram_mgr_init - init VRAM manager and DRM MM | |
37 | * | |
38 | * @man: TTM memory type manager | |
39 | * @p_size: maximum size of VRAM | |
40 | * | |
41 | * Allocate and initialize the VRAM manager. | |
42 | */ | |
43 | static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man, | |
44 | unsigned long p_size) | |
45 | { | |
46 | struct amdgpu_vram_mgr *mgr; | |
47 | ||
48 | mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); | |
49 | if (!mgr) | |
50 | return -ENOMEM; | |
51 | ||
52 | drm_mm_init(&mgr->mm, 0, p_size); | |
53 | spin_lock_init(&mgr->lock); | |
54 | man->priv = mgr; | |
55 | return 0; | |
56 | } | |
57 | ||
58 | /** | |
59 | * amdgpu_vram_mgr_fini - free and destroy VRAM manager | |
60 | * | |
61 | * @man: TTM memory type manager | |
62 | * | |
63 | * Destroy and free the VRAM manager, returns -EBUSY if ranges are still | |
64 | * allocated inside it. | |
65 | */ | |
66 | static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man) | |
67 | { | |
68 | struct amdgpu_vram_mgr *mgr = man->priv; | |
69 | ||
70 | spin_lock(&mgr->lock); | |
6a7f76e7 CK |
71 | drm_mm_takedown(&mgr->mm); |
72 | spin_unlock(&mgr->lock); | |
73 | kfree(mgr); | |
74 | man->priv = NULL; | |
75 | return 0; | |
76 | } | |
77 | ||
3c848bb3 CK |
78 | /** |
79 | * amdgpu_vram_mgr_vis_size - Calculate visible node size | |
80 | * | |
81 | * @adev: amdgpu device structure | |
82 | * @node: MM node structure | |
83 | * | |
84 | * Calculate how many bytes of the MM node are inside visible VRAM | |
85 | */ | |
86 | static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, | |
87 | struct drm_mm_node *node) | |
88 | { | |
89 | uint64_t start = node->start << PAGE_SHIFT; | |
90 | uint64_t end = (node->size + node->start) << PAGE_SHIFT; | |
91 | ||
92 | if (start >= adev->mc.visible_vram_size) | |
93 | return 0; | |
94 | ||
95 | return (end > adev->mc.visible_vram_size ? | |
96 | adev->mc.visible_vram_size : end) - start; | |
97 | } | |
98 | ||
e6f8a3c1 MD |
99 | /** |
100 | * amdgpu_vram_mgr_bo_invisible_size - CPU invisible BO size | |
101 | * | |
102 | * @bo: &amdgpu_bo buffer object (must be in VRAM) | |
103 | * | |
104 | * Returns: | |
105 | * How much of the given &amdgpu_bo buffer object lies in CPU invisible VRAM. | |
106 | */ | |
107 | u64 amdgpu_vram_mgr_bo_invisible_size(struct amdgpu_bo *bo) | |
108 | { | |
109 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) | |
110 | return amdgpu_bo_size(bo); | |
111 | ||
112 | return 0; | |
113 | } | |
114 | ||
6a7f76e7 CK |
115 | /** |
116 | * amdgpu_vram_mgr_new - allocate new ranges | |
117 | * | |
118 | * @man: TTM memory type manager | |
119 | * @tbo: TTM BO we need this range for | |
120 | * @place: placement flags and restrictions | |
121 | * @mem: the resulting mem object | |
122 | * | |
123 | * Allocate VRAM for the given BO. | |
124 | */ | |
125 | static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man, | |
126 | struct ttm_buffer_object *tbo, | |
127 | const struct ttm_place *place, | |
128 | struct ttm_mem_reg *mem) | |
129 | { | |
3c848bb3 | 130 | struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); |
6a7f76e7 CK |
131 | struct amdgpu_vram_mgr *mgr = man->priv; |
132 | struct drm_mm *mm = &mgr->mm; | |
133 | struct drm_mm_node *nodes; | |
4e64e553 | 134 | enum drm_mm_insert_mode mode; |
6a7f76e7 | 135 | unsigned long lpfn, num_nodes, pages_per_node, pages_left; |
3c848bb3 | 136 | uint64_t usage = 0, vis_usage = 0; |
6a7f76e7 CK |
137 | unsigned i; |
138 | int r; | |
139 | ||
140 | lpfn = place->lpfn; | |
141 | if (!lpfn) | |
142 | lpfn = man->size; | |
143 | ||
89bb5752 CK |
144 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS || |
145 | amdgpu_vram_page_split == -1) { | |
6a7f76e7 CK |
146 | pages_per_node = ~0ul; |
147 | num_nodes = 1; | |
148 | } else { | |
149 | pages_per_node = max((uint32_t)amdgpu_vram_page_split, | |
150 | mem->page_alignment); | |
151 | num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); | |
152 | } | |
153 | ||
df34f6f2 MD |
154 | nodes = kvmalloc_array(num_nodes, sizeof(*nodes), |
155 | GFP_KERNEL | __GFP_ZERO); | |
6a7f76e7 CK |
156 | if (!nodes) |
157 | return -ENOMEM; | |
158 | ||
4e64e553 CW |
159 | mode = DRM_MM_INSERT_BEST; |
160 | if (place->flags & TTM_PL_FLAG_TOPDOWN) | |
161 | mode = DRM_MM_INSERT_HIGH; | |
6a7f76e7 | 162 | |
89bb5752 | 163 | mem->start = 0; |
6a7f76e7 CK |
164 | pages_left = mem->num_pages; |
165 | ||
166 | spin_lock(&mgr->lock); | |
167 | for (i = 0; i < num_nodes; ++i) { | |
168 | unsigned long pages = min(pages_left, pages_per_node); | |
169 | uint32_t alignment = mem->page_alignment; | |
89bb5752 | 170 | unsigned long start; |
6a7f76e7 CK |
171 | |
172 | if (pages == pages_per_node) | |
173 | alignment = pages_per_node; | |
6a7f76e7 | 174 | |
4e64e553 CW |
175 | r = drm_mm_insert_node_in_range(mm, &nodes[i], |
176 | pages, alignment, 0, | |
177 | place->fpfn, lpfn, | |
178 | mode); | |
6a7f76e7 CK |
179 | if (unlikely(r)) |
180 | goto error; | |
181 | ||
3c848bb3 CK |
182 | usage += nodes[i].size << PAGE_SHIFT; |
183 | vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); | |
184 | ||
89bb5752 CK |
185 | /* Calculate a virtual BO start address to easily check if |
186 | * everything is CPU accessible. | |
187 | */ | |
188 | start = nodes[i].start + nodes[i].size; | |
189 | if (start > mem->num_pages) | |
190 | start -= mem->num_pages; | |
191 | else | |
192 | start = 0; | |
193 | mem->start = max(mem->start, start); | |
6a7f76e7 CK |
194 | pages_left -= pages; |
195 | } | |
196 | spin_unlock(&mgr->lock); | |
197 | ||
3c848bb3 CK |
198 | atomic64_add(usage, &mgr->usage); |
199 | atomic64_add(vis_usage, &mgr->vis_usage); | |
200 | ||
6a7f76e7 CK |
201 | mem->mm_node = nodes; |
202 | ||
203 | return 0; | |
204 | ||
205 | error: | |
206 | while (i--) | |
207 | drm_mm_remove_node(&nodes[i]); | |
208 | spin_unlock(&mgr->lock); | |
209 | ||
df34f6f2 | 210 | kvfree(nodes); |
6a7f76e7 CK |
211 | return r == -ENOSPC ? 0 : r; |
212 | } | |
213 | ||
214 | /** | |
215 | * amdgpu_vram_mgr_del - free ranges | |
216 | * | |
217 | * @man: TTM memory type manager | |
218 | * @tbo: TTM BO we need this range for | |
219 | * @place: placement flags and restrictions | |
220 | * @mem: TTM memory object | |
221 | * | |
222 | * Free the allocated VRAM again. | |
223 | */ | |
224 | static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man, | |
225 | struct ttm_mem_reg *mem) | |
226 | { | |
3c848bb3 | 227 | struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); |
6a7f76e7 CK |
228 | struct amdgpu_vram_mgr *mgr = man->priv; |
229 | struct drm_mm_node *nodes = mem->mm_node; | |
3c848bb3 | 230 | uint64_t usage = 0, vis_usage = 0; |
6a7f76e7 CK |
231 | unsigned pages = mem->num_pages; |
232 | ||
233 | if (!mem->mm_node) | |
234 | return; | |
235 | ||
236 | spin_lock(&mgr->lock); | |
237 | while (pages) { | |
238 | pages -= nodes->size; | |
239 | drm_mm_remove_node(nodes); | |
3c848bb3 CK |
240 | usage += nodes->size << PAGE_SHIFT; |
241 | vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); | |
6a7f76e7 CK |
242 | ++nodes; |
243 | } | |
244 | spin_unlock(&mgr->lock); | |
245 | ||
3c848bb3 CK |
246 | atomic64_sub(usage, &mgr->usage); |
247 | atomic64_sub(vis_usage, &mgr->vis_usage); | |
248 | ||
df34f6f2 | 249 | kvfree(mem->mm_node); |
6a7f76e7 CK |
250 | mem->mm_node = NULL; |
251 | } | |
252 | ||
3c848bb3 CK |
253 | /** |
254 | * amdgpu_vram_mgr_usage - how many bytes are used in this domain | |
255 | * | |
256 | * @man: TTM memory type manager | |
257 | * | |
258 | * Returns how many bytes are used in this domain. | |
259 | */ | |
260 | uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man) | |
261 | { | |
262 | struct amdgpu_vram_mgr *mgr = man->priv; | |
263 | ||
264 | return atomic64_read(&mgr->usage); | |
265 | } | |
266 | ||
267 | /** | |
268 | * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part | |
269 | * | |
270 | * @man: TTM memory type manager | |
271 | * | |
272 | * Returns how many bytes are used in the visible part of VRAM | |
273 | */ | |
274 | uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man) | |
275 | { | |
276 | struct amdgpu_vram_mgr *mgr = man->priv; | |
277 | ||
278 | return atomic64_read(&mgr->vis_usage); | |
279 | } | |
280 | ||
6a7f76e7 CK |
281 | /** |
282 | * amdgpu_vram_mgr_debug - dump VRAM table | |
283 | * | |
284 | * @man: TTM memory type manager | |
373533f8 | 285 | * @printer: DRM printer to use |
6a7f76e7 CK |
286 | * |
287 | * Dump the table content using printk. | |
288 | */ | |
289 | static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man, | |
373533f8 | 290 | struct drm_printer *printer) |
6a7f76e7 CK |
291 | { |
292 | struct amdgpu_vram_mgr *mgr = man->priv; | |
293 | ||
294 | spin_lock(&mgr->lock); | |
373533f8 | 295 | drm_mm_print(&mgr->mm, printer); |
6a7f76e7 | 296 | spin_unlock(&mgr->lock); |
97cbb284 CK |
297 | |
298 | drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", | |
3c848bb3 CK |
299 | man->size, amdgpu_vram_mgr_usage(man) >> 20, |
300 | amdgpu_vram_mgr_vis_usage(man) >> 20); | |
6a7f76e7 CK |
301 | } |
302 | ||
303 | const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = { | |
2a9d6d26 KC |
304 | .init = amdgpu_vram_mgr_init, |
305 | .takedown = amdgpu_vram_mgr_fini, | |
306 | .get_node = amdgpu_vram_mgr_new, | |
307 | .put_node = amdgpu_vram_mgr_del, | |
308 | .debug = amdgpu_vram_mgr_debug | |
6a7f76e7 | 309 | }; |