]>
Commit | Line | Data |
---|---|---|
6a7f76e7 CK |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Christian König | |
23 | */ | |
24 | ||
25 | #include <drm/drmP.h> | |
26 | #include "amdgpu.h" | |
27 | ||
28 | struct amdgpu_vram_mgr { | |
29 | struct drm_mm mm; | |
30 | spinlock_t lock; | |
3c848bb3 CK |
31 | atomic64_t usage; |
32 | atomic64_t vis_usage; | |
6a7f76e7 CK |
33 | }; |
34 | ||
35 | /** | |
36 | * amdgpu_vram_mgr_init - init VRAM manager and DRM MM | |
37 | * | |
38 | * @man: TTM memory type manager | |
39 | * @p_size: maximum size of VRAM | |
40 | * | |
41 | * Allocate and initialize the VRAM manager. | |
42 | */ | |
43 | static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man, | |
44 | unsigned long p_size) | |
45 | { | |
46 | struct amdgpu_vram_mgr *mgr; | |
47 | ||
48 | mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); | |
49 | if (!mgr) | |
50 | return -ENOMEM; | |
51 | ||
52 | drm_mm_init(&mgr->mm, 0, p_size); | |
53 | spin_lock_init(&mgr->lock); | |
54 | man->priv = mgr; | |
55 | return 0; | |
56 | } | |
57 | ||
58 | /** | |
59 | * amdgpu_vram_mgr_fini - free and destroy VRAM manager | |
60 | * | |
61 | * @man: TTM memory type manager | |
62 | * | |
63 | * Destroy and free the VRAM manager, returns -EBUSY if ranges are still | |
64 | * allocated inside it. | |
65 | */ | |
66 | static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man) | |
67 | { | |
68 | struct amdgpu_vram_mgr *mgr = man->priv; | |
69 | ||
70 | spin_lock(&mgr->lock); | |
71 | if (!drm_mm_clean(&mgr->mm)) { | |
72 | spin_unlock(&mgr->lock); | |
73 | return -EBUSY; | |
74 | } | |
75 | ||
76 | drm_mm_takedown(&mgr->mm); | |
77 | spin_unlock(&mgr->lock); | |
78 | kfree(mgr); | |
79 | man->priv = NULL; | |
80 | return 0; | |
81 | } | |
82 | ||
3c848bb3 CK |
83 | /** |
84 | * amdgpu_vram_mgr_vis_size - Calculate visible node size | |
85 | * | |
86 | * @adev: amdgpu device structure | |
87 | * @node: MM node structure | |
88 | * | |
89 | * Calculate how many bytes of the MM node are inside visible VRAM | |
90 | */ | |
91 | static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, | |
92 | struct drm_mm_node *node) | |
93 | { | |
94 | uint64_t start = node->start << PAGE_SHIFT; | |
95 | uint64_t end = (node->size + node->start) << PAGE_SHIFT; | |
96 | ||
97 | if (start >= adev->mc.visible_vram_size) | |
98 | return 0; | |
99 | ||
100 | return (end > adev->mc.visible_vram_size ? | |
101 | adev->mc.visible_vram_size : end) - start; | |
102 | } | |
103 | ||
6a7f76e7 CK |
104 | /** |
105 | * amdgpu_vram_mgr_new - allocate new ranges | |
106 | * | |
107 | * @man: TTM memory type manager | |
108 | * @tbo: TTM BO we need this range for | |
109 | * @place: placement flags and restrictions | |
110 | * @mem: the resulting mem object | |
111 | * | |
112 | * Allocate VRAM for the given BO. | |
113 | */ | |
114 | static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man, | |
115 | struct ttm_buffer_object *tbo, | |
116 | const struct ttm_place *place, | |
117 | struct ttm_mem_reg *mem) | |
118 | { | |
3c848bb3 | 119 | struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); |
6a7f76e7 CK |
120 | struct amdgpu_vram_mgr *mgr = man->priv; |
121 | struct drm_mm *mm = &mgr->mm; | |
122 | struct drm_mm_node *nodes; | |
4e64e553 | 123 | enum drm_mm_insert_mode mode; |
6a7f76e7 | 124 | unsigned long lpfn, num_nodes, pages_per_node, pages_left; |
3c848bb3 | 125 | uint64_t usage = 0, vis_usage = 0; |
6a7f76e7 CK |
126 | unsigned i; |
127 | int r; | |
128 | ||
129 | lpfn = place->lpfn; | |
130 | if (!lpfn) | |
131 | lpfn = man->size; | |
132 | ||
89bb5752 CK |
133 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS || |
134 | amdgpu_vram_page_split == -1) { | |
6a7f76e7 CK |
135 | pages_per_node = ~0ul; |
136 | num_nodes = 1; | |
137 | } else { | |
138 | pages_per_node = max((uint32_t)amdgpu_vram_page_split, | |
139 | mem->page_alignment); | |
140 | num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); | |
141 | } | |
142 | ||
143 | nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_KERNEL); | |
144 | if (!nodes) | |
145 | return -ENOMEM; | |
146 | ||
4e64e553 CW |
147 | mode = DRM_MM_INSERT_BEST; |
148 | if (place->flags & TTM_PL_FLAG_TOPDOWN) | |
149 | mode = DRM_MM_INSERT_HIGH; | |
6a7f76e7 | 150 | |
89bb5752 | 151 | mem->start = 0; |
6a7f76e7 CK |
152 | pages_left = mem->num_pages; |
153 | ||
154 | spin_lock(&mgr->lock); | |
155 | for (i = 0; i < num_nodes; ++i) { | |
156 | unsigned long pages = min(pages_left, pages_per_node); | |
157 | uint32_t alignment = mem->page_alignment; | |
89bb5752 | 158 | unsigned long start; |
6a7f76e7 CK |
159 | |
160 | if (pages == pages_per_node) | |
161 | alignment = pages_per_node; | |
6a7f76e7 | 162 | |
4e64e553 CW |
163 | r = drm_mm_insert_node_in_range(mm, &nodes[i], |
164 | pages, alignment, 0, | |
165 | place->fpfn, lpfn, | |
166 | mode); | |
6a7f76e7 CK |
167 | if (unlikely(r)) |
168 | goto error; | |
169 | ||
3c848bb3 CK |
170 | usage += nodes[i].size << PAGE_SHIFT; |
171 | vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); | |
172 | ||
89bb5752 CK |
173 | /* Calculate a virtual BO start address to easily check if |
174 | * everything is CPU accessible. | |
175 | */ | |
176 | start = nodes[i].start + nodes[i].size; | |
177 | if (start > mem->num_pages) | |
178 | start -= mem->num_pages; | |
179 | else | |
180 | start = 0; | |
181 | mem->start = max(mem->start, start); | |
6a7f76e7 CK |
182 | pages_left -= pages; |
183 | } | |
184 | spin_unlock(&mgr->lock); | |
185 | ||
3c848bb3 CK |
186 | atomic64_add(usage, &mgr->usage); |
187 | atomic64_add(vis_usage, &mgr->vis_usage); | |
188 | ||
6a7f76e7 CK |
189 | mem->mm_node = nodes; |
190 | ||
191 | return 0; | |
192 | ||
193 | error: | |
194 | while (i--) | |
195 | drm_mm_remove_node(&nodes[i]); | |
196 | spin_unlock(&mgr->lock); | |
197 | ||
198 | kfree(nodes); | |
199 | return r == -ENOSPC ? 0 : r; | |
200 | } | |
201 | ||
202 | /** | |
203 | * amdgpu_vram_mgr_del - free ranges | |
204 | * | |
205 | * @man: TTM memory type manager | |
206 | * @tbo: TTM BO we need this range for | |
207 | * @place: placement flags and restrictions | |
208 | * @mem: TTM memory object | |
209 | * | |
210 | * Free the allocated VRAM again. | |
211 | */ | |
212 | static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man, | |
213 | struct ttm_mem_reg *mem) | |
214 | { | |
3c848bb3 | 215 | struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); |
6a7f76e7 CK |
216 | struct amdgpu_vram_mgr *mgr = man->priv; |
217 | struct drm_mm_node *nodes = mem->mm_node; | |
3c848bb3 | 218 | uint64_t usage = 0, vis_usage = 0; |
6a7f76e7 CK |
219 | unsigned pages = mem->num_pages; |
220 | ||
221 | if (!mem->mm_node) | |
222 | return; | |
223 | ||
224 | spin_lock(&mgr->lock); | |
225 | while (pages) { | |
226 | pages -= nodes->size; | |
227 | drm_mm_remove_node(nodes); | |
3c848bb3 CK |
228 | usage += nodes->size << PAGE_SHIFT; |
229 | vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); | |
6a7f76e7 CK |
230 | ++nodes; |
231 | } | |
232 | spin_unlock(&mgr->lock); | |
233 | ||
3c848bb3 CK |
234 | atomic64_sub(usage, &mgr->usage); |
235 | atomic64_sub(vis_usage, &mgr->vis_usage); | |
236 | ||
6a7f76e7 CK |
237 | kfree(mem->mm_node); |
238 | mem->mm_node = NULL; | |
239 | } | |
240 | ||
3c848bb3 CK |
241 | /** |
242 | * amdgpu_vram_mgr_usage - how many bytes are used in this domain | |
243 | * | |
244 | * @man: TTM memory type manager | |
245 | * | |
246 | * Returns how many bytes are used in this domain. | |
247 | */ | |
248 | uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man) | |
249 | { | |
250 | struct amdgpu_vram_mgr *mgr = man->priv; | |
251 | ||
252 | return atomic64_read(&mgr->usage); | |
253 | } | |
254 | ||
255 | /** | |
256 | * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part | |
257 | * | |
258 | * @man: TTM memory type manager | |
259 | * | |
260 | * Returns how many bytes are used in the visible part of VRAM | |
261 | */ | |
262 | uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man) | |
263 | { | |
264 | struct amdgpu_vram_mgr *mgr = man->priv; | |
265 | ||
266 | return atomic64_read(&mgr->vis_usage); | |
267 | } | |
268 | ||
6a7f76e7 CK |
269 | /** |
270 | * amdgpu_vram_mgr_debug - dump VRAM table | |
271 | * | |
272 | * @man: TTM memory type manager | |
373533f8 | 273 | * @printer: DRM printer to use |
6a7f76e7 CK |
274 | * |
275 | * Dump the table content using printk. | |
276 | */ | |
277 | static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man, | |
373533f8 | 278 | struct drm_printer *printer) |
6a7f76e7 CK |
279 | { |
280 | struct amdgpu_vram_mgr *mgr = man->priv; | |
281 | ||
282 | spin_lock(&mgr->lock); | |
373533f8 | 283 | drm_mm_print(&mgr->mm, printer); |
6a7f76e7 | 284 | spin_unlock(&mgr->lock); |
97cbb284 CK |
285 | |
286 | drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", | |
3c848bb3 CK |
287 | man->size, amdgpu_vram_mgr_usage(man) >> 20, |
288 | amdgpu_vram_mgr_vis_usage(man) >> 20); | |
6a7f76e7 CK |
289 | } |
290 | ||
291 | const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = { | |
2a9d6d26 KC |
292 | .init = amdgpu_vram_mgr_init, |
293 | .takedown = amdgpu_vram_mgr_fini, | |
294 | .get_node = amdgpu_vram_mgr_new, | |
295 | .put_node = amdgpu_vram_mgr_del, | |
296 | .debug = amdgpu_vram_mgr_debug | |
6a7f76e7 | 297 | }; |