]>
Commit | Line | Data |
---|---|---|
dff96888 | 1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
fb1d9738 JB |
2 | /************************************************************************** |
3 | * | |
dff96888 | 4 | * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA |
fb1d9738 JB |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |
7 | * copy of this software and associated documentation files (the | |
8 | * "Software"), to deal in the Software without restriction, including | |
9 | * without limitation the rights to use, copy, modify, merge, publish, | |
10 | * distribute, sub license, and/or sell copies of the Software, and to | |
11 | * permit persons to whom the Software is furnished to do so, subject to | |
12 | * the following conditions: | |
13 | * | |
14 | * The above copyright notice and this permission notice (including the | |
15 | * next paragraph) shall be included in all copies or substantial portions | |
16 | * of the Software. | |
17 | * | |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
25 | * | |
26 | **************************************************************************/ | |
27 | ||
28 | #include "vmwgfx_drv.h" | |
760285e7 DH |
29 | #include <drm/ttm/ttm_bo_driver.h> |
30 | #include <drm/ttm/ttm_placement.h> | |
fb1d9738 | 31 | |
9036f8c7 | 32 | static const struct ttm_place vram_placement_flags = { |
f1217ed0 CK |
33 | .fpfn = 0, |
34 | .lpfn = 0, | |
48e07c23 | 35 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 36 | .flags = 0 |
f1217ed0 | 37 | }; |
fb1d9738 | 38 | |
9036f8c7 | 39 | static const struct ttm_place sys_placement_flags = { |
f1217ed0 CK |
40 | .fpfn = 0, |
41 | .lpfn = 0, | |
48e07c23 | 42 | .mem_type = TTM_PL_SYSTEM, |
ce65b874 | 43 | .flags = 0 |
f1217ed0 | 44 | }; |
3530bdc3 | 45 | |
9036f8c7 | 46 | static const struct ttm_place gmr_placement_flags = { |
f1217ed0 CK |
47 | .fpfn = 0, |
48 | .lpfn = 0, | |
48e07c23 | 49 | .mem_type = VMW_PL_GMR, |
ce65b874 | 50 | .flags = 0 |
f1217ed0 | 51 | }; |
d991ef03 | 52 | |
9036f8c7 | 53 | static const struct ttm_place mob_placement_flags = { |
fb1d9738 JB |
54 | .fpfn = 0, |
55 | .lpfn = 0, | |
48e07c23 | 56 | .mem_type = VMW_PL_MOB, |
ce65b874 | 57 | .flags = 0 |
f1217ed0 CK |
58 | }; |
59 | ||
60 | struct ttm_placement vmw_vram_placement = { | |
fb1d9738 JB |
61 | .num_placement = 1, |
62 | .placement = &vram_placement_flags, | |
63 | .num_busy_placement = 1, | |
64 | .busy_placement = &vram_placement_flags | |
65 | }; | |
66 | ||
9036f8c7 | 67 | static const struct ttm_place vram_gmr_placement_flags[] = { |
f1217ed0 CK |
68 | { |
69 | .fpfn = 0, | |
70 | .lpfn = 0, | |
48e07c23 | 71 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 72 | .flags = 0 |
f1217ed0 CK |
73 | }, { |
74 | .fpfn = 0, | |
75 | .lpfn = 0, | |
48e07c23 | 76 | .mem_type = VMW_PL_GMR, |
ce65b874 | 77 | .flags = 0 |
f1217ed0 | 78 | } |
135cba0d TH |
79 | }; |
80 | ||
9036f8c7 | 81 | static const struct ttm_place gmr_vram_placement_flags[] = { |
f1217ed0 CK |
82 | { |
83 | .fpfn = 0, | |
84 | .lpfn = 0, | |
48e07c23 | 85 | .mem_type = VMW_PL_GMR, |
ce65b874 | 86 | .flags = 0 |
f1217ed0 CK |
87 | }, { |
88 | .fpfn = 0, | |
89 | .lpfn = 0, | |
48e07c23 | 90 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 91 | .flags = 0 |
f1217ed0 | 92 | } |
5bb39e81 TH |
93 | }; |
94 | ||
208ae0a6 ZR |
95 | static const struct ttm_place vmw_sys_placement_flags = { |
96 | .fpfn = 0, | |
97 | .lpfn = 0, | |
98 | .mem_type = VMW_PL_SYSTEM, | |
99 | .flags = 0 | |
100 | }; | |
101 | ||
135cba0d | 102 | struct ttm_placement vmw_vram_gmr_placement = { |
135cba0d TH |
103 | .num_placement = 2, |
104 | .placement = vram_gmr_placement_flags, | |
105 | .num_busy_placement = 1, | |
106 | .busy_placement = &gmr_placement_flags | |
107 | }; | |
108 | ||
8ba5152a | 109 | struct ttm_placement vmw_vram_sys_placement = { |
8ba5152a TH |
110 | .num_placement = 1, |
111 | .placement = &vram_placement_flags, | |
112 | .num_busy_placement = 1, | |
113 | .busy_placement = &sys_placement_flags | |
114 | }; | |
115 | ||
fb1d9738 | 116 | struct ttm_placement vmw_sys_placement = { |
fb1d9738 JB |
117 | .num_placement = 1, |
118 | .placement = &sys_placement_flags, | |
119 | .num_busy_placement = 1, | |
120 | .busy_placement = &sys_placement_flags | |
121 | }; | |
122 | ||
208ae0a6 ZR |
123 | struct ttm_placement vmw_pt_sys_placement = { |
124 | .num_placement = 1, | |
125 | .placement = &vmw_sys_placement_flags, | |
126 | .num_busy_placement = 1, | |
127 | .busy_placement = &vmw_sys_placement_flags | |
d991ef03 JB |
128 | }; |
129 | ||
ef86cfee TH |
130 | static const struct ttm_place nonfixed_placement_flags[] = { |
131 | { | |
132 | .fpfn = 0, | |
133 | .lpfn = 0, | |
48e07c23 | 134 | .mem_type = TTM_PL_SYSTEM, |
ce65b874 | 135 | .flags = 0 |
ef86cfee TH |
136 | }, { |
137 | .fpfn = 0, | |
138 | .lpfn = 0, | |
48e07c23 | 139 | .mem_type = VMW_PL_GMR, |
ce65b874 | 140 | .flags = 0 |
ef86cfee TH |
141 | }, { |
142 | .fpfn = 0, | |
143 | .lpfn = 0, | |
48e07c23 | 144 | .mem_type = VMW_PL_MOB, |
ce65b874 | 145 | .flags = 0 |
ef86cfee TH |
146 | } |
147 | }; | |
148 | ||
5bb39e81 | 149 | struct ttm_placement vmw_srf_placement = { |
5bb39e81 TH |
150 | .num_placement = 1, |
151 | .num_busy_placement = 2, | |
152 | .placement = &gmr_placement_flags, | |
153 | .busy_placement = gmr_vram_placement_flags | |
154 | }; | |
155 | ||
6da768aa | 156 | struct ttm_placement vmw_mob_placement = { |
6da768aa TH |
157 | .num_placement = 1, |
158 | .num_busy_placement = 1, | |
159 | .placement = &mob_placement_flags, | |
160 | .busy_placement = &mob_placement_flags | |
161 | }; | |
162 | ||
ef86cfee TH |
163 | struct ttm_placement vmw_nonfixed_placement = { |
164 | .num_placement = 3, | |
165 | .placement = nonfixed_placement_flags, | |
166 | .num_busy_placement = 1, | |
167 | .busy_placement = &sys_placement_flags | |
168 | }; | |
169 | ||
649bf3ca | 170 | struct vmw_ttm_tt { |
e34b8fee | 171 | struct ttm_tt dma_ttm; |
135cba0d TH |
172 | struct vmw_private *dev_priv; |
173 | int gmr_id; | |
6da768aa TH |
174 | struct vmw_mob *mob; |
175 | int mem_type; | |
d92d9851 TH |
176 | struct sg_table sgt; |
177 | struct vmw_sg_table vsgt; | |
178 | uint64_t sg_alloc_size; | |
179 | bool mapped; | |
0b988ca1 | 180 | bool bound; |
fb1d9738 JB |
181 | }; |
182 | ||
308d17ef TH |
183 | const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt); |
184 | ||
d92d9851 | 185 | /** |
2cd80dbd ZR |
186 | * __vmw_piter_non_sg_next: Helper functions to advance |
187 | * a struct vmw_piter iterator. | |
d92d9851 TH |
188 | * |
189 | * @viter: Pointer to the iterator. | |
190 | * | |
191 | * These functions return false if past the end of the list, | |
192 | * true otherwise. Functions are selected depending on the current | |
193 | * DMA mapping mode. | |
194 | */ | |
195 | static bool __vmw_piter_non_sg_next(struct vmw_piter *viter) | |
196 | { | |
197 | return ++(viter->i) < viter->num_pages; | |
198 | } | |
199 | ||
200 | static bool __vmw_piter_sg_next(struct vmw_piter *viter) | |
201 | { | |
8dc39cfc TH |
202 | bool ret = __vmw_piter_non_sg_next(viter); |
203 | ||
204 | return __sg_page_iter_dma_next(&viter->iter) && ret; | |
d92d9851 TH |
205 | } |
206 | ||
207 | ||
d92d9851 TH |
208 | static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter) |
209 | { | |
210 | return viter->addrs[viter->i]; | |
211 | } | |
212 | ||
213 | static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter) | |
214 | { | |
8dc39cfc | 215 | return sg_page_iter_dma_address(&viter->iter); |
d92d9851 TH |
216 | } |
217 | ||
218 | ||
219 | /** | |
220 | * vmw_piter_start - Initialize a struct vmw_piter. | |
221 | * | |
222 | * @viter: Pointer to the iterator to initialize | |
223 | * @vsgt: Pointer to a struct vmw_sg_table to initialize from | |
a38feeaa | 224 | * @p_offset: Pointer offset used to update current array position |
d92d9851 TH |
225 | * |
226 | * Note that we're following the convention of __sg_page_iter_start, so that | |
227 | * the iterator doesn't point to a valid page after initialization; it has | |
228 | * to be advanced one step first. | |
229 | */ | |
230 | void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt, | |
231 | unsigned long p_offset) | |
232 | { | |
233 | viter->i = p_offset - 1; | |
234 | viter->num_pages = vsgt->num_pages; | |
8dc39cfc | 235 | viter->pages = vsgt->pages; |
d92d9851 | 236 | switch (vsgt->mode) { |
d92d9851 TH |
237 | case vmw_dma_alloc_coherent: |
238 | viter->next = &__vmw_piter_non_sg_next; | |
239 | viter->dma_address = &__vmw_piter_dma_addr; | |
d92d9851 TH |
240 | viter->addrs = vsgt->addrs; |
241 | break; | |
242 | case vmw_dma_map_populate: | |
243 | case vmw_dma_map_bind: | |
244 | viter->next = &__vmw_piter_sg_next; | |
245 | viter->dma_address = &__vmw_piter_sg_addr; | |
8dc39cfc | 246 | __sg_page_iter_start(&viter->iter.base, vsgt->sgt->sgl, |
d92d9851 TH |
247 | vsgt->sgt->orig_nents, p_offset); |
248 | break; | |
249 | default: | |
250 | BUG(); | |
251 | } | |
252 | } | |
253 | ||
254 | /** | |
255 | * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for | |
256 | * TTM pages | |
257 | * | |
258 | * @vmw_tt: Pointer to a struct vmw_ttm_backend | |
259 | * | |
260 | * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma. | |
261 | */ | |
262 | static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt) | |
263 | { | |
9703bb32 | 264 | struct device *dev = vmw_tt->dev_priv->drm.dev; |
d92d9851 | 265 | |
c915c2cb | 266 | dma_unmap_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0); |
d92d9851 TH |
267 | vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents; |
268 | } | |
269 | ||
270 | /** | |
271 | * vmw_ttm_map_for_dma - map TTM pages to get device addresses | |
272 | * | |
273 | * @vmw_tt: Pointer to a struct vmw_ttm_backend | |
274 | * | |
275 | * This function is used to get device addresses from the kernel DMA layer. | |
276 | * However, it's violating the DMA API in that when this operation has been | |
277 | * performed, it's illegal for the CPU to write to the pages without first | |
278 | * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is | |
279 | * therefore only legal to call this function if we know that the function | |
280 | * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most | |
281 | * a CPU write buffer flush. | |
282 | */ | |
283 | static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt) | |
284 | { | |
9703bb32 | 285 | struct device *dev = vmw_tt->dev_priv->drm.dev; |
d92d9851 | 286 | |
c915c2cb | 287 | return dma_map_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0); |
d92d9851 TH |
288 | } |
289 | ||
290 | /** | |
291 | * vmw_ttm_map_dma - Make sure TTM pages are visible to the device | |
292 | * | |
293 | * @vmw_tt: Pointer to a struct vmw_ttm_tt | |
294 | * | |
295 | * Select the correct function for and make sure the TTM pages are | |
296 | * visible to the device. Allocate storage for the device mappings. | |
297 | * If a mapping has already been performed, indicated by the storage | |
298 | * pointer being non NULL, the function returns success. | |
299 | */ | |
300 | static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) | |
301 | { | |
302 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
303 | struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); | |
304 | struct vmw_sg_table *vsgt = &vmw_tt->vsgt; | |
279c01f6 RH |
305 | struct ttm_operation_ctx ctx = { |
306 | .interruptible = true, | |
307 | .no_wait_gpu = false | |
308 | }; | |
d92d9851 TH |
309 | struct vmw_piter iter; |
310 | dma_addr_t old; | |
311 | int ret = 0; | |
312 | static size_t sgl_size; | |
313 | static size_t sgt_size; | |
314 | ||
315 | if (vmw_tt->mapped) | |
316 | return 0; | |
317 | ||
318 | vsgt->mode = dev_priv->map_mode; | |
e34b8fee CK |
319 | vsgt->pages = vmw_tt->dma_ttm.pages; |
320 | vsgt->num_pages = vmw_tt->dma_ttm.num_pages; | |
d92d9851 TH |
321 | vsgt->addrs = vmw_tt->dma_ttm.dma_address; |
322 | vsgt->sgt = &vmw_tt->sgt; | |
323 | ||
324 | switch (dev_priv->map_mode) { | |
325 | case vmw_dma_map_bind: | |
326 | case vmw_dma_map_populate: | |
327 | if (unlikely(!sgl_size)) { | |
328 | sgl_size = ttm_round_pot(sizeof(struct scatterlist)); | |
329 | sgt_size = ttm_round_pot(sizeof(struct sg_table)); | |
330 | } | |
331 | vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages; | |
279c01f6 | 332 | ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, &ctx); |
d92d9851 TH |
333 | if (unlikely(ret != 0)) |
334 | return ret; | |
335 | ||
90e7a6de MG |
336 | ret = sg_alloc_table_from_pages_segment( |
337 | &vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, | |
338 | (unsigned long)vsgt->num_pages << PAGE_SHIFT, | |
339 | dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL); | |
340 | if (ret) | |
d92d9851 TH |
341 | goto out_sg_alloc_fail; |
342 | ||
c915c2cb | 343 | if (vsgt->num_pages > vmw_tt->sgt.orig_nents) { |
d92d9851 TH |
344 | uint64_t over_alloc = |
345 | sgl_size * (vsgt->num_pages - | |
c915c2cb | 346 | vmw_tt->sgt.orig_nents); |
d92d9851 TH |
347 | |
348 | ttm_mem_global_free(glob, over_alloc); | |
349 | vmw_tt->sg_alloc_size -= over_alloc; | |
350 | } | |
351 | ||
352 | ret = vmw_ttm_map_for_dma(vmw_tt); | |
353 | if (unlikely(ret != 0)) | |
354 | goto out_map_fail; | |
355 | ||
356 | break; | |
357 | default: | |
358 | break; | |
359 | } | |
360 | ||
361 | old = ~((dma_addr_t) 0); | |
362 | vmw_tt->vsgt.num_regions = 0; | |
363 | for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) { | |
364 | dma_addr_t cur = vmw_piter_dma_addr(&iter); | |
365 | ||
366 | if (cur != old + PAGE_SIZE) | |
367 | vmw_tt->vsgt.num_regions++; | |
368 | old = cur; | |
369 | } | |
370 | ||
371 | vmw_tt->mapped = true; | |
372 | return 0; | |
373 | ||
374 | out_map_fail: | |
375 | sg_free_table(vmw_tt->vsgt.sgt); | |
376 | vmw_tt->vsgt.sgt = NULL; | |
377 | out_sg_alloc_fail: | |
378 | ttm_mem_global_free(glob, vmw_tt->sg_alloc_size); | |
379 | return ret; | |
380 | } | |
381 | ||
382 | /** | |
383 | * vmw_ttm_unmap_dma - Tear down any TTM page device mappings | |
384 | * | |
385 | * @vmw_tt: Pointer to a struct vmw_ttm_tt | |
386 | * | |
387 | * Tear down any previously set up device DMA mappings and free | |
388 | * any storage space allocated for them. If there are no mappings set up, | |
389 | * this function is a NOP. | |
390 | */ | |
391 | static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt) | |
392 | { | |
393 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
394 | ||
395 | if (!vmw_tt->vsgt.sgt) | |
396 | return; | |
397 | ||
398 | switch (dev_priv->map_mode) { | |
399 | case vmw_dma_map_bind: | |
400 | case vmw_dma_map_populate: | |
401 | vmw_ttm_unmap_from_dma(vmw_tt); | |
402 | sg_free_table(vmw_tt->vsgt.sgt); | |
403 | vmw_tt->vsgt.sgt = NULL; | |
404 | ttm_mem_global_free(vmw_mem_glob(dev_priv), | |
405 | vmw_tt->sg_alloc_size); | |
406 | break; | |
407 | default: | |
408 | break; | |
409 | } | |
410 | vmw_tt->mapped = false; | |
411 | } | |
412 | ||
0fd53cfb TH |
413 | /** |
414 | * vmw_bo_sg_table - Return a struct vmw_sg_table object for a | |
415 | * TTM buffer object | |
416 | * | |
417 | * @bo: Pointer to a struct ttm_buffer_object | |
418 | * | |
419 | * Returns a pointer to a struct vmw_sg_table object. The object should | |
420 | * not be freed after use. | |
421 | * Note that for the device addresses to be valid, the buffer object must | |
422 | * either be reserved or pinned. | |
423 | */ | |
424 | const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo) | |
425 | { | |
426 | struct vmw_ttm_tt *vmw_tt = | |
e34b8fee | 427 | container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm); |
0fd53cfb TH |
428 | |
429 | return &vmw_tt->vsgt; | |
430 | } | |
431 | ||
432 | ||
8af8a109 | 433 | static int vmw_ttm_bind(struct ttm_device *bdev, |
0a667b50 | 434 | struct ttm_tt *ttm, struct ttm_resource *bo_mem) |
fb1d9738 | 435 | { |
d92d9851 | 436 | struct vmw_ttm_tt *vmw_be = |
e34b8fee | 437 | container_of(ttm, struct vmw_ttm_tt, dma_ttm); |
0b988ca1 DA |
438 | int ret = 0; |
439 | ||
440 | if (!bo_mem) | |
441 | return -EINVAL; | |
442 | ||
443 | if (vmw_be->bound) | |
444 | return 0; | |
d92d9851 TH |
445 | |
446 | ret = vmw_ttm_map_dma(vmw_be); | |
447 | if (unlikely(ret != 0)) | |
448 | return ret; | |
135cba0d TH |
449 | |
450 | vmw_be->gmr_id = bo_mem->start; | |
6da768aa | 451 | vmw_be->mem_type = bo_mem->mem_type; |
135cba0d | 452 | |
6da768aa TH |
453 | switch (bo_mem->mem_type) { |
454 | case VMW_PL_GMR: | |
0b988ca1 | 455 | ret = vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt, |
6da768aa | 456 | ttm->num_pages, vmw_be->gmr_id); |
0b988ca1 | 457 | break; |
6da768aa TH |
458 | case VMW_PL_MOB: |
459 | if (unlikely(vmw_be->mob == NULL)) { | |
460 | vmw_be->mob = | |
461 | vmw_mob_create(ttm->num_pages); | |
462 | if (unlikely(vmw_be->mob == NULL)) | |
463 | return -ENOMEM; | |
464 | } | |
465 | ||
0b988ca1 | 466 | ret = vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob, |
0fd53cfb | 467 | &vmw_be->vsgt, ttm->num_pages, |
6da768aa | 468 | vmw_be->gmr_id); |
0b988ca1 | 469 | break; |
208ae0a6 ZR |
470 | case VMW_PL_SYSTEM: |
471 | /* Nothing to be done for a system bind */ | |
472 | break; | |
6da768aa TH |
473 | default: |
474 | BUG(); | |
475 | } | |
0b988ca1 DA |
476 | vmw_be->bound = true; |
477 | return ret; | |
fb1d9738 JB |
478 | } |
479 | ||
8af8a109 | 480 | static void vmw_ttm_unbind(struct ttm_device *bdev, |
0a667b50 | 481 | struct ttm_tt *ttm) |
fb1d9738 | 482 | { |
d92d9851 | 483 | struct vmw_ttm_tt *vmw_be = |
e34b8fee | 484 | container_of(ttm, struct vmw_ttm_tt, dma_ttm); |
135cba0d | 485 | |
0b988ca1 DA |
486 | if (!vmw_be->bound) |
487 | return; | |
488 | ||
6da768aa TH |
489 | switch (vmw_be->mem_type) { |
490 | case VMW_PL_GMR: | |
491 | vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id); | |
492 | break; | |
493 | case VMW_PL_MOB: | |
494 | vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob); | |
495 | break; | |
208ae0a6 ZR |
496 | case VMW_PL_SYSTEM: |
497 | break; | |
6da768aa TH |
498 | default: |
499 | BUG(); | |
500 | } | |
d92d9851 TH |
501 | |
502 | if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind) | |
503 | vmw_ttm_unmap_dma(vmw_be); | |
0b988ca1 | 504 | vmw_be->bound = false; |
fb1d9738 JB |
505 | } |
506 | ||
6da768aa | 507 | |
8af8a109 | 508 | static void vmw_ttm_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) |
fb1d9738 | 509 | { |
d92d9851 | 510 | struct vmw_ttm_tt *vmw_be = |
e34b8fee | 511 | container_of(ttm, struct vmw_ttm_tt, dma_ttm); |
d92d9851 | 512 | |
37bff654 | 513 | vmw_ttm_unbind(bdev, ttm); |
7626168f | 514 | ttm_tt_destroy_common(bdev, ttm); |
d92d9851 TH |
515 | vmw_ttm_unmap_dma(vmw_be); |
516 | if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) | |
e34b8fee | 517 | ttm_tt_fini(&vmw_be->dma_ttm); |
d92d9851 TH |
518 | else |
519 | ttm_tt_fini(ttm); | |
6da768aa TH |
520 | |
521 | if (vmw_be->mob) | |
522 | vmw_mob_destroy(vmw_be->mob); | |
523 | ||
fb1d9738 JB |
524 | kfree(vmw_be); |
525 | } | |
526 | ||
0fd53cfb | 527 | |
8af8a109 | 528 | static int vmw_ttm_populate(struct ttm_device *bdev, |
0a667b50 | 529 | struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) |
d92d9851 | 530 | { |
f07069da CK |
531 | unsigned int i; |
532 | int ret; | |
533 | ||
8567d515 | 534 | /* TODO: maybe completely drop this ? */ |
7eec9151 | 535 | if (ttm_tt_is_populated(ttm)) |
d92d9851 TH |
536 | return 0; |
537 | ||
f07069da CK |
538 | ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); |
539 | if (ret) | |
540 | return ret; | |
541 | ||
542 | for (i = 0; i < ttm->num_pages; ++i) { | |
543 | ret = ttm_mem_global_alloc_page(&ttm_mem_glob, ttm->pages[i], | |
544 | PAGE_SIZE, ctx); | |
545 | if (ret) | |
546 | goto error; | |
547 | } | |
548 | return 0; | |
549 | ||
550 | error: | |
551 | while (i--) | |
552 | ttm_mem_global_free_page(&ttm_mem_glob, ttm->pages[i], | |
553 | PAGE_SIZE); | |
554 | ttm_pool_free(&bdev->pool, ttm); | |
555 | return ret; | |
d92d9851 TH |
556 | } |
557 | ||
8af8a109 | 558 | static void vmw_ttm_unpopulate(struct ttm_device *bdev, |
0a667b50 | 559 | struct ttm_tt *ttm) |
d92d9851 TH |
560 | { |
561 | struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt, | |
e34b8fee | 562 | dma_ttm); |
f07069da | 563 | unsigned int i; |
6da768aa TH |
564 | |
565 | if (vmw_tt->mob) { | |
566 | vmw_mob_destroy(vmw_tt->mob); | |
567 | vmw_tt->mob = NULL; | |
568 | } | |
569 | ||
d92d9851 | 570 | vmw_ttm_unmap_dma(vmw_tt); |
f07069da CK |
571 | |
572 | for (i = 0; i < ttm->num_pages; ++i) | |
573 | ttm_mem_global_free_page(&ttm_mem_glob, ttm->pages[i], | |
574 | PAGE_SIZE); | |
575 | ||
8567d515 | 576 | ttm_pool_free(&bdev->pool, ttm); |
d92d9851 TH |
577 | } |
578 | ||
dde5da23 CK |
579 | static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo, |
580 | uint32_t page_flags) | |
fb1d9738 | 581 | { |
649bf3ca | 582 | struct vmw_ttm_tt *vmw_be; |
d92d9851 | 583 | int ret; |
fb1d9738 | 584 | |
d92d9851 | 585 | vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL); |
fb1d9738 JB |
586 | if (!vmw_be) |
587 | return NULL; | |
588 | ||
dde5da23 | 589 | vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev); |
6da768aa | 590 | vmw_be->mob = NULL; |
fb1d9738 | 591 | |
d92d9851 | 592 | if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) |
4c515bb1 CK |
593 | ret = ttm_sg_tt_init(&vmw_be->dma_ttm, bo, page_flags, |
594 | ttm_cached); | |
d92d9851 | 595 | else |
e34b8fee | 596 | ret = ttm_tt_init(&vmw_be->dma_ttm, bo, page_flags, |
1b4ea4c5 | 597 | ttm_cached); |
d92d9851 TH |
598 | if (unlikely(ret != 0)) |
599 | goto out_no_init; | |
600 | ||
e34b8fee | 601 | return &vmw_be->dma_ttm; |
d92d9851 TH |
602 | out_no_init: |
603 | kfree(vmw_be); | |
604 | return NULL; | |
fb1d9738 JB |
605 | } |
606 | ||
8227622f | 607 | static void vmw_evict_flags(struct ttm_buffer_object *bo, |
fb1d9738 JB |
608 | struct ttm_placement *placement) |
609 | { | |
610 | *placement = vmw_sys_placement; | |
611 | } | |
612 | ||
8af8a109 | 613 | static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) |
96bf8b87 | 614 | { |
96bf8b87 JG |
615 | struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev); |
616 | ||
96bf8b87 JG |
617 | switch (mem->mem_type) { |
618 | case TTM_PL_SYSTEM: | |
208ae0a6 | 619 | case VMW_PL_SYSTEM: |
135cba0d | 620 | case VMW_PL_GMR: |
6da768aa | 621 | case VMW_PL_MOB: |
96bf8b87 JG |
622 | return 0; |
623 | case TTM_PL_VRAM: | |
54d04ea8 CK |
624 | mem->bus.offset = (mem->start << PAGE_SHIFT) + |
625 | dev_priv->vram_start; | |
96bf8b87 | 626 | mem->bus.is_iomem = true; |
1cf65c45 | 627 | mem->bus.caching = ttm_cached; |
96bf8b87 JG |
628 | break; |
629 | default: | |
630 | return -EINVAL; | |
631 | } | |
632 | return 0; | |
633 | } | |
634 | ||
6da768aa TH |
635 | /** |
636 | * vmw_move_notify - TTM move_notify_callback | |
637 | * | |
fd11a3c0 | 638 | * @bo: The TTM buffer object about to move. |
6cf9dc23 CK |
639 | * @old_mem: The old memory where we move from |
640 | * @new_mem: The struct ttm_resource indicating to what memory | |
fd11a3c0 | 641 | * region the move is taking place. |
6da768aa TH |
642 | * |
643 | * Calls move_notify for all subsystems needing it. | |
644 | * (currently only resources). | |
645 | */ | |
646 | static void vmw_move_notify(struct ttm_buffer_object *bo, | |
6cf9dc23 CK |
647 | struct ttm_resource *old_mem, |
648 | struct ttm_resource *new_mem) | |
6da768aa | 649 | { |
6cf9dc23 CK |
650 | vmw_bo_move_notify(bo, new_mem); |
651 | vmw_query_move_notify(bo, old_mem, new_mem); | |
6da768aa TH |
652 | } |
653 | ||
654 | ||
655 | /** | |
656 | * vmw_swap_notify - TTM move_notify_callback | |
657 | * | |
fd11a3c0 | 658 | * @bo: The TTM buffer object about to be swapped out. |
6da768aa TH |
659 | */ |
660 | static void vmw_swap_notify(struct ttm_buffer_object *bo) | |
661 | { | |
e9431ea5 | 662 | vmw_bo_swap_notify(bo); |
f08c86c3 | 663 | (void) ttm_bo_wait(bo, false, false); |
6da768aa TH |
664 | } |
665 | ||
208ae0a6 ZR |
666 | static bool vmw_memtype_is_system(uint32_t mem_type) |
667 | { | |
668 | return mem_type == TTM_PL_SYSTEM || mem_type == VMW_PL_SYSTEM; | |
669 | } | |
670 | ||
bcff5d3e DA |
671 | static int vmw_move(struct ttm_buffer_object *bo, |
672 | bool evict, | |
673 | struct ttm_operation_ctx *ctx, | |
ebdf5651 DA |
674 | struct ttm_resource *new_mem, |
675 | struct ttm_place *hop) | |
bcff5d3e | 676 | { |
d3116756 | 677 | struct ttm_resource_manager *old_man = ttm_manager_type(bo->bdev, bo->resource->mem_type); |
bcff5d3e | 678 | struct ttm_resource_manager *new_man = ttm_manager_type(bo->bdev, new_mem->mem_type); |
c37d951c | 679 | int ret; |
bcff5d3e | 680 | |
208ae0a6 | 681 | if (new_man->use_tt && !vmw_memtype_is_system(new_mem->mem_type)) { |
bfe5e585 DA |
682 | ret = vmw_ttm_bind(bo->bdev, bo->ttm, new_mem); |
683 | if (ret) | |
684 | return ret; | |
685 | } | |
686 | ||
d3116756 | 687 | vmw_move_notify(bo, bo->resource, new_mem); |
6d820003 | 688 | |
bcff5d3e | 689 | if (old_man->use_tt && new_man->use_tt) { |
208ae0a6 | 690 | if (vmw_memtype_is_system(bo->resource->mem_type)) { |
5b7a2c92 | 691 | ttm_bo_move_null(bo, new_mem); |
bcff5d3e DA |
692 | return 0; |
693 | } | |
29a1d482 | 694 | ret = ttm_bo_wait_ctx(bo, ctx); |
c37d951c | 695 | if (ret) |
6d820003 | 696 | goto fail; |
29a1d482 DA |
697 | |
698 | vmw_ttm_unbind(bo->bdev, bo->ttm); | |
bfa3357e | 699 | ttm_resource_free(bo, &bo->resource); |
c37d951c DA |
700 | ttm_bo_assign_mem(bo, new_mem); |
701 | return 0; | |
bcff5d3e | 702 | } else { |
6d820003 DA |
703 | ret = ttm_bo_move_memcpy(bo, ctx, new_mem); |
704 | if (ret) | |
705 | goto fail; | |
bcff5d3e | 706 | } |
6d820003 DA |
707 | return 0; |
708 | fail: | |
d3116756 | 709 | vmw_move_notify(bo, new_mem, bo->resource); |
6d820003 | 710 | return ret; |
bcff5d3e | 711 | } |
6da768aa | 712 | |
8af8a109 | 713 | struct ttm_device_funcs vmw_bo_driver = { |
649bf3ca | 714 | .ttm_tt_create = &vmw_ttm_tt_create, |
d92d9851 TH |
715 | .ttm_tt_populate = &vmw_ttm_populate, |
716 | .ttm_tt_unpopulate = &vmw_ttm_unpopulate, | |
debf8ab9 | 717 | .ttm_tt_destroy = &vmw_ttm_destroy, |
a2ab19fe | 718 | .eviction_valuable = ttm_bo_eviction_valuable, |
fb1d9738 | 719 | .evict_flags = vmw_evict_flags, |
bcff5d3e | 720 | .move = vmw_move, |
6da768aa | 721 | .swap_notify = vmw_swap_notify, |
96bf8b87 | 722 | .io_mem_reserve = &vmw_ttm_io_mem_reserve, |
fb1d9738 | 723 | }; |
56dc01f1 DA |
724 | |
725 | int vmw_bo_create_and_populate(struct vmw_private *dev_priv, | |
726 | unsigned long bo_size, | |
727 | struct ttm_buffer_object **bo_p) | |
728 | { | |
729 | struct ttm_operation_ctx ctx = { | |
730 | .interruptible = false, | |
731 | .no_wait_gpu = false | |
732 | }; | |
733 | struct ttm_buffer_object *bo; | |
734 | int ret; | |
735 | ||
b254557c | 736 | ret = vmw_bo_create_kernel(dev_priv, bo_size, |
208ae0a6 | 737 | &vmw_pt_sys_placement, |
b254557c | 738 | &bo); |
56dc01f1 DA |
739 | if (unlikely(ret != 0)) |
740 | return ret; | |
741 | ||
742 | ret = ttm_bo_reserve(bo, false, true, NULL); | |
743 | BUG_ON(ret != 0); | |
0a667b50 | 744 | ret = vmw_ttm_populate(bo->bdev, bo->ttm, &ctx); |
a2d6ddc4 DA |
745 | if (likely(ret == 0)) { |
746 | struct vmw_ttm_tt *vmw_tt = | |
e34b8fee | 747 | container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm); |
a2d6ddc4 DA |
748 | ret = vmw_ttm_map_dma(vmw_tt); |
749 | } | |
56dc01f1 DA |
750 | |
751 | ttm_bo_unreserve(bo); | |
752 | ||
753 | if (likely(ret == 0)) | |
754 | *bo_p = bo; | |
755 | return ret; | |
756 | } |