]>
Commit | Line | Data |
---|---|---|
76aaf220 DV |
1 | /* |
2 | * Copyright © 2010 Daniel Vetter | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
760285e7 DH |
25 | #include <drm/drmP.h> |
26 | #include <drm/i915_drm.h> | |
76aaf220 DV |
27 | #include "i915_drv.h" |
28 | #include "i915_trace.h" | |
29 | #include "intel_drv.h" | |
30 | ||
1d2a314c DV |
31 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
32 | static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, | |
33 | unsigned first_entry, | |
34 | unsigned num_entries) | |
35 | { | |
1d2a314c DV |
36 | uint32_t *pt_vaddr; |
37 | uint32_t scratch_pte; | |
7bddb01f DV |
38 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
39 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; | |
40 | unsigned last_pte, i; | |
1d2a314c DV |
41 | |
42 | scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); | |
43 | scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; | |
44 | ||
7bddb01f DV |
45 | while (num_entries) { |
46 | last_pte = first_pte + num_entries; | |
47 | if (last_pte > I915_PPGTT_PT_ENTRIES) | |
48 | last_pte = I915_PPGTT_PT_ENTRIES; | |
49 | ||
50 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); | |
1d2a314c | 51 | |
7bddb01f DV |
52 | for (i = first_pte; i < last_pte; i++) |
53 | pt_vaddr[i] = scratch_pte; | |
1d2a314c DV |
54 | |
55 | kunmap_atomic(pt_vaddr); | |
1d2a314c | 56 | |
7bddb01f DV |
57 | num_entries -= last_pte - first_pte; |
58 | first_pte = 0; | |
59 | act_pd++; | |
60 | } | |
1d2a314c DV |
61 | } |
62 | ||
63 | int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) | |
64 | { | |
65 | struct drm_i915_private *dev_priv = dev->dev_private; | |
66 | struct i915_hw_ppgtt *ppgtt; | |
1d2a314c | 67 | unsigned first_pd_entry_in_global_pt; |
1d2a314c DV |
68 | int i; |
69 | int ret = -ENOMEM; | |
70 | ||
71 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 | |
72 | * entries. For aliasing ppgtt support we just steal them at the end for | |
73 | * now. */ | |
9a0f938b | 74 | first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES; |
1d2a314c DV |
75 | |
76 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); | |
77 | if (!ppgtt) | |
78 | return ret; | |
79 | ||
8f2c59f0 | 80 | ppgtt->dev = dev; |
1d2a314c DV |
81 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
82 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, | |
83 | GFP_KERNEL); | |
84 | if (!ppgtt->pt_pages) | |
85 | goto err_ppgtt; | |
86 | ||
87 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
88 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); | |
89 | if (!ppgtt->pt_pages[i]) | |
90 | goto err_pt_alloc; | |
91 | } | |
92 | ||
93 | if (dev_priv->mm.gtt->needs_dmar) { | |
94 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) | |
95 | *ppgtt->num_pd_entries, | |
96 | GFP_KERNEL); | |
97 | if (!ppgtt->pt_dma_addr) | |
98 | goto err_pt_alloc; | |
1d2a314c | 99 | |
211c568b DV |
100 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
101 | dma_addr_t pt_addr; | |
102 | ||
1d2a314c DV |
103 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], |
104 | 0, 4096, | |
105 | PCI_DMA_BIDIRECTIONAL); | |
106 | ||
107 | if (pci_dma_mapping_error(dev->pdev, | |
108 | pt_addr)) { | |
109 | ret = -EIO; | |
110 | goto err_pd_pin; | |
111 | ||
112 | } | |
113 | ppgtt->pt_dma_addr[i] = pt_addr; | |
211c568b | 114 | } |
1d2a314c | 115 | } |
1d2a314c DV |
116 | |
117 | ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; | |
118 | ||
119 | i915_ppgtt_clear_range(ppgtt, 0, | |
120 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); | |
121 | ||
122 | ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t); | |
123 | ||
124 | dev_priv->mm.aliasing_ppgtt = ppgtt; | |
125 | ||
126 | return 0; | |
127 | ||
128 | err_pd_pin: | |
129 | if (ppgtt->pt_dma_addr) { | |
130 | for (i--; i >= 0; i--) | |
131 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
132 | 4096, PCI_DMA_BIDIRECTIONAL); | |
133 | } | |
134 | err_pt_alloc: | |
135 | kfree(ppgtt->pt_dma_addr); | |
136 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
137 | if (ppgtt->pt_pages[i]) | |
138 | __free_page(ppgtt->pt_pages[i]); | |
139 | } | |
140 | kfree(ppgtt->pt_pages); | |
141 | err_ppgtt: | |
142 | kfree(ppgtt); | |
143 | ||
144 | return ret; | |
145 | } | |
146 | ||
147 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) | |
148 | { | |
149 | struct drm_i915_private *dev_priv = dev->dev_private; | |
150 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; | |
151 | int i; | |
152 | ||
153 | if (!ppgtt) | |
154 | return; | |
155 | ||
156 | if (ppgtt->pt_dma_addr) { | |
157 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
158 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
159 | 4096, PCI_DMA_BIDIRECTIONAL); | |
160 | } | |
161 | ||
162 | kfree(ppgtt->pt_dma_addr); | |
163 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
164 | __free_page(ppgtt->pt_pages[i]); | |
165 | kfree(ppgtt->pt_pages); | |
166 | kfree(ppgtt); | |
167 | } | |
168 | ||
7bddb01f | 169 | static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt, |
9da3da66 | 170 | const struct sg_table *pages, |
7bddb01f DV |
171 | unsigned first_entry, |
172 | uint32_t pte_flags) | |
173 | { | |
174 | uint32_t *pt_vaddr, pte; | |
175 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; | |
176 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; | |
177 | unsigned i, j, m, segment_len; | |
178 | dma_addr_t page_addr; | |
179 | struct scatterlist *sg; | |
180 | ||
181 | /* init sg walking */ | |
9da3da66 | 182 | sg = pages->sgl; |
7bddb01f DV |
183 | i = 0; |
184 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; | |
185 | m = 0; | |
186 | ||
9da3da66 | 187 | while (i < pages->nents) { |
7bddb01f DV |
188 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); |
189 | ||
190 | for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { | |
191 | page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); | |
192 | pte = GEN6_PTE_ADDR_ENCODE(page_addr); | |
193 | pt_vaddr[j] = pte | pte_flags; | |
194 | ||
195 | /* grab the next page */ | |
9da3da66 CW |
196 | if (++m == segment_len) { |
197 | if (++i == pages->nents) | |
7bddb01f DV |
198 | break; |
199 | ||
9da3da66 | 200 | sg = sg_next(sg); |
7bddb01f DV |
201 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
202 | m = 0; | |
203 | } | |
204 | } | |
205 | ||
206 | kunmap_atomic(pt_vaddr); | |
207 | ||
208 | first_pte = 0; | |
209 | act_pd++; | |
210 | } | |
211 | } | |
212 | ||
7bddb01f DV |
213 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
214 | struct drm_i915_gem_object *obj, | |
215 | enum i915_cache_level cache_level) | |
216 | { | |
7bddb01f DV |
217 | uint32_t pte_flags = GEN6_PTE_VALID; |
218 | ||
219 | switch (cache_level) { | |
220 | case I915_CACHE_LLC_MLC: | |
8693607a | 221 | /* Haswell doesn't set L3 this way */ |
8f2c59f0 | 222 | if (IS_HASWELL(ppgtt->dev)) |
8693607a BW |
223 | pte_flags |= GEN6_PTE_CACHE_LLC; |
224 | else | |
225 | pte_flags |= GEN6_PTE_CACHE_LLC_MLC; | |
7bddb01f DV |
226 | break; |
227 | case I915_CACHE_LLC: | |
228 | pte_flags |= GEN6_PTE_CACHE_LLC; | |
229 | break; | |
230 | case I915_CACHE_NONE: | |
8f2c59f0 | 231 | if (IS_HASWELL(ppgtt->dev)) |
a843af18 DV |
232 | pte_flags |= HSW_PTE_UNCACHED; |
233 | else | |
234 | pte_flags |= GEN6_PTE_UNCACHED; | |
7bddb01f DV |
235 | break; |
236 | default: | |
237 | BUG(); | |
238 | } | |
239 | ||
9da3da66 | 240 | i915_ppgtt_insert_sg_entries(ppgtt, |
2f745ad3 | 241 | obj->pages, |
9da3da66 CW |
242 | obj->gtt_space->start >> PAGE_SHIFT, |
243 | pte_flags); | |
7bddb01f DV |
244 | } |
245 | ||
246 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, | |
247 | struct drm_i915_gem_object *obj) | |
248 | { | |
249 | i915_ppgtt_clear_range(ppgtt, | |
250 | obj->gtt_space->start >> PAGE_SHIFT, | |
251 | obj->base.size >> PAGE_SHIFT); | |
252 | } | |
253 | ||
93dfb40c CW |
254 | /* XXX kill agp_type! */ |
255 | static unsigned int cache_level_to_agp_type(struct drm_device *dev, | |
256 | enum i915_cache_level cache_level) | |
257 | { | |
258 | switch (cache_level) { | |
259 | case I915_CACHE_LLC_MLC: | |
93dfb40c CW |
260 | /* Older chipsets do not have this extra level of CPU |
261 | * cacheing, so fallthrough and request the PTE simply | |
262 | * as cached. | |
263 | */ | |
8693607a BW |
264 | if (INTEL_INFO(dev)->gen >= 6 && !IS_HASWELL(dev)) |
265 | return AGP_USER_CACHED_MEMORY_LLC_MLC; | |
93dfb40c CW |
266 | case I915_CACHE_LLC: |
267 | return AGP_USER_CACHED_MEMORY; | |
268 | default: | |
269 | case I915_CACHE_NONE: | |
270 | return AGP_USER_MEMORY; | |
271 | } | |
272 | } | |
273 | ||
5c042287 BW |
274 | static bool do_idling(struct drm_i915_private *dev_priv) |
275 | { | |
276 | bool ret = dev_priv->mm.interruptible; | |
277 | ||
278 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { | |
279 | dev_priv->mm.interruptible = false; | |
b2da9fe5 | 280 | if (i915_gpu_idle(dev_priv->dev)) { |
5c042287 BW |
281 | DRM_ERROR("Couldn't idle GPU\n"); |
282 | /* Wait a bit, in hopes it avoids the hang */ | |
283 | udelay(10); | |
284 | } | |
285 | } | |
286 | ||
287 | return ret; | |
288 | } | |
289 | ||
290 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) | |
291 | { | |
292 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) | |
293 | dev_priv->mm.interruptible = interruptible; | |
294 | } | |
295 | ||
76aaf220 DV |
296 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
297 | { | |
298 | struct drm_i915_private *dev_priv = dev->dev_private; | |
05394f39 | 299 | struct drm_i915_gem_object *obj; |
76aaf220 | 300 | |
bee4a186 CW |
301 | /* First fill our portion of the GTT with scratch pages */ |
302 | intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, | |
303 | (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); | |
304 | ||
6c085a72 | 305 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
a8e93126 | 306 | i915_gem_clflush_object(obj); |
74163907 | 307 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
76aaf220 DV |
308 | } |
309 | ||
76aaf220 DV |
310 | intel_gtt_chipset_flush(); |
311 | } | |
7c2e6fdf | 312 | |
74163907 | 313 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
7c2e6fdf | 314 | { |
9da3da66 | 315 | if (obj->has_dma_mapping) |
74163907 | 316 | return 0; |
9da3da66 CW |
317 | |
318 | if (!dma_map_sg(&obj->base.dev->pdev->dev, | |
319 | obj->pages->sgl, obj->pages->nents, | |
320 | PCI_DMA_BIDIRECTIONAL)) | |
321 | return -ENOSPC; | |
322 | ||
323 | return 0; | |
7c2e6fdf DV |
324 | } |
325 | ||
74163907 DV |
326 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
327 | enum i915_cache_level cache_level) | |
d5bd1449 CW |
328 | { |
329 | struct drm_device *dev = obj->base.dev; | |
d5bd1449 CW |
330 | unsigned int agp_type = cache_level_to_agp_type(dev, cache_level); |
331 | ||
2f745ad3 | 332 | intel_gtt_insert_sg_entries(obj->pages, |
9da3da66 CW |
333 | obj->gtt_space->start >> PAGE_SHIFT, |
334 | agp_type); | |
74898d7e | 335 | obj->has_global_gtt_mapping = 1; |
d5bd1449 CW |
336 | } |
337 | ||
05394f39 | 338 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
74163907 DV |
339 | { |
340 | intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, | |
341 | obj->base.size >> PAGE_SHIFT); | |
74898d7e DV |
342 | |
343 | obj->has_global_gtt_mapping = 0; | |
74163907 DV |
344 | } |
345 | ||
346 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) | |
7c2e6fdf | 347 | { |
5c042287 BW |
348 | struct drm_device *dev = obj->base.dev; |
349 | struct drm_i915_private *dev_priv = dev->dev_private; | |
350 | bool interruptible; | |
351 | ||
352 | interruptible = do_idling(dev_priv); | |
353 | ||
9da3da66 CW |
354 | if (!obj->has_dma_mapping) |
355 | dma_unmap_sg(&dev->pdev->dev, | |
356 | obj->pages->sgl, obj->pages->nents, | |
357 | PCI_DMA_BIDIRECTIONAL); | |
5c042287 BW |
358 | |
359 | undo_idling(dev_priv, interruptible); | |
7c2e6fdf | 360 | } |
644ec02b | 361 | |
42d6ab48 CW |
362 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
363 | unsigned long color, | |
364 | unsigned long *start, | |
365 | unsigned long *end) | |
366 | { | |
367 | if (node->color != color) | |
368 | *start += 4096; | |
369 | ||
370 | if (!list_empty(&node->node_list)) { | |
371 | node = list_entry(node->node_list.next, | |
372 | struct drm_mm_node, | |
373 | node_list); | |
374 | if (node->allocated && node->color != color) | |
375 | *end -= 4096; | |
376 | } | |
377 | } | |
378 | ||
644ec02b DV |
379 | void i915_gem_init_global_gtt(struct drm_device *dev, |
380 | unsigned long start, | |
381 | unsigned long mappable_end, | |
382 | unsigned long end) | |
383 | { | |
384 | drm_i915_private_t *dev_priv = dev->dev_private; | |
385 | ||
d1dd20a9 DV |
386 | /* Substract the guard page ... */ |
387 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); | |
42d6ab48 CW |
388 | if (!HAS_LLC(dev)) |
389 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; | |
644ec02b DV |
390 | |
391 | dev_priv->mm.gtt_start = start; | |
392 | dev_priv->mm.gtt_mappable_end = mappable_end; | |
393 | dev_priv->mm.gtt_end = end; | |
394 | dev_priv->mm.gtt_total = end - start; | |
395 | dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; | |
396 | ||
d1dd20a9 | 397 | /* ... but ensure that we clear the entire range. */ |
644ec02b DV |
398 | intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); |
399 | } |