]>
Commit | Line | Data |
---|---|---|
920cf419 CW |
1 | /* |
2 | * Copyright © 2014-2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include <drm/drmP.h> | |
26 | #include <drm/i915_drm.h> | |
27 | #include "i915_drv.h" | |
28 | ||
29 | #define QUIET (__GFP_NORETRY | __GFP_NOWARN) | |
30 | ||
31 | /* convert swiotlb segment size into sensible units (pages)! */ | |
32 | #define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT) | |
33 | ||
34 | static void internal_free_pages(struct sg_table *st) | |
35 | { | |
36 | struct scatterlist *sg; | |
37 | ||
4703b047 CW |
38 | for (sg = st->sgl; sg; sg = __sg_next(sg)) { |
39 | if (sg_page(sg)) | |
40 | __free_pages(sg_page(sg), get_order(sg->length)); | |
41 | } | |
920cf419 CW |
42 | |
43 | sg_free_table(st); | |
44 | kfree(st); | |
45 | } | |
46 | ||
b91b09ee | 47 | static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) |
920cf419 CW |
48 | { |
49 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
920cf419 CW |
50 | struct sg_table *st; |
51 | struct scatterlist *sg; | |
a5c08166 | 52 | unsigned int sg_mask; |
bb96dcf5 | 53 | unsigned int npages; |
920cf419 CW |
54 | int max_order; |
55 | gfp_t gfp; | |
56 | ||
920cf419 CW |
57 | max_order = MAX_ORDER; |
58 | #ifdef CONFIG_SWIOTLB | |
5584f1b1 JG |
59 | if (swiotlb_nr_tbl()) { |
60 | unsigned int max_segment; | |
61 | ||
62 | max_segment = swiotlb_max_segment(); | |
63 | if (max_segment) { | |
64 | max_segment = max_t(unsigned int, max_segment, | |
65 | PAGE_SIZE) >> PAGE_SHIFT; | |
66 | max_order = min(max_order, ilog2(max_segment)); | |
67 | } | |
68 | } | |
920cf419 CW |
69 | #endif |
70 | ||
71 | gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; | |
c0f86832 | 72 | if (IS_I965GM(i915) || IS_I965G(i915)) { |
920cf419 CW |
73 | /* 965gm cannot relocate objects above 4GiB. */ |
74 | gfp &= ~__GFP_HIGHMEM; | |
75 | gfp |= __GFP_DMA32; | |
76 | } | |
77 | ||
bb96dcf5 CW |
78 | create_st: |
79 | st = kmalloc(sizeof(*st), GFP_KERNEL); | |
80 | if (!st) | |
b91b09ee | 81 | return -ENOMEM; |
bb96dcf5 CW |
82 | |
83 | npages = obj->base.size / PAGE_SIZE; | |
84 | if (sg_alloc_table(st, npages, GFP_KERNEL)) { | |
85 | kfree(st); | |
b91b09ee | 86 | return -ENOMEM; |
bb96dcf5 CW |
87 | } |
88 | ||
89 | sg = st->sgl; | |
90 | st->nents = 0; | |
a5c08166 | 91 | sg_mask = 0; |
bb96dcf5 | 92 | |
920cf419 CW |
93 | do { |
94 | int order = min(fls(npages) - 1, max_order); | |
95 | struct page *page; | |
96 | ||
97 | do { | |
98 | page = alloc_pages(gfp | (order ? QUIET : 0), order); | |
99 | if (page) | |
100 | break; | |
101 | if (!order--) | |
102 | goto err; | |
103 | ||
104 | /* Limit subsequent allocations as well */ | |
105 | max_order = order; | |
106 | } while (1); | |
107 | ||
108 | sg_set_page(sg, page, PAGE_SIZE << order, 0); | |
a5c08166 | 109 | sg_mask |= PAGE_SIZE << order; |
920cf419 CW |
110 | st->nents++; |
111 | ||
112 | npages -= 1 << order; | |
113 | if (!npages) { | |
114 | sg_mark_end(sg); | |
115 | break; | |
116 | } | |
117 | ||
118 | sg = __sg_next(sg); | |
119 | } while (1); | |
920cf419 | 120 | |
bb96dcf5 CW |
121 | if (i915_gem_gtt_prepare_pages(obj, st)) { |
122 | /* Failed to dma-map try again with single page sg segments */ | |
123 | if (get_order(st->sgl->length)) { | |
124 | internal_free_pages(st); | |
125 | max_order = 0; | |
126 | goto create_st; | |
127 | } | |
920cf419 | 128 | goto err; |
bb96dcf5 | 129 | } |
920cf419 CW |
130 | |
131 | /* Mark the pages as dontneed whilst they are still pinned. As soon | |
132 | * as they are unpinned they are allowed to be reaped by the shrinker, | |
133 | * and the caller is expected to repopulate - the contents of this | |
134 | * object are only valid whilst active and pinned. | |
135 | */ | |
a4f5ea64 | 136 | obj->mm.madv = I915_MADV_DONTNEED; |
b91b09ee | 137 | |
a5c08166 | 138 | __i915_gem_object_set_pages(obj, st, sg_mask); |
b91b09ee MA |
139 | |
140 | return 0; | |
920cf419 CW |
141 | |
142 | err: | |
4703b047 | 143 | sg_set_page(sg, NULL, 0, 0); |
920cf419 CW |
144 | sg_mark_end(sg); |
145 | internal_free_pages(st); | |
b91b09ee MA |
146 | |
147 | return -ENOMEM; | |
920cf419 CW |
148 | } |
149 | ||
03ac84f1 CW |
150 | static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, |
151 | struct sg_table *pages) | |
920cf419 | 152 | { |
03ac84f1 CW |
153 | i915_gem_gtt_finish_pages(obj, pages); |
154 | internal_free_pages(pages); | |
920cf419 | 155 | |
a4f5ea64 CW |
156 | obj->mm.dirty = false; |
157 | obj->mm.madv = I915_MADV_WILLNEED; | |
920cf419 CW |
158 | } |
159 | ||
160 | static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { | |
3599a91c TU |
161 | .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | |
162 | I915_GEM_OBJECT_IS_SHRINKABLE, | |
920cf419 CW |
163 | .get_pages = i915_gem_object_get_pages_internal, |
164 | .put_pages = i915_gem_object_put_pages_internal, | |
165 | }; | |
166 | ||
167 | /** | |
168 | * Creates a new object that wraps some internal memory for private use. | |
169 | * This object is not backed by swappable storage, and as such its contents | |
170 | * are volatile and only valid whilst pinned. If the object is reaped by the | |
171 | * shrinker, its pages and data will be discarded. Equally, it is not a full | |
172 | * GEM object and so not valid for access from userspace. This makes it useful | |
173 | * for hardware interfaces like ringbuffers (which are pinned from the time | |
174 | * the request is written to the time the hardware stops accessing it), but | |
175 | * not for contexts (which need to be preserved when not active for later | |
176 | * reuse). Note that it is not cleared upon allocation. | |
177 | */ | |
178 | struct drm_i915_gem_object * | |
179 | i915_gem_object_create_internal(struct drm_i915_private *i915, | |
fcd46e53 | 180 | phys_addr_t size) |
920cf419 CW |
181 | { |
182 | struct drm_i915_gem_object *obj; | |
b8f55be6 | 183 | unsigned int cache_level; |
920cf419 | 184 | |
fcd46e53 | 185 | GEM_BUG_ON(!size); |
bf6b2030 | 186 | GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); |
fcd46e53 CW |
187 | |
188 | if (overflows_type(size, obj->base.size)) | |
189 | return ERR_PTR(-E2BIG); | |
190 | ||
187685cb | 191 | obj = i915_gem_object_alloc(i915); |
920cf419 CW |
192 | if (!obj) |
193 | return ERR_PTR(-ENOMEM); | |
194 | ||
195 | drm_gem_private_object_init(&i915->drm, &obj->base, size); | |
196 | i915_gem_object_init(obj, &i915_gem_object_internal_ops); | |
197 | ||
920cf419 | 198 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
e27ab73d | 199 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
b8f55be6 CW |
200 | |
201 | cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; | |
202 | i915_gem_object_set_cache_coherency(obj, cache_level); | |
920cf419 CW |
203 | |
204 | return obj; | |
205 | } |