]>
Commit | Line | Data |
---|---|---|
7198e6b0 RC |
1 | /* |
2 | * Copyright (C) 2013 Red Hat | |
3 | * Author: Rob Clark <robdclark@gmail.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License version 2 as published by | |
7 | * the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | */ | |
17 | ||
18 | #include "msm_drv.h" | |
19 | #include "msm_gpu.h" | |
20 | #include "msm_gem.h" | |
21 | ||
22 | /* | |
23 | * Cmdstream submission: | |
24 | */ | |
25 | ||
26 | #define BO_INVALID_FLAGS ~(MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE) | |
27 | /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ | |
28 | #define BO_VALID 0x8000 | |
29 | #define BO_LOCKED 0x4000 | |
30 | #define BO_PINNED 0x2000 | |
31 | ||
32 | static inline void __user *to_user_ptr(u64 address) | |
33 | { | |
34 | return (void __user *)(uintptr_t)address; | |
35 | } | |
36 | ||
37 | static struct msm_gem_submit *submit_create(struct drm_device *dev, | |
38 | struct msm_gpu *gpu, int nr) | |
39 | { | |
40 | struct msm_gem_submit *submit; | |
41 | int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0])); | |
42 | ||
43 | submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); | |
44 | if (submit) { | |
45 | submit->dev = dev; | |
46 | submit->gpu = gpu; | |
47 | ||
48 | /* initially, until copy_from_user() and bo lookup succeeds: */ | |
49 | submit->nr_bos = 0; | |
50 | submit->nr_cmds = 0; | |
51 | ||
52 | INIT_LIST_HEAD(&submit->bo_list); | |
53 | ww_acquire_init(&submit->ticket, &reservation_ww_class); | |
54 | } | |
55 | ||
56 | return submit; | |
57 | } | |
58 | ||
59 | static int submit_lookup_objects(struct msm_gem_submit *submit, | |
60 | struct drm_msm_gem_submit *args, struct drm_file *file) | |
61 | { | |
62 | unsigned i; | |
63 | int ret = 0; | |
64 | ||
65 | spin_lock(&file->table_lock); | |
66 | ||
67 | for (i = 0; i < args->nr_bos; i++) { | |
68 | struct drm_msm_gem_submit_bo submit_bo; | |
69 | struct drm_gem_object *obj; | |
70 | struct msm_gem_object *msm_obj; | |
71 | void __user *userptr = | |
72 | to_user_ptr(args->bos + (i * sizeof(submit_bo))); | |
73 | ||
74 | ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo)); | |
75 | if (ret) { | |
76 | ret = -EFAULT; | |
77 | goto out_unlock; | |
78 | } | |
79 | ||
80 | if (submit_bo.flags & BO_INVALID_FLAGS) { | |
19872533 | 81 | DRM_ERROR("invalid flags: %x\n", submit_bo.flags); |
7198e6b0 RC |
82 | ret = -EINVAL; |
83 | goto out_unlock; | |
84 | } | |
85 | ||
86 | submit->bos[i].flags = submit_bo.flags; | |
87 | /* in validate_objects() we figure out if this is true: */ | |
88 | submit->bos[i].iova = submit_bo.presumed; | |
89 | ||
90 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
91 | * all under single table_lock just hit object_idr directly: | |
92 | */ | |
93 | obj = idr_find(&file->object_idr, submit_bo.handle); | |
94 | if (!obj) { | |
19872533 | 95 | DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i); |
7198e6b0 RC |
96 | ret = -EINVAL; |
97 | goto out_unlock; | |
98 | } | |
99 | ||
100 | msm_obj = to_msm_bo(obj); | |
101 | ||
102 | if (!list_empty(&msm_obj->submit_entry)) { | |
19872533 | 103 | DRM_ERROR("handle %u at index %u already on submit list\n", |
7198e6b0 RC |
104 | submit_bo.handle, i); |
105 | ret = -EINVAL; | |
106 | goto out_unlock; | |
107 | } | |
108 | ||
109 | drm_gem_object_reference(obj); | |
110 | ||
111 | submit->bos[i].obj = msm_obj; | |
112 | ||
113 | list_add_tail(&msm_obj->submit_entry, &submit->bo_list); | |
114 | } | |
115 | ||
116 | out_unlock: | |
117 | submit->nr_bos = i; | |
118 | spin_unlock(&file->table_lock); | |
119 | ||
120 | return ret; | |
121 | } | |
122 | ||
123 | static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i) | |
124 | { | |
125 | struct msm_gem_object *msm_obj = submit->bos[i].obj; | |
126 | ||
127 | if (submit->bos[i].flags & BO_PINNED) | |
128 | msm_gem_put_iova(&msm_obj->base, submit->gpu->id); | |
129 | ||
130 | if (submit->bos[i].flags & BO_LOCKED) | |
131 | ww_mutex_unlock(&msm_obj->resv->lock); | |
132 | ||
133 | if (!(submit->bos[i].flags & BO_VALID)) | |
134 | submit->bos[i].iova = 0; | |
135 | ||
136 | submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED); | |
137 | } | |
138 | ||
139 | /* This is where we make sure all the bo's are reserved and pin'd: */ | |
140 | static int submit_validate_objects(struct msm_gem_submit *submit) | |
141 | { | |
142 | int contended, slow_locked = -1, i, ret = 0; | |
143 | ||
144 | retry: | |
145 | submit->valid = true; | |
146 | ||
147 | for (i = 0; i < submit->nr_bos; i++) { | |
148 | struct msm_gem_object *msm_obj = submit->bos[i].obj; | |
149 | uint32_t iova; | |
150 | ||
151 | if (slow_locked == i) | |
152 | slow_locked = -1; | |
153 | ||
154 | contended = i; | |
155 | ||
156 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
157 | ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock, | |
158 | &submit->ticket); | |
159 | if (ret) | |
160 | goto fail; | |
161 | submit->bos[i].flags |= BO_LOCKED; | |
162 | } | |
163 | ||
164 | ||
165 | /* if locking succeeded, pin bo: */ | |
c2703b13 | 166 | ret = msm_gem_get_iova_locked(&msm_obj->base, |
7198e6b0 RC |
167 | submit->gpu->id, &iova); |
168 | ||
169 | /* this would break the logic in the fail path.. there is no | |
170 | * reason for this to happen, but just to be on the safe side | |
171 | * let's notice if this starts happening in the future: | |
172 | */ | |
173 | WARN_ON(ret == -EDEADLK); | |
174 | ||
175 | if (ret) | |
176 | goto fail; | |
177 | ||
178 | submit->bos[i].flags |= BO_PINNED; | |
179 | ||
180 | if (iova == submit->bos[i].iova) { | |
181 | submit->bos[i].flags |= BO_VALID; | |
182 | } else { | |
183 | submit->bos[i].iova = iova; | |
184 | submit->bos[i].flags &= ~BO_VALID; | |
185 | submit->valid = false; | |
186 | } | |
187 | } | |
188 | ||
189 | ww_acquire_done(&submit->ticket); | |
190 | ||
191 | return 0; | |
192 | ||
193 | fail: | |
194 | for (; i >= 0; i--) | |
195 | submit_unlock_unpin_bo(submit, i); | |
196 | ||
197 | if (slow_locked > 0) | |
198 | submit_unlock_unpin_bo(submit, slow_locked); | |
199 | ||
200 | if (ret == -EDEADLK) { | |
201 | struct msm_gem_object *msm_obj = submit->bos[contended].obj; | |
202 | /* we lost out in a seqno race, lock and retry.. */ | |
203 | ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock, | |
204 | &submit->ticket); | |
205 | if (!ret) { | |
206 | submit->bos[contended].flags |= BO_LOCKED; | |
207 | slow_locked = contended; | |
208 | goto retry; | |
209 | } | |
210 | } | |
211 | ||
212 | return ret; | |
213 | } | |
214 | ||
215 | static int submit_bo(struct msm_gem_submit *submit, uint32_t idx, | |
216 | struct msm_gem_object **obj, uint32_t *iova, bool *valid) | |
217 | { | |
218 | if (idx >= submit->nr_bos) { | |
19872533 RC |
219 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", |
220 | idx, submit->nr_bos); | |
221 | return -EINVAL; | |
7198e6b0 RC |
222 | } |
223 | ||
224 | if (obj) | |
225 | *obj = submit->bos[idx].obj; | |
226 | if (iova) | |
227 | *iova = submit->bos[idx].iova; | |
228 | if (valid) | |
229 | *valid = !!(submit->bos[idx].flags & BO_VALID); | |
230 | ||
231 | return 0; | |
232 | } | |
233 | ||
234 | /* process the reloc's and patch up the cmdstream as needed: */ | |
235 | static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj, | |
236 | uint32_t offset, uint32_t nr_relocs, uint64_t relocs) | |
237 | { | |
238 | uint32_t i, last_offset = 0; | |
239 | uint32_t *ptr; | |
240 | int ret; | |
241 | ||
242 | if (offset % 4) { | |
19872533 | 243 | DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset); |
7198e6b0 RC |
244 | return -EINVAL; |
245 | } | |
246 | ||
247 | /* For now, just map the entire thing. Eventually we probably | |
248 | * to do it page-by-page, w/ kmap() if not vmap()d.. | |
249 | */ | |
c2703b13 | 250 | ptr = msm_gem_vaddr_locked(&obj->base); |
7198e6b0 RC |
251 | |
252 | if (IS_ERR(ptr)) { | |
253 | ret = PTR_ERR(ptr); | |
254 | DBG("failed to map: %d", ret); | |
255 | return ret; | |
256 | } | |
257 | ||
258 | for (i = 0; i < nr_relocs; i++) { | |
259 | struct drm_msm_gem_submit_reloc submit_reloc; | |
260 | void __user *userptr = | |
261 | to_user_ptr(relocs + (i * sizeof(submit_reloc))); | |
262 | uint32_t iova, off; | |
263 | bool valid; | |
264 | ||
265 | ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc)); | |
266 | if (ret) | |
267 | return -EFAULT; | |
268 | ||
269 | if (submit_reloc.submit_offset % 4) { | |
19872533 | 270 | DRM_ERROR("non-aligned reloc offset: %u\n", |
7198e6b0 RC |
271 | submit_reloc.submit_offset); |
272 | return -EINVAL; | |
273 | } | |
274 | ||
275 | /* offset in dwords: */ | |
276 | off = submit_reloc.submit_offset / 4; | |
277 | ||
278 | if ((off >= (obj->base.size / 4)) || | |
279 | (off < last_offset)) { | |
19872533 | 280 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); |
7198e6b0 RC |
281 | return -EINVAL; |
282 | } | |
283 | ||
284 | ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid); | |
285 | if (ret) | |
286 | return ret; | |
287 | ||
288 | if (valid) | |
289 | continue; | |
290 | ||
291 | iova += submit_reloc.reloc_offset; | |
292 | ||
293 | if (submit_reloc.shift < 0) | |
294 | iova >>= -submit_reloc.shift; | |
295 | else | |
296 | iova <<= submit_reloc.shift; | |
297 | ||
298 | ptr[off] = iova | submit_reloc.or; | |
299 | ||
300 | last_offset = off; | |
301 | } | |
302 | ||
303 | return 0; | |
304 | } | |
305 | ||
306 | static void submit_cleanup(struct msm_gem_submit *submit, bool fail) | |
307 | { | |
308 | unsigned i; | |
309 | ||
7198e6b0 RC |
310 | for (i = 0; i < submit->nr_bos; i++) { |
311 | struct msm_gem_object *msm_obj = submit->bos[i].obj; | |
312 | submit_unlock_unpin_bo(submit, i); | |
313 | list_del_init(&msm_obj->submit_entry); | |
314 | drm_gem_object_unreference(&msm_obj->base); | |
315 | } | |
7198e6b0 RC |
316 | |
317 | ww_acquire_fini(&submit->ticket); | |
318 | kfree(submit); | |
319 | } | |
320 | ||
321 | int msm_ioctl_gem_submit(struct drm_device *dev, void *data, | |
322 | struct drm_file *file) | |
323 | { | |
324 | struct msm_drm_private *priv = dev->dev_private; | |
325 | struct drm_msm_gem_submit *args = data; | |
326 | struct msm_file_private *ctx = file->driver_priv; | |
327 | struct msm_gem_submit *submit; | |
328 | struct msm_gpu *gpu; | |
329 | unsigned i; | |
330 | int ret; | |
331 | ||
332 | /* for now, we just have 3d pipe.. eventually this would need to | |
333 | * be more clever to dispatch to appropriate gpu module: | |
334 | */ | |
335 | if (args->pipe != MSM_PIPE_3D0) | |
336 | return -EINVAL; | |
337 | ||
338 | gpu = priv->gpu; | |
339 | ||
340 | if (args->nr_cmds > MAX_CMDS) | |
341 | return -EINVAL; | |
342 | ||
c2703b13 RC |
343 | mutex_lock(&dev->struct_mutex); |
344 | ||
7198e6b0 RC |
345 | submit = submit_create(dev, gpu, args->nr_bos); |
346 | if (!submit) { | |
347 | ret = -ENOMEM; | |
348 | goto out; | |
349 | } | |
350 | ||
351 | ret = submit_lookup_objects(submit, args, file); | |
352 | if (ret) | |
353 | goto out; | |
354 | ||
355 | ret = submit_validate_objects(submit); | |
356 | if (ret) | |
357 | goto out; | |
358 | ||
359 | for (i = 0; i < args->nr_cmds; i++) { | |
360 | struct drm_msm_gem_submit_cmd submit_cmd; | |
361 | void __user *userptr = | |
362 | to_user_ptr(args->cmds + (i * sizeof(submit_cmd))); | |
363 | struct msm_gem_object *msm_obj; | |
364 | uint32_t iova; | |
365 | ||
366 | ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd)); | |
367 | if (ret) { | |
368 | ret = -EFAULT; | |
369 | goto out; | |
370 | } | |
371 | ||
372 | ret = submit_bo(submit, submit_cmd.submit_idx, | |
373 | &msm_obj, &iova, NULL); | |
374 | if (ret) | |
375 | goto out; | |
376 | ||
377 | if (submit_cmd.size % 4) { | |
19872533 | 378 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", |
7198e6b0 RC |
379 | submit_cmd.size); |
380 | ret = -EINVAL; | |
381 | goto out; | |
382 | } | |
383 | ||
19872533 RC |
384 | if ((submit_cmd.size + submit_cmd.submit_offset) >= |
385 | msm_obj->base.size) { | |
386 | DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size); | |
7198e6b0 RC |
387 | ret = -EINVAL; |
388 | goto out; | |
389 | } | |
390 | ||
391 | submit->cmd[i].type = submit_cmd.type; | |
392 | submit->cmd[i].size = submit_cmd.size / 4; | |
393 | submit->cmd[i].iova = iova + submit_cmd.submit_offset; | |
394 | ||
395 | if (submit->valid) | |
396 | continue; | |
397 | ||
398 | ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset, | |
399 | submit_cmd.nr_relocs, submit_cmd.relocs); | |
400 | if (ret) | |
401 | goto out; | |
402 | } | |
403 | ||
404 | submit->nr_cmds = i; | |
405 | ||
406 | ret = msm_gpu_submit(gpu, submit, ctx); | |
407 | ||
408 | args->fence = submit->fence; | |
409 | ||
410 | out: | |
411 | if (submit) | |
412 | submit_cleanup(submit, !!ret); | |
c2703b13 | 413 | mutex_unlock(&dev->struct_mutex); |
7198e6b0 RC |
414 | return ret; |
415 | } |