]>
Commit | Line | Data |
---|---|---|
a8c21a54 T |
1 | /* |
2 | * Copyright (C) 2015 Etnaviv Project | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License version 2 as published by | |
6 | * the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | * more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License along with | |
14 | * this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | */ | |
16 | ||
9ad59fea | 17 | #include <linux/dma-fence-array.h> |
a8c21a54 | 18 | #include <linux/reservation.h> |
9ad59fea | 19 | #include <linux/sync_file.h> |
ea1f5729 | 20 | #include "etnaviv_cmdbuf.h" |
a8c21a54 T |
21 | #include "etnaviv_drv.h" |
22 | #include "etnaviv_gpu.h" | |
23 | #include "etnaviv_gem.h" | |
c8e4a7fd | 24 | #include "etnaviv_perfmon.h" |
e93b6dee | 25 | #include "etnaviv_sched.h" |
a8c21a54 T |
26 | |
27 | /* | |
28 | * Cmdstream submission: | |
29 | */ | |
30 | ||
31 | #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) | |
32 | /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ | |
33 | #define BO_LOCKED 0x4000 | |
34 | #define BO_PINNED 0x2000 | |
35 | ||
a8c21a54 | 36 | static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, |
ef146c00 | 37 | struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs) |
a8c21a54 T |
38 | { |
39 | struct etnaviv_gem_submit *submit; | |
ef146c00 | 40 | size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit)); |
a8c21a54 | 41 | |
c5283723 LS |
42 | submit = kzalloc(sz, GFP_KERNEL); |
43 | if (!submit) | |
44 | return NULL; | |
a8c21a54 | 45 | |
ef146c00 LS |
46 | submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request), |
47 | GFP_KERNEL); | |
48 | if (!submit->pmrs) { | |
49 | kfree(submit); | |
50 | return NULL; | |
51 | } | |
52 | submit->nr_pmrs = nr_pmrs; | |
53 | ||
c5283723 | 54 | submit->gpu = gpu; |
e0329e6c | 55 | kref_init(&submit->refcount); |
a8c21a54 | 56 | |
a8c21a54 T |
57 | return submit; |
58 | } | |
59 | ||
60 | static int submit_lookup_objects(struct etnaviv_gem_submit *submit, | |
61 | struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, | |
62 | unsigned nr_bos) | |
63 | { | |
64 | struct drm_etnaviv_gem_submit_bo *bo; | |
65 | unsigned i; | |
66 | int ret = 0; | |
67 | ||
68 | spin_lock(&file->table_lock); | |
69 | ||
70 | for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { | |
71 | struct drm_gem_object *obj; | |
72 | ||
73 | if (bo->flags & BO_INVALID_FLAGS) { | |
74 | DRM_ERROR("invalid flags: %x\n", bo->flags); | |
75 | ret = -EINVAL; | |
76 | goto out_unlock; | |
77 | } | |
78 | ||
79 | submit->bos[i].flags = bo->flags; | |
80 | ||
81 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
82 | * all under single table_lock just hit object_idr directly: | |
83 | */ | |
84 | obj = idr_find(&file->object_idr, bo->handle); | |
85 | if (!obj) { | |
86 | DRM_ERROR("invalid handle %u at index %u\n", | |
87 | bo->handle, i); | |
88 | ret = -EINVAL; | |
89 | goto out_unlock; | |
90 | } | |
91 | ||
92 | /* | |
93 | * Take a refcount on the object. The file table lock | |
94 | * prevents the object_idr's refcount on this being dropped. | |
95 | */ | |
23d1dd03 | 96 | drm_gem_object_get(obj); |
a8c21a54 T |
97 | |
98 | submit->bos[i].obj = to_etnaviv_bo(obj); | |
99 | } | |
100 | ||
101 | out_unlock: | |
102 | submit->nr_bos = i; | |
103 | spin_unlock(&file->table_lock); | |
104 | ||
105 | return ret; | |
106 | } | |
107 | ||
108 | static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) | |
109 | { | |
110 | if (submit->bos[i].flags & BO_LOCKED) { | |
111 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
112 | ||
113 | ww_mutex_unlock(&etnaviv_obj->resv->lock); | |
114 | submit->bos[i].flags &= ~BO_LOCKED; | |
115 | } | |
116 | } | |
117 | ||
08301d73 LS |
118 | static int submit_lock_objects(struct etnaviv_gem_submit *submit, |
119 | struct ww_acquire_ctx *ticket) | |
a8c21a54 T |
120 | { |
121 | int contended, slow_locked = -1, i, ret = 0; | |
122 | ||
123 | retry: | |
124 | for (i = 0; i < submit->nr_bos; i++) { | |
125 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
126 | ||
127 | if (slow_locked == i) | |
128 | slow_locked = -1; | |
129 | ||
130 | contended = i; | |
131 | ||
132 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
133 | ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock, | |
08301d73 | 134 | ticket); |
a8c21a54 T |
135 | if (ret == -EALREADY) |
136 | DRM_ERROR("BO at index %u already on submit list\n", | |
137 | i); | |
138 | if (ret) | |
139 | goto fail; | |
140 | submit->bos[i].flags |= BO_LOCKED; | |
141 | } | |
142 | } | |
143 | ||
08301d73 | 144 | ww_acquire_done(ticket); |
a8c21a54 T |
145 | |
146 | return 0; | |
147 | ||
148 | fail: | |
149 | for (; i >= 0; i--) | |
150 | submit_unlock_object(submit, i); | |
151 | ||
152 | if (slow_locked > 0) | |
153 | submit_unlock_object(submit, slow_locked); | |
154 | ||
155 | if (ret == -EDEADLK) { | |
156 | struct etnaviv_gem_object *etnaviv_obj; | |
157 | ||
158 | etnaviv_obj = submit->bos[contended].obj; | |
159 | ||
160 | /* we lost out in a seqno race, lock and retry.. */ | |
161 | ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock, | |
08301d73 | 162 | ticket); |
a8c21a54 T |
163 | if (!ret) { |
164 | submit->bos[contended].flags |= BO_LOCKED; | |
165 | slow_locked = contended; | |
166 | goto retry; | |
167 | } | |
168 | } | |
169 | ||
170 | return ret; | |
171 | } | |
172 | ||
683da226 | 173 | static int submit_fence_sync(struct etnaviv_gem_submit *submit) |
a8c21a54 | 174 | { |
a8c21a54 T |
175 | int i, ret = 0; |
176 | ||
177 | for (i = 0; i < submit->nr_bos; i++) { | |
683da226 LS |
178 | struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; |
179 | struct reservation_object *robj = bo->obj->resv; | |
a8c21a54 | 180 | |
683da226 LS |
181 | if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) { |
182 | ret = reservation_object_reserve_shared(robj); | |
183 | if (ret) | |
184 | return ret; | |
185 | } | |
186 | ||
187 | if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT) | |
188 | continue; | |
189 | ||
190 | if (bo->flags & ETNA_SUBMIT_BO_WRITE) { | |
191 | ret = reservation_object_get_fences_rcu(robj, &bo->excl, | |
192 | &bo->nr_shared, | |
193 | &bo->shared); | |
194 | if (ret) | |
195 | return ret; | |
196 | } else { | |
197 | bo->excl = reservation_object_get_excl_rcu(robj); | |
198 | } | |
a8c21a54 | 199 | |
9efabd73 LS |
200 | } |
201 | ||
a8c21a54 T |
202 | return ret; |
203 | } | |
204 | ||
0236efe9 LS |
205 | static void submit_attach_object_fences(struct etnaviv_gem_submit *submit) |
206 | { | |
207 | int i; | |
208 | ||
209 | for (i = 0; i < submit->nr_bos; i++) { | |
210 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
211 | ||
212 | if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) | |
213 | reservation_object_add_excl_fence(etnaviv_obj->resv, | |
10009ea2 | 214 | submit->out_fence); |
0236efe9 LS |
215 | else |
216 | reservation_object_add_shared_fence(etnaviv_obj->resv, | |
10009ea2 | 217 | submit->out_fence); |
0236efe9 LS |
218 | |
219 | submit_unlock_object(submit, i); | |
220 | } | |
221 | } | |
222 | ||
a8c21a54 T |
223 | static int submit_pin_objects(struct etnaviv_gem_submit *submit) |
224 | { | |
225 | int i, ret = 0; | |
226 | ||
227 | for (i = 0; i < submit->nr_bos; i++) { | |
228 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
b6325f40 | 229 | struct etnaviv_vram_mapping *mapping; |
a8c21a54 | 230 | |
b6325f40 RK |
231 | mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base, |
232 | submit->gpu); | |
233 | if (IS_ERR(mapping)) { | |
234 | ret = PTR_ERR(mapping); | |
a8c21a54 | 235 | break; |
b6325f40 | 236 | } |
5b223e94 | 237 | atomic_inc(&etnaviv_obj->gpu_active); |
a8c21a54 T |
238 | |
239 | submit->bos[i].flags |= BO_PINNED; | |
b6325f40 | 240 | submit->bos[i].mapping = mapping; |
a8c21a54 T |
241 | } |
242 | ||
243 | return ret; | |
244 | } | |
245 | ||
246 | static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, | |
8779aa8f | 247 | struct etnaviv_gem_submit_bo **bo) |
a8c21a54 T |
248 | { |
249 | if (idx >= submit->nr_bos) { | |
250 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", | |
251 | idx, submit->nr_bos); | |
252 | return -EINVAL; | |
253 | } | |
254 | ||
8779aa8f | 255 | *bo = &submit->bos[idx]; |
a8c21a54 T |
256 | |
257 | return 0; | |
258 | } | |
259 | ||
260 | /* process the reloc's and patch up the cmdstream as needed: */ | |
261 | static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, | |
262 | u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, | |
263 | u32 nr_relocs) | |
264 | { | |
265 | u32 i, last_offset = 0; | |
266 | u32 *ptr = stream; | |
267 | int ret; | |
268 | ||
269 | for (i = 0; i < nr_relocs; i++) { | |
270 | const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; | |
8779aa8f RK |
271 | struct etnaviv_gem_submit_bo *bo; |
272 | u32 off; | |
a8c21a54 T |
273 | |
274 | if (unlikely(r->flags)) { | |
275 | DRM_ERROR("invalid reloc flags\n"); | |
276 | return -EINVAL; | |
277 | } | |
278 | ||
279 | if (r->submit_offset % 4) { | |
280 | DRM_ERROR("non-aligned reloc offset: %u\n", | |
281 | r->submit_offset); | |
282 | return -EINVAL; | |
283 | } | |
284 | ||
285 | /* offset in dwords: */ | |
286 | off = r->submit_offset / 4; | |
287 | ||
288 | if ((off >= size ) || | |
289 | (off < last_offset)) { | |
290 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); | |
291 | return -EINVAL; | |
292 | } | |
293 | ||
8779aa8f | 294 | ret = submit_bo(submit, r->reloc_idx, &bo); |
a8c21a54 T |
295 | if (ret) |
296 | return ret; | |
297 | ||
d6f756e0 WL |
298 | if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) { |
299 | DRM_ERROR("relocation %u outside object\n", i); | |
a8c21a54 T |
300 | return -EINVAL; |
301 | } | |
302 | ||
8779aa8f | 303 | ptr[off] = bo->mapping->iova + r->reloc_offset; |
a8c21a54 T |
304 | |
305 | last_offset = off; | |
306 | } | |
307 | ||
308 | return 0; | |
309 | } | |
310 | ||
c8e4a7fd | 311 | static int submit_perfmon_validate(struct etnaviv_gem_submit *submit, |
ef146c00 | 312 | u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs) |
c8e4a7fd CG |
313 | { |
314 | u32 i; | |
315 | ||
ef146c00 | 316 | for (i = 0; i < submit->nr_pmrs; i++) { |
c8e4a7fd CG |
317 | const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i; |
318 | struct etnaviv_gem_submit_bo *bo; | |
319 | int ret; | |
320 | ||
321 | ret = submit_bo(submit, r->read_idx, &bo); | |
322 | if (ret) | |
323 | return ret; | |
324 | ||
325 | /* at offset 0 a sequence number gets stored used for userspace sync */ | |
326 | if (r->read_offset == 0) { | |
327 | DRM_ERROR("perfmon request: offset is 0"); | |
328 | return -EINVAL; | |
329 | } | |
330 | ||
331 | if (r->read_offset >= bo->obj->base.size - sizeof(u32)) { | |
332 | DRM_ERROR("perfmon request: offset %u outside object", i); | |
333 | return -EINVAL; | |
334 | } | |
335 | ||
336 | if (r->flags & ~(ETNA_PM_PROCESS_PRE | ETNA_PM_PROCESS_POST)) { | |
337 | DRM_ERROR("perfmon request: flags are not valid"); | |
338 | return -EINVAL; | |
339 | } | |
340 | ||
ef146c00 | 341 | if (etnaviv_pm_req_validate(r, exec_state)) { |
c8e4a7fd CG |
342 | DRM_ERROR("perfmon request: domain or signal not valid"); |
343 | return -EINVAL; | |
344 | } | |
345 | ||
ef146c00 LS |
346 | submit->pmrs[i].flags = r->flags; |
347 | submit->pmrs[i].domain = r->domain; | |
348 | submit->pmrs[i].signal = r->signal; | |
349 | submit->pmrs[i].sequence = r->sequence; | |
350 | submit->pmrs[i].offset = r->read_offset; | |
351 | submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base); | |
c8e4a7fd CG |
352 | } |
353 | ||
354 | return 0; | |
355 | } | |
356 | ||
e0329e6c | 357 | static void submit_cleanup(struct kref *kref) |
a8c21a54 | 358 | { |
e0329e6c LS |
359 | struct etnaviv_gem_submit *submit = |
360 | container_of(kref, struct etnaviv_gem_submit, refcount); | |
a8c21a54 T |
361 | unsigned i; |
362 | ||
8bda1516 LS |
363 | if (submit->runtime_resumed) |
364 | pm_runtime_put_autosuspend(submit->gpu->dev); | |
365 | ||
2f9225db LS |
366 | if (submit->cmdbuf.suballoc) |
367 | etnaviv_cmdbuf_free(&submit->cmdbuf); | |
368 | ||
a8c21a54 T |
369 | for (i = 0; i < submit->nr_bos; i++) { |
370 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
371 | ||
33a63e68 LS |
372 | /* unpin all objects */ |
373 | if (submit->bos[i].flags & BO_PINNED) { | |
374 | etnaviv_gem_mapping_unreference(submit->bos[i].mapping); | |
5b223e94 | 375 | atomic_dec(&etnaviv_obj->gpu_active); |
33a63e68 LS |
376 | submit->bos[i].mapping = NULL; |
377 | submit->bos[i].flags &= ~BO_PINNED; | |
378 | } | |
379 | ||
0236efe9 | 380 | /* if the GPU submit failed, objects might still be locked */ |
a8c21a54 | 381 | submit_unlock_object(submit, i); |
23d1dd03 | 382 | drm_gem_object_put_unlocked(&etnaviv_obj->base); |
a8c21a54 T |
383 | } |
384 | ||
5b223e94 LS |
385 | wake_up_all(&submit->gpu->fence_event); |
386 | ||
9efabd73 LS |
387 | if (submit->in_fence) |
388 | dma_fence_put(submit->in_fence); | |
e93b6dee LS |
389 | if (submit->out_fence) { |
390 | /* first remove from IDR, so fence can not be found anymore */ | |
391 | mutex_lock(&submit->gpu->fence_idr_lock); | |
392 | idr_remove(&submit->gpu->fence_idr, submit->out_fence_id); | |
393 | mutex_unlock(&submit->gpu->fence_idr_lock); | |
10009ea2 | 394 | dma_fence_put(submit->out_fence); |
e93b6dee | 395 | } |
ef146c00 | 396 | kfree(submit->pmrs); |
a8c21a54 T |
397 | kfree(submit); |
398 | } | |
399 | ||
e0329e6c LS |
400 | void etnaviv_submit_put(struct etnaviv_gem_submit *submit) |
401 | { | |
402 | kref_put(&submit->refcount, submit_cleanup); | |
403 | } | |
404 | ||
a8c21a54 T |
405 | int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, |
406 | struct drm_file *file) | |
407 | { | |
e93b6dee | 408 | struct etnaviv_file_private *ctx = file->driver_priv; |
a8c21a54 T |
409 | struct etnaviv_drm_private *priv = dev->dev_private; |
410 | struct drm_etnaviv_gem_submit *args = data; | |
411 | struct drm_etnaviv_gem_submit_reloc *relocs; | |
c8e4a7fd | 412 | struct drm_etnaviv_gem_submit_pmr *pmrs; |
a8c21a54 T |
413 | struct drm_etnaviv_gem_submit_bo *bos; |
414 | struct etnaviv_gem_submit *submit; | |
a8c21a54 | 415 | struct etnaviv_gpu *gpu; |
78ec187f | 416 | struct sync_file *sync_file = NULL; |
08301d73 | 417 | struct ww_acquire_ctx ticket; |
78ec187f | 418 | int out_fence_fd = -1; |
a8c21a54 T |
419 | void *stream; |
420 | int ret; | |
421 | ||
422 | if (args->pipe >= ETNA_MAX_PIPES) | |
423 | return -EINVAL; | |
424 | ||
425 | gpu = priv->gpu[args->pipe]; | |
426 | if (!gpu) | |
427 | return -ENXIO; | |
428 | ||
429 | if (args->stream_size % 4) { | |
430 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", | |
431 | args->stream_size); | |
432 | return -EINVAL; | |
433 | } | |
434 | ||
435 | if (args->exec_state != ETNA_PIPE_3D && | |
436 | args->exec_state != ETNA_PIPE_2D && | |
437 | args->exec_state != ETNA_PIPE_VG) { | |
438 | DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); | |
439 | return -EINVAL; | |
440 | } | |
441 | ||
9ad59fea PZ |
442 | if (args->flags & ~ETNA_SUBMIT_FLAGS) { |
443 | DRM_ERROR("invalid flags: 0x%x\n", args->flags); | |
444 | return -EINVAL; | |
445 | } | |
446 | ||
a8c21a54 T |
447 | /* |
448 | * Copy the command submission and bo array to kernel space in | |
449 | * one go, and do this outside of any locks. | |
450 | */ | |
2098105e MH |
451 | bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL); |
452 | relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL); | |
c8e4a7fd | 453 | pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL); |
2098105e | 454 | stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL); |
2f9225db | 455 | if (!bos || !relocs || !pmrs || !stream) { |
a8c21a54 T |
456 | ret = -ENOMEM; |
457 | goto err_submit_cmds; | |
458 | } | |
459 | ||
3ed605bc | 460 | ret = copy_from_user(bos, u64_to_user_ptr(args->bos), |
a8c21a54 T |
461 | args->nr_bos * sizeof(*bos)); |
462 | if (ret) { | |
463 | ret = -EFAULT; | |
464 | goto err_submit_cmds; | |
465 | } | |
466 | ||
3ed605bc | 467 | ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs), |
a8c21a54 T |
468 | args->nr_relocs * sizeof(*relocs)); |
469 | if (ret) { | |
470 | ret = -EFAULT; | |
471 | goto err_submit_cmds; | |
472 | } | |
473 | ||
c8e4a7fd CG |
474 | ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs), |
475 | args->nr_pmrs * sizeof(*pmrs)); | |
476 | if (ret) { | |
477 | ret = -EFAULT; | |
478 | goto err_submit_cmds; | |
479 | } | |
c8e4a7fd | 480 | |
3ed605bc | 481 | ret = copy_from_user(stream, u64_to_user_ptr(args->stream), |
a8c21a54 T |
482 | args->stream_size); |
483 | if (ret) { | |
484 | ret = -EFAULT; | |
485 | goto err_submit_cmds; | |
486 | } | |
487 | ||
78ec187f PZ |
488 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
489 | out_fence_fd = get_unused_fd_flags(O_CLOEXEC); | |
490 | if (out_fence_fd < 0) { | |
491 | ret = out_fence_fd; | |
492 | goto err_submit_cmds; | |
493 | } | |
494 | } | |
495 | ||
08301d73 LS |
496 | ww_acquire_init(&ticket, &reservation_ww_class); |
497 | ||
ef146c00 | 498 | submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs); |
a8c21a54 T |
499 | if (!submit) { |
500 | ret = -ENOMEM; | |
08301d73 | 501 | goto err_submit_ww_acquire; |
a8c21a54 T |
502 | } |
503 | ||
2f9225db LS |
504 | ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf, |
505 | ALIGN(args->stream_size, 8) + 8); | |
506 | if (ret) | |
507 | goto err_submit_objects; | |
508 | ||
509 | submit->cmdbuf.ctx = file->driver_priv; | |
797b0159 | 510 | submit->exec_state = args->exec_state; |
9ad59fea PZ |
511 | submit->flags = args->flags; |
512 | ||
a8c21a54 T |
513 | ret = submit_lookup_objects(submit, file, bos, args->nr_bos); |
514 | if (ret) | |
515 | goto err_submit_objects; | |
516 | ||
a8c21a54 T |
517 | if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, |
518 | relocs, args->nr_relocs)) { | |
519 | ret = -EINVAL; | |
520 | goto err_submit_objects; | |
521 | } | |
522 | ||
9ad59fea | 523 | if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) { |
9efabd73 LS |
524 | submit->in_fence = sync_file_get_fence(args->fence_fd); |
525 | if (!submit->in_fence) { | |
9ad59fea PZ |
526 | ret = -EINVAL; |
527 | goto err_submit_objects; | |
528 | } | |
9ad59fea PZ |
529 | } |
530 | ||
a8c21a54 T |
531 | ret = submit_pin_objects(submit); |
532 | if (ret) | |
33a63e68 | 533 | goto err_submit_objects; |
a8c21a54 T |
534 | |
535 | ret = submit_reloc(submit, stream, args->stream_size / 4, | |
536 | relocs, args->nr_relocs); | |
537 | if (ret) | |
33a63e68 | 538 | goto err_submit_objects; |
a8c21a54 | 539 | |
ef146c00 | 540 | ret = submit_perfmon_validate(submit, args->exec_state, pmrs); |
c8e4a7fd | 541 | if (ret) |
33a63e68 | 542 | goto err_submit_objects; |
c8e4a7fd | 543 | |
2f9225db | 544 | memcpy(submit->cmdbuf.vaddr, stream, args->stream_size); |
a8c21a54 | 545 | |
e0580254 LS |
546 | ret = submit_lock_objects(submit, &ticket); |
547 | if (ret) | |
548 | goto err_submit_objects; | |
549 | ||
550 | ret = submit_fence_sync(submit); | |
551 | if (ret) | |
552 | goto err_submit_objects; | |
553 | ||
e93b6dee | 554 | ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit); |
5a642e6b | 555 | if (ret) |
33a63e68 | 556 | goto err_submit_objects; |
5a642e6b | 557 | |
0236efe9 LS |
558 | submit_attach_object_fences(submit); |
559 | ||
78ec187f PZ |
560 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
561 | /* | |
562 | * This can be improved: ideally we want to allocate the sync | |
563 | * file before kicking off the GPU job and just attach the | |
564 | * fence to the sync file here, eliminating the ENOMEM | |
565 | * possibility at this stage. | |
566 | */ | |
10009ea2 | 567 | sync_file = sync_file_create(submit->out_fence); |
78ec187f PZ |
568 | if (!sync_file) { |
569 | ret = -ENOMEM; | |
33a63e68 | 570 | goto err_submit_objects; |
78ec187f PZ |
571 | } |
572 | fd_install(out_fence_fd, sync_file->file); | |
573 | } | |
574 | ||
575 | args->fence_fd = out_fence_fd; | |
8bc4d885 | 576 | args->fence = submit->out_fence_id; |
a8c21a54 | 577 | |
a8c21a54 | 578 | err_submit_objects: |
e0329e6c | 579 | etnaviv_submit_put(submit); |
a8c21a54 | 580 | |
08301d73 LS |
581 | err_submit_ww_acquire: |
582 | ww_acquire_fini(&ticket); | |
583 | ||
a8c21a54 | 584 | err_submit_cmds: |
78ec187f PZ |
585 | if (ret && (out_fence_fd >= 0)) |
586 | put_unused_fd(out_fence_fd); | |
a8c21a54 | 587 | if (stream) |
2098105e | 588 | kvfree(stream); |
a8c21a54 | 589 | if (bos) |
2098105e | 590 | kvfree(bos); |
a8c21a54 | 591 | if (relocs) |
2098105e | 592 | kvfree(relocs); |
c8e4a7fd CG |
593 | if (pmrs) |
594 | kvfree(pmrs); | |
a8c21a54 T |
595 | |
596 | return ret; | |
597 | } |