]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a8c21a54 T |
2 | /* |
3 | * Copyright (C) 2015 Etnaviv Project | |
a8c21a54 T |
4 | */ |
5 | ||
9ad59fea | 6 | #include <linux/dma-fence-array.h> |
a8c21a54 | 7 | #include <linux/reservation.h> |
9ad59fea | 8 | #include <linux/sync_file.h> |
ea1f5729 | 9 | #include "etnaviv_cmdbuf.h" |
a8c21a54 T |
10 | #include "etnaviv_drv.h" |
11 | #include "etnaviv_gpu.h" | |
12 | #include "etnaviv_gem.h" | |
c8e4a7fd | 13 | #include "etnaviv_perfmon.h" |
e93b6dee | 14 | #include "etnaviv_sched.h" |
a8c21a54 T |
15 | |
16 | /* | |
17 | * Cmdstream submission: | |
18 | */ | |
19 | ||
20 | #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) | |
21 | /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ | |
22 | #define BO_LOCKED 0x4000 | |
23 | #define BO_PINNED 0x2000 | |
24 | ||
a8c21a54 | 25 | static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, |
ef146c00 | 26 | struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs) |
a8c21a54 T |
27 | { |
28 | struct etnaviv_gem_submit *submit; | |
ef146c00 | 29 | size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit)); |
a8c21a54 | 30 | |
c5283723 LS |
31 | submit = kzalloc(sz, GFP_KERNEL); |
32 | if (!submit) | |
33 | return NULL; | |
a8c21a54 | 34 | |
ef146c00 LS |
35 | submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request), |
36 | GFP_KERNEL); | |
37 | if (!submit->pmrs) { | |
38 | kfree(submit); | |
39 | return NULL; | |
40 | } | |
41 | submit->nr_pmrs = nr_pmrs; | |
42 | ||
c5283723 | 43 | submit->gpu = gpu; |
e0329e6c | 44 | kref_init(&submit->refcount); |
a8c21a54 | 45 | |
a8c21a54 T |
46 | return submit; |
47 | } | |
48 | ||
49 | static int submit_lookup_objects(struct etnaviv_gem_submit *submit, | |
50 | struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, | |
51 | unsigned nr_bos) | |
52 | { | |
53 | struct drm_etnaviv_gem_submit_bo *bo; | |
54 | unsigned i; | |
55 | int ret = 0; | |
56 | ||
57 | spin_lock(&file->table_lock); | |
58 | ||
59 | for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { | |
60 | struct drm_gem_object *obj; | |
61 | ||
62 | if (bo->flags & BO_INVALID_FLAGS) { | |
63 | DRM_ERROR("invalid flags: %x\n", bo->flags); | |
64 | ret = -EINVAL; | |
65 | goto out_unlock; | |
66 | } | |
67 | ||
68 | submit->bos[i].flags = bo->flags; | |
69 | ||
70 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
71 | * all under single table_lock just hit object_idr directly: | |
72 | */ | |
73 | obj = idr_find(&file->object_idr, bo->handle); | |
74 | if (!obj) { | |
75 | DRM_ERROR("invalid handle %u at index %u\n", | |
76 | bo->handle, i); | |
77 | ret = -EINVAL; | |
78 | goto out_unlock; | |
79 | } | |
80 | ||
81 | /* | |
82 | * Take a refcount on the object. The file table lock | |
83 | * prevents the object_idr's refcount on this being dropped. | |
84 | */ | |
23d1dd03 | 85 | drm_gem_object_get(obj); |
a8c21a54 T |
86 | |
87 | submit->bos[i].obj = to_etnaviv_bo(obj); | |
88 | } | |
89 | ||
90 | out_unlock: | |
91 | submit->nr_bos = i; | |
92 | spin_unlock(&file->table_lock); | |
93 | ||
94 | return ret; | |
95 | } | |
96 | ||
97 | static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) | |
98 | { | |
99 | if (submit->bos[i].flags & BO_LOCKED) { | |
fa238ea1 | 100 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
a8c21a54 | 101 | |
fa238ea1 | 102 | ww_mutex_unlock(&obj->resv->lock); |
a8c21a54 T |
103 | submit->bos[i].flags &= ~BO_LOCKED; |
104 | } | |
105 | } | |
106 | ||
08301d73 LS |
107 | static int submit_lock_objects(struct etnaviv_gem_submit *submit, |
108 | struct ww_acquire_ctx *ticket) | |
a8c21a54 T |
109 | { |
110 | int contended, slow_locked = -1, i, ret = 0; | |
111 | ||
112 | retry: | |
113 | for (i = 0; i < submit->nr_bos; i++) { | |
fa238ea1 | 114 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
a8c21a54 T |
115 | |
116 | if (slow_locked == i) | |
117 | slow_locked = -1; | |
118 | ||
119 | contended = i; | |
120 | ||
121 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
fa238ea1 | 122 | ret = ww_mutex_lock_interruptible(&obj->resv->lock, |
08301d73 | 123 | ticket); |
a8c21a54 T |
124 | if (ret == -EALREADY) |
125 | DRM_ERROR("BO at index %u already on submit list\n", | |
126 | i); | |
127 | if (ret) | |
128 | goto fail; | |
129 | submit->bos[i].flags |= BO_LOCKED; | |
130 | } | |
131 | } | |
132 | ||
08301d73 | 133 | ww_acquire_done(ticket); |
a8c21a54 T |
134 | |
135 | return 0; | |
136 | ||
137 | fail: | |
138 | for (; i >= 0; i--) | |
139 | submit_unlock_object(submit, i); | |
140 | ||
141 | if (slow_locked > 0) | |
142 | submit_unlock_object(submit, slow_locked); | |
143 | ||
144 | if (ret == -EDEADLK) { | |
fa238ea1 | 145 | struct drm_gem_object *obj; |
a8c21a54 | 146 | |
fa238ea1 | 147 | obj = &submit->bos[contended].obj->base; |
a8c21a54 T |
148 | |
149 | /* we lost out in a seqno race, lock and retry.. */ | |
fa238ea1 | 150 | ret = ww_mutex_lock_slow_interruptible(&obj->resv->lock, |
08301d73 | 151 | ticket); |
a8c21a54 T |
152 | if (!ret) { |
153 | submit->bos[contended].flags |= BO_LOCKED; | |
154 | slow_locked = contended; | |
155 | goto retry; | |
156 | } | |
157 | } | |
158 | ||
159 | return ret; | |
160 | } | |
161 | ||
683da226 | 162 | static int submit_fence_sync(struct etnaviv_gem_submit *submit) |
a8c21a54 | 163 | { |
a8c21a54 T |
164 | int i, ret = 0; |
165 | ||
166 | for (i = 0; i < submit->nr_bos; i++) { | |
683da226 | 167 | struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; |
fa238ea1 | 168 | struct reservation_object *robj = bo->obj->base.resv; |
a8c21a54 | 169 | |
683da226 | 170 | if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) { |
ca05359f | 171 | ret = reservation_object_reserve_shared(robj, 1); |
683da226 LS |
172 | if (ret) |
173 | return ret; | |
174 | } | |
175 | ||
176 | if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT) | |
177 | continue; | |
178 | ||
179 | if (bo->flags & ETNA_SUBMIT_BO_WRITE) { | |
180 | ret = reservation_object_get_fences_rcu(robj, &bo->excl, | |
181 | &bo->nr_shared, | |
182 | &bo->shared); | |
183 | if (ret) | |
184 | return ret; | |
185 | } else { | |
186 | bo->excl = reservation_object_get_excl_rcu(robj); | |
187 | } | |
a8c21a54 | 188 | |
9efabd73 LS |
189 | } |
190 | ||
a8c21a54 T |
191 | return ret; |
192 | } | |
193 | ||
0236efe9 LS |
194 | static void submit_attach_object_fences(struct etnaviv_gem_submit *submit) |
195 | { | |
196 | int i; | |
197 | ||
198 | for (i = 0; i < submit->nr_bos; i++) { | |
fa238ea1 | 199 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
0236efe9 LS |
200 | |
201 | if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) | |
fa238ea1 | 202 | reservation_object_add_excl_fence(obj->resv, |
10009ea2 | 203 | submit->out_fence); |
0236efe9 | 204 | else |
fa238ea1 | 205 | reservation_object_add_shared_fence(obj->resv, |
10009ea2 | 206 | submit->out_fence); |
0236efe9 LS |
207 | |
208 | submit_unlock_object(submit, i); | |
209 | } | |
210 | } | |
211 | ||
a8c21a54 T |
212 | static int submit_pin_objects(struct etnaviv_gem_submit *submit) |
213 | { | |
214 | int i, ret = 0; | |
215 | ||
216 | for (i = 0; i < submit->nr_bos; i++) { | |
217 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
b6325f40 | 218 | struct etnaviv_vram_mapping *mapping; |
a8c21a54 | 219 | |
b6325f40 RK |
220 | mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base, |
221 | submit->gpu); | |
222 | if (IS_ERR(mapping)) { | |
223 | ret = PTR_ERR(mapping); | |
a8c21a54 | 224 | break; |
b6325f40 | 225 | } |
5b223e94 | 226 | atomic_inc(&etnaviv_obj->gpu_active); |
a8c21a54 T |
227 | |
228 | submit->bos[i].flags |= BO_PINNED; | |
b6325f40 | 229 | submit->bos[i].mapping = mapping; |
a8c21a54 T |
230 | } |
231 | ||
232 | return ret; | |
233 | } | |
234 | ||
235 | static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, | |
8779aa8f | 236 | struct etnaviv_gem_submit_bo **bo) |
a8c21a54 T |
237 | { |
238 | if (idx >= submit->nr_bos) { | |
239 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", | |
240 | idx, submit->nr_bos); | |
241 | return -EINVAL; | |
242 | } | |
243 | ||
8779aa8f | 244 | *bo = &submit->bos[idx]; |
a8c21a54 T |
245 | |
246 | return 0; | |
247 | } | |
248 | ||
249 | /* process the reloc's and patch up the cmdstream as needed: */ | |
250 | static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, | |
251 | u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, | |
252 | u32 nr_relocs) | |
253 | { | |
254 | u32 i, last_offset = 0; | |
255 | u32 *ptr = stream; | |
256 | int ret; | |
257 | ||
258 | for (i = 0; i < nr_relocs; i++) { | |
259 | const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; | |
8779aa8f RK |
260 | struct etnaviv_gem_submit_bo *bo; |
261 | u32 off; | |
a8c21a54 T |
262 | |
263 | if (unlikely(r->flags)) { | |
264 | DRM_ERROR("invalid reloc flags\n"); | |
265 | return -EINVAL; | |
266 | } | |
267 | ||
268 | if (r->submit_offset % 4) { | |
269 | DRM_ERROR("non-aligned reloc offset: %u\n", | |
270 | r->submit_offset); | |
271 | return -EINVAL; | |
272 | } | |
273 | ||
274 | /* offset in dwords: */ | |
275 | off = r->submit_offset / 4; | |
276 | ||
277 | if ((off >= size ) || | |
278 | (off < last_offset)) { | |
279 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); | |
280 | return -EINVAL; | |
281 | } | |
282 | ||
8779aa8f | 283 | ret = submit_bo(submit, r->reloc_idx, &bo); |
a8c21a54 T |
284 | if (ret) |
285 | return ret; | |
286 | ||
d6f756e0 WL |
287 | if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) { |
288 | DRM_ERROR("relocation %u outside object\n", i); | |
a8c21a54 T |
289 | return -EINVAL; |
290 | } | |
291 | ||
8779aa8f | 292 | ptr[off] = bo->mapping->iova + r->reloc_offset; |
a8c21a54 T |
293 | |
294 | last_offset = off; | |
295 | } | |
296 | ||
297 | return 0; | |
298 | } | |
299 | ||
c8e4a7fd | 300 | static int submit_perfmon_validate(struct etnaviv_gem_submit *submit, |
ef146c00 | 301 | u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs) |
c8e4a7fd CG |
302 | { |
303 | u32 i; | |
304 | ||
ef146c00 | 305 | for (i = 0; i < submit->nr_pmrs; i++) { |
c8e4a7fd CG |
306 | const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i; |
307 | struct etnaviv_gem_submit_bo *bo; | |
308 | int ret; | |
309 | ||
310 | ret = submit_bo(submit, r->read_idx, &bo); | |
311 | if (ret) | |
312 | return ret; | |
313 | ||
314 | /* at offset 0 a sequence number gets stored used for userspace sync */ | |
315 | if (r->read_offset == 0) { | |
316 | DRM_ERROR("perfmon request: offset is 0"); | |
317 | return -EINVAL; | |
318 | } | |
319 | ||
320 | if (r->read_offset >= bo->obj->base.size - sizeof(u32)) { | |
321 | DRM_ERROR("perfmon request: offset %u outside object", i); | |
322 | return -EINVAL; | |
323 | } | |
324 | ||
325 | if (r->flags & ~(ETNA_PM_PROCESS_PRE | ETNA_PM_PROCESS_POST)) { | |
326 | DRM_ERROR("perfmon request: flags are not valid"); | |
327 | return -EINVAL; | |
328 | } | |
329 | ||
ef146c00 | 330 | if (etnaviv_pm_req_validate(r, exec_state)) { |
c8e4a7fd CG |
331 | DRM_ERROR("perfmon request: domain or signal not valid"); |
332 | return -EINVAL; | |
333 | } | |
334 | ||
ef146c00 LS |
335 | submit->pmrs[i].flags = r->flags; |
336 | submit->pmrs[i].domain = r->domain; | |
337 | submit->pmrs[i].signal = r->signal; | |
338 | submit->pmrs[i].sequence = r->sequence; | |
339 | submit->pmrs[i].offset = r->read_offset; | |
340 | submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base); | |
c8e4a7fd CG |
341 | } |
342 | ||
343 | return 0; | |
344 | } | |
345 | ||
e0329e6c | 346 | static void submit_cleanup(struct kref *kref) |
a8c21a54 | 347 | { |
e0329e6c LS |
348 | struct etnaviv_gem_submit *submit = |
349 | container_of(kref, struct etnaviv_gem_submit, refcount); | |
a8c21a54 T |
350 | unsigned i; |
351 | ||
8bda1516 LS |
352 | if (submit->runtime_resumed) |
353 | pm_runtime_put_autosuspend(submit->gpu->dev); | |
354 | ||
2f9225db LS |
355 | if (submit->cmdbuf.suballoc) |
356 | etnaviv_cmdbuf_free(&submit->cmdbuf); | |
357 | ||
a8c21a54 T |
358 | for (i = 0; i < submit->nr_bos; i++) { |
359 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
360 | ||
33a63e68 LS |
361 | /* unpin all objects */ |
362 | if (submit->bos[i].flags & BO_PINNED) { | |
363 | etnaviv_gem_mapping_unreference(submit->bos[i].mapping); | |
5b223e94 | 364 | atomic_dec(&etnaviv_obj->gpu_active); |
33a63e68 LS |
365 | submit->bos[i].mapping = NULL; |
366 | submit->bos[i].flags &= ~BO_PINNED; | |
367 | } | |
368 | ||
0236efe9 | 369 | /* if the GPU submit failed, objects might still be locked */ |
a8c21a54 | 370 | submit_unlock_object(submit, i); |
23d1dd03 | 371 | drm_gem_object_put_unlocked(&etnaviv_obj->base); |
a8c21a54 T |
372 | } |
373 | ||
5b223e94 LS |
374 | wake_up_all(&submit->gpu->fence_event); |
375 | ||
9efabd73 LS |
376 | if (submit->in_fence) |
377 | dma_fence_put(submit->in_fence); | |
e93b6dee LS |
378 | if (submit->out_fence) { |
379 | /* first remove from IDR, so fence can not be found anymore */ | |
a0780bb1 | 380 | mutex_lock(&submit->gpu->fence_lock); |
e93b6dee | 381 | idr_remove(&submit->gpu->fence_idr, submit->out_fence_id); |
a0780bb1 | 382 | mutex_unlock(&submit->gpu->fence_lock); |
10009ea2 | 383 | dma_fence_put(submit->out_fence); |
e93b6dee | 384 | } |
ef146c00 | 385 | kfree(submit->pmrs); |
a8c21a54 T |
386 | kfree(submit); |
387 | } | |
388 | ||
e0329e6c LS |
389 | void etnaviv_submit_put(struct etnaviv_gem_submit *submit) |
390 | { | |
391 | kref_put(&submit->refcount, submit_cleanup); | |
392 | } | |
393 | ||
a8c21a54 T |
394 | int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, |
395 | struct drm_file *file) | |
396 | { | |
e93b6dee | 397 | struct etnaviv_file_private *ctx = file->driver_priv; |
a8c21a54 T |
398 | struct etnaviv_drm_private *priv = dev->dev_private; |
399 | struct drm_etnaviv_gem_submit *args = data; | |
400 | struct drm_etnaviv_gem_submit_reloc *relocs; | |
c8e4a7fd | 401 | struct drm_etnaviv_gem_submit_pmr *pmrs; |
a8c21a54 T |
402 | struct drm_etnaviv_gem_submit_bo *bos; |
403 | struct etnaviv_gem_submit *submit; | |
a8c21a54 | 404 | struct etnaviv_gpu *gpu; |
78ec187f | 405 | struct sync_file *sync_file = NULL; |
08301d73 | 406 | struct ww_acquire_ctx ticket; |
78ec187f | 407 | int out_fence_fd = -1; |
a8c21a54 T |
408 | void *stream; |
409 | int ret; | |
410 | ||
411 | if (args->pipe >= ETNA_MAX_PIPES) | |
412 | return -EINVAL; | |
413 | ||
414 | gpu = priv->gpu[args->pipe]; | |
415 | if (!gpu) | |
416 | return -ENXIO; | |
417 | ||
418 | if (args->stream_size % 4) { | |
419 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", | |
420 | args->stream_size); | |
421 | return -EINVAL; | |
422 | } | |
423 | ||
424 | if (args->exec_state != ETNA_PIPE_3D && | |
425 | args->exec_state != ETNA_PIPE_2D && | |
426 | args->exec_state != ETNA_PIPE_VG) { | |
427 | DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); | |
428 | return -EINVAL; | |
429 | } | |
430 | ||
9ad59fea PZ |
431 | if (args->flags & ~ETNA_SUBMIT_FLAGS) { |
432 | DRM_ERROR("invalid flags: 0x%x\n", args->flags); | |
433 | return -EINVAL; | |
434 | } | |
435 | ||
a8c21a54 T |
436 | /* |
437 | * Copy the command submission and bo array to kernel space in | |
438 | * one go, and do this outside of any locks. | |
439 | */ | |
2098105e MH |
440 | bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL); |
441 | relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL); | |
c8e4a7fd | 442 | pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL); |
2098105e | 443 | stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL); |
2f9225db | 444 | if (!bos || !relocs || !pmrs || !stream) { |
a8c21a54 T |
445 | ret = -ENOMEM; |
446 | goto err_submit_cmds; | |
447 | } | |
448 | ||
3ed605bc | 449 | ret = copy_from_user(bos, u64_to_user_ptr(args->bos), |
a8c21a54 T |
450 | args->nr_bos * sizeof(*bos)); |
451 | if (ret) { | |
452 | ret = -EFAULT; | |
453 | goto err_submit_cmds; | |
454 | } | |
455 | ||
3ed605bc | 456 | ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs), |
a8c21a54 T |
457 | args->nr_relocs * sizeof(*relocs)); |
458 | if (ret) { | |
459 | ret = -EFAULT; | |
460 | goto err_submit_cmds; | |
461 | } | |
462 | ||
c8e4a7fd CG |
463 | ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs), |
464 | args->nr_pmrs * sizeof(*pmrs)); | |
465 | if (ret) { | |
466 | ret = -EFAULT; | |
467 | goto err_submit_cmds; | |
468 | } | |
c8e4a7fd | 469 | |
3ed605bc | 470 | ret = copy_from_user(stream, u64_to_user_ptr(args->stream), |
a8c21a54 T |
471 | args->stream_size); |
472 | if (ret) { | |
473 | ret = -EFAULT; | |
474 | goto err_submit_cmds; | |
475 | } | |
476 | ||
78ec187f PZ |
477 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
478 | out_fence_fd = get_unused_fd_flags(O_CLOEXEC); | |
479 | if (out_fence_fd < 0) { | |
480 | ret = out_fence_fd; | |
481 | goto err_submit_cmds; | |
482 | } | |
483 | } | |
484 | ||
08301d73 LS |
485 | ww_acquire_init(&ticket, &reservation_ww_class); |
486 | ||
ef146c00 | 487 | submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs); |
a8c21a54 T |
488 | if (!submit) { |
489 | ret = -ENOMEM; | |
08301d73 | 490 | goto err_submit_ww_acquire; |
a8c21a54 T |
491 | } |
492 | ||
2f9225db LS |
493 | ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf, |
494 | ALIGN(args->stream_size, 8) + 8); | |
495 | if (ret) | |
496 | goto err_submit_objects; | |
497 | ||
72ac64b8 | 498 | submit->ctx = file->driver_priv; |
797b0159 | 499 | submit->exec_state = args->exec_state; |
9ad59fea PZ |
500 | submit->flags = args->flags; |
501 | ||
a8c21a54 T |
502 | ret = submit_lookup_objects(submit, file, bos, args->nr_bos); |
503 | if (ret) | |
504 | goto err_submit_objects; | |
505 | ||
a8c21a54 T |
506 | if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, |
507 | relocs, args->nr_relocs)) { | |
508 | ret = -EINVAL; | |
509 | goto err_submit_objects; | |
510 | } | |
511 | ||
9ad59fea | 512 | if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) { |
9efabd73 LS |
513 | submit->in_fence = sync_file_get_fence(args->fence_fd); |
514 | if (!submit->in_fence) { | |
9ad59fea PZ |
515 | ret = -EINVAL; |
516 | goto err_submit_objects; | |
517 | } | |
9ad59fea PZ |
518 | } |
519 | ||
a8c21a54 T |
520 | ret = submit_pin_objects(submit); |
521 | if (ret) | |
33a63e68 | 522 | goto err_submit_objects; |
a8c21a54 T |
523 | |
524 | ret = submit_reloc(submit, stream, args->stream_size / 4, | |
525 | relocs, args->nr_relocs); | |
526 | if (ret) | |
33a63e68 | 527 | goto err_submit_objects; |
a8c21a54 | 528 | |
ef146c00 | 529 | ret = submit_perfmon_validate(submit, args->exec_state, pmrs); |
c8e4a7fd | 530 | if (ret) |
33a63e68 | 531 | goto err_submit_objects; |
c8e4a7fd | 532 | |
2f9225db | 533 | memcpy(submit->cmdbuf.vaddr, stream, args->stream_size); |
a8c21a54 | 534 | |
e0580254 LS |
535 | ret = submit_lock_objects(submit, &ticket); |
536 | if (ret) | |
537 | goto err_submit_objects; | |
538 | ||
539 | ret = submit_fence_sync(submit); | |
540 | if (ret) | |
541 | goto err_submit_objects; | |
542 | ||
e93b6dee | 543 | ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit); |
5a642e6b | 544 | if (ret) |
33a63e68 | 545 | goto err_submit_objects; |
5a642e6b | 546 | |
0236efe9 LS |
547 | submit_attach_object_fences(submit); |
548 | ||
78ec187f PZ |
549 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
550 | /* | |
551 | * This can be improved: ideally we want to allocate the sync | |
552 | * file before kicking off the GPU job and just attach the | |
553 | * fence to the sync file here, eliminating the ENOMEM | |
554 | * possibility at this stage. | |
555 | */ | |
10009ea2 | 556 | sync_file = sync_file_create(submit->out_fence); |
78ec187f PZ |
557 | if (!sync_file) { |
558 | ret = -ENOMEM; | |
33a63e68 | 559 | goto err_submit_objects; |
78ec187f PZ |
560 | } |
561 | fd_install(out_fence_fd, sync_file->file); | |
562 | } | |
563 | ||
564 | args->fence_fd = out_fence_fd; | |
8bc4d885 | 565 | args->fence = submit->out_fence_id; |
a8c21a54 | 566 | |
a8c21a54 | 567 | err_submit_objects: |
e0329e6c | 568 | etnaviv_submit_put(submit); |
a8c21a54 | 569 | |
08301d73 LS |
570 | err_submit_ww_acquire: |
571 | ww_acquire_fini(&ticket); | |
572 | ||
a8c21a54 | 573 | err_submit_cmds: |
78ec187f PZ |
574 | if (ret && (out_fence_fd >= 0)) |
575 | put_unused_fd(out_fence_fd); | |
a8c21a54 | 576 | if (stream) |
2098105e | 577 | kvfree(stream); |
a8c21a54 | 578 | if (bos) |
2098105e | 579 | kvfree(bos); |
a8c21a54 | 580 | if (relocs) |
2098105e | 581 | kvfree(relocs); |
c8e4a7fd CG |
582 | if (pmrs) |
583 | kvfree(pmrs); | |
a8c21a54 T |
584 | |
585 | return ret; | |
586 | } |