]>
Commit | Line | Data |
---|---|---|
1802d0be | 1 | // SPDX-License-Identifier: GPL-2.0-only |
b3dfbdf2 | 2 | /* |
f54d1867 | 3 | * dma-fence-array: aggregate fences to be waited together |
b3dfbdf2 GP |
4 | * |
5 | * Copyright (C) 2016 Collabora Ltd | |
6 | * Copyright (C) 2016 Advanced Micro Devices, Inc. | |
7 | * Authors: | |
8 | * Gustavo Padovan <gustavo@padovan.org> | |
9 | * Christian König <christian.koenig@amd.com> | |
b3dfbdf2 GP |
10 | */ |
11 | ||
12 | #include <linux/export.h> | |
13 | #include <linux/slab.h> | |
f54d1867 | 14 | #include <linux/dma-fence-array.h> |
b3dfbdf2 | 15 | |
1f70b8b8 CW |
16 | #define PENDING_ERROR 1 |
17 | ||
f54d1867 | 18 | static const char *dma_fence_array_get_driver_name(struct dma_fence *fence) |
b3dfbdf2 | 19 | { |
f54d1867 | 20 | return "dma_fence_array"; |
b3dfbdf2 GP |
21 | } |
22 | ||
f54d1867 | 23 | static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence) |
b3dfbdf2 GP |
24 | { |
25 | return "unbound"; | |
26 | } | |
27 | ||
1f70b8b8 CW |
28 | static void dma_fence_array_set_pending_error(struct dma_fence_array *array, |
29 | int error) | |
30 | { | |
31 | /* | |
32 | * Propagate the first error reported by any of our fences, but only | |
33 | * before we ourselves are signaled. | |
34 | */ | |
35 | if (error) | |
36 | cmpxchg(&array->base.error, PENDING_ERROR, error); | |
37 | } | |
38 | ||
39 | static void dma_fence_array_clear_pending_error(struct dma_fence_array *array) | |
40 | { | |
41 | /* Clear the error flag if not actually set. */ | |
42 | cmpxchg(&array->base.error, PENDING_ERROR, 0); | |
43 | } | |
44 | ||
03e4e0a9 CW |
45 | static void irq_dma_fence_array_work(struct irq_work *wrk) |
46 | { | |
47 | struct dma_fence_array *array = container_of(wrk, typeof(*array), work); | |
48 | ||
1f70b8b8 CW |
49 | dma_fence_array_clear_pending_error(array); |
50 | ||
03e4e0a9 CW |
51 | dma_fence_signal(&array->base); |
52 | dma_fence_put(&array->base); | |
53 | } | |
54 | ||
f54d1867 CW |
55 | static void dma_fence_array_cb_func(struct dma_fence *f, |
56 | struct dma_fence_cb *cb) | |
b3dfbdf2 | 57 | { |
f54d1867 CW |
58 | struct dma_fence_array_cb *array_cb = |
59 | container_of(cb, struct dma_fence_array_cb, cb); | |
60 | struct dma_fence_array *array = array_cb->array; | |
b3dfbdf2 | 61 | |
1f70b8b8 CW |
62 | dma_fence_array_set_pending_error(array, f->error); |
63 | ||
b3dfbdf2 | 64 | if (atomic_dec_and_test(&array->num_pending)) |
03e4e0a9 CW |
65 | irq_work_queue(&array->work); |
66 | else | |
67 | dma_fence_put(&array->base); | |
b3dfbdf2 GP |
68 | } |
69 | ||
f54d1867 | 70 | static bool dma_fence_array_enable_signaling(struct dma_fence *fence) |
b3dfbdf2 | 71 | { |
f54d1867 CW |
72 | struct dma_fence_array *array = to_dma_fence_array(fence); |
73 | struct dma_fence_array_cb *cb = (void *)(&array[1]); | |
b3dfbdf2 GP |
74 | unsigned i; |
75 | ||
76 | for (i = 0; i < array->num_fences; ++i) { | |
77 | cb[i].array = array; | |
f7104568 CK |
78 | /* |
79 | * As we may report that the fence is signaled before all | |
80 | * callbacks are complete, we need to take an additional | |
81 | * reference count on the array so that we do not free it too | |
82 | * early. The core fence handling will only hold the reference | |
83 | * until we signal the array as complete (but that is now | |
84 | * insufficient). | |
85 | */ | |
f54d1867 CW |
86 | dma_fence_get(&array->base); |
87 | if (dma_fence_add_callback(array->fences[i], &cb[i].cb, | |
88 | dma_fence_array_cb_func)) { | |
1f70b8b8 CW |
89 | int error = array->fences[i]->error; |
90 | ||
91 | dma_fence_array_set_pending_error(array, error); | |
f54d1867 | 92 | dma_fence_put(&array->base); |
1f70b8b8 CW |
93 | if (atomic_dec_and_test(&array->num_pending)) { |
94 | dma_fence_array_clear_pending_error(array); | |
b3dfbdf2 | 95 | return false; |
1f70b8b8 | 96 | } |
f7104568 | 97 | } |
b3dfbdf2 GP |
98 | } |
99 | ||
100 | return true; | |
101 | } | |
102 | ||
f54d1867 | 103 | static bool dma_fence_array_signaled(struct dma_fence *fence) |
b3dfbdf2 | 104 | { |
f54d1867 | 105 | struct dma_fence_array *array = to_dma_fence_array(fence); |
b3dfbdf2 | 106 | |
fab86220 TH |
107 | if (atomic_read(&array->num_pending) > 0) |
108 | return false; | |
109 | ||
110 | dma_fence_array_clear_pending_error(array); | |
111 | return true; | |
b3dfbdf2 GP |
112 | } |
113 | ||
f54d1867 | 114 | static void dma_fence_array_release(struct dma_fence *fence) |
b3dfbdf2 | 115 | { |
f54d1867 | 116 | struct dma_fence_array *array = to_dma_fence_array(fence); |
b3dfbdf2 GP |
117 | unsigned i; |
118 | ||
119 | for (i = 0; i < array->num_fences; ++i) | |
f54d1867 | 120 | dma_fence_put(array->fences[i]); |
b3dfbdf2 GP |
121 | |
122 | kfree(array->fences); | |
f54d1867 | 123 | dma_fence_free(fence); |
b3dfbdf2 GP |
124 | } |
125 | ||
f54d1867 CW |
126 | const struct dma_fence_ops dma_fence_array_ops = { |
127 | .get_driver_name = dma_fence_array_get_driver_name, | |
128 | .get_timeline_name = dma_fence_array_get_timeline_name, | |
129 | .enable_signaling = dma_fence_array_enable_signaling, | |
130 | .signaled = dma_fence_array_signaled, | |
f54d1867 | 131 | .release = dma_fence_array_release, |
b3dfbdf2 | 132 | }; |
f54d1867 | 133 | EXPORT_SYMBOL(dma_fence_array_ops); |
b3dfbdf2 GP |
134 | |
135 | /** | |
f54d1867 | 136 | * dma_fence_array_create - Create a custom fence array |
f7104568 CK |
137 | * @num_fences: [in] number of fences to add in the array |
138 | * @fences: [in] array containing the fences | |
139 | * @context: [in] fence context to use | |
140 | * @seqno: [in] sequence number to use | |
68acb6af | 141 | * @signal_on_any: [in] signal on any fence in the array |
b3dfbdf2 | 142 | * |
f54d1867 CW |
143 | * Allocate a dma_fence_array object and initialize the base fence with |
144 | * dma_fence_init(). | |
b3dfbdf2 GP |
145 | * In case of error it returns NULL. |
146 | * | |
68acb6af | 147 | * The caller should allocate the fences array with num_fences size |
b3dfbdf2 | 148 | * and fill it with the fences it wants to add to the object. Ownership of this |
f54d1867 | 149 | * array is taken and dma_fence_put() is used on each fence on release. |
f7104568 CK |
150 | * |
151 | * If @signal_on_any is true the fence array signals if any fence in the array | |
152 | * signals, otherwise it signals when all fences in the array signal. | |
b3dfbdf2 | 153 | */ |
f54d1867 CW |
154 | struct dma_fence_array *dma_fence_array_create(int num_fences, |
155 | struct dma_fence **fences, | |
156 | u64 context, unsigned seqno, | |
157 | bool signal_on_any) | |
b3dfbdf2 | 158 | { |
f54d1867 | 159 | struct dma_fence_array *array; |
b3dfbdf2 GP |
160 | size_t size = sizeof(*array); |
161 | ||
162 | /* Allocate the callback structures behind the array. */ | |
f54d1867 | 163 | size += num_fences * sizeof(struct dma_fence_array_cb); |
b3dfbdf2 GP |
164 | array = kzalloc(size, GFP_KERNEL); |
165 | if (!array) | |
166 | return NULL; | |
167 | ||
168 | spin_lock_init(&array->lock); | |
f54d1867 CW |
169 | dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock, |
170 | context, seqno); | |
03e4e0a9 | 171 | init_irq_work(&array->work, irq_dma_fence_array_work); |
b3dfbdf2 GP |
172 | |
173 | array->num_fences = num_fences; | |
f7104568 | 174 | atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences); |
b3dfbdf2 GP |
175 | array->fences = fences; |
176 | ||
1f70b8b8 CW |
177 | array->base.error = PENDING_ERROR; |
178 | ||
b3dfbdf2 GP |
179 | return array; |
180 | } | |
f54d1867 | 181 | EXPORT_SYMBOL(dma_fence_array_create); |
d5b72a21 PZ |
182 | |
183 | /** | |
184 | * dma_fence_match_context - Check if all fences are from the given context | |
185 | * @fence: [in] fence or fence array | |
186 | * @context: [in] fence context to check all fences against | |
187 | * | |
188 | * Checks the provided fence or, for a fence array, all fences in the array | |
189 | * against the given context. Returns false if any fence is from a different | |
190 | * context. | |
191 | */ | |
192 | bool dma_fence_match_context(struct dma_fence *fence, u64 context) | |
193 | { | |
194 | struct dma_fence_array *array = to_dma_fence_array(fence); | |
195 | unsigned i; | |
196 | ||
197 | if (!dma_fence_is_array(fence)) | |
198 | return fence->context == context; | |
199 | ||
200 | for (i = 0; i < array->num_fences; i++) { | |
201 | if (array->fences[i]->context != context) | |
202 | return false; | |
203 | } | |
204 | ||
205 | return true; | |
206 | } | |
207 | EXPORT_SYMBOL(dma_fence_match_context); |