]>
Commit | Line | Data |
---|---|---|
735d37b5 BW |
1 | /* |
2 | * Handle async block request by crypto hardware engine. | |
3 | * | |
4 | * Copyright (C) 2016 Linaro, Inc. | |
5 | * | |
6 | * Author: Baolin Wang <baolin.wang@linaro.org> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify it | |
9 | * under the terms of the GNU General Public License as published by the Free | |
10 | * Software Foundation; either version 2 of the License, or (at your option) | |
11 | * any later version. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <linux/err.h> | |
16 | #include <linux/delay.h> | |
2589ad84 | 17 | #include <crypto/engine.h> |
735d37b5 BW |
18 | #include "internal.h" |
19 | ||
20 | #define CRYPTO_ENGINE_MAX_QLEN 10 | |
21 | ||
22 | void crypto_finalize_request(struct crypto_engine *engine, | |
23 | struct ablkcipher_request *req, int err); | |
24 | ||
25 | /** | |
26 | * crypto_pump_requests - dequeue one request from engine queue to process | |
27 | * @engine: the hardware engine | |
28 | * @in_kthread: true if we are in the context of the request pump thread | |
29 | * | |
30 | * This function checks if there is any request in the engine queue that | |
31 | * needs processing and if so call out to the driver to initialize hardware | |
32 | * and handle each request. | |
33 | */ | |
34 | static void crypto_pump_requests(struct crypto_engine *engine, | |
35 | bool in_kthread) | |
36 | { | |
37 | struct crypto_async_request *async_req, *backlog; | |
38 | struct ablkcipher_request *req; | |
39 | unsigned long flags; | |
40 | bool was_busy = false; | |
41 | int ret; | |
42 | ||
43 | spin_lock_irqsave(&engine->queue_lock, flags); | |
44 | ||
45 | /* Make sure we are not already running a request */ | |
46 | if (engine->cur_req) | |
47 | goto out; | |
48 | ||
49 | /* If another context is idling then defer */ | |
50 | if (engine->idling) { | |
51 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
52 | goto out; | |
53 | } | |
54 | ||
55 | /* Check if the engine queue is idle */ | |
56 | if (!crypto_queue_len(&engine->queue) || !engine->running) { | |
57 | if (!engine->busy) | |
58 | goto out; | |
59 | ||
60 | /* Only do teardown in the thread */ | |
61 | if (!in_kthread) { | |
62 | queue_kthread_work(&engine->kworker, | |
63 | &engine->pump_requests); | |
64 | goto out; | |
65 | } | |
66 | ||
67 | engine->busy = false; | |
68 | engine->idling = true; | |
69 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
70 | ||
71 | if (engine->unprepare_crypt_hardware && | |
72 | engine->unprepare_crypt_hardware(engine)) | |
73 | pr_err("failed to unprepare crypt hardware\n"); | |
74 | ||
75 | spin_lock_irqsave(&engine->queue_lock, flags); | |
76 | engine->idling = false; | |
77 | goto out; | |
78 | } | |
79 | ||
80 | /* Get the fist request from the engine queue to handle */ | |
81 | backlog = crypto_get_backlog(&engine->queue); | |
82 | async_req = crypto_dequeue_request(&engine->queue); | |
83 | if (!async_req) | |
84 | goto out; | |
85 | ||
86 | req = ablkcipher_request_cast(async_req); | |
87 | ||
88 | engine->cur_req = req; | |
89 | if (backlog) | |
90 | backlog->complete(backlog, -EINPROGRESS); | |
91 | ||
92 | if (engine->busy) | |
93 | was_busy = true; | |
94 | else | |
95 | engine->busy = true; | |
96 | ||
97 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
98 | ||
99 | /* Until here we get the request need to be encrypted successfully */ | |
100 | if (!was_busy && engine->prepare_crypt_hardware) { | |
101 | ret = engine->prepare_crypt_hardware(engine); | |
102 | if (ret) { | |
103 | pr_err("failed to prepare crypt hardware\n"); | |
104 | goto req_err; | |
105 | } | |
106 | } | |
107 | ||
108 | if (engine->prepare_request) { | |
109 | ret = engine->prepare_request(engine, engine->cur_req); | |
110 | if (ret) { | |
111 | pr_err("failed to prepare request: %d\n", ret); | |
112 | goto req_err; | |
113 | } | |
114 | engine->cur_req_prepared = true; | |
115 | } | |
116 | ||
117 | ret = engine->crypt_one_request(engine, engine->cur_req); | |
118 | if (ret) { | |
119 | pr_err("failed to crypt one request from queue\n"); | |
120 | goto req_err; | |
121 | } | |
122 | return; | |
123 | ||
124 | req_err: | |
125 | crypto_finalize_request(engine, engine->cur_req, ret); | |
126 | return; | |
127 | ||
128 | out: | |
129 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
130 | } | |
131 | ||
132 | static void crypto_pump_work(struct kthread_work *work) | |
133 | { | |
134 | struct crypto_engine *engine = | |
135 | container_of(work, struct crypto_engine, pump_requests); | |
136 | ||
137 | crypto_pump_requests(engine, true); | |
138 | } | |
139 | ||
140 | /** | |
141 | * crypto_transfer_request - transfer the new request into the engine queue | |
142 | * @engine: the hardware engine | |
143 | * @req: the request need to be listed into the engine queue | |
144 | */ | |
145 | int crypto_transfer_request(struct crypto_engine *engine, | |
146 | struct ablkcipher_request *req, bool need_pump) | |
147 | { | |
148 | unsigned long flags; | |
149 | int ret; | |
150 | ||
151 | spin_lock_irqsave(&engine->queue_lock, flags); | |
152 | ||
153 | if (!engine->running) { | |
154 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
155 | return -ESHUTDOWN; | |
156 | } | |
157 | ||
158 | ret = ablkcipher_enqueue_request(&engine->queue, req); | |
159 | ||
160 | if (!engine->busy && need_pump) | |
161 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
162 | ||
163 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
164 | return ret; | |
165 | } | |
166 | EXPORT_SYMBOL_GPL(crypto_transfer_request); | |
167 | ||
168 | /** | |
169 | * crypto_transfer_request_to_engine - transfer one request to list into the | |
170 | * engine queue | |
171 | * @engine: the hardware engine | |
172 | * @req: the request need to be listed into the engine queue | |
173 | */ | |
174 | int crypto_transfer_request_to_engine(struct crypto_engine *engine, | |
175 | struct ablkcipher_request *req) | |
176 | { | |
177 | return crypto_transfer_request(engine, req, true); | |
178 | } | |
179 | EXPORT_SYMBOL_GPL(crypto_transfer_request_to_engine); | |
180 | ||
181 | /** | |
182 | * crypto_finalize_request - finalize one request if the request is done | |
183 | * @engine: the hardware engine | |
184 | * @req: the request need to be finalized | |
185 | * @err: error number | |
186 | */ | |
187 | void crypto_finalize_request(struct crypto_engine *engine, | |
188 | struct ablkcipher_request *req, int err) | |
189 | { | |
190 | unsigned long flags; | |
191 | bool finalize_cur_req = false; | |
192 | int ret; | |
193 | ||
194 | spin_lock_irqsave(&engine->queue_lock, flags); | |
195 | if (engine->cur_req == req) | |
196 | finalize_cur_req = true; | |
197 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
198 | ||
199 | if (finalize_cur_req) { | |
200 | if (engine->cur_req_prepared && engine->unprepare_request) { | |
201 | ret = engine->unprepare_request(engine, req); | |
202 | if (ret) | |
203 | pr_err("failed to unprepare request\n"); | |
204 | } | |
205 | ||
206 | spin_lock_irqsave(&engine->queue_lock, flags); | |
207 | engine->cur_req = NULL; | |
208 | engine->cur_req_prepared = false; | |
209 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
210 | } | |
211 | ||
212 | req->base.complete(&req->base, err); | |
213 | ||
214 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
215 | } | |
216 | EXPORT_SYMBOL_GPL(crypto_finalize_request); | |
217 | ||
218 | /** | |
219 | * crypto_engine_start - start the hardware engine | |
220 | * @engine: the hardware engine need to be started | |
221 | * | |
222 | * Return 0 on success, else on fail. | |
223 | */ | |
224 | int crypto_engine_start(struct crypto_engine *engine) | |
225 | { | |
226 | unsigned long flags; | |
227 | ||
228 | spin_lock_irqsave(&engine->queue_lock, flags); | |
229 | ||
230 | if (engine->running || engine->busy) { | |
231 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
232 | return -EBUSY; | |
233 | } | |
234 | ||
235 | engine->running = true; | |
236 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
237 | ||
238 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
239 | ||
240 | return 0; | |
241 | } | |
242 | EXPORT_SYMBOL_GPL(crypto_engine_start); | |
243 | ||
244 | /** | |
245 | * crypto_engine_stop - stop the hardware engine | |
246 | * @engine: the hardware engine need to be stopped | |
247 | * | |
248 | * Return 0 on success, else on fail. | |
249 | */ | |
250 | int crypto_engine_stop(struct crypto_engine *engine) | |
251 | { | |
252 | unsigned long flags; | |
253 | unsigned limit = 500; | |
254 | int ret = 0; | |
255 | ||
256 | spin_lock_irqsave(&engine->queue_lock, flags); | |
257 | ||
258 | /* | |
259 | * If the engine queue is not empty or the engine is on busy state, | |
260 | * we need to wait for a while to pump the requests of engine queue. | |
261 | */ | |
262 | while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) { | |
263 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
264 | msleep(20); | |
265 | spin_lock_irqsave(&engine->queue_lock, flags); | |
266 | } | |
267 | ||
268 | if (crypto_queue_len(&engine->queue) || engine->busy) | |
269 | ret = -EBUSY; | |
270 | else | |
271 | engine->running = false; | |
272 | ||
273 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
274 | ||
275 | if (ret) | |
276 | pr_warn("could not stop engine\n"); | |
277 | ||
278 | return ret; | |
279 | } | |
280 | EXPORT_SYMBOL_GPL(crypto_engine_stop); | |
281 | ||
282 | /** | |
283 | * crypto_engine_alloc_init - allocate crypto hardware engine structure and | |
284 | * initialize it. | |
285 | * @dev: the device attached with one hardware engine | |
286 | * @rt: whether this queue is set to run as a realtime task | |
287 | * | |
288 | * This must be called from context that can sleep. | |
289 | * Return: the crypto engine structure on success, else NULL. | |
290 | */ | |
291 | struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) | |
292 | { | |
293 | struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; | |
294 | struct crypto_engine *engine; | |
295 | ||
296 | if (!dev) | |
297 | return NULL; | |
298 | ||
299 | engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL); | |
300 | if (!engine) | |
301 | return NULL; | |
302 | ||
303 | engine->rt = rt; | |
304 | engine->running = false; | |
305 | engine->busy = false; | |
306 | engine->idling = false; | |
307 | engine->cur_req_prepared = false; | |
308 | engine->priv_data = dev; | |
309 | snprintf(engine->name, sizeof(engine->name), | |
310 | "%s-engine", dev_name(dev)); | |
311 | ||
312 | crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN); | |
313 | spin_lock_init(&engine->queue_lock); | |
314 | ||
315 | init_kthread_worker(&engine->kworker); | |
316 | engine->kworker_task = kthread_run(kthread_worker_fn, | |
317 | &engine->kworker, "%s", | |
318 | engine->name); | |
319 | if (IS_ERR(engine->kworker_task)) { | |
320 | dev_err(dev, "failed to create crypto request pump task\n"); | |
321 | return NULL; | |
322 | } | |
323 | init_kthread_work(&engine->pump_requests, crypto_pump_work); | |
324 | ||
325 | if (engine->rt) { | |
326 | dev_info(dev, "will run requests pump with realtime priority\n"); | |
327 | sched_setscheduler(engine->kworker_task, SCHED_FIFO, ¶m); | |
328 | } | |
329 | ||
330 | return engine; | |
331 | } | |
332 | EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); | |
333 | ||
334 | /** | |
335 | * crypto_engine_exit - free the resources of hardware engine when exit | |
336 | * @engine: the hardware engine need to be freed | |
337 | * | |
338 | * Return 0 for success. | |
339 | */ | |
340 | int crypto_engine_exit(struct crypto_engine *engine) | |
341 | { | |
342 | int ret; | |
343 | ||
344 | ret = crypto_engine_stop(engine); | |
345 | if (ret) | |
346 | return ret; | |
347 | ||
348 | flush_kthread_worker(&engine->kworker); | |
349 | kthread_stop(engine->kworker_task); | |
350 | ||
351 | return 0; | |
352 | } | |
353 | EXPORT_SYMBOL_GPL(crypto_engine_exit); | |
354 | ||
355 | MODULE_LICENSE("GPL"); | |
356 | MODULE_DESCRIPTION("Crypto hardware engine framework"); |