]>
Commit | Line | Data |
---|---|---|
1b44c5a6 AT |
1 | /* |
2 | * Copyright (C) 2017 Marvell | |
3 | * | |
4 | * Antoine Tenart <antoine.tenart@free-electrons.com> | |
5 | * | |
6 | * This file is licensed under the terms of the GNU General Public | |
7 | * License version 2. This program is licensed "as is" without any | |
8 | * warranty of any kind, whether express or implied. | |
9 | */ | |
10 | ||
11 | #include <linux/clk.h> | |
12 | #include <linux/device.h> | |
13 | #include <linux/dma-mapping.h> | |
14 | #include <linux/dmapool.h> | |
15 | #include <linux/firmware.h> | |
16 | #include <linux/interrupt.h> | |
17 | #include <linux/module.h> | |
18 | #include <linux/of_platform.h> | |
19 | #include <linux/of_irq.h> | |
20 | #include <linux/platform_device.h> | |
21 | #include <linux/workqueue.h> | |
22 | ||
23 | #include <crypto/internal/hash.h> | |
24 | #include <crypto/internal/skcipher.h> | |
25 | ||
26 | #include "safexcel.h" | |
27 | ||
28 | static u32 max_rings = EIP197_MAX_RINGS; | |
29 | module_param(max_rings, uint, 0644); | |
30 | MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); | |
31 | ||
32 | static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) | |
33 | { | |
34 | u32 val, htable_offset; | |
35 | int i; | |
36 | ||
37 | /* Enable the record cache memory access */ | |
38 | val = readl(priv->base + EIP197_CS_RAM_CTRL); | |
39 | val &= ~EIP197_TRC_ENABLE_MASK; | |
40 | val |= EIP197_TRC_ENABLE_0; | |
41 | writel(val, priv->base + EIP197_CS_RAM_CTRL); | |
42 | ||
43 | /* Clear all ECC errors */ | |
44 | writel(0, priv->base + EIP197_TRC_ECCCTRL); | |
45 | ||
46 | /* | |
47 | * Make sure the cache memory is accessible by taking record cache into | |
48 | * reset. | |
49 | */ | |
50 | val = readl(priv->base + EIP197_TRC_PARAMS); | |
51 | val |= EIP197_TRC_PARAMS_SW_RESET; | |
52 | val &= ~EIP197_TRC_PARAMS_DATA_ACCESS; | |
53 | writel(val, priv->base + EIP197_TRC_PARAMS); | |
54 | ||
55 | /* Clear all records */ | |
56 | for (i = 0; i < EIP197_CS_RC_MAX; i++) { | |
57 | u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; | |
58 | ||
59 | writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | | |
60 | EIP197_CS_RC_PREV(EIP197_RC_NULL), | |
61 | priv->base + offset); | |
62 | ||
63 | val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1); | |
64 | if (i == 0) | |
65 | val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); | |
66 | else if (i == EIP197_CS_RC_MAX - 1) | |
67 | val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); | |
68 | writel(val, priv->base + offset + sizeof(u32)); | |
69 | } | |
70 | ||
71 | /* Clear the hash table entries */ | |
72 | htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE; | |
73 | for (i = 0; i < 64; i++) | |
74 | writel(GENMASK(29, 0), | |
75 | priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32)); | |
76 | ||
77 | /* Disable the record cache memory access */ | |
78 | val = readl(priv->base + EIP197_CS_RAM_CTRL); | |
79 | val &= ~EIP197_TRC_ENABLE_MASK; | |
80 | writel(val, priv->base + EIP197_CS_RAM_CTRL); | |
81 | ||
82 | /* Write head and tail pointers of the record free chain */ | |
83 | val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | | |
84 | EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1); | |
85 | writel(val, priv->base + EIP197_TRC_FREECHAIN); | |
86 | ||
87 | /* Configure the record cache #1 */ | |
88 | val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) | | |
89 | EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX); | |
90 | writel(val, priv->base + EIP197_TRC_PARAMS2); | |
91 | ||
92 | /* Configure the record cache #2 */ | |
93 | val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) | | |
94 | EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | | |
95 | EIP197_TRC_PARAMS_HTABLE_SZ(2); | |
96 | writel(val, priv->base + EIP197_TRC_PARAMS); | |
97 | } | |
98 | ||
99 | static void eip197_write_firmware(struct safexcel_crypto_priv *priv, | |
100 | const struct firmware *fw, u32 ctrl, | |
101 | u32 prog_en) | |
102 | { | |
103 | const u32 *data = (const u32 *)fw->data; | |
104 | u32 val; | |
105 | int i; | |
106 | ||
107 | /* Reset the engine to make its program memory accessible */ | |
108 | writel(EIP197_PE_ICE_x_CTRL_SW_RESET | | |
109 | EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | | |
110 | EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, | |
111 | priv->base + ctrl); | |
112 | ||
113 | /* Enable access to the program memory */ | |
114 | writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL); | |
115 | ||
116 | /* Write the firmware */ | |
117 | for (i = 0; i < fw->size / sizeof(u32); i++) | |
118 | writel(be32_to_cpu(data[i]), | |
119 | priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); | |
120 | ||
121 | /* Disable access to the program memory */ | |
122 | writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL); | |
123 | ||
124 | /* Release engine from reset */ | |
125 | val = readl(priv->base + ctrl); | |
126 | val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET; | |
127 | writel(val, priv->base + ctrl); | |
128 | } | |
129 | ||
130 | static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) | |
131 | { | |
132 | const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; | |
133 | const struct firmware *fw[FW_NB]; | |
134 | int i, j, ret = 0; | |
135 | u32 val; | |
136 | ||
137 | for (i = 0; i < FW_NB; i++) { | |
138 | ret = request_firmware(&fw[i], fw_name[i], priv->dev); | |
139 | if (ret) { | |
140 | dev_err(priv->dev, | |
141 | "Failed to request firmware %s (%d)\n", | |
142 | fw_name[i], ret); | |
143 | goto release_fw; | |
144 | } | |
145 | } | |
146 | ||
147 | /* Clear the scratchpad memory */ | |
148 | val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL); | |
149 | val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | | |
150 | EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | | |
151 | EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | | |
152 | EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; | |
153 | writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL); | |
154 | ||
155 | memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0, | |
156 | EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32)); | |
157 | ||
158 | eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL, | |
159 | EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN); | |
160 | ||
161 | eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL, | |
162 | EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN); | |
163 | ||
164 | release_fw: | |
165 | for (j = 0; j < i; j++) | |
166 | release_firmware(fw[j]); | |
167 | ||
168 | return ret; | |
169 | } | |
170 | ||
171 | static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) | |
172 | { | |
173 | u32 hdw, cd_size_rnd, val; | |
174 | int i; | |
175 | ||
176 | hdw = readl(priv->base + EIP197_HIA_OPTIONS); | |
177 | hdw &= GENMASK(27, 25); | |
178 | hdw >>= 25; | |
179 | ||
180 | cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw; | |
181 | ||
182 | for (i = 0; i < priv->config.rings; i++) { | |
183 | /* ring base address */ | |
184 | writel(lower_32_bits(priv->ring[i].cdr.base_dma), | |
185 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); | |
186 | writel(upper_32_bits(priv->ring[i].cdr.base_dma), | |
187 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); | |
188 | ||
189 | writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | | |
190 | priv->config.cd_size, | |
191 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE); | |
192 | writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) | | |
193 | (EIP197_FETCH_COUNT * priv->config.cd_offset), | |
194 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); | |
195 | ||
196 | /* Configure DMA tx control */ | |
197 | val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); | |
198 | val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); | |
199 | writel(val, | |
200 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG); | |
201 | ||
202 | /* clear any pending interrupt */ | |
203 | writel(GENMASK(5, 0), | |
204 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT); | |
205 | } | |
206 | ||
207 | return 0; | |
208 | } | |
209 | ||
210 | static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) | |
211 | { | |
212 | u32 hdw, rd_size_rnd, val; | |
213 | int i; | |
214 | ||
215 | hdw = readl(priv->base + EIP197_HIA_OPTIONS); | |
216 | hdw &= GENMASK(27, 25); | |
217 | hdw >>= 25; | |
218 | ||
219 | rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw; | |
220 | ||
221 | for (i = 0; i < priv->config.rings; i++) { | |
222 | /* ring base address */ | |
223 | writel(lower_32_bits(priv->ring[i].rdr.base_dma), | |
224 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); | |
225 | writel(upper_32_bits(priv->ring[i].rdr.base_dma), | |
226 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); | |
227 | ||
228 | writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | | |
229 | priv->config.rd_size, | |
230 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE); | |
231 | ||
232 | writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) | | |
233 | (EIP197_FETCH_COUNT * priv->config.rd_offset), | |
234 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); | |
235 | ||
236 | /* Configure DMA tx control */ | |
237 | val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); | |
238 | val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); | |
239 | val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG; | |
240 | writel(val, | |
241 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG); | |
242 | ||
243 | /* clear any pending interrupt */ | |
244 | writel(GENMASK(7, 0), | |
245 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT); | |
246 | ||
247 | /* enable ring interrupt */ | |
248 | val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); | |
249 | val |= EIP197_RDR_IRQ(i); | |
250 | writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); | |
251 | } | |
252 | ||
253 | return 0; | |
254 | } | |
255 | ||
256 | static int safexcel_hw_init(struct safexcel_crypto_priv *priv) | |
257 | { | |
258 | u32 version, val; | |
259 | int i, ret; | |
260 | ||
261 | /* Determine endianess and configure byte swap */ | |
262 | version = readl(priv->base + EIP197_HIA_VERSION); | |
263 | val = readl(priv->base + EIP197_HIA_MST_CTRL); | |
264 | ||
265 | if ((version & 0xffff) == EIP197_HIA_VERSION_BE) | |
266 | val |= EIP197_MST_CTRL_BYTE_SWAP; | |
267 | else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE) | |
268 | val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24); | |
269 | ||
270 | writel(val, priv->base + EIP197_HIA_MST_CTRL); | |
271 | ||
272 | ||
273 | /* Configure wr/rd cache values */ | |
274 | writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | | |
275 | EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), | |
276 | priv->base + EIP197_MST_CTRL); | |
277 | ||
278 | /* Interrupts reset */ | |
279 | ||
280 | /* Disable all global interrupts */ | |
281 | writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL); | |
282 | ||
283 | /* Clear any pending interrupt */ | |
284 | writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK); | |
285 | ||
286 | /* Data Fetch Engine configuration */ | |
287 | ||
288 | /* Reset all DFE threads */ | |
289 | writel(EIP197_DxE_THR_CTRL_RESET_PE, | |
290 | priv->base + EIP197_HIA_DFE_THR_CTRL); | |
291 | ||
292 | /* Reset HIA input interface arbiter */ | |
293 | writel(EIP197_HIA_RA_PE_CTRL_RESET, | |
294 | priv->base + EIP197_HIA_RA_PE_CTRL); | |
295 | ||
296 | /* DMA transfer size to use */ | |
297 | val = EIP197_HIA_DFE_CFG_DIS_DEBUG; | |
298 | val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); | |
299 | val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); | |
300 | val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); | |
301 | val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); | |
302 | writel(val, priv->base + EIP197_HIA_DFE_CFG); | |
303 | ||
304 | /* Leave the DFE threads reset state */ | |
305 | writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL); | |
306 | ||
307 | /* Configure the procesing engine thresholds */ | |
308 | writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9), | |
309 | priv->base + EIP197_PE_IN_DBUF_THRES); | |
310 | writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7), | |
311 | priv->base + EIP197_PE_IN_TBUF_THRES); | |
312 | ||
313 | /* enable HIA input interface arbiter and rings */ | |
314 | writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0), | |
315 | priv->base + EIP197_HIA_RA_PE_CTRL); | |
316 | ||
317 | /* Data Store Engine configuration */ | |
318 | ||
319 | /* Reset all DSE threads */ | |
320 | writel(EIP197_DxE_THR_CTRL_RESET_PE, | |
321 | priv->base + EIP197_HIA_DSE_THR_CTRL); | |
322 | ||
323 | /* Wait for all DSE threads to complete */ | |
324 | while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) & | |
325 | GENMASK(15, 12)) != GENMASK(15, 12)) | |
326 | ; | |
327 | ||
328 | /* DMA transfer size to use */ | |
329 | val = EIP197_HIA_DSE_CFG_DIS_DEBUG; | |
330 | val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); | |
39ba1bb4 | 331 | val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); |
ee1fd870 | 332 | val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE; |
c87925bf | 333 | val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; |
1b44c5a6 AT |
334 | writel(val, priv->base + EIP197_HIA_DSE_CFG); |
335 | ||
336 | /* Leave the DSE threads reset state */ | |
337 | writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL); | |
338 | ||
339 | /* Configure the procesing engine thresholds */ | |
340 | writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8), | |
341 | priv->base + EIP197_PE_OUT_DBUF_THRES); | |
342 | ||
343 | /* Processing Engine configuration */ | |
344 | ||
345 | /* H/W capabilities selection */ | |
346 | val = EIP197_FUNCTION_RSVD; | |
347 | val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; | |
348 | val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC; | |
349 | val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1; | |
350 | val |= EIP197_ALG_SHA2; | |
351 | writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN); | |
352 | ||
353 | /* Command Descriptor Rings prepare */ | |
354 | for (i = 0; i < priv->config.rings; i++) { | |
355 | /* Clear interrupts for this ring */ | |
356 | writel(GENMASK(31, 0), | |
357 | priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i)); | |
358 | ||
359 | /* Disable external triggering */ | |
360 | writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG); | |
361 | ||
362 | /* Clear the pending prepared counter */ | |
363 | writel(EIP197_xDR_PREP_CLR_COUNT, | |
364 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); | |
365 | ||
366 | /* Clear the pending processed counter */ | |
367 | writel(EIP197_xDR_PROC_CLR_COUNT, | |
368 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT); | |
369 | ||
370 | writel(0, | |
371 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR); | |
372 | writel(0, | |
373 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR); | |
374 | ||
375 | writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, | |
376 | priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE); | |
377 | } | |
378 | ||
379 | /* Result Descriptor Ring prepare */ | |
380 | for (i = 0; i < priv->config.rings; i++) { | |
381 | /* Disable external triggering*/ | |
382 | writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG); | |
383 | ||
384 | /* Clear the pending prepared counter */ | |
385 | writel(EIP197_xDR_PREP_CLR_COUNT, | |
386 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); | |
387 | ||
388 | /* Clear the pending processed counter */ | |
389 | writel(EIP197_xDR_PROC_CLR_COUNT, | |
390 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT); | |
391 | ||
392 | writel(0, | |
393 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR); | |
394 | writel(0, | |
395 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR); | |
396 | ||
397 | /* Ring size */ | |
398 | writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, | |
399 | priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE); | |
400 | } | |
401 | ||
402 | /* Enable command descriptor rings */ | |
403 | writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), | |
404 | priv->base + EIP197_HIA_DFE_THR_CTRL); | |
405 | ||
406 | /* Enable result descriptor rings */ | |
407 | writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), | |
408 | priv->base + EIP197_HIA_DSE_THR_CTRL); | |
409 | ||
410 | /* Clear any HIA interrupt */ | |
411 | writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK); | |
412 | ||
413 | eip197_trc_cache_init(priv); | |
414 | ||
415 | ret = eip197_load_firmwares(priv); | |
416 | if (ret) | |
417 | return ret; | |
418 | ||
419 | safexcel_hw_setup_cdesc_rings(priv); | |
420 | safexcel_hw_setup_rdesc_rings(priv); | |
421 | ||
422 | return 0; | |
423 | } | |
424 | ||
86671abb | 425 | void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) |
1b44c5a6 AT |
426 | { |
427 | struct crypto_async_request *req, *backlog; | |
428 | struct safexcel_context *ctx; | |
429 | struct safexcel_request *request; | |
86671abb | 430 | int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; |
1b44c5a6 | 431 | |
5eb0cc66 AT |
432 | priv->ring[ring].need_dequeue = false; |
433 | ||
1b44c5a6 | 434 | do { |
86671abb | 435 | spin_lock_bh(&priv->ring[ring].queue_lock); |
86671abb | 436 | backlog = crypto_get_backlog(&priv->ring[ring].queue); |
b1deb47a | 437 | req = crypto_dequeue_request(&priv->ring[ring].queue); |
86671abb | 438 | spin_unlock_bh(&priv->ring[ring].queue_lock); |
1b44c5a6 AT |
439 | |
440 | if (!req) | |
441 | goto finalize; | |
442 | ||
443 | request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req)); | |
c5acabd3 AT |
444 | if (!request) { |
445 | spin_lock_bh(&priv->ring[ring].queue_lock); | |
446 | crypto_enqueue_request(&priv->ring[ring].queue, req); | |
447 | spin_unlock_bh(&priv->ring[ring].queue_lock); | |
448 | ||
449 | priv->ring[ring].need_dequeue = true; | |
450 | goto finalize; | |
451 | } | |
1b44c5a6 AT |
452 | |
453 | ctx = crypto_tfm_ctx(req->tfm); | |
86671abb | 454 | ret = ctx->send(req, ring, request, &commands, &results); |
1b44c5a6 AT |
455 | if (ret) { |
456 | kfree(request); | |
c5acabd3 | 457 | req->complete(req, ret); |
86671abb | 458 | priv->ring[ring].need_dequeue = true; |
c5acabd3 | 459 | goto finalize; |
1b44c5a6 AT |
460 | } |
461 | ||
462 | if (backlog) | |
463 | backlog->complete(backlog, -EINPROGRESS); | |
464 | ||
86671abb AT |
465 | spin_lock_bh(&priv->ring[ring].egress_lock); |
466 | list_add_tail(&request->list, &priv->ring[ring].list); | |
467 | spin_unlock_bh(&priv->ring[ring].egress_lock); | |
1b44c5a6 | 468 | |
86671abb AT |
469 | cdesc += commands; |
470 | rdesc += results; | |
471 | } while (nreq++ < EIP197_MAX_BATCH_SZ); | |
1b44c5a6 AT |
472 | |
473 | finalize: | |
86671abb AT |
474 | if (nreq == EIP197_MAX_BATCH_SZ) |
475 | priv->ring[ring].need_dequeue = true; | |
476 | else if (!nreq) | |
1b44c5a6 AT |
477 | return; |
478 | ||
86671abb | 479 | spin_lock_bh(&priv->ring[ring].lock); |
1b44c5a6 | 480 | |
86671abb AT |
481 | /* Configure when we want an interrupt */ |
482 | writel(EIP197_HIA_RDR_THRESH_PKT_MODE | | |
483 | EIP197_HIA_RDR_THRESH_PROC_PKT(nreq), | |
484 | priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); | |
1b44c5a6 | 485 | |
86671abb AT |
486 | /* let the RDR know we have pending descriptors */ |
487 | writel((rdesc * priv->config.rd_offset) << 2, | |
488 | priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT); | |
1b44c5a6 | 489 | |
86671abb AT |
490 | /* let the CDR know we have pending descriptors */ |
491 | writel((cdesc * priv->config.cd_offset) << 2, | |
492 | priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT); | |
1b44c5a6 | 493 | |
86671abb | 494 | spin_unlock_bh(&priv->ring[ring].lock); |
1b44c5a6 AT |
495 | } |
496 | ||
497 | void safexcel_free_context(struct safexcel_crypto_priv *priv, | |
498 | struct crypto_async_request *req, | |
499 | int result_sz) | |
500 | { | |
501 | struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm); | |
502 | ||
503 | if (ctx->result_dma) | |
504 | dma_unmap_single(priv->dev, ctx->result_dma, result_sz, | |
505 | DMA_FROM_DEVICE); | |
506 | ||
507 | if (ctx->cache) { | |
508 | dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz, | |
509 | DMA_TO_DEVICE); | |
510 | kfree(ctx->cache); | |
511 | ctx->cache = NULL; | |
512 | ctx->cache_sz = 0; | |
513 | } | |
514 | } | |
515 | ||
516 | void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) | |
517 | { | |
518 | struct safexcel_command_desc *cdesc; | |
519 | ||
520 | /* Acknowledge the command descriptors */ | |
521 | do { | |
522 | cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); | |
523 | if (IS_ERR(cdesc)) { | |
524 | dev_err(priv->dev, | |
525 | "Could not retrieve the command descriptor\n"); | |
526 | return; | |
527 | } | |
528 | } while (!cdesc->last_seg); | |
529 | } | |
530 | ||
531 | void safexcel_inv_complete(struct crypto_async_request *req, int error) | |
532 | { | |
533 | struct safexcel_inv_result *result = req->data; | |
534 | ||
535 | if (error == -EINPROGRESS) | |
536 | return; | |
537 | ||
538 | result->error = error; | |
539 | complete(&result->completion); | |
540 | } | |
541 | ||
542 | int safexcel_invalidate_cache(struct crypto_async_request *async, | |
543 | struct safexcel_context *ctx, | |
544 | struct safexcel_crypto_priv *priv, | |
545 | dma_addr_t ctxr_dma, int ring, | |
546 | struct safexcel_request *request) | |
547 | { | |
548 | struct safexcel_command_desc *cdesc; | |
549 | struct safexcel_result_desc *rdesc; | |
550 | int ret = 0; | |
551 | ||
552 | spin_lock_bh(&priv->ring[ring].egress_lock); | |
553 | ||
554 | /* Prepare command descriptor */ | |
555 | cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma); | |
556 | if (IS_ERR(cdesc)) { | |
557 | ret = PTR_ERR(cdesc); | |
558 | goto unlock; | |
559 | } | |
560 | ||
561 | cdesc->control_data.type = EIP197_TYPE_EXTENDED; | |
562 | cdesc->control_data.options = 0; | |
563 | cdesc->control_data.refresh = 0; | |
564 | cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; | |
565 | ||
566 | /* Prepare result descriptor */ | |
567 | rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); | |
568 | ||
569 | if (IS_ERR(rdesc)) { | |
570 | ret = PTR_ERR(rdesc); | |
571 | goto cdesc_rollback; | |
572 | } | |
573 | ||
574 | request->req = async; | |
575 | goto unlock; | |
576 | ||
577 | cdesc_rollback: | |
578 | safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); | |
579 | ||
580 | unlock: | |
581 | spin_unlock_bh(&priv->ring[ring].egress_lock); | |
582 | return ret; | |
583 | } | |
584 | ||
585 | static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, | |
586 | int ring) | |
587 | { | |
588 | struct safexcel_request *sreq; | |
589 | struct safexcel_context *ctx; | |
590 | int ret, i, nreq, ndesc = 0; | |
591 | bool should_complete; | |
592 | ||
593 | nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); | |
594 | nreq >>= 24; | |
595 | nreq &= GENMASK(6, 0); | |
596 | if (!nreq) | |
597 | return; | |
598 | ||
599 | for (i = 0; i < nreq; i++) { | |
600 | spin_lock_bh(&priv->ring[ring].egress_lock); | |
601 | sreq = list_first_entry(&priv->ring[ring].list, | |
602 | struct safexcel_request, list); | |
603 | list_del(&sreq->list); | |
604 | spin_unlock_bh(&priv->ring[ring].egress_lock); | |
605 | ||
606 | ctx = crypto_tfm_ctx(sreq->req->tfm); | |
607 | ndesc = ctx->handle_result(priv, ring, sreq->req, | |
608 | &should_complete, &ret); | |
609 | if (ndesc < 0) { | |
0a02dcca | 610 | kfree(sreq); |
1b44c5a6 AT |
611 | dev_err(priv->dev, "failed to handle result (%d)", ndesc); |
612 | return; | |
613 | } | |
614 | ||
615 | writel(EIP197_xDR_PROC_xD_PKT(1) | | |
616 | EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset), | |
617 | priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT); | |
618 | ||
619 | if (should_complete) { | |
620 | local_bh_disable(); | |
621 | sreq->req->complete(sreq->req, ret); | |
622 | local_bh_enable(); | |
623 | } | |
624 | ||
625 | kfree(sreq); | |
626 | } | |
627 | } | |
628 | ||
629 | static void safexcel_handle_result_work(struct work_struct *work) | |
630 | { | |
631 | struct safexcel_work_data *data = | |
632 | container_of(work, struct safexcel_work_data, work); | |
633 | struct safexcel_crypto_priv *priv = data->priv; | |
634 | ||
635 | safexcel_handle_result_descriptor(priv, data->ring); | |
636 | ||
5eb0cc66 | 637 | if (priv->ring[data->ring].need_dequeue) |
86671abb | 638 | safexcel_dequeue(data->priv, data->ring); |
1b44c5a6 AT |
639 | } |
640 | ||
641 | struct safexcel_ring_irq_data { | |
642 | struct safexcel_crypto_priv *priv; | |
643 | int ring; | |
644 | }; | |
645 | ||
646 | static irqreturn_t safexcel_irq_ring(int irq, void *data) | |
647 | { | |
648 | struct safexcel_ring_irq_data *irq_data = data; | |
649 | struct safexcel_crypto_priv *priv = irq_data->priv; | |
650 | int ring = irq_data->ring; | |
651 | u32 status, stat; | |
652 | ||
653 | status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); | |
654 | if (!status) | |
655 | return IRQ_NONE; | |
656 | ||
657 | /* RDR interrupts */ | |
658 | if (status & EIP197_RDR_IRQ(ring)) { | |
659 | stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); | |
660 | ||
661 | if (unlikely(stat & EIP197_xDR_ERR)) { | |
662 | /* | |
663 | * Fatal error, the RDR is unusable and must be | |
664 | * reinitialized. This should not happen under | |
665 | * normal circumstances. | |
666 | */ | |
667 | dev_err(priv->dev, "RDR: fatal error."); | |
668 | } else if (likely(stat & EIP197_xDR_THRESH)) { | |
669 | queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work); | |
670 | } | |
671 | ||
672 | /* ACK the interrupts */ | |
673 | writel(stat & 0xff, | |
674 | priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT); | |
675 | } | |
676 | ||
677 | /* ACK the interrupts */ | |
678 | writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring)); | |
679 | ||
680 | return IRQ_HANDLED; | |
681 | } | |
682 | ||
683 | static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name, | |
684 | irq_handler_t handler, | |
685 | struct safexcel_ring_irq_data *ring_irq_priv) | |
686 | { | |
687 | int ret, irq = platform_get_irq_byname(pdev, name); | |
688 | ||
689 | if (irq < 0) { | |
690 | dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name); | |
691 | return irq; | |
692 | } | |
693 | ||
694 | ret = devm_request_irq(&pdev->dev, irq, handler, 0, | |
695 | dev_name(&pdev->dev), ring_irq_priv); | |
696 | if (ret) { | |
697 | dev_err(&pdev->dev, "unable to request IRQ %d\n", irq); | |
698 | return ret; | |
699 | } | |
700 | ||
701 | return irq; | |
702 | } | |
703 | ||
704 | static struct safexcel_alg_template *safexcel_algs[] = { | |
705 | &safexcel_alg_ecb_aes, | |
706 | &safexcel_alg_cbc_aes, | |
707 | &safexcel_alg_sha1, | |
708 | &safexcel_alg_sha224, | |
709 | &safexcel_alg_sha256, | |
710 | &safexcel_alg_hmac_sha1, | |
711 | }; | |
712 | ||
713 | static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) | |
714 | { | |
715 | int i, j, ret = 0; | |
716 | ||
717 | for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { | |
718 | safexcel_algs[i]->priv = priv; | |
719 | ||
720 | if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) | |
721 | ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); | |
722 | else | |
723 | ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); | |
724 | ||
725 | if (ret) | |
726 | goto fail; | |
727 | } | |
728 | ||
729 | return 0; | |
730 | ||
731 | fail: | |
732 | for (j = 0; j < i; j++) { | |
733 | if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) | |
734 | crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); | |
735 | else | |
736 | crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); | |
737 | } | |
738 | ||
739 | return ret; | |
740 | } | |
741 | ||
742 | static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) | |
743 | { | |
744 | int i; | |
745 | ||
746 | for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { | |
747 | if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) | |
748 | crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); | |
749 | else | |
750 | crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); | |
751 | } | |
752 | } | |
753 | ||
754 | static void safexcel_configure(struct safexcel_crypto_priv *priv) | |
755 | { | |
756 | u32 val, mask; | |
757 | ||
758 | val = readl(priv->base + EIP197_HIA_OPTIONS); | |
759 | val = (val & GENMASK(27, 25)) >> 25; | |
760 | mask = BIT(val) - 1; | |
761 | ||
762 | val = readl(priv->base + EIP197_HIA_OPTIONS); | |
763 | priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); | |
764 | ||
765 | priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); | |
766 | priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; | |
767 | ||
768 | priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); | |
769 | priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; | |
770 | } | |
771 | ||
772 | static int safexcel_probe(struct platform_device *pdev) | |
773 | { | |
774 | struct device *dev = &pdev->dev; | |
775 | struct resource *res; | |
776 | struct safexcel_crypto_priv *priv; | |
1b44c5a6 AT |
777 | int i, ret; |
778 | ||
779 | priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); | |
780 | if (!priv) | |
781 | return -ENOMEM; | |
782 | ||
783 | priv->dev = dev; | |
784 | ||
785 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
786 | priv->base = devm_ioremap_resource(dev, res); | |
787 | if (IS_ERR(priv->base)) { | |
788 | dev_err(dev, "failed to get resource\n"); | |
789 | return PTR_ERR(priv->base); | |
790 | } | |
791 | ||
792 | priv->clk = of_clk_get(dev->of_node, 0); | |
793 | if (!IS_ERR(priv->clk)) { | |
794 | ret = clk_prepare_enable(priv->clk); | |
795 | if (ret) { | |
796 | dev_err(dev, "unable to enable clk (%d)\n", ret); | |
797 | return ret; | |
798 | } | |
799 | } else { | |
800 | /* The clock isn't mandatory */ | |
801 | if (PTR_ERR(priv->clk) == -EPROBE_DEFER) | |
802 | return -EPROBE_DEFER; | |
803 | } | |
804 | ||
a2069aac | 805 | ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); |
1b44c5a6 AT |
806 | if (ret) |
807 | goto err_clk; | |
808 | ||
809 | priv->context_pool = dmam_pool_create("safexcel-context", dev, | |
810 | sizeof(struct safexcel_context_record), | |
811 | 1, 0); | |
812 | if (!priv->context_pool) { | |
813 | ret = -ENOMEM; | |
814 | goto err_clk; | |
815 | } | |
816 | ||
817 | safexcel_configure(priv); | |
818 | ||
819 | for (i = 0; i < priv->config.rings; i++) { | |
820 | char irq_name[6] = {0}; /* "ringX\0" */ | |
821 | char wq_name[9] = {0}; /* "wq_ringX\0" */ | |
822 | int irq; | |
823 | struct safexcel_ring_irq_data *ring_irq; | |
824 | ||
825 | ret = safexcel_init_ring_descriptors(priv, | |
826 | &priv->ring[i].cdr, | |
827 | &priv->ring[i].rdr); | |
828 | if (ret) | |
829 | goto err_clk; | |
830 | ||
831 | ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); | |
832 | if (!ring_irq) { | |
833 | ret = -ENOMEM; | |
834 | goto err_clk; | |
835 | } | |
836 | ||
837 | ring_irq->priv = priv; | |
838 | ring_irq->ring = i; | |
839 | ||
840 | snprintf(irq_name, 6, "ring%d", i); | |
841 | irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, | |
842 | ring_irq); | |
b7d65fe1 CJ |
843 | if (irq < 0) { |
844 | ret = irq; | |
1b44c5a6 | 845 | goto err_clk; |
b7d65fe1 | 846 | } |
1b44c5a6 AT |
847 | |
848 | priv->ring[i].work_data.priv = priv; | |
849 | priv->ring[i].work_data.ring = i; | |
850 | INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work); | |
851 | ||
852 | snprintf(wq_name, 9, "wq_ring%d", i); | |
853 | priv->ring[i].workqueue = create_singlethread_workqueue(wq_name); | |
854 | if (!priv->ring[i].workqueue) { | |
855 | ret = -ENOMEM; | |
856 | goto err_clk; | |
857 | } | |
858 | ||
86671abb AT |
859 | crypto_init_queue(&priv->ring[i].queue, |
860 | EIP197_DEFAULT_RING_SIZE); | |
861 | ||
1b44c5a6 AT |
862 | INIT_LIST_HEAD(&priv->ring[i].list); |
863 | spin_lock_init(&priv->ring[i].lock); | |
864 | spin_lock_init(&priv->ring[i].egress_lock); | |
86671abb | 865 | spin_lock_init(&priv->ring[i].queue_lock); |
1b44c5a6 AT |
866 | } |
867 | ||
868 | platform_set_drvdata(pdev, priv); | |
869 | atomic_set(&priv->ring_used, 0); | |
870 | ||
1b44c5a6 AT |
871 | ret = safexcel_hw_init(priv); |
872 | if (ret) { | |
873 | dev_err(dev, "EIP h/w init failed (%d)\n", ret); | |
874 | goto err_clk; | |
875 | } | |
876 | ||
877 | ret = safexcel_register_algorithms(priv); | |
878 | if (ret) { | |
879 | dev_err(dev, "Failed to register algorithms (%d)\n", ret); | |
880 | goto err_clk; | |
881 | } | |
882 | ||
883 | return 0; | |
884 | ||
885 | err_clk: | |
886 | clk_disable_unprepare(priv->clk); | |
887 | return ret; | |
888 | } | |
889 | ||
890 | ||
891 | static int safexcel_remove(struct platform_device *pdev) | |
892 | { | |
893 | struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); | |
894 | int i; | |
895 | ||
896 | safexcel_unregister_algorithms(priv); | |
897 | clk_disable_unprepare(priv->clk); | |
898 | ||
899 | for (i = 0; i < priv->config.rings; i++) | |
900 | destroy_workqueue(priv->ring[i].workqueue); | |
901 | ||
902 | return 0; | |
903 | } | |
904 | ||
905 | static const struct of_device_id safexcel_of_match_table[] = { | |
906 | { .compatible = "inside-secure,safexcel-eip197" }, | |
907 | {}, | |
908 | }; | |
909 | ||
910 | ||
911 | static struct platform_driver crypto_safexcel = { | |
912 | .probe = safexcel_probe, | |
913 | .remove = safexcel_remove, | |
914 | .driver = { | |
915 | .name = "crypto-safexcel", | |
916 | .of_match_table = safexcel_of_match_table, | |
917 | }, | |
918 | }; | |
919 | module_platform_driver(crypto_safexcel); | |
920 | ||
921 | MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>"); | |
922 | MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); | |
923 | MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); | |
924 | MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197"); | |
925 | MODULE_LICENSE("GPL v2"); |