]>
Commit | Line | Data |
---|---|---|
301422e3 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1b44c5a6 AT |
2 | /* |
3 | * Copyright (C) 2017 Marvell | |
4 | * | |
5 | * Antoine Tenart <antoine.tenart@free-electrons.com> | |
1b44c5a6 AT |
6 | */ |
7 | ||
8 | #include <linux/clk.h> | |
9 | #include <linux/device.h> | |
10 | #include <linux/dma-mapping.h> | |
11 | #include <linux/dmapool.h> | |
12 | #include <linux/firmware.h> | |
13 | #include <linux/interrupt.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/of_platform.h> | |
16 | #include <linux/of_irq.h> | |
17 | #include <linux/platform_device.h> | |
18 | #include <linux/workqueue.h> | |
19 | ||
f6beaea3 | 20 | #include <crypto/internal/aead.h> |
1b44c5a6 AT |
21 | #include <crypto/internal/hash.h> |
22 | #include <crypto/internal/skcipher.h> | |
23 | ||
24 | #include "safexcel.h" | |
25 | ||
26 | static u32 max_rings = EIP197_MAX_RINGS; | |
27 | module_param(max_rings, uint, 0644); | |
28 | MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); | |
29 | ||
30 | static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv) | |
31 | { | |
32 | u32 val, htable_offset; | |
fbd0a7c9 OH |
33 | int i, cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc; |
34 | ||
35 | if (priv->version == EIP197B) { | |
36 | cs_rc_max = EIP197B_CS_RC_MAX; | |
37 | cs_ht_wc = EIP197B_CS_HT_WC; | |
38 | cs_trc_rec_wc = EIP197B_CS_TRC_REC_WC; | |
39 | cs_trc_lg_rec_wc = EIP197B_CS_TRC_LG_REC_WC; | |
40 | } else { | |
41 | cs_rc_max = EIP197D_CS_RC_MAX; | |
42 | cs_ht_wc = EIP197D_CS_HT_WC; | |
43 | cs_trc_rec_wc = EIP197D_CS_TRC_REC_WC; | |
44 | cs_trc_lg_rec_wc = EIP197D_CS_TRC_LG_REC_WC; | |
45 | } | |
1b44c5a6 AT |
46 | |
47 | /* Enable the record cache memory access */ | |
48 | val = readl(priv->base + EIP197_CS_RAM_CTRL); | |
49 | val &= ~EIP197_TRC_ENABLE_MASK; | |
50 | val |= EIP197_TRC_ENABLE_0; | |
51 | writel(val, priv->base + EIP197_CS_RAM_CTRL); | |
52 | ||
53 | /* Clear all ECC errors */ | |
54 | writel(0, priv->base + EIP197_TRC_ECCCTRL); | |
55 | ||
56 | /* | |
57 | * Make sure the cache memory is accessible by taking record cache into | |
58 | * reset. | |
59 | */ | |
60 | val = readl(priv->base + EIP197_TRC_PARAMS); | |
61 | val |= EIP197_TRC_PARAMS_SW_RESET; | |
62 | val &= ~EIP197_TRC_PARAMS_DATA_ACCESS; | |
63 | writel(val, priv->base + EIP197_TRC_PARAMS); | |
64 | ||
65 | /* Clear all records */ | |
fbd0a7c9 | 66 | for (i = 0; i < cs_rc_max; i++) { |
1b44c5a6 AT |
67 | u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; |
68 | ||
69 | writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | | |
70 | EIP197_CS_RC_PREV(EIP197_RC_NULL), | |
71 | priv->base + offset); | |
72 | ||
73 | val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1); | |
74 | if (i == 0) | |
75 | val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); | |
fbd0a7c9 | 76 | else if (i == cs_rc_max - 1) |
1b44c5a6 AT |
77 | val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); |
78 | writel(val, priv->base + offset + sizeof(u32)); | |
79 | } | |
80 | ||
81 | /* Clear the hash table entries */ | |
fbd0a7c9 OH |
82 | htable_offset = cs_rc_max * EIP197_CS_RC_SIZE; |
83 | for (i = 0; i < cs_ht_wc; i++) | |
1b44c5a6 AT |
84 | writel(GENMASK(29, 0), |
85 | priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32)); | |
86 | ||
87 | /* Disable the record cache memory access */ | |
88 | val = readl(priv->base + EIP197_CS_RAM_CTRL); | |
89 | val &= ~EIP197_TRC_ENABLE_MASK; | |
90 | writel(val, priv->base + EIP197_CS_RAM_CTRL); | |
91 | ||
92 | /* Write head and tail pointers of the record free chain */ | |
93 | val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | | |
fbd0a7c9 | 94 | EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1); |
1b44c5a6 AT |
95 | writel(val, priv->base + EIP197_TRC_FREECHAIN); |
96 | ||
97 | /* Configure the record cache #1 */ | |
fbd0a7c9 OH |
98 | val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) | |
99 | EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max); | |
1b44c5a6 AT |
100 | writel(val, priv->base + EIP197_TRC_PARAMS2); |
101 | ||
102 | /* Configure the record cache #2 */ | |
fbd0a7c9 | 103 | val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) | |
1b44c5a6 AT |
104 | EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | |
105 | EIP197_TRC_PARAMS_HTABLE_SZ(2); | |
106 | writel(val, priv->base + EIP197_TRC_PARAMS); | |
107 | } | |
108 | ||
109 | static void eip197_write_firmware(struct safexcel_crypto_priv *priv, | |
367571e4 | 110 | const struct firmware *fw, int pe, u32 ctrl, |
1b44c5a6 AT |
111 | u32 prog_en) |
112 | { | |
113 | const u32 *data = (const u32 *)fw->data; | |
114 | u32 val; | |
115 | int i; | |
116 | ||
117 | /* Reset the engine to make its program memory accessible */ | |
118 | writel(EIP197_PE_ICE_x_CTRL_SW_RESET | | |
119 | EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | | |
120 | EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, | |
871df319 | 121 | EIP197_PE(priv) + ctrl); |
1b44c5a6 AT |
122 | |
123 | /* Enable access to the program memory */ | |
367571e4 | 124 | writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); |
1b44c5a6 AT |
125 | |
126 | /* Write the firmware */ | |
127 | for (i = 0; i < fw->size / sizeof(u32); i++) | |
128 | writel(be32_to_cpu(data[i]), | |
129 | priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32)); | |
130 | ||
131 | /* Disable access to the program memory */ | |
367571e4 | 132 | writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); |
1b44c5a6 AT |
133 | |
134 | /* Release engine from reset */ | |
871df319 | 135 | val = readl(EIP197_PE(priv) + ctrl); |
1b44c5a6 | 136 | val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET; |
871df319 | 137 | writel(val, EIP197_PE(priv) + ctrl); |
1b44c5a6 AT |
138 | } |
139 | ||
140 | static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) | |
141 | { | |
142 | const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; | |
143 | const struct firmware *fw[FW_NB]; | |
5eb09118 | 144 | char fw_path[31], *dir = NULL; |
367571e4 | 145 | int i, j, ret = 0, pe; |
1b44c5a6 AT |
146 | u32 val; |
147 | ||
5eb09118 AT |
148 | switch (priv->version) { |
149 | case EIP197B: | |
150 | dir = "eip197b"; | |
151 | break; | |
152 | case EIP197D: | |
153 | dir = "eip197d"; | |
154 | break; | |
155 | default: | |
156 | /* No firmware is required */ | |
157 | return 0; | |
158 | } | |
159 | ||
1b44c5a6 | 160 | for (i = 0; i < FW_NB; i++) { |
5eb09118 | 161 | snprintf(fw_path, 31, "inside-secure/%s/%s", dir, fw_name[i]); |
043ecc63 | 162 | ret = request_firmware(&fw[i], fw_path, priv->dev); |
1b44c5a6 | 163 | if (ret) { |
5eb09118 AT |
164 | if (priv->version != EIP197B) |
165 | goto release_fw; | |
166 | ||
167 | /* Fallback to the old firmware location for the | |
168 | * EIP197b. | |
169 | */ | |
043ecc63 AT |
170 | ret = request_firmware(&fw[i], fw_name[i], priv->dev); |
171 | if (ret) { | |
172 | dev_err(priv->dev, | |
173 | "Failed to request firmware %s (%d)\n", | |
174 | fw_name[i], ret); | |
175 | goto release_fw; | |
176 | } | |
1b44c5a6 | 177 | } |
043ecc63 | 178 | } |
1b44c5a6 | 179 | |
367571e4 OH |
180 | for (pe = 0; pe < priv->config.pes; pe++) { |
181 | /* Clear the scratchpad memory */ | |
182 | val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); | |
183 | val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | | |
184 | EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | | |
185 | EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | | |
186 | EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; | |
187 | writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); | |
188 | ||
189 | memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM(pe), 0, | |
190 | EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32)); | |
191 | ||
192 | eip197_write_firmware(priv, fw[FW_IFPP], pe, | |
193 | EIP197_PE_ICE_FPP_CTRL(pe), | |
194 | EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN); | |
195 | ||
196 | eip197_write_firmware(priv, fw[FW_IPUE], pe, | |
197 | EIP197_PE_ICE_PUE_CTRL(pe), | |
198 | EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN); | |
199 | } | |
1b44c5a6 AT |
200 | |
201 | release_fw: | |
202 | for (j = 0; j < i; j++) | |
203 | release_firmware(fw[j]); | |
204 | ||
205 | return ret; | |
206 | } | |
207 | ||
208 | static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) | |
209 | { | |
210 | u32 hdw, cd_size_rnd, val; | |
211 | int i; | |
212 | ||
871df319 | 213 | hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); |
1b44c5a6 AT |
214 | hdw &= GENMASK(27, 25); |
215 | hdw >>= 25; | |
216 | ||
217 | cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw; | |
218 | ||
219 | for (i = 0; i < priv->config.rings; i++) { | |
220 | /* ring base address */ | |
221 | writel(lower_32_bits(priv->ring[i].cdr.base_dma), | |
871df319 | 222 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); |
1b44c5a6 | 223 | writel(upper_32_bits(priv->ring[i].cdr.base_dma), |
871df319 | 224 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); |
1b44c5a6 AT |
225 | |
226 | writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) | | |
227 | priv->config.cd_size, | |
871df319 | 228 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); |
1b44c5a6 AT |
229 | writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) | |
230 | (EIP197_FETCH_COUNT * priv->config.cd_offset), | |
871df319 | 231 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); |
1b44c5a6 AT |
232 | |
233 | /* Configure DMA tx control */ | |
234 | val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); | |
235 | val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); | |
871df319 | 236 | writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); |
1b44c5a6 AT |
237 | |
238 | /* clear any pending interrupt */ | |
239 | writel(GENMASK(5, 0), | |
871df319 | 240 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); |
1b44c5a6 AT |
241 | } |
242 | ||
243 | return 0; | |
244 | } | |
245 | ||
246 | static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) | |
247 | { | |
248 | u32 hdw, rd_size_rnd, val; | |
249 | int i; | |
250 | ||
871df319 | 251 | hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); |
1b44c5a6 AT |
252 | hdw &= GENMASK(27, 25); |
253 | hdw >>= 25; | |
254 | ||
255 | rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw; | |
256 | ||
257 | for (i = 0; i < priv->config.rings; i++) { | |
258 | /* ring base address */ | |
259 | writel(lower_32_bits(priv->ring[i].rdr.base_dma), | |
871df319 | 260 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); |
1b44c5a6 | 261 | writel(upper_32_bits(priv->ring[i].rdr.base_dma), |
871df319 | 262 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); |
1b44c5a6 AT |
263 | |
264 | writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) | | |
265 | priv->config.rd_size, | |
871df319 | 266 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); |
1b44c5a6 AT |
267 | |
268 | writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) | | |
269 | (EIP197_FETCH_COUNT * priv->config.rd_offset), | |
871df319 | 270 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); |
1b44c5a6 AT |
271 | |
272 | /* Configure DMA tx control */ | |
273 | val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); | |
274 | val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); | |
d9107087 | 275 | val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF; |
1b44c5a6 | 276 | writel(val, |
871df319 | 277 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); |
1b44c5a6 AT |
278 | |
279 | /* clear any pending interrupt */ | |
280 | writel(GENMASK(7, 0), | |
871df319 | 281 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); |
1b44c5a6 AT |
282 | |
283 | /* enable ring interrupt */ | |
871df319 | 284 | val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); |
1b44c5a6 | 285 | val |= EIP197_RDR_IRQ(i); |
871df319 | 286 | writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); |
1b44c5a6 AT |
287 | } |
288 | ||
289 | return 0; | |
290 | } | |
291 | ||
292 | static int safexcel_hw_init(struct safexcel_crypto_priv *priv) | |
293 | { | |
294 | u32 version, val; | |
367571e4 | 295 | int i, ret, pe; |
1b44c5a6 AT |
296 | |
297 | /* Determine endianess and configure byte swap */ | |
871df319 AT |
298 | version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION); |
299 | val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); | |
1b44c5a6 AT |
300 | |
301 | if ((version & 0xffff) == EIP197_HIA_VERSION_BE) | |
302 | val |= EIP197_MST_CTRL_BYTE_SWAP; | |
303 | else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE) | |
304 | val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24); | |
305 | ||
63b94278 OH |
306 | /* For EIP197 set maximum number of TX commands to 2^5 = 32 */ |
307 | if (priv->version == EIP197B || priv->version == EIP197D) | |
308 | val |= EIP197_MST_CTRL_TX_MAX_CMD(5); | |
309 | ||
871df319 | 310 | writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); |
1b44c5a6 AT |
311 | |
312 | /* Configure wr/rd cache values */ | |
313 | writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | | |
314 | EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), | |
871df319 | 315 | EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); |
1b44c5a6 AT |
316 | |
317 | /* Interrupts reset */ | |
318 | ||
319 | /* Disable all global interrupts */ | |
871df319 | 320 | writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); |
1b44c5a6 AT |
321 | |
322 | /* Clear any pending interrupt */ | |
871df319 | 323 | writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); |
1b44c5a6 | 324 | |
367571e4 OH |
325 | /* Processing Engine configuration */ |
326 | for (pe = 0; pe < priv->config.pes; pe++) { | |
327 | /* Data Fetch Engine configuration */ | |
1b44c5a6 | 328 | |
367571e4 OH |
329 | /* Reset all DFE threads */ |
330 | writel(EIP197_DxE_THR_CTRL_RESET_PE, | |
331 | EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); | |
1b44c5a6 | 332 | |
5eb09118 | 333 | if (priv->version == EIP197B || priv->version == EIP197D) { |
367571e4 OH |
334 | /* Reset HIA input interface arbiter */ |
335 | writel(EIP197_HIA_RA_PE_CTRL_RESET, | |
336 | EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); | |
337 | } | |
1b44c5a6 | 338 | |
367571e4 OH |
339 | /* DMA transfer size to use */ |
340 | val = EIP197_HIA_DFE_CFG_DIS_DEBUG; | |
341 | val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) | | |
342 | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); | |
343 | val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | | |
344 | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); | |
345 | val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); | |
346 | val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); | |
347 | writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe)); | |
348 | ||
349 | /* Leave the DFE threads reset state */ | |
350 | writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); | |
351 | ||
352 | /* Configure the processing engine thresholds */ | |
353 | writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | | |
354 | EIP197_PE_IN_xBUF_THRES_MAX(9), | |
355 | EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe)); | |
356 | writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | | |
357 | EIP197_PE_IN_xBUF_THRES_MAX(7), | |
358 | EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe)); | |
359 | ||
5eb09118 | 360 | if (priv->version == EIP197B || priv->version == EIP197D) { |
367571e4 OH |
361 | /* enable HIA input interface arbiter and rings */ |
362 | writel(EIP197_HIA_RA_PE_CTRL_EN | | |
363 | GENMASK(priv->config.rings - 1, 0), | |
364 | EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); | |
365 | } | |
1b44c5a6 | 366 | |
367571e4 OH |
367 | /* Data Store Engine configuration */ |
368 | ||
369 | /* Reset all DSE threads */ | |
370 | writel(EIP197_DxE_THR_CTRL_RESET_PE, | |
371 | EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); | |
372 | ||
373 | /* Wait for all DSE threads to complete */ | |
374 | while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) & | |
375 | GENMASK(15, 12)) != GENMASK(15, 12)) | |
376 | ; | |
377 | ||
378 | /* DMA transfer size to use */ | |
379 | val = EIP197_HIA_DSE_CFG_DIS_DEBUG; | |
380 | val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | | |
381 | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8); | |
382 | val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); | |
383 | val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; | |
384 | /* FIXME: instability issues can occur for EIP97 but disabling it impact | |
385 | * performances. | |
386 | */ | |
5eb09118 | 387 | if (priv->version == EIP197B || priv->version == EIP197D) |
367571e4 OH |
388 | val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; |
389 | writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe)); | |
390 | ||
391 | /* Leave the DSE threads reset state */ | |
392 | writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); | |
393 | ||
394 | /* Configure the procesing engine thresholds */ | |
395 | writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | | |
396 | EIP197_PE_OUT_DBUF_THRES_MAX(8), | |
397 | EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe)); | |
398 | ||
399 | /* Processing Engine configuration */ | |
400 | ||
401 | /* H/W capabilities selection */ | |
402 | val = EIP197_FUNCTION_RSVD; | |
403 | val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; | |
404 | val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT; | |
a7dea8c0 | 405 | val |= EIP197_ALG_DES_ECB | EIP197_ALG_DES_CBC; |
62469879 | 406 | val |= EIP197_ALG_3DES_ECB | EIP197_ALG_3DES_CBC; |
367571e4 | 407 | val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC; |
b471e4b9 | 408 | val |= EIP197_ALG_MD5 | EIP197_ALG_HMAC_MD5; |
367571e4 OH |
409 | val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1; |
410 | val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2; | |
411 | writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe)); | |
871df319 | 412 | } |
1b44c5a6 | 413 | |
1b44c5a6 AT |
414 | /* Command Descriptor Rings prepare */ |
415 | for (i = 0; i < priv->config.rings; i++) { | |
416 | /* Clear interrupts for this ring */ | |
417 | writel(GENMASK(31, 0), | |
871df319 | 418 | EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); |
1b44c5a6 AT |
419 | |
420 | /* Disable external triggering */ | |
871df319 | 421 | writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); |
1b44c5a6 AT |
422 | |
423 | /* Clear the pending prepared counter */ | |
424 | writel(EIP197_xDR_PREP_CLR_COUNT, | |
871df319 | 425 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); |
1b44c5a6 AT |
426 | |
427 | /* Clear the pending processed counter */ | |
428 | writel(EIP197_xDR_PROC_CLR_COUNT, | |
871df319 | 429 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); |
1b44c5a6 AT |
430 | |
431 | writel(0, | |
871df319 | 432 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); |
1b44c5a6 | 433 | writel(0, |
871df319 | 434 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); |
1b44c5a6 AT |
435 | |
436 | writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2, | |
871df319 | 437 | EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); |
1b44c5a6 AT |
438 | } |
439 | ||
440 | /* Result Descriptor Ring prepare */ | |
441 | for (i = 0; i < priv->config.rings; i++) { | |
442 | /* Disable external triggering*/ | |
871df319 | 443 | writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); |
1b44c5a6 AT |
444 | |
445 | /* Clear the pending prepared counter */ | |
446 | writel(EIP197_xDR_PREP_CLR_COUNT, | |
871df319 | 447 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); |
1b44c5a6 AT |
448 | |
449 | /* Clear the pending processed counter */ | |
450 | writel(EIP197_xDR_PROC_CLR_COUNT, | |
871df319 | 451 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); |
1b44c5a6 AT |
452 | |
453 | writel(0, | |
871df319 | 454 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); |
1b44c5a6 | 455 | writel(0, |
871df319 | 456 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); |
1b44c5a6 AT |
457 | |
458 | /* Ring size */ | |
459 | writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2, | |
871df319 | 460 | EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); |
1b44c5a6 AT |
461 | } |
462 | ||
367571e4 OH |
463 | for (pe = 0; pe < priv->config.pes; pe++) { |
464 | /* Enable command descriptor rings */ | |
465 | writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), | |
466 | EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); | |
1b44c5a6 | 467 | |
367571e4 OH |
468 | /* Enable result descriptor rings */ |
469 | writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), | |
470 | EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); | |
471 | } | |
1b44c5a6 AT |
472 | |
473 | /* Clear any HIA interrupt */ | |
871df319 | 474 | writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); |
1b44c5a6 | 475 | |
5eb09118 | 476 | if (priv->version == EIP197B || priv->version == EIP197D) { |
871df319 | 477 | eip197_trc_cache_init(priv); |
1b44c5a6 | 478 | |
871df319 AT |
479 | ret = eip197_load_firmwares(priv); |
480 | if (ret) | |
481 | return ret; | |
482 | } | |
1b44c5a6 AT |
483 | |
484 | safexcel_hw_setup_cdesc_rings(priv); | |
485 | safexcel_hw_setup_rdesc_rings(priv); | |
486 | ||
487 | return 0; | |
488 | } | |
489 | ||
dc7e28a3 | 490 | /* Called with ring's lock taken */ |
f7268c53 AT |
491 | static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv, |
492 | int ring) | |
dc7e28a3 | 493 | { |
f7268c53 | 494 | int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ); |
dc7e28a3 AT |
495 | |
496 | if (!coal) | |
f7268c53 | 497 | return; |
dc7e28a3 AT |
498 | |
499 | /* Configure when we want an interrupt */ | |
500 | writel(EIP197_HIA_RDR_THRESH_PKT_MODE | | |
501 | EIP197_HIA_RDR_THRESH_PROC_PKT(coal), | |
871df319 | 502 | EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); |
dc7e28a3 AT |
503 | } |
504 | ||
86671abb | 505 | void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) |
1b44c5a6 AT |
506 | { |
507 | struct crypto_async_request *req, *backlog; | |
508 | struct safexcel_context *ctx; | |
86671abb | 509 | int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; |
1b44c5a6 | 510 | |
8732b298 AT |
511 | /* If a request wasn't properly dequeued because of a lack of resources, |
512 | * proceeded it first, | |
513 | */ | |
514 | req = priv->ring[ring].req; | |
515 | backlog = priv->ring[ring].backlog; | |
516 | if (req) | |
517 | goto handle_req; | |
518 | ||
dc7e28a3 | 519 | while (true) { |
86671abb | 520 | spin_lock_bh(&priv->ring[ring].queue_lock); |
86671abb | 521 | backlog = crypto_get_backlog(&priv->ring[ring].queue); |
b1deb47a | 522 | req = crypto_dequeue_request(&priv->ring[ring].queue); |
86671abb | 523 | spin_unlock_bh(&priv->ring[ring].queue_lock); |
1b44c5a6 | 524 | |
8732b298 AT |
525 | if (!req) { |
526 | priv->ring[ring].req = NULL; | |
527 | priv->ring[ring].backlog = NULL; | |
1b44c5a6 | 528 | goto finalize; |
8732b298 | 529 | } |
1b44c5a6 | 530 | |
8732b298 | 531 | handle_req: |
1b44c5a6 | 532 | ctx = crypto_tfm_ctx(req->tfm); |
9744fec9 OH |
533 | ret = ctx->send(req, ring, &commands, &results); |
534 | if (ret) | |
8732b298 | 535 | goto request_failed; |
1b44c5a6 AT |
536 | |
537 | if (backlog) | |
538 | backlog->complete(backlog, -EINPROGRESS); | |
539 | ||
95831cea AT |
540 | /* In case the send() helper did not issue any command to push |
541 | * to the engine because the input data was cached, continue to | |
542 | * dequeue other requests as this is valid and not an error. | |
543 | */ | |
9744fec9 | 544 | if (!commands && !results) |
95831cea | 545 | continue; |
1b44c5a6 | 546 | |
86671abb AT |
547 | cdesc += commands; |
548 | rdesc += results; | |
dc7e28a3 AT |
549 | nreq++; |
550 | } | |
1b44c5a6 | 551 | |
8732b298 AT |
552 | request_failed: |
553 | /* Not enough resources to handle all the requests. Bail out and save | |
554 | * the request and the backlog for the next dequeue call (per-ring). | |
555 | */ | |
556 | priv->ring[ring].req = req; | |
557 | priv->ring[ring].backlog = backlog; | |
558 | ||
1b44c5a6 | 559 | finalize: |
8472e778 | 560 | if (!nreq) |
1b44c5a6 AT |
561 | return; |
562 | ||
9744fec9 | 563 | spin_lock_bh(&priv->ring[ring].lock); |
1b44c5a6 | 564 | |
f7268c53 AT |
565 | priv->ring[ring].requests += nreq; |
566 | ||
dc7e28a3 | 567 | if (!priv->ring[ring].busy) { |
f7268c53 | 568 | safexcel_try_push_requests(priv, ring); |
e1d24c0b | 569 | priv->ring[ring].busy = true; |
dc7e28a3 AT |
570 | } |
571 | ||
9744fec9 | 572 | spin_unlock_bh(&priv->ring[ring].lock); |
1b44c5a6 | 573 | |
86671abb AT |
574 | /* let the RDR know we have pending descriptors */ |
575 | writel((rdesc * priv->config.rd_offset) << 2, | |
871df319 | 576 | EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); |
1b44c5a6 | 577 | |
86671abb AT |
578 | /* let the CDR know we have pending descriptors */ |
579 | writel((cdesc * priv->config.cd_offset) << 2, | |
871df319 | 580 | EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); |
1b44c5a6 AT |
581 | } |
582 | ||
bdfd1909 AT |
583 | inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, |
584 | struct safexcel_result_desc *rdesc) | |
585 | { | |
586 | if (likely(!rdesc->result_data.error_code)) | |
587 | return 0; | |
588 | ||
589 | if (rdesc->result_data.error_code & 0x407f) { | |
590 | /* Fatal error (bits 0-7, 14) */ | |
591 | dev_err(priv->dev, | |
592 | "cipher: result: result descriptor error (%d)\n", | |
593 | rdesc->result_data.error_code); | |
594 | return -EIO; | |
595 | } else if (rdesc->result_data.error_code == BIT(9)) { | |
596 | /* Authentication failed */ | |
597 | return -EBADMSG; | |
598 | } | |
599 | ||
600 | /* All other non-fatal errors */ | |
601 | return -EINVAL; | |
602 | } | |
603 | ||
9744fec9 OH |
604 | inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, |
605 | int ring, | |
606 | struct safexcel_result_desc *rdesc, | |
607 | struct crypto_async_request *req) | |
608 | { | |
609 | int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc); | |
610 | ||
611 | priv->ring[ring].rdr_req[i] = req; | |
612 | } | |
613 | ||
614 | inline struct crypto_async_request * | |
615 | safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring) | |
616 | { | |
617 | int i = safexcel_ring_first_rdr_index(priv, ring); | |
618 | ||
619 | return priv->ring[ring].rdr_req[i]; | |
620 | } | |
621 | ||
1b44c5a6 AT |
622 | void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) |
623 | { | |
624 | struct safexcel_command_desc *cdesc; | |
625 | ||
626 | /* Acknowledge the command descriptors */ | |
627 | do { | |
628 | cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); | |
629 | if (IS_ERR(cdesc)) { | |
630 | dev_err(priv->dev, | |
631 | "Could not retrieve the command descriptor\n"); | |
632 | return; | |
633 | } | |
634 | } while (!cdesc->last_seg); | |
635 | } | |
636 | ||
637 | void safexcel_inv_complete(struct crypto_async_request *req, int error) | |
638 | { | |
639 | struct safexcel_inv_result *result = req->data; | |
640 | ||
641 | if (error == -EINPROGRESS) | |
642 | return; | |
643 | ||
644 | result->error = error; | |
645 | complete(&result->completion); | |
646 | } | |
647 | ||
648 | int safexcel_invalidate_cache(struct crypto_async_request *async, | |
1b44c5a6 | 649 | struct safexcel_crypto_priv *priv, |
9744fec9 | 650 | dma_addr_t ctxr_dma, int ring) |
1b44c5a6 AT |
651 | { |
652 | struct safexcel_command_desc *cdesc; | |
653 | struct safexcel_result_desc *rdesc; | |
654 | int ret = 0; | |
655 | ||
1b44c5a6 AT |
656 | /* Prepare command descriptor */ |
657 | cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma); | |
9744fec9 OH |
658 | if (IS_ERR(cdesc)) |
659 | return PTR_ERR(cdesc); | |
1b44c5a6 AT |
660 | |
661 | cdesc->control_data.type = EIP197_TYPE_EXTENDED; | |
662 | cdesc->control_data.options = 0; | |
663 | cdesc->control_data.refresh = 0; | |
664 | cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; | |
665 | ||
666 | /* Prepare result descriptor */ | |
667 | rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); | |
668 | ||
669 | if (IS_ERR(rdesc)) { | |
670 | ret = PTR_ERR(rdesc); | |
671 | goto cdesc_rollback; | |
672 | } | |
673 | ||
9744fec9 OH |
674 | safexcel_rdr_req_set(priv, ring, rdesc, async); |
675 | ||
676 | return ret; | |
1b44c5a6 AT |
677 | |
678 | cdesc_rollback: | |
679 | safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); | |
680 | ||
1b44c5a6 AT |
681 | return ret; |
682 | } | |
683 | ||
684 | static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, | |
685 | int ring) | |
686 | { | |
9744fec9 | 687 | struct crypto_async_request *req; |
1b44c5a6 | 688 | struct safexcel_context *ctx; |
f7268c53 | 689 | int ret, i, nreq, ndesc, tot_descs, handled = 0; |
1b44c5a6 AT |
690 | bool should_complete; |
691 | ||
7f77f5a4 AT |
692 | handle_results: |
693 | tot_descs = 0; | |
694 | ||
871df319 | 695 | nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); |
7f77f5a4 AT |
696 | nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; |
697 | nreq &= EIP197_xDR_PROC_xD_PKT_MASK; | |
1b44c5a6 | 698 | if (!nreq) |
dc7e28a3 | 699 | goto requests_left; |
1b44c5a6 AT |
700 | |
701 | for (i = 0; i < nreq; i++) { | |
9744fec9 OH |
702 | req = safexcel_rdr_req_get(priv, ring); |
703 | ||
704 | ctx = crypto_tfm_ctx(req->tfm); | |
705 | ndesc = ctx->handle_result(priv, ring, req, | |
1b44c5a6 AT |
706 | &should_complete, &ret); |
707 | if (ndesc < 0) { | |
708 | dev_err(priv->dev, "failed to handle result (%d)", ndesc); | |
2313e9f7 | 709 | goto acknowledge; |
1b44c5a6 AT |
710 | } |
711 | ||
1b44c5a6 AT |
712 | if (should_complete) { |
713 | local_bh_disable(); | |
9744fec9 | 714 | req->complete(req, ret); |
1b44c5a6 AT |
715 | local_bh_enable(); |
716 | } | |
717 | ||
2313e9f7 | 718 | tot_descs += ndesc; |
f7268c53 | 719 | handled++; |
2313e9f7 AT |
720 | } |
721 | ||
722 | acknowledge: | |
723 | if (i) { | |
724 | writel(EIP197_xDR_PROC_xD_PKT(i) | | |
725 | EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), | |
871df319 | 726 | EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); |
1b44c5a6 | 727 | } |
dc7e28a3 | 728 | |
7f77f5a4 AT |
729 | /* If the number of requests overflowed the counter, try to proceed more |
730 | * requests. | |
731 | */ | |
732 | if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) | |
733 | goto handle_results; | |
734 | ||
dc7e28a3 | 735 | requests_left: |
9744fec9 | 736 | spin_lock_bh(&priv->ring[ring].lock); |
dc7e28a3 | 737 | |
f7268c53 AT |
738 | priv->ring[ring].requests -= handled; |
739 | safexcel_try_push_requests(priv, ring); | |
dc7e28a3 | 740 | |
f7268c53 | 741 | if (!priv->ring[ring].requests) |
dc7e28a3 AT |
742 | priv->ring[ring].busy = false; |
743 | ||
9744fec9 | 744 | spin_unlock_bh(&priv->ring[ring].lock); |
1b44c5a6 AT |
745 | } |
746 | ||
8472e778 AT |
747 | static void safexcel_dequeue_work(struct work_struct *work) |
748 | { | |
749 | struct safexcel_work_data *data = | |
750 | container_of(work, struct safexcel_work_data, work); | |
1b44c5a6 | 751 | |
8472e778 | 752 | safexcel_dequeue(data->priv, data->ring); |
1b44c5a6 AT |
753 | } |
754 | ||
755 | struct safexcel_ring_irq_data { | |
756 | struct safexcel_crypto_priv *priv; | |
757 | int ring; | |
758 | }; | |
759 | ||
760 | static irqreturn_t safexcel_irq_ring(int irq, void *data) | |
761 | { | |
762 | struct safexcel_ring_irq_data *irq_data = data; | |
763 | struct safexcel_crypto_priv *priv = irq_data->priv; | |
69ee4dd5 | 764 | int ring = irq_data->ring, rc = IRQ_NONE; |
1b44c5a6 AT |
765 | u32 status, stat; |
766 | ||
871df319 | 767 | status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); |
1b44c5a6 | 768 | if (!status) |
69ee4dd5 | 769 | return rc; |
1b44c5a6 AT |
770 | |
771 | /* RDR interrupts */ | |
772 | if (status & EIP197_RDR_IRQ(ring)) { | |
871df319 | 773 | stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); |
1b44c5a6 AT |
774 | |
775 | if (unlikely(stat & EIP197_xDR_ERR)) { | |
776 | /* | |
777 | * Fatal error, the RDR is unusable and must be | |
778 | * reinitialized. This should not happen under | |
779 | * normal circumstances. | |
780 | */ | |
781 | dev_err(priv->dev, "RDR: fatal error."); | |
782 | } else if (likely(stat & EIP197_xDR_THRESH)) { | |
69ee4dd5 | 783 | rc = IRQ_WAKE_THREAD; |
1b44c5a6 AT |
784 | } |
785 | ||
786 | /* ACK the interrupts */ | |
787 | writel(stat & 0xff, | |
871df319 | 788 | EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); |
1b44c5a6 AT |
789 | } |
790 | ||
791 | /* ACK the interrupts */ | |
871df319 | 792 | writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); |
1b44c5a6 | 793 | |
69ee4dd5 AT |
794 | return rc; |
795 | } | |
796 | ||
797 | static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) | |
798 | { | |
799 | struct safexcel_ring_irq_data *irq_data = data; | |
800 | struct safexcel_crypto_priv *priv = irq_data->priv; | |
801 | int ring = irq_data->ring; | |
802 | ||
803 | safexcel_handle_result_descriptor(priv, ring); | |
804 | ||
805 | queue_work(priv->ring[ring].workqueue, | |
806 | &priv->ring[ring].work_data.work); | |
807 | ||
1b44c5a6 AT |
808 | return IRQ_HANDLED; |
809 | } | |
810 | ||
811 | static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name, | |
812 | irq_handler_t handler, | |
69ee4dd5 | 813 | irq_handler_t threaded_handler, |
1b44c5a6 AT |
814 | struct safexcel_ring_irq_data *ring_irq_priv) |
815 | { | |
816 | int ret, irq = platform_get_irq_byname(pdev, name); | |
817 | ||
818 | if (irq < 0) { | |
819 | dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name); | |
820 | return irq; | |
821 | } | |
822 | ||
69ee4dd5 AT |
823 | ret = devm_request_threaded_irq(&pdev->dev, irq, handler, |
824 | threaded_handler, IRQF_ONESHOT, | |
825 | dev_name(&pdev->dev), ring_irq_priv); | |
1b44c5a6 AT |
826 | if (ret) { |
827 | dev_err(&pdev->dev, "unable to request IRQ %d\n", irq); | |
828 | return ret; | |
829 | } | |
830 | ||
831 | return irq; | |
832 | } | |
833 | ||
834 | static struct safexcel_alg_template *safexcel_algs[] = { | |
a7dea8c0 OH |
835 | &safexcel_alg_ecb_des, |
836 | &safexcel_alg_cbc_des, | |
62469879 OH |
837 | &safexcel_alg_ecb_des3_ede, |
838 | &safexcel_alg_cbc_des3_ede, | |
1b44c5a6 AT |
839 | &safexcel_alg_ecb_aes, |
840 | &safexcel_alg_cbc_aes, | |
293f89cf | 841 | &safexcel_alg_md5, |
1b44c5a6 AT |
842 | &safexcel_alg_sha1, |
843 | &safexcel_alg_sha224, | |
844 | &safexcel_alg_sha256, | |
9e46eafd | 845 | &safexcel_alg_sha384, |
b460edb6 | 846 | &safexcel_alg_sha512, |
b471e4b9 | 847 | &safexcel_alg_hmac_md5, |
1b44c5a6 | 848 | &safexcel_alg_hmac_sha1, |
3ad618d8 | 849 | &safexcel_alg_hmac_sha224, |
73f36ea7 | 850 | &safexcel_alg_hmac_sha256, |
1f5d5d98 | 851 | &safexcel_alg_hmac_sha384, |
0de54fb1 | 852 | &safexcel_alg_hmac_sha512, |
01ba061d | 853 | &safexcel_alg_authenc_hmac_sha1_cbc_aes, |
678b2878 | 854 | &safexcel_alg_authenc_hmac_sha224_cbc_aes, |
f6beaea3 | 855 | &safexcel_alg_authenc_hmac_sha256_cbc_aes, |
ea23cb53 | 856 | &safexcel_alg_authenc_hmac_sha384_cbc_aes, |
87eee125 | 857 | &safexcel_alg_authenc_hmac_sha512_cbc_aes, |
1b44c5a6 AT |
858 | }; |
859 | ||
860 | static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) | |
861 | { | |
862 | int i, j, ret = 0; | |
863 | ||
864 | for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { | |
865 | safexcel_algs[i]->priv = priv; | |
866 | ||
998d2abb AT |
867 | if (!(safexcel_algs[i]->engines & priv->version)) |
868 | continue; | |
869 | ||
1b44c5a6 AT |
870 | if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) |
871 | ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); | |
f6beaea3 AT |
872 | else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) |
873 | ret = crypto_register_aead(&safexcel_algs[i]->alg.aead); | |
1b44c5a6 AT |
874 | else |
875 | ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); | |
876 | ||
877 | if (ret) | |
878 | goto fail; | |
879 | } | |
880 | ||
881 | return 0; | |
882 | ||
883 | fail: | |
884 | for (j = 0; j < i; j++) { | |
998d2abb AT |
885 | if (!(safexcel_algs[j]->engines & priv->version)) |
886 | continue; | |
887 | ||
1b44c5a6 AT |
888 | if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) |
889 | crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); | |
f6beaea3 AT |
890 | else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD) |
891 | crypto_unregister_aead(&safexcel_algs[j]->alg.aead); | |
1b44c5a6 AT |
892 | else |
893 | crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); | |
894 | } | |
895 | ||
896 | return ret; | |
897 | } | |
898 | ||
899 | static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) | |
900 | { | |
901 | int i; | |
902 | ||
903 | for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { | |
998d2abb AT |
904 | if (!(safexcel_algs[i]->engines & priv->version)) |
905 | continue; | |
906 | ||
1b44c5a6 AT |
907 | if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) |
908 | crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); | |
f6beaea3 AT |
909 | else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) |
910 | crypto_unregister_aead(&safexcel_algs[i]->alg.aead); | |
1b44c5a6 AT |
911 | else |
912 | crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); | |
913 | } | |
914 | } | |
915 | ||
916 | static void safexcel_configure(struct safexcel_crypto_priv *priv) | |
917 | { | |
367571e4 | 918 | u32 val, mask = 0; |
1b44c5a6 | 919 | |
871df319 | 920 | val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); |
367571e4 OH |
921 | |
922 | /* Read number of PEs from the engine */ | |
923 | switch (priv->version) { | |
924 | case EIP197B: | |
5eb09118 | 925 | case EIP197D: |
367571e4 OH |
926 | mask = EIP197_N_PES_MASK; |
927 | break; | |
928 | default: | |
929 | mask = EIP97_N_PES_MASK; | |
930 | } | |
931 | priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask; | |
932 | ||
1b44c5a6 AT |
933 | val = (val & GENMASK(27, 25)) >> 25; |
934 | mask = BIT(val) - 1; | |
935 | ||
871df319 | 936 | val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS); |
1b44c5a6 AT |
937 | priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings); |
938 | ||
939 | priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32)); | |
940 | priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; | |
941 | ||
942 | priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32)); | |
943 | priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; | |
944 | } | |
945 | ||
871df319 AT |
946 | static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) |
947 | { | |
948 | struct safexcel_register_offsets *offsets = &priv->offsets; | |
949 | ||
5eb09118 AT |
950 | switch (priv->version) { |
951 | case EIP197B: | |
952 | case EIP197D: | |
871df319 AT |
953 | offsets->hia_aic = EIP197_HIA_AIC_BASE; |
954 | offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; | |
955 | offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; | |
956 | offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; | |
957 | offsets->hia_dfe = EIP197_HIA_DFE_BASE; | |
958 | offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; | |
959 | offsets->hia_dse = EIP197_HIA_DSE_BASE; | |
960 | offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; | |
961 | offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; | |
962 | offsets->pe = EIP197_PE_BASE; | |
5eb09118 AT |
963 | break; |
964 | case EIP97IES: | |
871df319 AT |
965 | offsets->hia_aic = EIP97_HIA_AIC_BASE; |
966 | offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; | |
967 | offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; | |
968 | offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; | |
969 | offsets->hia_dfe = EIP97_HIA_DFE_BASE; | |
970 | offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; | |
971 | offsets->hia_dse = EIP97_HIA_DSE_BASE; | |
972 | offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; | |
973 | offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; | |
974 | offsets->pe = EIP97_PE_BASE; | |
5eb09118 | 975 | break; |
871df319 AT |
976 | } |
977 | } | |
978 | ||
1b44c5a6 AT |
979 | static int safexcel_probe(struct platform_device *pdev) |
980 | { | |
981 | struct device *dev = &pdev->dev; | |
982 | struct resource *res; | |
983 | struct safexcel_crypto_priv *priv; | |
1b44c5a6 AT |
984 | int i, ret; |
985 | ||
986 | priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); | |
987 | if (!priv) | |
988 | return -ENOMEM; | |
989 | ||
990 | priv->dev = dev; | |
871df319 AT |
991 | priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev); |
992 | ||
5eb09118 | 993 | if (priv->version == EIP197B || priv->version == EIP197D) |
53c83e91 AT |
994 | priv->flags |= EIP197_TRC_CACHE; |
995 | ||
871df319 | 996 | safexcel_init_register_offsets(priv); |
1b44c5a6 AT |
997 | |
998 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
999 | priv->base = devm_ioremap_resource(dev, res); | |
1000 | if (IS_ERR(priv->base)) { | |
1001 | dev_err(dev, "failed to get resource\n"); | |
1002 | return PTR_ERR(priv->base); | |
1003 | } | |
1004 | ||
f962eb46 | 1005 | priv->clk = devm_clk_get(&pdev->dev, NULL); |
5b376896 GC |
1006 | ret = PTR_ERR_OR_ZERO(priv->clk); |
1007 | /* The clock isn't mandatory */ | |
1008 | if (ret != -ENOENT) { | |
1009 | if (ret) | |
1010 | return ret; | |
1011 | ||
1b44c5a6 AT |
1012 | ret = clk_prepare_enable(priv->clk); |
1013 | if (ret) { | |
1014 | dev_err(dev, "unable to enable clk (%d)\n", ret); | |
1015 | return ret; | |
1016 | } | |
1b44c5a6 AT |
1017 | } |
1018 | ||
1d17cbfb GC |
1019 | priv->reg_clk = devm_clk_get(&pdev->dev, "reg"); |
1020 | ret = PTR_ERR_OR_ZERO(priv->reg_clk); | |
1021 | /* The clock isn't mandatory */ | |
1022 | if (ret != -ENOENT) { | |
1023 | if (ret) | |
1024 | goto err_core_clk; | |
1025 | ||
1026 | ret = clk_prepare_enable(priv->reg_clk); | |
1027 | if (ret) { | |
1028 | dev_err(dev, "unable to enable reg clk (%d)\n", ret); | |
1029 | goto err_core_clk; | |
1030 | } | |
1031 | } | |
1032 | ||
a2069aac | 1033 | ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); |
1b44c5a6 | 1034 | if (ret) |
1d17cbfb | 1035 | goto err_reg_clk; |
1b44c5a6 AT |
1036 | |
1037 | priv->context_pool = dmam_pool_create("safexcel-context", dev, | |
1038 | sizeof(struct safexcel_context_record), | |
1039 | 1, 0); | |
1040 | if (!priv->context_pool) { | |
1041 | ret = -ENOMEM; | |
1d17cbfb | 1042 | goto err_reg_clk; |
1b44c5a6 AT |
1043 | } |
1044 | ||
1045 | safexcel_configure(priv); | |
1046 | ||
329e0989 KC |
1047 | priv->ring = devm_kcalloc(dev, priv->config.rings, |
1048 | sizeof(*priv->ring), | |
18e0e95b OH |
1049 | GFP_KERNEL); |
1050 | if (!priv->ring) { | |
1051 | ret = -ENOMEM; | |
1052 | goto err_reg_clk; | |
1053 | } | |
1054 | ||
1b44c5a6 AT |
1055 | for (i = 0; i < priv->config.rings; i++) { |
1056 | char irq_name[6] = {0}; /* "ringX\0" */ | |
1057 | char wq_name[9] = {0}; /* "wq_ringX\0" */ | |
1058 | int irq; | |
1059 | struct safexcel_ring_irq_data *ring_irq; | |
1060 | ||
1061 | ret = safexcel_init_ring_descriptors(priv, | |
1062 | &priv->ring[i].cdr, | |
1063 | &priv->ring[i].rdr); | |
1064 | if (ret) | |
1d17cbfb | 1065 | goto err_reg_clk; |
1b44c5a6 | 1066 | |
329e0989 KC |
1067 | priv->ring[i].rdr_req = devm_kcalloc(dev, |
1068 | EIP197_DEFAULT_RING_SIZE, | |
1069 | sizeof(priv->ring[i].rdr_req), | |
9744fec9 OH |
1070 | GFP_KERNEL); |
1071 | if (!priv->ring[i].rdr_req) { | |
1072 | ret = -ENOMEM; | |
1073 | goto err_reg_clk; | |
1074 | } | |
1075 | ||
1b44c5a6 AT |
1076 | ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); |
1077 | if (!ring_irq) { | |
1078 | ret = -ENOMEM; | |
1d17cbfb | 1079 | goto err_reg_clk; |
1b44c5a6 AT |
1080 | } |
1081 | ||
1082 | ring_irq->priv = priv; | |
1083 | ring_irq->ring = i; | |
1084 | ||
1085 | snprintf(irq_name, 6, "ring%d", i); | |
1086 | irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, | |
69ee4dd5 | 1087 | safexcel_irq_ring_thread, |
1b44c5a6 | 1088 | ring_irq); |
b7d65fe1 CJ |
1089 | if (irq < 0) { |
1090 | ret = irq; | |
1d17cbfb | 1091 | goto err_reg_clk; |
b7d65fe1 | 1092 | } |
1b44c5a6 AT |
1093 | |
1094 | priv->ring[i].work_data.priv = priv; | |
1095 | priv->ring[i].work_data.ring = i; | |
8472e778 | 1096 | INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work); |
1b44c5a6 AT |
1097 | |
1098 | snprintf(wq_name, 9, "wq_ring%d", i); | |
1099 | priv->ring[i].workqueue = create_singlethread_workqueue(wq_name); | |
1100 | if (!priv->ring[i].workqueue) { | |
1101 | ret = -ENOMEM; | |
1d17cbfb | 1102 | goto err_reg_clk; |
1b44c5a6 AT |
1103 | } |
1104 | ||
f7268c53 | 1105 | priv->ring[i].requests = 0; |
dc7e28a3 AT |
1106 | priv->ring[i].busy = false; |
1107 | ||
86671abb AT |
1108 | crypto_init_queue(&priv->ring[i].queue, |
1109 | EIP197_DEFAULT_RING_SIZE); | |
1110 | ||
1b44c5a6 | 1111 | spin_lock_init(&priv->ring[i].lock); |
86671abb | 1112 | spin_lock_init(&priv->ring[i].queue_lock); |
1b44c5a6 AT |
1113 | } |
1114 | ||
1115 | platform_set_drvdata(pdev, priv); | |
1116 | atomic_set(&priv->ring_used, 0); | |
1117 | ||
1b44c5a6 AT |
1118 | ret = safexcel_hw_init(priv); |
1119 | if (ret) { | |
1120 | dev_err(dev, "EIP h/w init failed (%d)\n", ret); | |
1d17cbfb | 1121 | goto err_reg_clk; |
1b44c5a6 AT |
1122 | } |
1123 | ||
1124 | ret = safexcel_register_algorithms(priv); | |
1125 | if (ret) { | |
1126 | dev_err(dev, "Failed to register algorithms (%d)\n", ret); | |
1d17cbfb | 1127 | goto err_reg_clk; |
1b44c5a6 AT |
1128 | } |
1129 | ||
1130 | return 0; | |
1131 | ||
1d17cbfb GC |
1132 | err_reg_clk: |
1133 | clk_disable_unprepare(priv->reg_clk); | |
1134 | err_core_clk: | |
1b44c5a6 AT |
1135 | clk_disable_unprepare(priv->clk); |
1136 | return ret; | |
1137 | } | |
1138 | ||
f1edf29a OH |
1139 | static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) |
1140 | { | |
1141 | int i; | |
1142 | ||
1143 | for (i = 0; i < priv->config.rings; i++) { | |
1144 | /* clear any pending interrupt */ | |
1145 | writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); | |
1146 | writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); | |
1147 | ||
1148 | /* Reset the CDR base address */ | |
1149 | writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); | |
1150 | writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); | |
1151 | ||
1152 | /* Reset the RDR base address */ | |
1153 | writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); | |
1154 | writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); | |
1155 | } | |
1156 | } | |
1b44c5a6 AT |
1157 | |
1158 | static int safexcel_remove(struct platform_device *pdev) | |
1159 | { | |
1160 | struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); | |
1161 | int i; | |
1162 | ||
1163 | safexcel_unregister_algorithms(priv); | |
f1edf29a OH |
1164 | safexcel_hw_reset_rings(priv); |
1165 | ||
1b44c5a6 AT |
1166 | clk_disable_unprepare(priv->clk); |
1167 | ||
1168 | for (i = 0; i < priv->config.rings; i++) | |
1169 | destroy_workqueue(priv->ring[i].workqueue); | |
1170 | ||
1171 | return 0; | |
1172 | } | |
1173 | ||
1174 | static const struct of_device_id safexcel_of_match_table[] = { | |
871df319 | 1175 | { |
7d8f52a3 AT |
1176 | .compatible = "inside-secure,safexcel-eip97ies", |
1177 | .data = (void *)EIP97IES, | |
1178 | }, | |
1179 | { | |
1180 | .compatible = "inside-secure,safexcel-eip197b", | |
1181 | .data = (void *)EIP197B, | |
1182 | }, | |
5eb09118 AT |
1183 | { |
1184 | .compatible = "inside-secure,safexcel-eip197d", | |
1185 | .data = (void *)EIP197D, | |
1186 | }, | |
7d8f52a3 AT |
1187 | { |
1188 | /* Deprecated. Kept for backward compatibility. */ | |
871df319 | 1189 | .compatible = "inside-secure,safexcel-eip97", |
7d8f52a3 | 1190 | .data = (void *)EIP97IES, |
871df319 AT |
1191 | }, |
1192 | { | |
7d8f52a3 | 1193 | /* Deprecated. Kept for backward compatibility. */ |
871df319 | 1194 | .compatible = "inside-secure,safexcel-eip197", |
7d8f52a3 | 1195 | .data = (void *)EIP197B, |
871df319 | 1196 | }, |
1b44c5a6 AT |
1197 | {}, |
1198 | }; | |
1199 | ||
1200 | ||
1201 | static struct platform_driver crypto_safexcel = { | |
1202 | .probe = safexcel_probe, | |
1203 | .remove = safexcel_remove, | |
1204 | .driver = { | |
1205 | .name = "crypto-safexcel", | |
1206 | .of_match_table = safexcel_of_match_table, | |
1207 | }, | |
1208 | }; | |
1209 | module_platform_driver(crypto_safexcel); | |
1210 | ||
1211 | MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>"); | |
1212 | MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); | |
1213 | MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); | |
1214 | MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197"); | |
1215 | MODULE_LICENSE("GPL v2"); |