]>
Commit | Line | Data |
---|---|---|
f88fc122 BB |
1 | /* |
2 | * Copyright 2017 ATMEL | |
3 | * Copyright 2017 Free Electrons | |
4 | * | |
5 | * Author: Boris Brezillon <boris.brezillon@free-electrons.com> | |
6 | * | |
7 | * Derived from the atmel_nand.c driver which contained the following | |
8 | * copyrights: | |
9 | * | |
10 | * Copyright 2003 Rick Bronson | |
11 | * | |
12 | * Derived from drivers/mtd/nand/autcpu12.c | |
13 | * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) | |
14 | * | |
15 | * Derived from drivers/mtd/spia.c | |
16 | * Copyright 2000 Steven J. Hill (sjhill@cotw.com) | |
17 | * | |
18 | * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 | |
19 | * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 | |
20 | * | |
21 | * Derived from Das U-Boot source code | |
22 | * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) | |
23 | * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas | |
24 | * | |
25 | * Add Programmable Multibit ECC support for various AT91 SoC | |
26 | * Copyright 2012 ATMEL, Hong Xu | |
27 | * | |
28 | * Add Nand Flash Controller support for SAMA5 SoC | |
29 | * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) | |
30 | * | |
31 | * This program is free software; you can redistribute it and/or modify | |
32 | * it under the terms of the GNU General Public License version 2 as | |
33 | * published by the Free Software Foundation. | |
34 | * | |
35 | * The PMECC is an hardware assisted BCH engine, which means part of the | |
36 | * ECC algorithm is left to the software. The hardware/software repartition | |
37 | * is explained in the "PMECC Controller Functional Description" chapter in | |
38 | * Atmel datasheets, and some of the functions in this file are directly | |
39 | * implementing the algorithms described in the "Software Implementation" | |
40 | * sub-section. | |
41 | * | |
42 | * TODO: it seems that the software BCH implementation in lib/bch.c is already | |
43 | * providing some of the logic we are implementing here. It would be smart | |
44 | * to expose the needed lib/bch.c helpers/functions and re-use them here. | |
45 | */ | |
46 | ||
47 | #include <linux/genalloc.h> | |
48 | #include <linux/iopoll.h> | |
49 | #include <linux/module.h> | |
50 | #include <linux/mtd/nand.h> | |
51 | #include <linux/of_irq.h> | |
52 | #include <linux/of_platform.h> | |
53 | #include <linux/platform_device.h> | |
54 | #include <linux/slab.h> | |
55 | ||
56 | #include "pmecc.h" | |
57 | ||
58 | /* Galois field dimension */ | |
59 | #define PMECC_GF_DIMENSION_13 13 | |
60 | #define PMECC_GF_DIMENSION_14 14 | |
61 | ||
62 | /* Primitive Polynomial used by PMECC */ | |
63 | #define PMECC_GF_13_PRIMITIVE_POLY 0x201b | |
64 | #define PMECC_GF_14_PRIMITIVE_POLY 0x4443 | |
65 | ||
66 | #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 | |
67 | #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 | |
68 | ||
69 | /* Time out value for reading PMECC status register */ | |
70 | #define PMECC_MAX_TIMEOUT_MS 100 | |
71 | ||
72 | /* PMECC Register Definitions */ | |
73 | #define ATMEL_PMECC_CFG 0x0 | |
74 | #define PMECC_CFG_BCH_STRENGTH(x) (x) | |
75 | #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) | |
76 | #define PMECC_CFG_SECTOR512 (0 << 4) | |
77 | #define PMECC_CFG_SECTOR1024 (1 << 4) | |
78 | #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) | |
79 | #define PMECC_CFG_READ_OP (0 << 12) | |
80 | #define PMECC_CFG_WRITE_OP (1 << 12) | |
81 | #define PMECC_CFG_SPARE_ENABLE BIT(16) | |
82 | #define PMECC_CFG_AUTO_ENABLE BIT(20) | |
83 | ||
84 | #define ATMEL_PMECC_SAREA 0x4 | |
85 | #define ATMEL_PMECC_SADDR 0x8 | |
86 | #define ATMEL_PMECC_EADDR 0xc | |
87 | ||
88 | #define ATMEL_PMECC_CLK 0x10 | |
89 | #define PMECC_CLK_133MHZ (2 << 0) | |
90 | ||
91 | #define ATMEL_PMECC_CTRL 0x14 | |
92 | #define PMECC_CTRL_RST BIT(0) | |
93 | #define PMECC_CTRL_DATA BIT(1) | |
94 | #define PMECC_CTRL_USER BIT(2) | |
95 | #define PMECC_CTRL_ENABLE BIT(4) | |
96 | #define PMECC_CTRL_DISABLE BIT(5) | |
97 | ||
98 | #define ATMEL_PMECC_SR 0x18 | |
99 | #define PMECC_SR_BUSY BIT(0) | |
100 | #define PMECC_SR_ENABLE BIT(4) | |
101 | ||
102 | #define ATMEL_PMECC_IER 0x1c | |
103 | #define ATMEL_PMECC_IDR 0x20 | |
104 | #define ATMEL_PMECC_IMR 0x24 | |
105 | #define ATMEL_PMECC_ISR 0x28 | |
106 | #define PMECC_ERROR_INT BIT(0) | |
107 | ||
108 | #define ATMEL_PMECC_ECC(sector, n) \ | |
109 | ((((sector) + 1) * 0x40) + (n)) | |
110 | ||
111 | #define ATMEL_PMECC_REM(sector, n) \ | |
112 | ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) | |
113 | ||
114 | /* PMERRLOC Register Definitions */ | |
115 | #define ATMEL_PMERRLOC_ELCFG 0x0 | |
116 | #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) | |
117 | #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) | |
118 | #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) | |
119 | ||
120 | #define ATMEL_PMERRLOC_ELPRIM 0x4 | |
121 | #define ATMEL_PMERRLOC_ELEN 0x8 | |
122 | #define ATMEL_PMERRLOC_ELDIS 0xc | |
123 | #define PMERRLOC_DISABLE BIT(0) | |
124 | ||
125 | #define ATMEL_PMERRLOC_ELSR 0x10 | |
126 | #define PMERRLOC_ELSR_BUSY BIT(0) | |
127 | ||
128 | #define ATMEL_PMERRLOC_ELIER 0x14 | |
129 | #define ATMEL_PMERRLOC_ELIDR 0x18 | |
130 | #define ATMEL_PMERRLOC_ELIMR 0x1c | |
131 | #define ATMEL_PMERRLOC_ELISR 0x20 | |
132 | #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) | |
133 | #define PMERRLOC_CALC_DONE BIT(0) | |
134 | ||
135 | #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) | |
136 | ||
137 | #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) | |
138 | ||
139 | struct atmel_pmecc_gf_tables { | |
140 | u16 *alpha_to; | |
141 | u16 *index_of; | |
142 | }; | |
143 | ||
144 | struct atmel_pmecc_caps { | |
145 | const int *strengths; | |
146 | int nstrengths; | |
147 | int el_offset; | |
148 | bool correct_erased_chunks; | |
149 | }; | |
150 | ||
151 | struct atmel_pmecc { | |
152 | struct device *dev; | |
153 | const struct atmel_pmecc_caps *caps; | |
154 | ||
155 | struct { | |
156 | void __iomem *base; | |
157 | void __iomem *errloc; | |
158 | } regs; | |
159 | ||
160 | struct mutex lock; | |
161 | }; | |
162 | ||
163 | struct atmel_pmecc_user_conf_cache { | |
164 | u32 cfg; | |
165 | u32 sarea; | |
166 | u32 saddr; | |
167 | u32 eaddr; | |
168 | }; | |
169 | ||
170 | struct atmel_pmecc_user { | |
171 | struct atmel_pmecc_user_conf_cache cache; | |
172 | struct atmel_pmecc *pmecc; | |
173 | const struct atmel_pmecc_gf_tables *gf_tables; | |
174 | int eccbytes; | |
175 | s16 *partial_syn; | |
176 | s16 *si; | |
177 | s16 *lmu; | |
178 | s16 *smu; | |
179 | s32 *mu; | |
180 | s32 *dmu; | |
181 | s32 *delta; | |
182 | u32 isr; | |
183 | }; | |
184 | ||
185 | static DEFINE_MUTEX(pmecc_gf_tables_lock); | |
186 | static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; | |
187 | static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; | |
188 | ||
189 | static inline int deg(unsigned int poly) | |
190 | { | |
191 | /* polynomial degree is the most-significant bit index */ | |
192 | return fls(poly) - 1; | |
193 | } | |
194 | ||
195 | static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, | |
196 | struct atmel_pmecc_gf_tables *gf_tables) | |
197 | { | |
198 | unsigned int i, x = 1; | |
199 | const unsigned int k = BIT(deg(poly)); | |
200 | unsigned int nn = BIT(mm) - 1; | |
201 | ||
202 | /* primitive polynomial must be of degree m */ | |
203 | if (k != (1u << mm)) | |
204 | return -EINVAL; | |
205 | ||
206 | for (i = 0; i < nn; i++) { | |
207 | gf_tables->alpha_to[i] = x; | |
208 | gf_tables->index_of[x] = i; | |
209 | if (i && (x == 1)) | |
210 | /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ | |
211 | return -EINVAL; | |
212 | x <<= 1; | |
213 | if (x & k) | |
214 | x ^= poly; | |
215 | } | |
216 | gf_tables->alpha_to[nn] = 1; | |
217 | gf_tables->index_of[0] = 0; | |
218 | ||
219 | return 0; | |
220 | } | |
221 | ||
222 | static const struct atmel_pmecc_gf_tables * | |
223 | atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) | |
224 | { | |
225 | struct atmel_pmecc_gf_tables *gf_tables; | |
226 | unsigned int poly, degree, table_size; | |
227 | int ret; | |
228 | ||
229 | if (req->ecc.sectorsize == 512) { | |
230 | degree = PMECC_GF_DIMENSION_13; | |
231 | poly = PMECC_GF_13_PRIMITIVE_POLY; | |
232 | table_size = PMECC_LOOKUP_TABLE_SIZE_512; | |
233 | } else { | |
234 | degree = PMECC_GF_DIMENSION_14; | |
235 | poly = PMECC_GF_14_PRIMITIVE_POLY; | |
236 | table_size = PMECC_LOOKUP_TABLE_SIZE_1024; | |
237 | } | |
238 | ||
239 | gf_tables = kzalloc(sizeof(*gf_tables) + | |
240 | (2 * table_size * sizeof(u16)), | |
241 | GFP_KERNEL); | |
242 | if (!gf_tables) | |
243 | return ERR_PTR(-ENOMEM); | |
244 | ||
245 | gf_tables->alpha_to = (void *)(gf_tables + 1); | |
246 | gf_tables->index_of = gf_tables->alpha_to + table_size; | |
247 | ||
248 | ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); | |
249 | if (ret) { | |
250 | kfree(gf_tables); | |
251 | return ERR_PTR(ret); | |
252 | } | |
253 | ||
254 | return gf_tables; | |
255 | } | |
256 | ||
257 | static const struct atmel_pmecc_gf_tables * | |
258 | atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) | |
259 | { | |
260 | const struct atmel_pmecc_gf_tables **gf_tables, *ret; | |
261 | ||
262 | mutex_lock(&pmecc_gf_tables_lock); | |
263 | if (req->ecc.sectorsize == 512) | |
264 | gf_tables = &pmecc_gf_tables_512; | |
265 | else | |
266 | gf_tables = &pmecc_gf_tables_1024; | |
267 | ||
268 | ret = *gf_tables; | |
269 | ||
270 | if (!ret) { | |
271 | ret = atmel_pmecc_create_gf_tables(req); | |
272 | if (!IS_ERR(ret)) | |
273 | *gf_tables = ret; | |
274 | } | |
275 | mutex_unlock(&pmecc_gf_tables_lock); | |
276 | ||
277 | return ret; | |
278 | } | |
279 | ||
280 | static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, | |
281 | struct atmel_pmecc_user_req *req) | |
282 | { | |
283 | int i, max_eccbytes, eccbytes = 0, eccstrength = 0; | |
284 | ||
285 | if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) | |
286 | return -EINVAL; | |
287 | ||
288 | if (req->ecc.ooboffset >= 0 && | |
289 | req->ecc.ooboffset + req->ecc.bytes > req->oobsize) | |
290 | return -EINVAL; | |
291 | ||
292 | if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { | |
293 | if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) | |
294 | return -EINVAL; | |
295 | ||
296 | if (req->pagesize > 512) | |
297 | req->ecc.sectorsize = 1024; | |
298 | else | |
299 | req->ecc.sectorsize = 512; | |
300 | } | |
301 | ||
302 | if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) | |
303 | return -EINVAL; | |
304 | ||
305 | if (req->pagesize % req->ecc.sectorsize) | |
306 | return -EINVAL; | |
307 | ||
308 | req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; | |
309 | ||
310 | max_eccbytes = req->ecc.bytes; | |
311 | ||
312 | for (i = 0; i < pmecc->caps->nstrengths; i++) { | |
313 | int nbytes, strength = pmecc->caps->strengths[i]; | |
314 | ||
315 | if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && | |
316 | strength < req->ecc.strength) | |
317 | continue; | |
318 | ||
319 | nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), | |
320 | 8); | |
321 | nbytes *= req->ecc.nsectors; | |
322 | ||
323 | if (nbytes > max_eccbytes) | |
324 | break; | |
325 | ||
326 | eccstrength = strength; | |
327 | eccbytes = nbytes; | |
328 | ||
329 | if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) | |
330 | break; | |
331 | } | |
332 | ||
333 | if (!eccstrength) | |
334 | return -EINVAL; | |
335 | ||
336 | req->ecc.bytes = eccbytes; | |
337 | req->ecc.strength = eccstrength; | |
338 | ||
339 | if (req->ecc.ooboffset < 0) | |
340 | req->ecc.ooboffset = req->oobsize - eccbytes; | |
341 | ||
342 | return 0; | |
343 | } | |
344 | ||
345 | struct atmel_pmecc_user * | |
346 | atmel_pmecc_create_user(struct atmel_pmecc *pmecc, | |
347 | struct atmel_pmecc_user_req *req) | |
348 | { | |
349 | struct atmel_pmecc_user *user; | |
350 | const struct atmel_pmecc_gf_tables *gf_tables; | |
351 | int strength, size, ret; | |
352 | ||
353 | ret = atmel_pmecc_prepare_user_req(pmecc, req); | |
354 | if (ret) | |
355 | return ERR_PTR(ret); | |
356 | ||
357 | size = sizeof(*user); | |
358 | size = ALIGN(size, sizeof(u16)); | |
359 | /* Reserve space for partial_syn, si and smu */ | |
360 | size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * | |
361 | (2 + req->ecc.strength + 2); | |
362 | /* Reserve space for lmu. */ | |
363 | size += (req->ecc.strength + 1) * sizeof(u16); | |
364 | /* Reserve space for mu, dmu and delta. */ | |
365 | size = ALIGN(size, sizeof(s32)); | |
366 | size += (req->ecc.strength + 1) * sizeof(s32); | |
367 | ||
368 | user = kzalloc(size, GFP_KERNEL); | |
369 | if (!user) | |
370 | return ERR_PTR(-ENOMEM); | |
371 | ||
372 | user->pmecc = pmecc; | |
373 | ||
374 | user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); | |
375 | user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); | |
376 | user->lmu = user->si + ((2 * req->ecc.strength) + 1); | |
377 | user->smu = user->lmu + (req->ecc.strength + 1); | |
378 | user->mu = (s32 *)PTR_ALIGN(user->smu + | |
379 | (((2 * req->ecc.strength) + 1) * | |
380 | (req->ecc.strength + 2)), | |
381 | sizeof(s32)); | |
382 | user->dmu = user->mu + req->ecc.strength + 1; | |
383 | user->delta = user->dmu + req->ecc.strength + 1; | |
384 | ||
385 | gf_tables = atmel_pmecc_get_gf_tables(req); | |
386 | if (IS_ERR(gf_tables)) { | |
387 | kfree(user); | |
388 | return ERR_CAST(gf_tables); | |
389 | } | |
390 | ||
391 | user->gf_tables = gf_tables; | |
392 | ||
393 | user->eccbytes = req->ecc.bytes / req->ecc.nsectors; | |
394 | ||
395 | for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { | |
396 | if (pmecc->caps->strengths[strength] == req->ecc.strength) | |
397 | break; | |
398 | } | |
399 | ||
400 | user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | | |
401 | PMECC_CFG_NSECTORS(req->ecc.nsectors); | |
402 | ||
403 | if (req->ecc.sectorsize == 1024) | |
404 | user->cache.cfg |= PMECC_CFG_SECTOR1024; | |
405 | ||
406 | user->cache.sarea = req->oobsize - 1; | |
407 | user->cache.saddr = req->ecc.ooboffset; | |
408 | user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; | |
409 | ||
410 | return user; | |
411 | } | |
412 | EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); | |
413 | ||
414 | void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user) | |
415 | { | |
416 | kfree(user); | |
417 | } | |
418 | EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user); | |
419 | ||
420 | static int get_strength(struct atmel_pmecc_user *user) | |
421 | { | |
422 | const int *strengths = user->pmecc->caps->strengths; | |
423 | ||
424 | return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; | |
425 | } | |
426 | ||
427 | static int get_sectorsize(struct atmel_pmecc_user *user) | |
428 | { | |
429 | return user->cache.cfg & PMECC_LOOKUP_TABLE_SIZE_1024 ? 1024 : 512; | |
430 | } | |
431 | ||
432 | static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) | |
433 | { | |
434 | int strength = get_strength(user); | |
435 | u32 value; | |
436 | int i; | |
437 | ||
438 | /* Fill odd syndromes */ | |
439 | for (i = 0; i < strength; i++) { | |
440 | value = readl_relaxed(user->pmecc->regs.base + | |
441 | ATMEL_PMECC_REM(sector, i / 2)); | |
442 | if (i & 1) | |
443 | value >>= 16; | |
444 | ||
445 | user->partial_syn[(2 * i) + 1] = value; | |
446 | } | |
447 | } | |
448 | ||
449 | static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) | |
450 | { | |
451 | int degree = get_sectorsize(user) == 512 ? 13 : 14; | |
452 | int cw_len = BIT(degree) - 1; | |
453 | int strength = get_strength(user); | |
454 | s16 *alpha_to = user->gf_tables->alpha_to; | |
455 | s16 *index_of = user->gf_tables->index_of; | |
456 | s16 *partial_syn = user->partial_syn; | |
457 | s16 *si; | |
458 | int i, j; | |
459 | ||
460 | /* | |
461 | * si[] is a table that holds the current syndrome value, | |
462 | * an element of that table belongs to the field | |
463 | */ | |
464 | si = user->si; | |
465 | ||
466 | memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); | |
467 | ||
468 | /* Computation 2t syndromes based on S(x) */ | |
469 | /* Odd syndromes */ | |
470 | for (i = 1; i < 2 * strength; i += 2) { | |
471 | for (j = 0; j < degree; j++) { | |
472 | if (partial_syn[i] & BIT(j)) | |
473 | si[i] = alpha_to[i * j] ^ si[i]; | |
474 | } | |
475 | } | |
476 | /* Even syndrome = (Odd syndrome) ** 2 */ | |
477 | for (i = 2, j = 1; j <= strength; i = ++j << 1) { | |
478 | if (si[j] == 0) { | |
479 | si[i] = 0; | |
480 | } else { | |
481 | s16 tmp; | |
482 | ||
483 | tmp = index_of[si[j]]; | |
484 | tmp = (tmp * 2) % cw_len; | |
485 | si[i] = alpha_to[tmp]; | |
486 | } | |
487 | } | |
488 | } | |
489 | ||
490 | static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) | |
491 | { | |
492 | s16 *lmu = user->lmu; | |
493 | s16 *si = user->si; | |
494 | s32 *mu = user->mu; | |
495 | s32 *dmu = user->dmu; | |
496 | s32 *delta = user->delta; | |
497 | int degree = get_sectorsize(user) == 512 ? 13 : 14; | |
498 | int cw_len = BIT(degree) - 1; | |
499 | int strength = get_strength(user); | |
500 | int num = 2 * strength + 1; | |
501 | s16 *index_of = user->gf_tables->index_of; | |
502 | s16 *alpha_to = user->gf_tables->alpha_to; | |
503 | int i, j, k; | |
504 | u32 dmu_0_count, tmp; | |
505 | s16 *smu = user->smu; | |
506 | ||
507 | /* index of largest delta */ | |
508 | int ro; | |
509 | int largest; | |
510 | int diff; | |
511 | ||
512 | dmu_0_count = 0; | |
513 | ||
514 | /* First Row */ | |
515 | ||
516 | /* Mu */ | |
517 | mu[0] = -1; | |
518 | ||
519 | memset(smu, 0, sizeof(s16) * num); | |
520 | smu[0] = 1; | |
521 | ||
522 | /* discrepancy set to 1 */ | |
523 | dmu[0] = 1; | |
524 | /* polynom order set to 0 */ | |
525 | lmu[0] = 0; | |
526 | delta[0] = (mu[0] * 2 - lmu[0]) >> 1; | |
527 | ||
528 | /* Second Row */ | |
529 | ||
530 | /* Mu */ | |
531 | mu[1] = 0; | |
532 | /* Sigma(x) set to 1 */ | |
533 | memset(&smu[num], 0, sizeof(s16) * num); | |
534 | smu[num] = 1; | |
535 | ||
536 | /* discrepancy set to S1 */ | |
537 | dmu[1] = si[1]; | |
538 | ||
539 | /* polynom order set to 0 */ | |
540 | lmu[1] = 0; | |
541 | ||
542 | delta[1] = (mu[1] * 2 - lmu[1]) >> 1; | |
543 | ||
544 | /* Init the Sigma(x) last row */ | |
545 | memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); | |
546 | ||
547 | for (i = 1; i <= strength; i++) { | |
548 | mu[i + 1] = i << 1; | |
549 | /* Begin Computing Sigma (Mu+1) and L(mu) */ | |
550 | /* check if discrepancy is set to 0 */ | |
551 | if (dmu[i] == 0) { | |
552 | dmu_0_count++; | |
553 | ||
554 | tmp = ((strength - (lmu[i] >> 1) - 1) / 2); | |
555 | if ((strength - (lmu[i] >> 1) - 1) & 0x1) | |
556 | tmp += 2; | |
557 | else | |
558 | tmp += 1; | |
559 | ||
560 | if (dmu_0_count == tmp) { | |
561 | for (j = 0; j <= (lmu[i] >> 1) + 1; j++) | |
562 | smu[(strength + 1) * num + j] = | |
563 | smu[i * num + j]; | |
564 | ||
565 | lmu[strength + 1] = lmu[i]; | |
566 | return; | |
567 | } | |
568 | ||
569 | /* copy polynom */ | |
570 | for (j = 0; j <= lmu[i] >> 1; j++) | |
571 | smu[(i + 1) * num + j] = smu[i * num + j]; | |
572 | ||
573 | /* copy previous polynom order to the next */ | |
574 | lmu[i + 1] = lmu[i]; | |
575 | } else { | |
576 | ro = 0; | |
577 | largest = -1; | |
578 | /* find largest delta with dmu != 0 */ | |
579 | for (j = 0; j < i; j++) { | |
580 | if ((dmu[j]) && (delta[j] > largest)) { | |
581 | largest = delta[j]; | |
582 | ro = j; | |
583 | } | |
584 | } | |
585 | ||
586 | /* compute difference */ | |
587 | diff = (mu[i] - mu[ro]); | |
588 | ||
589 | /* Compute degree of the new smu polynomial */ | |
590 | if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) | |
591 | lmu[i + 1] = lmu[i]; | |
592 | else | |
593 | lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; | |
594 | ||
595 | /* Init smu[i+1] with 0 */ | |
596 | for (k = 0; k < num; k++) | |
597 | smu[(i + 1) * num + k] = 0; | |
598 | ||
599 | /* Compute smu[i+1] */ | |
600 | for (k = 0; k <= lmu[ro] >> 1; k++) { | |
601 | s16 a, b, c; | |
602 | ||
603 | if (!(smu[ro * num + k] && dmu[i])) | |
604 | continue; | |
605 | ||
606 | a = index_of[dmu[i]]; | |
607 | b = index_of[dmu[ro]]; | |
608 | c = index_of[smu[ro * num + k]]; | |
609 | tmp = a + (cw_len - b) + c; | |
610 | a = alpha_to[tmp % cw_len]; | |
611 | smu[(i + 1) * num + (k + diff)] = a; | |
612 | } | |
613 | ||
614 | for (k = 0; k <= lmu[i] >> 1; k++) | |
615 | smu[(i + 1) * num + k] ^= smu[i * num + k]; | |
616 | } | |
617 | ||
618 | /* End Computing Sigma (Mu+1) and L(mu) */ | |
619 | /* In either case compute delta */ | |
620 | delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; | |
621 | ||
622 | /* Do not compute discrepancy for the last iteration */ | |
623 | if (i >= strength) | |
624 | continue; | |
625 | ||
626 | for (k = 0; k <= (lmu[i + 1] >> 1); k++) { | |
627 | tmp = 2 * (i - 1); | |
628 | if (k == 0) { | |
629 | dmu[i + 1] = si[tmp + 3]; | |
630 | } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { | |
631 | s16 a, b, c; | |
632 | ||
633 | a = index_of[smu[(i + 1) * num + k]]; | |
634 | b = si[2 * (i - 1) + 3 - k]; | |
635 | c = index_of[b]; | |
636 | tmp = a + c; | |
637 | tmp %= cw_len; | |
638 | dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; | |
639 | } | |
640 | } | |
641 | } | |
642 | } | |
643 | ||
644 | static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) | |
645 | { | |
646 | int sector_size = get_sectorsize(user); | |
647 | int degree = sector_size == 512 ? 13 : 14; | |
648 | struct atmel_pmecc *pmecc = user->pmecc; | |
649 | int strength = get_strength(user); | |
650 | int ret, roots_nbr, i, err_nbr = 0; | |
651 | int num = (2 * strength) + 1; | |
652 | s16 *smu = user->smu; | |
653 | u32 val; | |
654 | ||
655 | writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); | |
656 | ||
657 | for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { | |
658 | writel_relaxed(smu[(strength + 1) * num + i], | |
659 | pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); | |
660 | err_nbr++; | |
661 | } | |
662 | ||
663 | val = (err_nbr - 1) << 16; | |
664 | if (sector_size == 1024) | |
665 | val |= 1; | |
666 | ||
667 | writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); | |
668 | writel((sector_size * 8) + (degree * strength), | |
669 | pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); | |
670 | ||
671 | ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + | |
672 | ATMEL_PMERRLOC_ELISR, | |
673 | val, val & PMERRLOC_CALC_DONE, 0, | |
674 | PMECC_MAX_TIMEOUT_MS * 1000); | |
675 | if (ret) { | |
676 | dev_err(pmecc->dev, | |
677 | "PMECC: Timeout to calculate error location.\n"); | |
678 | return ret; | |
679 | } | |
680 | ||
681 | roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; | |
682 | /* Number of roots == degree of smu hence <= cap */ | |
683 | if (roots_nbr == user->lmu[strength + 1] >> 1) | |
684 | return err_nbr - 1; | |
685 | ||
686 | /* | |
687 | * Number of roots does not match the degree of smu | |
688 | * unable to correct error. | |
689 | */ | |
690 | return -EBADMSG; | |
691 | } | |
692 | ||
693 | int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, | |
694 | void *data, void *ecc) | |
695 | { | |
696 | struct atmel_pmecc *pmecc = user->pmecc; | |
697 | int sectorsize = get_sectorsize(user); | |
698 | int eccbytes = user->eccbytes; | |
699 | int i, nerrors; | |
700 | ||
701 | if (!(user->isr & BIT(sector))) | |
702 | return 0; | |
703 | ||
704 | atmel_pmecc_gen_syndrome(user, sector); | |
705 | atmel_pmecc_substitute(user); | |
706 | atmel_pmecc_get_sigma(user); | |
707 | ||
708 | nerrors = atmel_pmecc_err_location(user); | |
709 | if (nerrors < 0) | |
710 | return nerrors; | |
711 | ||
712 | for (i = 0; i < nerrors; i++) { | |
713 | const char *area; | |
714 | int byte, bit; | |
715 | u32 errpos; | |
716 | u8 *ptr; | |
717 | ||
718 | errpos = readl_relaxed(pmecc->regs.errloc + | |
719 | ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); | |
720 | errpos--; | |
721 | ||
722 | byte = errpos / 8; | |
723 | bit = errpos % 8; | |
724 | ||
725 | if (byte < sectorsize) { | |
726 | ptr = data + byte; | |
727 | area = "data"; | |
728 | } else if (byte < sectorsize + eccbytes) { | |
729 | ptr = ecc + byte - sectorsize; | |
730 | area = "ECC"; | |
731 | } else { | |
732 | dev_dbg(pmecc->dev, | |
733 | "Invalid errpos value (%d, max is %d)\n", | |
734 | errpos, (sectorsize + eccbytes) * 8); | |
735 | return -EINVAL; | |
736 | } | |
737 | ||
738 | dev_dbg(pmecc->dev, | |
739 | "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", | |
740 | area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); | |
741 | ||
742 | *ptr ^= BIT(bit); | |
743 | } | |
744 | ||
745 | return nerrors; | |
746 | } | |
747 | EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); | |
748 | ||
749 | bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) | |
750 | { | |
751 | return user->pmecc->caps->correct_erased_chunks; | |
752 | } | |
753 | EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); | |
754 | ||
755 | void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, | |
756 | int sector, void *ecc) | |
757 | { | |
758 | struct atmel_pmecc *pmecc = user->pmecc; | |
759 | u8 *ptr = ecc; | |
760 | int i; | |
761 | ||
762 | for (i = 0; i < user->eccbytes; i++) | |
763 | ptr[i] = readb_relaxed(pmecc->regs.base + | |
764 | ATMEL_PMECC_ECC(sector, i)); | |
765 | } | |
766 | EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); | |
767 | ||
768 | int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) | |
769 | { | |
770 | struct atmel_pmecc *pmecc = user->pmecc; | |
771 | u32 cfg; | |
772 | ||
773 | if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { | |
774 | dev_err(pmecc->dev, "Bad ECC operation!"); | |
775 | return -EINVAL; | |
776 | } | |
777 | ||
778 | mutex_lock(&user->pmecc->lock); | |
779 | ||
780 | cfg = user->cache.cfg; | |
781 | if (op == NAND_ECC_WRITE) | |
782 | cfg |= PMECC_CFG_WRITE_OP; | |
783 | else | |
784 | cfg |= PMECC_CFG_AUTO_ENABLE; | |
785 | ||
786 | writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); | |
787 | writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); | |
788 | writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); | |
789 | writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); | |
790 | ||
791 | writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
792 | writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
793 | ||
794 | return 0; | |
795 | } | |
796 | EXPORT_SYMBOL_GPL(atmel_pmecc_enable); | |
797 | ||
798 | void atmel_pmecc_disable(struct atmel_pmecc_user *user) | |
799 | { | |
800 | struct atmel_pmecc *pmecc = user->pmecc; | |
801 | ||
802 | writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
803 | writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
804 | mutex_unlock(&user->pmecc->lock); | |
805 | } | |
806 | EXPORT_SYMBOL_GPL(atmel_pmecc_disable); | |
807 | ||
808 | int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) | |
809 | { | |
810 | struct atmel_pmecc *pmecc = user->pmecc; | |
811 | u32 status; | |
812 | int ret; | |
813 | ||
814 | ret = readl_relaxed_poll_timeout(pmecc->regs.base + | |
815 | ATMEL_PMECC_SR, | |
816 | status, !(status & PMECC_SR_BUSY), 0, | |
817 | PMECC_MAX_TIMEOUT_MS * 1000); | |
818 | if (ret) { | |
819 | dev_err(pmecc->dev, | |
820 | "Timeout while waiting for PMECC ready.\n"); | |
821 | return ret; | |
822 | } | |
823 | ||
824 | user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); | |
825 | ||
826 | return 0; | |
827 | } | |
828 | EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); | |
829 | ||
830 | static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, | |
831 | const struct atmel_pmecc_caps *caps, | |
832 | int pmecc_res_idx, int errloc_res_idx) | |
833 | { | |
834 | struct device *dev = &pdev->dev; | |
835 | struct atmel_pmecc *pmecc; | |
836 | struct resource *res; | |
837 | ||
838 | pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); | |
839 | if (!pmecc) | |
840 | return ERR_PTR(-ENOMEM); | |
841 | ||
842 | pmecc->caps = caps; | |
843 | pmecc->dev = dev; | |
844 | mutex_init(&pmecc->lock); | |
845 | ||
846 | res = platform_get_resource(pdev, IORESOURCE_MEM, pmecc_res_idx); | |
847 | pmecc->regs.base = devm_ioremap_resource(dev, res); | |
848 | if (IS_ERR(pmecc->regs.base)) | |
849 | return ERR_CAST(pmecc->regs.base); | |
850 | ||
851 | res = platform_get_resource(pdev, IORESOURCE_MEM, errloc_res_idx); | |
852 | pmecc->regs.errloc = devm_ioremap_resource(dev, res); | |
853 | if (IS_ERR(pmecc->regs.errloc)) | |
854 | return ERR_CAST(pmecc->regs.errloc); | |
855 | ||
856 | /* Disable all interrupts before registering the PMECC handler. */ | |
857 | writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); | |
858 | ||
859 | /* Reset the ECC engine */ | |
860 | writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
861 | writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); | |
862 | ||
863 | return pmecc; | |
864 | } | |
865 | ||
866 | static void devm_atmel_pmecc_put(struct device *dev, void *res) | |
867 | { | |
868 | struct atmel_pmecc **pmecc = res; | |
869 | ||
870 | put_device((*pmecc)->dev); | |
871 | } | |
872 | ||
873 | static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, | |
874 | struct device_node *np) | |
875 | { | |
876 | struct platform_device *pdev; | |
877 | struct atmel_pmecc *pmecc, **ptr; | |
878 | ||
879 | pdev = of_find_device_by_node(np); | |
880 | if (!pdev || !platform_get_drvdata(pdev)) | |
881 | return ERR_PTR(-EPROBE_DEFER); | |
882 | ||
883 | ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); | |
884 | if (!ptr) | |
885 | return ERR_PTR(-ENOMEM); | |
886 | ||
887 | get_device(&pdev->dev); | |
888 | pmecc = platform_get_drvdata(pdev); | |
889 | ||
890 | *ptr = pmecc; | |
891 | ||
892 | devres_add(userdev, ptr); | |
893 | ||
894 | return pmecc; | |
895 | } | |
896 | ||
897 | static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; | |
898 | ||
899 | static struct atmel_pmecc_caps at91sam9g45_caps = { | |
900 | .strengths = atmel_pmecc_strengths, | |
901 | .nstrengths = 5, | |
902 | .el_offset = 0x8c, | |
903 | }; | |
904 | ||
905 | static struct atmel_pmecc_caps sama5d4_caps = { | |
906 | .strengths = atmel_pmecc_strengths, | |
907 | .nstrengths = 5, | |
908 | .el_offset = 0x8c, | |
909 | .correct_erased_chunks = true, | |
910 | }; | |
911 | ||
912 | static struct atmel_pmecc_caps sama5d2_caps = { | |
913 | .strengths = atmel_pmecc_strengths, | |
914 | .nstrengths = 6, | |
915 | .el_offset = 0xac, | |
916 | .correct_erased_chunks = true, | |
917 | }; | |
918 | ||
919 | static const struct of_device_id atmel_pmecc_legacy_match[] = { | |
920 | { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, | |
921 | { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, | |
922 | { /* sentinel */ } | |
923 | }; | |
924 | ||
925 | struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) | |
926 | { | |
927 | struct atmel_pmecc *pmecc; | |
928 | struct device_node *np; | |
929 | ||
930 | if (!userdev) | |
931 | return ERR_PTR(-EINVAL); | |
932 | ||
933 | if (!userdev->of_node) | |
934 | return NULL; | |
935 | ||
936 | np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); | |
937 | if (np) { | |
938 | pmecc = atmel_pmecc_get_by_node(userdev, np); | |
939 | of_node_put(np); | |
940 | } else { | |
941 | /* | |
942 | * Support old DT bindings: in this case the PMECC iomem | |
943 | * resources are directly defined in the user pdev at position | |
944 | * 1 and 2. Extract all relevant information from there. | |
945 | */ | |
946 | struct platform_device *pdev = to_platform_device(userdev); | |
947 | const struct atmel_pmecc_caps *caps; | |
3aa09076 | 948 | const struct of_device_id *match; |
f88fc122 BB |
949 | |
950 | /* No PMECC engine available. */ | |
951 | if (!of_property_read_bool(userdev->of_node, | |
952 | "atmel,has-pmecc")) | |
953 | return NULL; | |
954 | ||
955 | caps = &at91sam9g45_caps; | |
956 | ||
3aa09076 BB |
957 | /* Find the caps associated to the NAND dev node. */ |
958 | match = of_match_node(atmel_pmecc_legacy_match, | |
959 | userdev->of_node); | |
960 | if (match && match->data) | |
961 | caps = match->data; | |
f88fc122 BB |
962 | |
963 | pmecc = atmel_pmecc_create(pdev, caps, 1, 2); | |
964 | } | |
965 | ||
966 | return pmecc; | |
967 | } | |
968 | EXPORT_SYMBOL(devm_atmel_pmecc_get); | |
969 | ||
970 | static const struct of_device_id atmel_pmecc_match[] = { | |
971 | { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, | |
972 | { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, | |
973 | { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, | |
974 | { /* sentinel */ } | |
975 | }; | |
976 | MODULE_DEVICE_TABLE(of, atmel_pmecc_match); | |
977 | ||
978 | static int atmel_pmecc_probe(struct platform_device *pdev) | |
979 | { | |
980 | struct device *dev = &pdev->dev; | |
981 | const struct atmel_pmecc_caps *caps; | |
982 | struct atmel_pmecc *pmecc; | |
983 | ||
984 | caps = of_device_get_match_data(&pdev->dev); | |
985 | if (!caps) { | |
986 | dev_err(dev, "Invalid caps\n"); | |
987 | return -EINVAL; | |
988 | } | |
989 | ||
990 | pmecc = atmel_pmecc_create(pdev, caps, 0, 1); | |
991 | if (IS_ERR(pmecc)) | |
992 | return PTR_ERR(pmecc); | |
993 | ||
994 | platform_set_drvdata(pdev, pmecc); | |
995 | ||
996 | return 0; | |
997 | } | |
998 | ||
999 | static struct platform_driver atmel_pmecc_driver = { | |
1000 | .driver = { | |
1001 | .name = "atmel-pmecc", | |
1002 | .of_match_table = of_match_ptr(atmel_pmecc_match), | |
1003 | }, | |
1004 | .probe = atmel_pmecc_probe, | |
1005 | }; | |
1006 | module_platform_driver(atmel_pmecc_driver); | |
1007 | ||
1008 | MODULE_LICENSE("GPL"); | |
1009 | MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); | |
1010 | MODULE_DESCRIPTION("PMECC engine driver"); | |
1011 | MODULE_ALIAS("platform:atmel_pmecc"); |