drivers/mtd/nand/mtk_ecc.c

   1 /*
   2  * MTK ECC controller driver.
   3  * Copyright (C) 2016  MediaTek Inc.
   4  * Authors:     Xiaolei Li              <xiaolei.li@mediatek.com>
   5  *              Jorge Ramirez-Ortiz     <jorge.ramirez-ortiz@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  */
  16
  17 #include <linux/platform_device.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/clk.h>
  21 #include <linux/module.h>
  22 #include <linux/iopoll.h>
  23 #include <linux/of.h>
  24 #include <linux/of_platform.h>
  25 #include <linux/mutex.h>
  26
  27 #include "mtk_ecc.h"
  28
  29 #define ECC_IDLE_MASK           BIT(0)
  30 #define ECC_IRQ_EN              BIT(0)
  31 #define ECC_PG_IRQ_SEL          BIT(1)
  32 #define ECC_OP_ENABLE           (1)
  33 #define ECC_OP_DISABLE          (0)
  34
  35 #define ECC_ENCCON              (0x00)
  36 #define ECC_ENCCNFG             (0x04)
  37 #define         ECC_MODE_SHIFT          (5)
  38 #define         ECC_MS_SHIFT            (16)
  39 #define ECC_ENCDIADDR           (0x08)
  40 #define ECC_ENCIDLE             (0x0C)
  41 #define ECC_ENCIRQ_EN           (0x80)
  42 #define ECC_ENCIRQ_STA          (0x84)
  43 #define ECC_DECCON              (0x100)
  44 #define ECC_DECCNFG             (0x104)
  45 #define         DEC_EMPTY_EN            BIT(31)
  46 #define         DEC_CNFG_CORRECT        (0x3 << 12)
  47 #define ECC_DECIDLE             (0x10C)
  48 #define ECC_DECENUM0            (0x114)
  49 #define ECC_DECDONE             (0x124)
  50 #define ECC_DECIRQ_EN           (0x200)
  51 #define ECC_DECIRQ_STA          (0x204)
  52
  53 #define ECC_TIMEOUT             (500000)
  54
  55 #define ECC_IDLE_REG(op)        ((op) == ECC_ENCODE ? ECC_ENCIDLE : ECC_DECIDLE)
  56 #define ECC_CTL_REG(op)         ((op) == ECC_ENCODE ? ECC_ENCCON : ECC_DECCON)
  57 #define ECC_IRQ_REG(op)         ((op) == ECC_ENCODE ? \
  58                                         ECC_ENCIRQ_EN : ECC_DECIRQ_EN)
  59
  60 struct mtk_ecc_caps {
  61         u32 err_mask;
  62         const u8 *ecc_strength;
  63         u8 num_ecc_strength;
  64         u32 encode_parity_reg0;
  65         int pg_irq_sel;
  66 };
  67
  68 struct mtk_ecc {
  69         struct device *dev;
  70         const struct mtk_ecc_caps *caps;
  71         void __iomem *regs;
  72         struct clk *clk;
  73
  74         struct completion done;
  75         struct mutex lock;
  76         u32 sectors;
  77
  78         u8 *eccdata;
  79 };
  80
  81 /* ecc strength that each IP supports */
  82 static const u8 ecc_strength_mt2701[] = {
  83         4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36,
  84         40, 44, 48, 52, 56, 60
  85 };
  86
  87 static const u8 ecc_strength_mt2712[] = {
  88         4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36,
  89         40, 44, 48, 52, 56, 60, 68, 72, 80
  90 };
  91
  92 static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
  93                                      enum mtk_ecc_operation op)
  94 {
  95         struct device *dev = ecc->dev;
  96         u32 val;
  97         int ret;
  98
  99         ret = readl_poll_timeout_atomic(ecc->regs + ECC_IDLE_REG(op), val,
 100                                         val & ECC_IDLE_MASK,
 101                                         10, ECC_TIMEOUT);
 102         if (ret)
 103                 dev_warn(dev, "%s NOT idle\n",
 104                          op == ECC_ENCODE ? "encoder" : "decoder");
 105 }
 106
 107 static irqreturn_t mtk_ecc_irq(int irq, void *id)
 108 {
 109         struct mtk_ecc *ecc = id;
 110         enum mtk_ecc_operation op;
 111         u32 dec, enc;
 112
 113         dec = readw(ecc->regs + ECC_DECIRQ_STA) & ECC_IRQ_EN;
 114         if (dec) {
 115                 op = ECC_DECODE;
 116                 dec = readw(ecc->regs + ECC_DECDONE);
 117                 if (dec & ecc->sectors) {
 118                         ecc->sectors = 0;
 119                         complete(&ecc->done);
 120                 } else {
 121                         return IRQ_HANDLED;
 122                 }
 123         } else {
 124                 enc = readl(ecc->regs + ECC_ENCIRQ_STA) & ECC_IRQ_EN;
 125                 if (enc) {
 126                         op = ECC_ENCODE;
 127                         complete(&ecc->done);
 128                 } else {
 129                         return IRQ_NONE;
 130                 }
 131         }
 132
 133         writel(0, ecc->regs + ECC_IRQ_REG(op));
 134
 135         return IRQ_HANDLED;
 136 }
 137
 138 static int mtk_ecc_config(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 139 {
 140         u32 ecc_bit, dec_sz, enc_sz;
 141         u32 reg, i;
 142
 143         for (i = 0; i < ecc->caps->num_ecc_strength; i++) {
 144                 if (ecc->caps->ecc_strength[i] == config->strength)
 145                         break;
 146         }
 147
 148         if (i == ecc->caps->num_ecc_strength) {
 149                 dev_err(ecc->dev, "invalid ecc strength %d\n",
 150                         config->strength);
 151                 return -EINVAL;
 152         }
 153
 154         ecc_bit = i;
 155
 156         if (config->op == ECC_ENCODE) {
 157                 /* configure ECC encoder (in bits) */
 158                 enc_sz = config->len << 3;
 159
 160                 reg = ecc_bit | (config->mode << ECC_MODE_SHIFT);
 161                 reg |= (enc_sz << ECC_MS_SHIFT);
 162                 writel(reg, ecc->regs + ECC_ENCCNFG);
 163
 164                 if (config->mode != ECC_NFI_MODE)
 165                         writel(lower_32_bits(config->addr),
 166                                ecc->regs + ECC_ENCDIADDR);
 167
 168         } else {
 169                 /* configure ECC decoder (in bits) */
 170                 dec_sz = (config->len << 3) +
 171                                         config->strength * ECC_PARITY_BITS;
 172
 173                 reg = ecc_bit | (config->mode << ECC_MODE_SHIFT);
 174                 reg |= (dec_sz << ECC_MS_SHIFT) | DEC_CNFG_CORRECT;
 175                 reg |= DEC_EMPTY_EN;
 176                 writel(reg, ecc->regs + ECC_DECCNFG);
 177
 178                 if (config->sectors)
 179                         ecc->sectors = 1 << (config->sectors - 1);
 180         }
 181
 182         return 0;
 183 }
 184
 185 void mtk_ecc_get_stats(struct mtk_ecc *ecc, struct mtk_ecc_stats *stats,
 186                        int sectors)
 187 {
 188         u32 offset, i, err;
 189         u32 bitflips = 0;
 190
 191         stats->corrected = 0;
 192         stats->failed = 0;
 193
 194         for (i = 0; i < sectors; i++) {
 195                 offset = (i >> 2) << 2;
 196                 err = readl(ecc->regs + ECC_DECENUM0 + offset);
 197                 err = err >> ((i % 4) * 8);
 198                 err &= ecc->caps->err_mask;
 199                 if (err == ecc->caps->err_mask) {
 200                         /* uncorrectable errors */
 201                         stats->failed++;
 202                         continue;
 203                 }
 204
 205                 stats->corrected += err;
 206                 bitflips = max_t(u32, bitflips, err);
 207         }
 208
 209         stats->bitflips = bitflips;
 210 }
 211 EXPORT_SYMBOL(mtk_ecc_get_stats);
 212
 213 void mtk_ecc_release(struct mtk_ecc *ecc)
 214 {
 215         clk_disable_unprepare(ecc->clk);
 216         put_device(ecc->dev);
 217 }
 218 EXPORT_SYMBOL(mtk_ecc_release);
 219
 220 static void mtk_ecc_hw_init(struct mtk_ecc *ecc)
 221 {
 222         mtk_ecc_wait_idle(ecc, ECC_ENCODE);
 223         writew(ECC_OP_DISABLE, ecc->regs + ECC_ENCCON);
 224
 225         mtk_ecc_wait_idle(ecc, ECC_DECODE);
 226         writel(ECC_OP_DISABLE, ecc->regs + ECC_DECCON);
 227 }
 228
 229 static struct mtk_ecc *mtk_ecc_get(struct device_node *np)
 230 {
 231         struct platform_device *pdev;
 232         struct mtk_ecc *ecc;
 233
 234         pdev = of_find_device_by_node(np);
 235         if (!pdev || !platform_get_drvdata(pdev))
 236                 return ERR_PTR(-EPROBE_DEFER);
 237
 238         get_device(&pdev->dev);
 239         ecc = platform_get_drvdata(pdev);
 240         clk_prepare_enable(ecc->clk);
 241         mtk_ecc_hw_init(ecc);
 242
 243         return ecc;
 244 }
 245
 246 struct mtk_ecc *of_mtk_ecc_get(struct device_node *of_node)
 247 {
 248         struct mtk_ecc *ecc = NULL;
 249         struct device_node *np;
 250
 251         np = of_parse_phandle(of_node, "ecc-engine", 0);
 252         if (np) {
 253                 ecc = mtk_ecc_get(np);
 254                 of_node_put(np);
 255         }
 256
 257         return ecc;
 258 }
 259 EXPORT_SYMBOL(of_mtk_ecc_get);
 260
 261 int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 262 {
 263         enum mtk_ecc_operation op = config->op;
 264         u16 reg_val;
 265         int ret;
 266
 267         ret = mutex_lock_interruptible(&ecc->lock);
 268         if (ret) {
 269                 dev_err(ecc->dev, "interrupted when attempting to lock\n");
 270                 return ret;
 271         }
 272
 273         mtk_ecc_wait_idle(ecc, op);
 274
 275         ret = mtk_ecc_config(ecc, config);
 276         if (ret) {
 277                 mutex_unlock(&ecc->lock);
 278                 return ret;
 279         }
 280
 281         if (config->mode != ECC_NFI_MODE || op != ECC_ENCODE) {
 282                 init_completion(&ecc->done);
 283                 reg_val = ECC_IRQ_EN;
 284                 /*
 285                  * For ECC_NFI_MODE, if ecc->caps->pg_irq_sel is 1, then it
 286                  * means this chip can only generate one ecc irq during page
 287                  * read / write. If is 0, generate one ecc irq each ecc step.
 288                  */
 289                 if (ecc->caps->pg_irq_sel && config->mode == ECC_NFI_MODE)
 290                         reg_val |= ECC_PG_IRQ_SEL;
 291                 writew(reg_val, ecc->regs + ECC_IRQ_REG(op));
 292         }
 293
 294         writew(ECC_OP_ENABLE, ecc->regs + ECC_CTL_REG(op));
 295
 296         return 0;
 297 }
 298 EXPORT_SYMBOL(mtk_ecc_enable);
 299
 300 void mtk_ecc_disable(struct mtk_ecc *ecc)
 301 {
 302         enum mtk_ecc_operation op = ECC_ENCODE;
 303
 304         /* find out the running operation */
 305         if (readw(ecc->regs + ECC_CTL_REG(op)) != ECC_OP_ENABLE)
 306                 op = ECC_DECODE;
 307
 308         /* disable it */
 309         mtk_ecc_wait_idle(ecc, op);
 310         writew(0, ecc->regs + ECC_IRQ_REG(op));
 311         writew(ECC_OP_DISABLE, ecc->regs + ECC_CTL_REG(op));
 312
 313         mutex_unlock(&ecc->lock);
 314 }
 315 EXPORT_SYMBOL(mtk_ecc_disable);
 316
 317 int mtk_ecc_wait_done(struct mtk_ecc *ecc, enum mtk_ecc_operation op)
 318 {
 319         int ret;
 320
 321         ret = wait_for_completion_timeout(&ecc->done, msecs_to_jiffies(500));
 322         if (!ret) {
 323                 dev_err(ecc->dev, "%s timeout - interrupt did not arrive)\n",
 324                         (op == ECC_ENCODE) ? "encoder" : "decoder");
 325                 return -ETIMEDOUT;
 326         }
 327
 328         return 0;
 329 }
 330 EXPORT_SYMBOL(mtk_ecc_wait_done);
 331
 332 int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
 333                    u8 *data, u32 bytes)
 334 {
 335         dma_addr_t addr;
 336         u32 len;
 337         int ret;
 338
 339         addr = dma_map_single(ecc->dev, data, bytes, DMA_TO_DEVICE);
 340         ret = dma_mapping_error(ecc->dev, addr);
 341         if (ret) {
 342                 dev_err(ecc->dev, "dma mapping error\n");
 343                 return -EINVAL;
 344         }
 345
 346         config->op = ECC_ENCODE;
 347         config->addr = addr;
 348         ret = mtk_ecc_enable(ecc, config);
 349         if (ret) {
 350                 dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE);
 351                 return ret;
 352         }
 353
 354         ret = mtk_ecc_wait_done(ecc, ECC_ENCODE);
 355         if (ret)
 356                 goto timeout;
 357
 358         mtk_ecc_wait_idle(ecc, ECC_ENCODE);
 359
 360         /* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */
 361         len = (config->strength * ECC_PARITY_BITS + 7) >> 3;
 362
 363         /* write the parity bytes generated by the ECC back to temp buffer */
 364         __ioread32_copy(ecc->eccdata,
 365                         ecc->regs + ecc->caps->encode_parity_reg0,
 366                         round_up(len, 4));
 367
 368         /* copy into possibly unaligned OOB region with actual length */
 369         memcpy(data + bytes, ecc->eccdata, len);
 370 timeout:
 371
 372         dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE);
 373         mtk_ecc_disable(ecc);
 374
 375         return ret;
 376 }
 377 EXPORT_SYMBOL(mtk_ecc_encode);
 378
 379 void mtk_ecc_adjust_strength(struct mtk_ecc *ecc, u32 *p)
 380 {
 381         const u8 *ecc_strength = ecc->caps->ecc_strength;
 382         int i;
 383
 384         for (i = 0; i < ecc->caps->num_ecc_strength; i++) {
 385                 if (*p <= ecc_strength[i]) {
 386                         if (!i)
 387                                 *p = ecc_strength[i];
 388                         else if (*p != ecc_strength[i])
 389                                 *p = ecc_strength[i - 1];
 390                         return;
 391                 }
 392         }
 393
 394         *p = ecc_strength[ecc->caps->num_ecc_strength - 1];
 395 }
 396 EXPORT_SYMBOL(mtk_ecc_adjust_strength);
 397
 398 static const struct mtk_ecc_caps mtk_ecc_caps_mt2701 = {
 399         .err_mask = 0x3f,
 400         .ecc_strength = ecc_strength_mt2701,
 401         .num_ecc_strength = 20,
 402         .encode_parity_reg0 = 0x10,
 403         .pg_irq_sel = 0,
 404 };
 405
 406 static const struct mtk_ecc_caps mtk_ecc_caps_mt2712 = {
 407         .err_mask = 0x7f,
 408         .ecc_strength = ecc_strength_mt2712,
 409         .num_ecc_strength = 23,
 410         .encode_parity_reg0 = 0x300,
 411         .pg_irq_sel = 1,
 412 };
 413
 414 static const struct of_device_id mtk_ecc_dt_match[] = {
 415         {
 416                 .compatible = "mediatek,mt2701-ecc",
 417                 .data = &mtk_ecc_caps_mt2701,
 418         }, {
 419                 .compatible = "mediatek,mt2712-ecc",
 420                 .data = &mtk_ecc_caps_mt2712,
 421         },
 422         {},
 423 };
 424
 425 static int mtk_ecc_probe(struct platform_device *pdev)
 426 {
 427         struct device *dev = &pdev->dev;
 428         struct mtk_ecc *ecc;
 429         struct resource *res;
 430         const struct of_device_id *of_ecc_id = NULL;
 431         u32 max_eccdata_size;
 432         int irq, ret;
 433
 434         ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
 435         if (!ecc)
 436                 return -ENOMEM;
 437
 438         of_ecc_id = of_match_device(mtk_ecc_dt_match, &pdev->dev);
 439         if (!of_ecc_id)
 440                 return -ENODEV;
 441
 442         ecc->caps = of_ecc_id->data;
 443
 444         max_eccdata_size = ecc->caps->num_ecc_strength - 1;
 445         max_eccdata_size = ecc->caps->ecc_strength[max_eccdata_size];
 446         max_eccdata_size = (max_eccdata_size * ECC_PARITY_BITS + 7) >> 3;
 447         max_eccdata_size = round_up(max_eccdata_size, 4);
 448         ecc->eccdata = devm_kzalloc(dev, max_eccdata_size, GFP_KERNEL);
 449         if (!ecc->eccdata)
 450                 return -ENOMEM;
 451
 452         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 453         ecc->regs = devm_ioremap_resource(dev, res);
 454         if (IS_ERR(ecc->regs)) {
 455                 dev_err(dev, "failed to map regs: %ld\n", PTR_ERR(ecc->regs));
 456                 return PTR_ERR(ecc->regs);
 457         }
 458
 459         ecc->clk = devm_clk_get(dev, NULL);
 460         if (IS_ERR(ecc->clk)) {
 461                 dev_err(dev, "failed to get clock: %ld\n", PTR_ERR(ecc->clk));
 462                 return PTR_ERR(ecc->clk);
 463         }
 464
 465         irq = platform_get_irq(pdev, 0);
 466         if (irq < 0) {
 467                 dev_err(dev, "failed to get irq: %d\n", irq);
 468                 return irq;
 469         }
 470
 471         ret = dma_set_mask(dev, DMA_BIT_MASK(32));
 472         if (ret) {
 473                 dev_err(dev, "failed to set DMA mask\n");
 474                 return ret;
 475         }
 476
 477         ret = devm_request_irq(dev, irq, mtk_ecc_irq, 0x0, "mtk-ecc", ecc);
 478         if (ret) {
 479                 dev_err(dev, "failed to request irq\n");
 480                 return -EINVAL;
 481         }
 482
 483         ecc->dev = dev;
 484         mutex_init(&ecc->lock);
 485         platform_set_drvdata(pdev, ecc);
 486         dev_info(dev, "probed\n");
 487
 488         return 0;
 489 }
 490
 491 #ifdef CONFIG_PM_SLEEP
 492 static int mtk_ecc_suspend(struct device *dev)
 493 {
 494         struct mtk_ecc *ecc = dev_get_drvdata(dev);
 495
 496         clk_disable_unprepare(ecc->clk);
 497
 498         return 0;
 499 }
 500
 501 static int mtk_ecc_resume(struct device *dev)
 502 {
 503         struct mtk_ecc *ecc = dev_get_drvdata(dev);
 504         int ret;
 505
 506         ret = clk_prepare_enable(ecc->clk);
 507         if (ret) {
 508                 dev_err(dev, "failed to enable clk\n");
 509                 return ret;
 510         }
 511
 512         return 0;
 513 }
 514
 515 static SIMPLE_DEV_PM_OPS(mtk_ecc_pm_ops, mtk_ecc_suspend, mtk_ecc_resume);
 516 #endif
 517
 518 MODULE_DEVICE_TABLE(of, mtk_ecc_dt_match);
 519
 520 static struct platform_driver mtk_ecc_driver = {
 521         .probe  = mtk_ecc_probe,
 522         .driver = {
 523                 .name  = "mtk-ecc",
 524                 .of_match_table = of_match_ptr(mtk_ecc_dt_match),
 525 #ifdef CONFIG_PM_SLEEP
 526                 .pm = &mtk_ecc_pm_ops,
 527 #endif
 528         },
 529 };
 530
 531 module_platform_driver(mtk_ecc_driver);
 532
 533 MODULE_AUTHOR("Xiaolei Li <xiaolei.li@mediatek.com>");
 534 MODULE_DESCRIPTION("MTK Nand ECC Driver");
 535 MODULE_LICENSE("GPL");