drivers/pci/dwc/pci-imx6.c

   1 /*
   2  * PCIe host controller driver for Freescale i.MX6 SoCs
   3  *
   4  * Copyright (C) 2013 Kosagi
   5  *              http://www.kosagi.com
   6  *
   7  * Author: Sean Cross <xobs@kosagi.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/delay.h>
  16 #include <linux/gpio.h>
  17 #include <linux/kernel.h>
  18 #include <linux/mfd/syscon.h>
  19 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
  20 #include <linux/mfd/syscon/imx7-iomuxc-gpr.h>
  21 #include <linux/module.h>
  22 #include <linux/of_gpio.h>
  23 #include <linux/of_device.h>
  24 #include <linux/pci.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/regmap.h>
  27 #include <linux/resource.h>
  28 #include <linux/signal.h>
  29 #include <linux/types.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/reset.h>
  32
  33 #include "pcie-designware.h"
  34
  35 #define to_imx6_pcie(x) dev_get_drvdata((x)->dev)
  36
  37 enum imx6_pcie_variants {
  38         IMX6Q,
  39         IMX6SX,
  40         IMX6QP,
  41         IMX7D,
  42 };
  43
  44 struct imx6_pcie {
  45         struct dw_pcie          *pci;
  46         int                     reset_gpio;
  47         bool                    gpio_active_high;
  48         struct clk              *pcie_bus;
  49         struct clk              *pcie_phy;
  50         struct clk              *pcie_inbound_axi;
  51         struct clk              *pcie;
  52         struct regmap           *iomuxc_gpr;
  53         struct reset_control    *pciephy_reset;
  54         struct reset_control    *apps_reset;
  55         enum imx6_pcie_variants variant;
  56         u32                     tx_deemph_gen1;
  57         u32                     tx_deemph_gen2_3p5db;
  58         u32                     tx_deemph_gen2_6db;
  59         u32                     tx_swing_full;
  60         u32                     tx_swing_low;
  61         int                     link_gen;
  62 };
  63
  64 /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */
  65 #define PHY_PLL_LOCK_WAIT_MAX_RETRIES   2000
  66 #define PHY_PLL_LOCK_WAIT_USLEEP_MIN    50
  67 #define PHY_PLL_LOCK_WAIT_USLEEP_MAX    200
  68
  69 /* PCIe Root Complex registers (memory-mapped) */
  70 #define PCIE_RC_LCR                             0x7c
  71 #define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1        0x1
  72 #define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2        0x2
  73 #define PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK        0xf
  74
  75 #define PCIE_RC_LCSR                            0x80
  76
  77 /* PCIe Port Logic registers (memory-mapped) */
  78 #define PL_OFFSET 0x700
  79 #define PCIE_PL_PFLR (PL_OFFSET + 0x08)
  80 #define PCIE_PL_PFLR_LINK_STATE_MASK            (0x3f << 16)
  81 #define PCIE_PL_PFLR_FORCE_LINK                 (1 << 15)
  82 #define PCIE_PHY_DEBUG_R0 (PL_OFFSET + 0x28)
  83 #define PCIE_PHY_DEBUG_R1 (PL_OFFSET + 0x2c)
  84 #define PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING (1 << 29)
  85 #define PCIE_PHY_DEBUG_R1_XMLH_LINK_UP          (1 << 4)
  86
  87 #define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
  88 #define PCIE_PHY_CTRL_DATA_LOC 0
  89 #define PCIE_PHY_CTRL_CAP_ADR_LOC 16
  90 #define PCIE_PHY_CTRL_CAP_DAT_LOC 17
  91 #define PCIE_PHY_CTRL_WR_LOC 18
  92 #define PCIE_PHY_CTRL_RD_LOC 19
  93
  94 #define PCIE_PHY_STAT (PL_OFFSET + 0x110)
  95 #define PCIE_PHY_STAT_ACK_LOC 16
  96
  97 #define PCIE_LINK_WIDTH_SPEED_CONTROL   0x80C
  98 #define PORT_LOGIC_SPEED_CHANGE         (0x1 << 17)
  99
 100 /* PHY registers (not memory-mapped) */
 101 #define PCIE_PHY_RX_ASIC_OUT 0x100D
 102 #define PCIE_PHY_RX_ASIC_OUT_VALID      (1 << 0)
 103
 104 #define PHY_RX_OVRD_IN_LO 0x1005
 105 #define PHY_RX_OVRD_IN_LO_RX_DATA_EN (1 << 5)
 106 #define PHY_RX_OVRD_IN_LO_RX_PLL_EN (1 << 3)
 107
 108 static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, int exp_val)
 109 {
 110         struct dw_pcie *pci = imx6_pcie->pci;
 111         u32 val;
 112         u32 max_iterations = 10;
 113         u32 wait_counter = 0;
 114
 115         do {
 116                 val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
 117                 val = (val >> PCIE_PHY_STAT_ACK_LOC) & 0x1;
 118                 wait_counter++;
 119
 120                 if (val == exp_val)
 121                         return 0;
 122
 123                 udelay(1);
 124         } while (wait_counter < max_iterations);
 125
 126         return -ETIMEDOUT;
 127 }
 128
 129 static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr)
 130 {
 131         struct dw_pcie *pci = imx6_pcie->pci;
 132         u32 val;
 133         int ret;
 134
 135         val = addr << PCIE_PHY_CTRL_DATA_LOC;
 136         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 137
 138         val |= (0x1 << PCIE_PHY_CTRL_CAP_ADR_LOC);
 139         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 140
 141         ret = pcie_phy_poll_ack(imx6_pcie, 1);
 142         if (ret)
 143                 return ret;
 144
 145         val = addr << PCIE_PHY_CTRL_DATA_LOC;
 146         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 147
 148         return pcie_phy_poll_ack(imx6_pcie, 0);
 149 }
 150
 151 /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
 152 static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, int *data)
 153 {
 154         struct dw_pcie *pci = imx6_pcie->pci;
 155         u32 val, phy_ctl;
 156         int ret;
 157
 158         ret = pcie_phy_wait_ack(imx6_pcie, addr);
 159         if (ret)
 160                 return ret;
 161
 162         /* assert Read signal */
 163         phy_ctl = 0x1 << PCIE_PHY_CTRL_RD_LOC;
 164         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl);
 165
 166         ret = pcie_phy_poll_ack(imx6_pcie, 1);
 167         if (ret)
 168                 return ret;
 169
 170         val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
 171         *data = val & 0xffff;
 172
 173         /* deassert Read signal */
 174         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00);
 175
 176         return pcie_phy_poll_ack(imx6_pcie, 0);
 177 }
 178
 179 static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, int data)
 180 {
 181         struct dw_pcie *pci = imx6_pcie->pci;
 182         u32 var;
 183         int ret;
 184
 185         /* write addr */
 186         /* cap addr */
 187         ret = pcie_phy_wait_ack(imx6_pcie, addr);
 188         if (ret)
 189                 return ret;
 190
 191         var = data << PCIE_PHY_CTRL_DATA_LOC;
 192         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 193
 194         /* capture data */
 195         var |= (0x1 << PCIE_PHY_CTRL_CAP_DAT_LOC);
 196         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 197
 198         ret = pcie_phy_poll_ack(imx6_pcie, 1);
 199         if (ret)
 200                 return ret;
 201
 202         /* deassert cap data */
 203         var = data << PCIE_PHY_CTRL_DATA_LOC;
 204         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 205
 206         /* wait for ack de-assertion */
 207         ret = pcie_phy_poll_ack(imx6_pcie, 0);
 208         if (ret)
 209                 return ret;
 210
 211         /* assert wr signal */
 212         var = 0x1 << PCIE_PHY_CTRL_WR_LOC;
 213         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 214
 215         /* wait for ack */
 216         ret = pcie_phy_poll_ack(imx6_pcie, 1);
 217         if (ret)
 218                 return ret;
 219
 220         /* deassert wr signal */
 221         var = data << PCIE_PHY_CTRL_DATA_LOC;
 222         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 223
 224         /* wait for ack de-assertion */
 225         ret = pcie_phy_poll_ack(imx6_pcie, 0);
 226         if (ret)
 227                 return ret;
 228
 229         dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0);
 230
 231         return 0;
 232 }
 233
 234 static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie)
 235 {
 236         u32 tmp;
 237
 238         pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
 239         tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
 240                 PHY_RX_OVRD_IN_LO_RX_PLL_EN);
 241         pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
 242
 243         usleep_range(2000, 3000);
 244
 245         pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
 246         tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
 247                   PHY_RX_OVRD_IN_LO_RX_PLL_EN);
 248         pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
 249 }
 250
 251 /*  Added for PCI abort handling */
 252 static int imx6q_pcie_abort_handler(unsigned long addr,
 253                 unsigned int fsr, struct pt_regs *regs)
 254 {
 255         unsigned long pc = instruction_pointer(regs);
 256         unsigned long instr = *(unsigned long *)pc;
 257         int reg = (instr >> 12) & 15;
 258
 259         /*
 260          * If the instruction being executed was a read,
 261          * make it look like it read all-ones.
 262          */
 263         if ((instr & 0x0c100000) == 0x04100000) {
 264                 unsigned long val;
 265
 266                 if (instr & 0x00400000)
 267                         val = 255;
 268                 else
 269                         val = -1;
 270
 271                 regs->uregs[reg] = val;
 272                 regs->ARM_pc += 4;
 273                 return 0;
 274         }
 275
 276         if ((instr & 0x0e100090) == 0x00100090) {
 277                 regs->uregs[reg] = -1;
 278                 regs->ARM_pc += 4;
 279                 return 0;
 280         }
 281
 282         return 1;
 283 }
 284
 285 static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie)
 286 {
 287         switch (imx6_pcie->variant) {
 288         case IMX7D:
 289                 reset_control_assert(imx6_pcie->pciephy_reset);
 290                 reset_control_assert(imx6_pcie->apps_reset);
 291                 break;
 292         case IMX6SX:
 293                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 294                                    IMX6SX_GPR12_PCIE_TEST_POWERDOWN,
 295                                    IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
 296                 /* Force PCIe PHY reset */
 297                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
 298                                    IMX6SX_GPR5_PCIE_BTNRST_RESET,
 299                                    IMX6SX_GPR5_PCIE_BTNRST_RESET);
 300                 break;
 301         case IMX6QP:
 302                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 303                                    IMX6Q_GPR1_PCIE_SW_RST,
 304                                    IMX6Q_GPR1_PCIE_SW_RST);
 305                 break;
 306         case IMX6Q:
 307                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 308                                    IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);
 309                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 310                                    IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);
 311                 break;
 312         }
 313 }
 314
 315 static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie)
 316 {
 317         struct dw_pcie *pci = imx6_pcie->pci;
 318         struct device *dev = pci->dev;
 319         int ret = 0;
 320
 321         switch (imx6_pcie->variant) {
 322         case IMX6SX:
 323                 ret = clk_prepare_enable(imx6_pcie->pcie_inbound_axi);
 324                 if (ret) {
 325                         dev_err(dev, "unable to enable pcie_axi clock\n");
 326                         break;
 327                 }
 328
 329                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 330                                    IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0);
 331                 break;
 332         case IMX6QP:            /* FALLTHROUGH */
 333         case IMX6Q:
 334                 /* power up core phy and enable ref clock */
 335                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 336                                    IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18);
 337                 /*
 338                  * the async reset input need ref clock to sync internally,
 339                  * when the ref clock comes after reset, internal synced
 340                  * reset time is too short, cannot meet the requirement.
 341                  * add one ~10us delay here.
 342                  */
 343                 udelay(10);
 344                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 345                                    IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16);
 346                 break;
 347         case IMX7D:
 348                 break;
 349         }
 350
 351         return ret;
 352 }
 353
 354 static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie)
 355 {
 356         u32 val;
 357         unsigned int retries;
 358         struct device *dev = imx6_pcie->pci->dev;
 359
 360         for (retries = 0; retries < PHY_PLL_LOCK_WAIT_MAX_RETRIES; retries++) {
 361                 regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR22, &val);
 362
 363                 if (val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED)
 364                         return;
 365
 366                 usleep_range(PHY_PLL_LOCK_WAIT_USLEEP_MIN,
 367                              PHY_PLL_LOCK_WAIT_USLEEP_MAX);
 368         }
 369
 370         dev_err(dev, "PCIe PLL lock timeout\n");
 371 }
 372
 373 static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie)
 374 {
 375         struct dw_pcie *pci = imx6_pcie->pci;
 376         struct device *dev = pci->dev;
 377         int ret;
 378
 379         ret = clk_prepare_enable(imx6_pcie->pcie_phy);
 380         if (ret) {
 381                 dev_err(dev, "unable to enable pcie_phy clock\n");
 382                 return;
 383         }
 384
 385         ret = clk_prepare_enable(imx6_pcie->pcie_bus);
 386         if (ret) {
 387                 dev_err(dev, "unable to enable pcie_bus clock\n");
 388                 goto err_pcie_bus;
 389         }
 390
 391         ret = clk_prepare_enable(imx6_pcie->pcie);
 392         if (ret) {
 393                 dev_err(dev, "unable to enable pcie clock\n");
 394                 goto err_pcie;
 395         }
 396
 397         ret = imx6_pcie_enable_ref_clk(imx6_pcie);
 398         if (ret) {
 399                 dev_err(dev, "unable to enable pcie ref clock\n");
 400                 goto err_ref_clk;
 401         }
 402
 403         /* allow the clocks to stabilize */
 404         usleep_range(200, 500);
 405
 406         /* Some boards don't have PCIe reset GPIO. */
 407         if (gpio_is_valid(imx6_pcie->reset_gpio)) {
 408                 gpio_set_value_cansleep(imx6_pcie->reset_gpio,
 409                                         imx6_pcie->gpio_active_high);
 410                 msleep(100);
 411                 gpio_set_value_cansleep(imx6_pcie->reset_gpio,
 412                                         !imx6_pcie->gpio_active_high);
 413         }
 414
 415         switch (imx6_pcie->variant) {
 416         case IMX7D:
 417                 reset_control_deassert(imx6_pcie->pciephy_reset);
 418                 imx7d_pcie_wait_for_phy_pll_lock(imx6_pcie);
 419                 break;
 420         case IMX6SX:
 421                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
 422                                    IMX6SX_GPR5_PCIE_BTNRST_RESET, 0);
 423                 break;
 424         case IMX6QP:
 425                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
 426                                    IMX6Q_GPR1_PCIE_SW_RST, 0);
 427
 428                 usleep_range(200, 500);
 429                 break;
 430         case IMX6Q:             /* Nothing to do */
 431                 break;
 432         }
 433
 434         return;
 435
 436 err_ref_clk:
 437         clk_disable_unprepare(imx6_pcie->pcie);
 438 err_pcie:
 439         clk_disable_unprepare(imx6_pcie->pcie_bus);
 440 err_pcie_bus:
 441         clk_disable_unprepare(imx6_pcie->pcie_phy);
 442 }
 443
 444 static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie)
 445 {
 446         switch (imx6_pcie->variant) {
 447         case IMX7D:
 448                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 449                                    IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0);
 450                 break;
 451         case IMX6SX:
 452                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 453                                    IMX6SX_GPR12_PCIE_RX_EQ_MASK,
 454                                    IMX6SX_GPR12_PCIE_RX_EQ_2);
 455                 /* FALLTHROUGH */
 456         default:
 457                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 458                                    IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
 459
 460                 /* configure constant input signal to the pcie ctrl and phy */
 461                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 462                                    IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
 463
 464                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
 465                                    IMX6Q_GPR8_TX_DEEMPH_GEN1,
 466                                    imx6_pcie->tx_deemph_gen1 << 0);
 467                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
 468                                    IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,
 469                                    imx6_pcie->tx_deemph_gen2_3p5db << 6);
 470                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
 471                                    IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,
 472                                    imx6_pcie->tx_deemph_gen2_6db << 12);
 473                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
 474                                    IMX6Q_GPR8_TX_SWING_FULL,
 475                                    imx6_pcie->tx_swing_full << 18);
 476                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
 477                                    IMX6Q_GPR8_TX_SWING_LOW,
 478                                    imx6_pcie->tx_swing_low << 25);
 479                 break;
 480         }
 481
 482         regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 483                         IMX6Q_GPR12_DEVICE_TYPE, PCI_EXP_TYPE_ROOT_PORT << 12);
 484 }
 485
 486 static int imx6_pcie_wait_for_link(struct imx6_pcie *imx6_pcie)
 487 {
 488         struct dw_pcie *pci = imx6_pcie->pci;
 489         struct device *dev = pci->dev;
 490
 491         /* check if the link is up or not */
 492         if (!dw_pcie_wait_for_link(pci))
 493                 return 0;
 494
 495         dev_dbg(dev, "DEBUG_R0: 0x%08x, DEBUG_R1: 0x%08x\n",
 496                 dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R0),
 497                 dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R1));
 498         return -ETIMEDOUT;
 499 }
 500
 501 static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie)
 502 {
 503         struct dw_pcie *pci = imx6_pcie->pci;
 504         struct device *dev = pci->dev;
 505         u32 tmp;
 506         unsigned int retries;
 507
 508         for (retries = 0; retries < 200; retries++) {
 509                 tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 510                 /* Test if the speed change finished. */
 511                 if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
 512                         return 0;
 513                 usleep_range(100, 1000);
 514         }
 515
 516         dev_err(dev, "Speed change timeout\n");
 517         return -EINVAL;
 518 }
 519
 520 static irqreturn_t imx6_pcie_msi_handler(int irq, void *arg)
 521 {
 522         struct imx6_pcie *imx6_pcie = arg;
 523         struct dw_pcie *pci = imx6_pcie->pci;
 524         struct pcie_port *pp = &pci->pp;
 525
 526         return dw_handle_msi_irq(pp);
 527 }
 528
 529 static int imx6_pcie_establish_link(struct imx6_pcie *imx6_pcie)
 530 {
 531         struct dw_pcie *pci = imx6_pcie->pci;
 532         struct device *dev = pci->dev;
 533         u32 tmp;
 534         int ret;
 535
 536         /*
 537          * Force Gen1 operation when starting the link.  In case the link is
 538          * started in Gen2 mode, there is a possibility the devices on the
 539          * bus will not be detected at all.  This happens with PCIe switches.
 540          */
 541         tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCR);
 542         tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
 543         tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1;
 544         dw_pcie_writel_dbi(pci, PCIE_RC_LCR, tmp);
 545
 546         /* Start LTSSM. */
 547         if (imx6_pcie->variant == IMX7D)
 548                 reset_control_deassert(imx6_pcie->apps_reset);
 549         else
 550                 regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
 551                                    IMX6Q_GPR12_PCIE_CTL_2, 1 << 10);
 552
 553         ret = imx6_pcie_wait_for_link(imx6_pcie);
 554         if (ret)
 555                 goto err_reset_phy;
 556
 557         if (imx6_pcie->link_gen == 2) {
 558                 /* Allow Gen2 mode after the link is up. */
 559                 tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCR);
 560                 tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
 561                 tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2;
 562                 dw_pcie_writel_dbi(pci, PCIE_RC_LCR, tmp);
 563
 564                 /*
 565                  * Start Directed Speed Change so the best possible
 566                  * speed both link partners support can be negotiated.
 567                  */
 568                 tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 569                 tmp |= PORT_LOGIC_SPEED_CHANGE;
 570                 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
 571
 572                 if (imx6_pcie->variant != IMX7D) {
 573                         /*
 574                          * On i.MX7, DIRECT_SPEED_CHANGE behaves differently
 575                          * from i.MX6 family when no link speed transition
 576                          * occurs and we go Gen1 -> yep, Gen1. The difference
 577                          * is that, in such case, it will not be cleared by HW
 578                          * which will cause the following code to report false
 579                          * failure.
 580                          */
 581
 582                         ret = imx6_pcie_wait_for_speed_change(imx6_pcie);
 583                         if (ret) {
 584                                 dev_err(dev, "Failed to bring link up!\n");
 585                                 goto err_reset_phy;
 586                         }
 587                 }
 588
 589                 /* Make sure link training is finished as well! */
 590                 ret = imx6_pcie_wait_for_link(imx6_pcie);
 591                 if (ret) {
 592                         dev_err(dev, "Failed to bring link up!\n");
 593                         goto err_reset_phy;
 594                 }
 595         } else {
 596                 dev_info(dev, "Link: Gen2 disabled\n");
 597         }
 598
 599         tmp = dw_pcie_readl_dbi(pci, PCIE_RC_LCSR);
 600         dev_info(dev, "Link up, Gen%i\n", (tmp >> 16) & 0xf);
 601         return 0;
 602
 603 err_reset_phy:
 604         dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",
 605                 dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R0),
 606                 dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R1));
 607         imx6_pcie_reset_phy(imx6_pcie);
 608         return ret;
 609 }
 610
 611 static void imx6_pcie_host_init(struct pcie_port *pp)
 612 {
 613         struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 614         struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
 615
 616         imx6_pcie_assert_core_reset(imx6_pcie);
 617         imx6_pcie_init_phy(imx6_pcie);
 618         imx6_pcie_deassert_core_reset(imx6_pcie);
 619         dw_pcie_setup_rc(pp);
 620         imx6_pcie_establish_link(imx6_pcie);
 621
 622         if (IS_ENABLED(CONFIG_PCI_MSI))
 623                 dw_pcie_msi_init(pp);
 624 }
 625
 626 static int imx6_pcie_link_up(struct dw_pcie *pci)
 627 {
 628         return dw_pcie_readl_dbi(pci, PCIE_PHY_DEBUG_R1) &
 629                         PCIE_PHY_DEBUG_R1_XMLH_LINK_UP;
 630 }
 631
 632 static struct dw_pcie_host_ops imx6_pcie_host_ops = {
 633         .host_init = imx6_pcie_host_init,
 634 };
 635
 636 static int imx6_add_pcie_port(struct imx6_pcie *imx6_pcie,
 637                               struct platform_device *pdev)
 638 {
 639         struct dw_pcie *pci = imx6_pcie->pci;
 640         struct pcie_port *pp = &pci->pp;
 641         struct device *dev = &pdev->dev;
 642         int ret;
 643
 644         if (IS_ENABLED(CONFIG_PCI_MSI)) {
 645                 pp->msi_irq = platform_get_irq_byname(pdev, "msi");
 646                 if (pp->msi_irq <= 0) {
 647                         dev_err(dev, "failed to get MSI irq\n");
 648                         return -ENODEV;
 649                 }
 650
 651                 ret = devm_request_irq(dev, pp->msi_irq,
 652                                        imx6_pcie_msi_handler,
 653                                        IRQF_SHARED | IRQF_NO_THREAD,
 654                                        "mx6-pcie-msi", imx6_pcie);
 655                 if (ret) {
 656                         dev_err(dev, "failed to request MSI irq\n");
 657                         return ret;
 658                 }
 659         }
 660
 661         pp->root_bus_nr = -1;
 662         pp->ops = &imx6_pcie_host_ops;
 663
 664         ret = dw_pcie_host_init(pp);
 665         if (ret) {
 666                 dev_err(dev, "failed to initialize host\n");
 667                 return ret;
 668         }
 669
 670         return 0;
 671 }
 672
 673 static const struct dw_pcie_ops dw_pcie_ops = {
 674         .link_up = imx6_pcie_link_up,
 675 };
 676
 677 static int imx6_pcie_probe(struct platform_device *pdev)
 678 {
 679         struct device *dev = &pdev->dev;
 680         struct dw_pcie *pci;
 681         struct imx6_pcie *imx6_pcie;
 682         struct resource *dbi_base;
 683         struct device_node *node = dev->of_node;
 684         int ret;
 685
 686         imx6_pcie = devm_kzalloc(dev, sizeof(*imx6_pcie), GFP_KERNEL);
 687         if (!imx6_pcie)
 688                 return -ENOMEM;
 689
 690         pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
 691         if (!pci)
 692                 return -ENOMEM;
 693
 694         pci->dev = dev;
 695         pci->ops = &dw_pcie_ops;
 696
 697         imx6_pcie->pci = pci;
 698         imx6_pcie->variant =
 699                 (enum imx6_pcie_variants)of_device_get_match_data(dev);
 700
 701         dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 702         pci->dbi_base = devm_ioremap_resource(dev, dbi_base);
 703         if (IS_ERR(pci->dbi_base))
 704                 return PTR_ERR(pci->dbi_base);
 705
 706         /* Fetch GPIOs */
 707         imx6_pcie->reset_gpio = of_get_named_gpio(node, "reset-gpio", 0);
 708         imx6_pcie->gpio_active_high = of_property_read_bool(node,
 709                                                 "reset-gpio-active-high");
 710         if (gpio_is_valid(imx6_pcie->reset_gpio)) {
 711                 ret = devm_gpio_request_one(dev, imx6_pcie->reset_gpio,
 712                                 imx6_pcie->gpio_active_high ?
 713                                         GPIOF_OUT_INIT_HIGH :
 714                                         GPIOF_OUT_INIT_LOW,
 715                                 "PCIe reset");
 716                 if (ret) {
 717                         dev_err(dev, "unable to get reset gpio\n");
 718                         return ret;
 719                 }
 720         } else if (imx6_pcie->reset_gpio == -EPROBE_DEFER) {
 721                 return imx6_pcie->reset_gpio;
 722         }
 723
 724         /* Fetch clocks */
 725         imx6_pcie->pcie_phy = devm_clk_get(dev, "pcie_phy");
 726         if (IS_ERR(imx6_pcie->pcie_phy)) {
 727                 dev_err(dev, "pcie_phy clock source missing or invalid\n");
 728                 return PTR_ERR(imx6_pcie->pcie_phy);
 729         }
 730
 731         imx6_pcie->pcie_bus = devm_clk_get(dev, "pcie_bus");
 732         if (IS_ERR(imx6_pcie->pcie_bus)) {
 733                 dev_err(dev, "pcie_bus clock source missing or invalid\n");
 734                 return PTR_ERR(imx6_pcie->pcie_bus);
 735         }
 736
 737         imx6_pcie->pcie = devm_clk_get(dev, "pcie");
 738         if (IS_ERR(imx6_pcie->pcie)) {
 739                 dev_err(dev, "pcie clock source missing or invalid\n");
 740                 return PTR_ERR(imx6_pcie->pcie);
 741         }
 742
 743         switch (imx6_pcie->variant) {
 744         case IMX6SX:
 745                 imx6_pcie->pcie_inbound_axi = devm_clk_get(dev,
 746                                                            "pcie_inbound_axi");
 747                 if (IS_ERR(imx6_pcie->pcie_inbound_axi)) {
 748                         dev_err(dev, "pcie_inbound_axi clock missing or invalid\n");
 749                         return PTR_ERR(imx6_pcie->pcie_inbound_axi);
 750                 }
 751                 break;
 752         case IMX7D:
 753                 imx6_pcie->pciephy_reset = devm_reset_control_get(dev,
 754                                                                   "pciephy");
 755                 if (IS_ERR(imx6_pcie->pciephy_reset)) {
 756                         dev_err(dev, "Failed to get PCIEPHY reset control\n");
 757                         return PTR_ERR(imx6_pcie->pciephy_reset);
 758                 }
 759
 760                 imx6_pcie->apps_reset = devm_reset_control_get(dev, "apps");
 761                 if (IS_ERR(imx6_pcie->apps_reset)) {
 762                         dev_err(dev, "Failed to get PCIE APPS reset control\n");
 763                         return PTR_ERR(imx6_pcie->apps_reset);
 764                 }
 765                 break;
 766         default:
 767                 break;
 768         }
 769
 770         /* Grab GPR config register range */
 771         imx6_pcie->iomuxc_gpr =
 772                  syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr");
 773         if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
 774                 dev_err(dev, "unable to find iomuxc registers\n");
 775                 return PTR_ERR(imx6_pcie->iomuxc_gpr);
 776         }
 777
 778         /* Grab PCIe PHY Tx Settings */
 779         if (of_property_read_u32(node, "fsl,tx-deemph-gen1",
 780                                  &imx6_pcie->tx_deemph_gen1))
 781                 imx6_pcie->tx_deemph_gen1 = 0;
 782
 783         if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db",
 784                                  &imx6_pcie->tx_deemph_gen2_3p5db))
 785                 imx6_pcie->tx_deemph_gen2_3p5db = 0;
 786
 787         if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db",
 788                                  &imx6_pcie->tx_deemph_gen2_6db))
 789                 imx6_pcie->tx_deemph_gen2_6db = 20;
 790
 791         if (of_property_read_u32(node, "fsl,tx-swing-full",
 792                                  &imx6_pcie->tx_swing_full))
 793                 imx6_pcie->tx_swing_full = 127;
 794
 795         if (of_property_read_u32(node, "fsl,tx-swing-low",
 796                                  &imx6_pcie->tx_swing_low))
 797                 imx6_pcie->tx_swing_low = 127;
 798
 799         /* Limit link speed */
 800         ret = of_property_read_u32(node, "fsl,max-link-speed",
 801                                    &imx6_pcie->link_gen);
 802         if (ret)
 803                 imx6_pcie->link_gen = 1;
 804
 805         platform_set_drvdata(pdev, imx6_pcie);
 806
 807         ret = imx6_add_pcie_port(imx6_pcie, pdev);
 808         if (ret < 0)
 809                 return ret;
 810
 811         return 0;
 812 }
 813
 814 static void imx6_pcie_shutdown(struct platform_device *pdev)
 815 {
 816         struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev);
 817
 818         /* bring down link, so bootloader gets clean state in case of reboot */
 819         imx6_pcie_assert_core_reset(imx6_pcie);
 820 }
 821
 822 static const struct of_device_id imx6_pcie_of_match[] = {
 823         { .compatible = "fsl,imx6q-pcie",  .data = (void *)IMX6Q,  },
 824         { .compatible = "fsl,imx6sx-pcie", .data = (void *)IMX6SX, },
 825         { .compatible = "fsl,imx6qp-pcie", .data = (void *)IMX6QP, },
 826         { .compatible = "fsl,imx7d-pcie",  .data = (void *)IMX7D,  },
 827         {},
 828 };
 829
 830 static struct platform_driver imx6_pcie_driver = {
 831         .driver = {
 832                 .name   = "imx6q-pcie",
 833                 .of_match_table = imx6_pcie_of_match,
 834                 .suppress_bind_attrs = true,
 835         },
 836         .probe    = imx6_pcie_probe,
 837         .shutdown = imx6_pcie_shutdown,
 838 };
 839
 840 static int __init imx6_pcie_init(void)
 841 {
 842         /*
 843          * Since probe() can be deferred we need to make sure that
 844          * hook_fault_code is not called after __init memory is freed
 845          * by kernel and since imx6q_pcie_abort_handler() is a no-op,
 846          * we can install the handler here without risking it
 847          * accessing some uninitialized driver state.
 848          */
 849         hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0,
 850                         "external abort on non-linefetch");
 851
 852         return platform_driver_register(&imx6_pcie_driver);
 853 }
 854 device_initcall(imx6_pcie_init);