[mirror_ubuntu-focal-kernel.git] / drivers / pci / host / pci-imx6.c

/*
 * PCIe host controller driver for Freescale i.MX6 SoCs
 *
 * Copyright (C) 2013 Kosagi
 *		http://www.kosagi.com
 *
 * Author: Sean Cross <xobs@kosagi.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/gpio.h>
#include <linux/kernel.h>
#include <linux/mfd/syscon.h>
#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
#include <linux/module.h>
#include <linux/of_gpio.h>
#include <linux/pci.h>
#include <linux/platform_device.h>
#include <linux/regmap.h>
#include <linux/resource.h>
#include <linux/signal.h>
#include <linux/types.h>
#include <linux/interrupt.h>

#include "pcie-designware.h"

#define to_imx6_pcie(x)	container_of(x, struct imx6_pcie, pp)

struct imx6_pcie {
	int			reset_gpio;
	struct clk		*pcie_bus;
	struct clk		*pcie_phy;
	struct clk		*pcie;
	struct pcie_port	pp;
	struct regmap		*iomuxc_gpr;
	void __iomem		*mem_base;
};

/* PCIe Root Complex registers (memory-mapped) */
#define PCIE_RC_LCR				0x7c
#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1	0x1
#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2	0x2
#define PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK	0xf

#define PCIE_RC_LCSR				0x80

/* PCIe Port Logic registers (memory-mapped) */
#define PL_OFFSET 0x700
#define PCIE_PL_PFLR (PL_OFFSET + 0x08)
#define PCIE_PL_PFLR_LINK_STATE_MASK		(0x3f << 16)
#define PCIE_PL_PFLR_FORCE_LINK			(1 << 15)
#define PCIE_PHY_DEBUG_R0 (PL_OFFSET + 0x28)
#define PCIE_PHY_DEBUG_R1 (PL_OFFSET + 0x2c)
#define PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING	(1 << 29)
#define PCIE_PHY_DEBUG_R1_XMLH_LINK_UP		(1 << 4)

#define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
#define PCIE_PHY_CTRL_DATA_LOC 0
#define PCIE_PHY_CTRL_CAP_ADR_LOC 16
#define PCIE_PHY_CTRL_CAP_DAT_LOC 17
#define PCIE_PHY_CTRL_WR_LOC 18
#define PCIE_PHY_CTRL_RD_LOC 19

#define PCIE_PHY_STAT (PL_OFFSET + 0x110)
#define PCIE_PHY_STAT_ACK_LOC 16

#define PCIE_LINK_WIDTH_SPEED_CONTROL	0x80C
#define PORT_LOGIC_SPEED_CHANGE		(0x1 << 17)

/* PHY registers (not memory-mapped) */
#define PCIE_PHY_RX_ASIC_OUT 0x100D

#define PHY_RX_OVRD_IN_LO 0x1005
#define PHY_RX_OVRD_IN_LO_RX_DATA_EN (1 << 5)
#define PHY_RX_OVRD_IN_LO_RX_PLL_EN (1 << 3)

static int pcie_phy_poll_ack(void __iomem *dbi_base, int exp_val)
{
	u32 val;
	u32 max_iterations = 10;
	u32 wait_counter = 0;

	do {
		val = readl(dbi_base + PCIE_PHY_STAT);
		val = (val >> PCIE_PHY_STAT_ACK_LOC) & 0x1;
		wait_counter++;

		if (val == exp_val)
			return 0;

		udelay(1);
	} while (wait_counter < max_iterations);

	return -ETIMEDOUT;
}

static int pcie_phy_wait_ack(void __iomem *dbi_base, int addr)
{
	u32 val;
	int ret;

	val = addr << PCIE_PHY_CTRL_DATA_LOC;
	writel(val, dbi_base + PCIE_PHY_CTRL);

	val |= (0x1 << PCIE_PHY_CTRL_CAP_ADR_LOC);
	writel(val, dbi_base + PCIE_PHY_CTRL);

	ret = pcie_phy_poll_ack(dbi_base, 1);
	if (ret)
		return ret;

	val = addr << PCIE_PHY_CTRL_DATA_LOC;
	writel(val, dbi_base + PCIE_PHY_CTRL);

	ret = pcie_phy_poll_ack(dbi_base, 0);
	if (ret)
		return ret;

	return 0;
}

/* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
static int pcie_phy_read(void __iomem *dbi_base, int addr , int *data)
{
	u32 val, phy_ctl;
	int ret;

	ret = pcie_phy_wait_ack(dbi_base, addr);
	if (ret)
		return ret;

	/* assert Read signal */
	phy_ctl = 0x1 << PCIE_PHY_CTRL_RD_LOC;
	writel(phy_ctl, dbi_base + PCIE_PHY_CTRL);

	ret = pcie_phy_poll_ack(dbi_base, 1);
	if (ret)
		return ret;

	val = readl(dbi_base + PCIE_PHY_STAT);
	*data = val & 0xffff;

	/* deassert Read signal */
	writel(0x00, dbi_base + PCIE_PHY_CTRL);

	ret = pcie_phy_poll_ack(dbi_base, 0);
	if (ret)
		return ret;

	return 0;
}

static int pcie_phy_write(void __iomem *dbi_base, int addr, int data)
{
	u32 var;
	int ret;

	/* write addr */
	/* cap addr */
	ret = pcie_phy_wait_ack(dbi_base, addr);
	if (ret)
		return ret;

	var = data << PCIE_PHY_CTRL_DATA_LOC;
	writel(var, dbi_base + PCIE_PHY_CTRL);

	/* capture data */
	var |= (0x1 << PCIE_PHY_CTRL_CAP_DAT_LOC);
	writel(var, dbi_base + PCIE_PHY_CTRL);

	ret = pcie_phy_poll_ack(dbi_base, 1);
	if (ret)
		return ret;

	/* deassert cap data */
	var = data << PCIE_PHY_CTRL_DATA_LOC;
	writel(var, dbi_base + PCIE_PHY_CTRL);

	/* wait for ack de-assertion */
	ret = pcie_phy_poll_ack(dbi_base, 0);
	if (ret)
		return ret;

	/* assert wr signal */
	var = 0x1 << PCIE_PHY_CTRL_WR_LOC;
	writel(var, dbi_base + PCIE_PHY_CTRL);

	/* wait for ack */
	ret = pcie_phy_poll_ack(dbi_base, 1);
	if (ret)
		return ret;

	/* deassert wr signal */
	var = data << PCIE_PHY_CTRL_DATA_LOC;
	writel(var, dbi_base + PCIE_PHY_CTRL);

	/* wait for ack de-assertion */
	ret = pcie_phy_poll_ack(dbi_base, 0);
	if (ret)
		return ret;

	writel(0x0, dbi_base + PCIE_PHY_CTRL);

	return 0;
}

/*  Added for PCI abort handling */
static int imx6q_pcie_abort_handler(unsigned long addr,
		unsigned int fsr, struct pt_regs *regs)
{
	return 0;
}

static int imx6_pcie_assert_core_reset(struct pcie_port *pp)
{
	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
	u32 val, gpr1, gpr12;

	/*
	 * If the bootloader already enabled the link we need some special
	 * handling to get the core back into a state where it is safe to
	 * touch it for configuration.  As there is no dedicated reset signal
	 * wired up for MX6QDL, we need to manually force LTSSM into "detect"
	 * state before completely disabling LTSSM, which is a prerequisite
	 * for core configuration.
	 *
	 * If both LTSSM_ENABLE and REF_SSP_ENABLE are active we have a strong
	 * indication that the bootloader activated the link.
	 */
	regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, &gpr1);
	regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, &gpr12);

	if ((gpr1 & IMX6Q_GPR1_PCIE_REF_CLK_EN) &&
	    (gpr12 & IMX6Q_GPR12_PCIE_CTL_2)) {
		val = readl(pp->dbi_base + PCIE_PL_PFLR);
		val &= ~PCIE_PL_PFLR_LINK_STATE_MASK;
		val |= PCIE_PL_PFLR_FORCE_LINK;
		writel(val, pp->dbi_base + PCIE_PL_PFLR);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
				IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
	}

	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
			IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
			IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);

	return 0;
}

static int imx6_pcie_deassert_core_reset(struct pcie_port *pp)
{
	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
	int ret;

	ret = clk_prepare_enable(imx6_pcie->pcie_phy);
	if (ret) {
		dev_err(pp->dev, "unable to enable pcie_phy clock\n");
		goto err_pcie_phy;
	}

	ret = clk_prepare_enable(imx6_pcie->pcie_bus);
	if (ret) {
		dev_err(pp->dev, "unable to enable pcie_bus clock\n");
		goto err_pcie_bus;
	}

	ret = clk_prepare_enable(imx6_pcie->pcie);
	if (ret) {
		dev_err(pp->dev, "unable to enable pcie clock\n");
		goto err_pcie;
	}

	/* power up core phy and enable ref clock */
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
			IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18);
	/*
	 * the async reset input need ref clock to sync internally,
	 * when the ref clock comes after reset, internal synced
	 * reset time is too short, cannot meet the requirement.
	 * add one ~10us delay here.
	 */
	udelay(10);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
			IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16);

	/* allow the clocks to stabilize */
	usleep_range(200, 500);

	/* Some boards don't have PCIe reset GPIO. */
	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
		gpio_set_value(imx6_pcie->reset_gpio, 0);
		msleep(100);
		gpio_set_value(imx6_pcie->reset_gpio, 1);
	}
	return 0;

err_pcie:
	clk_disable_unprepare(imx6_pcie->pcie_bus);
err_pcie_bus:
	clk_disable_unprepare(imx6_pcie->pcie_phy);
err_pcie_phy:
	return ret;

}

static void imx6_pcie_init_phy(struct pcie_port *pp)
{
	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);

	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
			IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);

	/* configure constant input signal to the pcie ctrl and phy */
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
			IMX6Q_GPR12_DEVICE_TYPE, PCI_EXP_TYPE_ROOT_PORT << 12);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
			IMX6Q_GPR12_LOS_LEVEL, 9 << 4);

	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
			IMX6Q_GPR8_TX_DEEMPH_GEN1, 0 << 0);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
			IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB, 0 << 6);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
			IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB, 20 << 12);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
			IMX6Q_GPR8_TX_SWING_FULL, 127 << 18);
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
			IMX6Q_GPR8_TX_SWING_LOW, 127 << 25);
}

static int imx6_pcie_wait_for_link(struct pcie_port *pp)
{
	unsigned int retries;

	for (retries = 0; retries < 200; retries++) {
		if (dw_pcie_link_up(pp))
			return 0;
		usleep_range(100, 1000);
	}

	dev_err(pp->dev, "phy link never came up\n");
	dev_dbg(pp->dev, "DEBUG_R0: 0x%08x, DEBUG_R1: 0x%08x\n",
		readl(pp->dbi_base + PCIE_PHY_DEBUG_R0),
		readl(pp->dbi_base + PCIE_PHY_DEBUG_R1));
	return -EINVAL;
}

static int imx6_pcie_wait_for_speed_change(struct pcie_port *pp)
{
	u32 tmp;
	unsigned int retries;

	for (retries = 0; retries < 200; retries++) {
		tmp = readl(pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);
		/* Test if the speed change finished. */
		if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
			return 0;
		usleep_range(100, 1000);
	}

	dev_err(pp->dev, "Speed change timeout\n");
	return -EINVAL;
}

static irqreturn_t imx6_pcie_msi_handler(int irq, void *arg)
{
	struct pcie_port *pp = arg;

	return dw_handle_msi_irq(pp);
}

static int imx6_pcie_establish_link(struct pcie_port *pp)
{
	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
	u32 tmp;
	int ret;

	/*
	 * Force Gen1 operation when starting the link.  In case the link is
	 * started in Gen2 mode, there is a possibility the devices on the
	 * bus will not be detected at all.  This happens with PCIe switches.
	 */
	tmp = readl(pp->dbi_base + PCIE_RC_LCR);
	tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
	tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1;
	writel(tmp, pp->dbi_base + PCIE_RC_LCR);

	/* Start LTSSM. */
	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
			IMX6Q_GPR12_PCIE_CTL_2, 1 << 10);

	ret = imx6_pcie_wait_for_link(pp);
	if (ret)
		return ret;

	/* Allow Gen2 mode after the link is up. */
	tmp = readl(pp->dbi_base + PCIE_RC_LCR);
	tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
	tmp |= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2;
	writel(tmp, pp->dbi_base + PCIE_RC_LCR);

	/*
	 * Start Directed Speed Change so the best possible speed both link
	 * partners support can be negotiated.
	 */
	tmp = readl(pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);
	tmp |= PORT_LOGIC_SPEED_CHANGE;
	writel(tmp, pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);

	ret = imx6_pcie_wait_for_speed_change(pp);
	if (ret) {
		dev_err(pp->dev, "Failed to bring link up!\n");
		return ret;
	}

	/* Make sure link training is finished as well! */
	ret = imx6_pcie_wait_for_link(pp);
	if (ret) {
		dev_err(pp->dev, "Failed to bring link up!\n");
		return ret;
	}

	tmp = readl(pp->dbi_base + PCIE_RC_LCSR);
	dev_dbg(pp->dev, "Link up, Gen=%i\n", (tmp >> 16) & 0xf);
	return 0;
}

static void imx6_pcie_host_init(struct pcie_port *pp)
{
	imx6_pcie_assert_core_reset(pp);

	imx6_pcie_init_phy(pp);

	imx6_pcie_deassert_core_reset(pp);

	dw_pcie_setup_rc(pp);

	imx6_pcie_establish_link(pp);

	if (IS_ENABLED(CONFIG_PCI_MSI))
		dw_pcie_msi_init(pp);
}

static void imx6_pcie_reset_phy(struct pcie_port *pp)
{
	u32 tmp;

	pcie_phy_read(pp->dbi_base, PHY_RX_OVRD_IN_LO, &tmp);
	tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
		PHY_RX_OVRD_IN_LO_RX_PLL_EN);
	pcie_phy_write(pp->dbi_base, PHY_RX_OVRD_IN_LO, tmp);

	usleep_range(2000, 3000);

	pcie_phy_read(pp->dbi_base, PHY_RX_OVRD_IN_LO, &tmp);
	tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
		  PHY_RX_OVRD_IN_LO_RX_PLL_EN);
	pcie_phy_write(pp->dbi_base, PHY_RX_OVRD_IN_LO, tmp);
}

static int imx6_pcie_link_up(struct pcie_port *pp)
{
	u32 rc, debug_r0, rx_valid;
	int count = 5;

	/*
	 * Test if the PHY reports that the link is up and also that the LTSSM
	 * training finished. There are three possible states of the link when
	 * this code is called:
	 * 1) The link is DOWN (unlikely)
	 *     The link didn't come up yet for some reason. This usually means
	 *     we have a real problem somewhere. Reset the PHY and exit. This
	 *     state calls for inspection of the DEBUG registers.
	 * 2) The link is UP, but still in LTSSM training
	 *     Wait for the training to finish, which should take a very short
	 *     time. If the training does not finish, we have a problem and we
	 *     need to inspect the DEBUG registers. If the training does finish,
	 *     the link is up and operating correctly.
	 * 3) The link is UP and no longer in LTSSM training
	 *     The link is up and operating correctly.
	 */
	while (1) {
		rc = readl(pp->dbi_base + PCIE_PHY_DEBUG_R1);
		if (!(rc & PCIE_PHY_DEBUG_R1_XMLH_LINK_UP))
			break;
		if (!(rc & PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING))
			return 1;
		if (!count--)
			break;
		dev_dbg(pp->dev, "Link is up, but still in training\n");
		/*
		 * Wait a little bit, then re-check if the link finished
		 * the training.
		 */
		usleep_range(1000, 2000);
	}
	/*
	 * From L0, initiate MAC entry to gen2 if EP/RC supports gen2.
	 * Wait 2ms (LTSSM timeout is 24ms, PHY lock is ~5us in gen2).
	 * If (MAC/LTSSM.state == Recovery.RcvrLock)
	 * && (PHY/rx_valid==0) then pulse PHY/rx_reset. Transition
	 * to gen2 is stuck
	 */
	pcie_phy_read(pp->dbi_base, PCIE_PHY_RX_ASIC_OUT, &rx_valid);
	debug_r0 = readl(pp->dbi_base + PCIE_PHY_DEBUG_R0);

	if (rx_valid & 0x01)
		return 0;

	if ((debug_r0 & 0x3f) != 0x0d)
		return 0;

	dev_err(pp->dev, "transition to gen2 is stuck, reset PHY!\n");
	dev_dbg(pp->dev, "debug_r0=%08x debug_r1=%08x\n", debug_r0, rc);

	imx6_pcie_reset_phy(pp);

	return 0;
}

static struct pcie_host_ops imx6_pcie_host_ops = {
	.link_up = imx6_pcie_link_up,
	.host_init = imx6_pcie_host_init,
};

static int __init imx6_add_pcie_port(struct pcie_port *pp,
			struct platform_device *pdev)
{
	int ret;

	if (IS_ENABLED(CONFIG_PCI_MSI)) {
		pp->msi_irq = platform_get_irq_byname(pdev, "msi");
		if (pp->msi_irq <= 0) {
			dev_err(&pdev->dev, "failed to get MSI irq\n");
			return -ENODEV;
		}

		ret = devm_request_irq(&pdev->dev, pp->msi_irq,
				       imx6_pcie_msi_handler,
				       IRQF_SHARED, "mx6-pcie-msi", pp);
		if (ret) {
			dev_err(&pdev->dev, "failed to request MSI irq\n");
			return -ENODEV;
		}
	}

	pp->root_bus_nr = -1;
	pp->ops = &imx6_pcie_host_ops;

	ret = dw_pcie_host_init(pp);
	if (ret) {
		dev_err(&pdev->dev, "failed to initialize host\n");
		return ret;
	}

	return 0;
}

static int __init imx6_pcie_probe(struct platform_device *pdev)
{
	struct imx6_pcie *imx6_pcie;
	struct pcie_port *pp;
	struct device_node *np = pdev->dev.of_node;
	struct resource *dbi_base;
	int ret;

	imx6_pcie = devm_kzalloc(&pdev->dev, sizeof(*imx6_pcie), GFP_KERNEL);
	if (!imx6_pcie)
		return -ENOMEM;

	pp = &imx6_pcie->pp;
	pp->dev = &pdev->dev;

	/* Added for PCI abort handling */
	hook_fault_code(16 + 6, imx6q_pcie_abort_handler, SIGBUS, 0,
		"imprecise external abort");

	dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	pp->dbi_base = devm_ioremap_resource(&pdev->dev, dbi_base);
	if (IS_ERR(pp->dbi_base))
		return PTR_ERR(pp->dbi_base);

	/* Fetch GPIOs */
	imx6_pcie->reset_gpio = of_get_named_gpio(np, "reset-gpio", 0);
	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
		ret = devm_gpio_request_one(&pdev->dev, imx6_pcie->reset_gpio,
					    GPIOF_OUT_INIT_LOW, "PCIe reset");
		if (ret) {
			dev_err(&pdev->dev, "unable to get reset gpio\n");
			return ret;
		}
	}

	/* Fetch clocks */
	imx6_pcie->pcie_phy = devm_clk_get(&pdev->dev, "pcie_phy");
	if (IS_ERR(imx6_pcie->pcie_phy)) {
		dev_err(&pdev->dev,
			"pcie_phy clock source missing or invalid\n");
		return PTR_ERR(imx6_pcie->pcie_phy);
	}

	imx6_pcie->pcie_bus = devm_clk_get(&pdev->dev, "pcie_bus");
	if (IS_ERR(imx6_pcie->pcie_bus)) {
		dev_err(&pdev->dev,
			"pcie_bus clock source missing or invalid\n");
		return PTR_ERR(imx6_pcie->pcie_bus);
	}

	imx6_pcie->pcie = devm_clk_get(&pdev->dev, "pcie");
	if (IS_ERR(imx6_pcie->pcie)) {
		dev_err(&pdev->dev,
			"pcie clock source missing or invalid\n");
		return PTR_ERR(imx6_pcie->pcie);
	}

	/* Grab GPR config register range */
	imx6_pcie->iomuxc_gpr =
		 syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr");
	if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
		dev_err(&pdev->dev, "unable to find iomuxc registers\n");
		return PTR_ERR(imx6_pcie->iomuxc_gpr);
	}

	ret = imx6_add_pcie_port(pp, pdev);
	if (ret < 0)
		return ret;

	platform_set_drvdata(pdev, imx6_pcie);
	return 0;
}

static void imx6_pcie_shutdown(struct platform_device *pdev)
{
	struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev);

	/* bring down link, so bootloader gets clean state in case of reboot */
	imx6_pcie_assert_core_reset(&imx6_pcie->pp);
}

static const struct of_device_id imx6_pcie_of_match[] = {
	{ .compatible = "fsl,imx6q-pcie", },
	{},
};
MODULE_DEVICE_TABLE(of, imx6_pcie_of_match);

static struct platform_driver imx6_pcie_driver = {
	.driver = {
		.name	= "imx6q-pcie",
		.of_match_table = imx6_pcie_of_match,
	},
	.shutdown = imx6_pcie_shutdown,
};

/* Freescale PCIe driver does not allow module unload */

static int __init imx6_pcie_init(void)
{
	return platform_driver_probe(&imx6_pcie_driver, imx6_pcie_probe);
}
module_init(imx6_pcie_init);

MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
MODULE_DESCRIPTION("Freescale i.MX6 PCIe host controller driver");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
bb38919e SC	1	/*
	2	* PCIe host controller driver for Freescale i.MX6 SoCs
	3	*
	4	* Copyright (C) 2013 Kosagi
	5	* http://www.kosagi.com
	6	*
	7	* Author: Sean Cross <xobs@kosagi.com>
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License version 2 as
	11	* published by the Free Software Foundation.
	12	*/
	13
	14	#include <linux/clk.h>
	15	#include <linux/delay.h>
	16	#include <linux/gpio.h>
	17	#include <linux/kernel.h>
	18	#include <linux/mfd/syscon.h>
	19	#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
	20	#include <linux/module.h>
	21	#include <linux/of_gpio.h>
	22	#include <linux/pci.h>
	23	#include <linux/platform_device.h>
	24	#include <linux/regmap.h>
	25	#include <linux/resource.h>
	26	#include <linux/signal.h>
	27	#include <linux/types.h>
d1dc9749	28	#include <linux/interrupt.h>
bb38919e SC	29
	30	#include "pcie-designware.h"
	31
	32	#define to_imx6_pcie(x) container_of(x, struct imx6_pcie, pp)
	33
	34	struct imx6_pcie {
	35	int reset_gpio;
57526136 LS	36	struct clk *pcie_bus;
	37	struct clk *pcie_phy;
	38	struct clk *pcie;
bb38919e SC	39	struct pcie_port pp;
	40	struct regmap *iomuxc_gpr;
	41	void __iomem *mem_base;
	42	};
	43
fa33a6d8 MV	44	/* PCIe Root Complex registers (memory-mapped) */
	45	#define PCIE_RC_LCR 0x7c
	46	#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1 0x1
	47	#define PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2 0x2
	48	#define PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK 0xf
	49
2393f79c BH	50	#define PCIE_RC_LCSR 0x80
2393f79c BH	51
bb38919e SC	52	/* PCIe Port Logic registers (memory-mapped) */
bb38919e SC	53	#define PL_OFFSET 0x700
3e3e406e LS	54	#define PCIE_PL_PFLR (PL_OFFSET + 0x08)
	55	#define PCIE_PL_PFLR_LINK_STATE_MASK (0x3f << 16)
	56	#define PCIE_PL_PFLR_FORCE_LINK (1 << 15)
bb38919e SC	57	#define PCIE_PHY_DEBUG_R0 (PL_OFFSET + 0x28)
bb38919e SC	58	#define PCIE_PHY_DEBUG_R1 (PL_OFFSET + 0x2c)
7f9f40c0 MV	59	#define PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING (1 << 29)
7f9f40c0 MV	60	#define PCIE_PHY_DEBUG_R1_XMLH_LINK_UP (1 << 4)
bb38919e SC	61
	62	#define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
	63	#define PCIE_PHY_CTRL_DATA_LOC 0
	64	#define PCIE_PHY_CTRL_CAP_ADR_LOC 16
	65	#define PCIE_PHY_CTRL_CAP_DAT_LOC 17
	66	#define PCIE_PHY_CTRL_WR_LOC 18
	67	#define PCIE_PHY_CTRL_RD_LOC 19
	68
	69	#define PCIE_PHY_STAT (PL_OFFSET + 0x110)
	70	#define PCIE_PHY_STAT_ACK_LOC 16
	71
fa33a6d8 MV	72	#define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C
	73	#define PORT_LOGIC_SPEED_CHANGE (0x1 << 17)
	74
bb38919e SC	75	/* PHY registers (not memory-mapped) */
	76	#define PCIE_PHY_RX_ASIC_OUT 0x100D
	77
	78	#define PHY_RX_OVRD_IN_LO 0x1005
	79	#define PHY_RX_OVRD_IN_LO_RX_DATA_EN (1 << 5)
	80	#define PHY_RX_OVRD_IN_LO_RX_PLL_EN (1 << 3)
	81
	82	static int pcie_phy_poll_ack(void __iomem *dbi_base, int exp_val)
	83	{
	84	u32 val;
	85	u32 max_iterations = 10;
	86	u32 wait_counter = 0;
	87
	88	do {
	89	val = readl(dbi_base + PCIE_PHY_STAT);
	90	val = (val >> PCIE_PHY_STAT_ACK_LOC) & 0x1;
	91	wait_counter++;
	92
	93	if (val == exp_val)
	94	return 0;
	95
	96	udelay(1);
	97	} while (wait_counter < max_iterations);
	98
	99	return -ETIMEDOUT;
	100	}
	101
	102	static int pcie_phy_wait_ack(void __iomem *dbi_base, int addr)
	103	{
	104	u32 val;
	105	int ret;
	106
	107	val = addr << PCIE_PHY_CTRL_DATA_LOC;
	108	writel(val, dbi_base + PCIE_PHY_CTRL);
	109
	110	val \|= (0x1 << PCIE_PHY_CTRL_CAP_ADR_LOC);
	111	writel(val, dbi_base + PCIE_PHY_CTRL);
	112
	113	ret = pcie_phy_poll_ack(dbi_base, 1);
	114	if (ret)
	115	return ret;
	116
	117	val = addr << PCIE_PHY_CTRL_DATA_LOC;
	118	writel(val, dbi_base + PCIE_PHY_CTRL);
	119
	120	ret = pcie_phy_poll_ack(dbi_base, 0);
	121	if (ret)
	122	return ret;
	123
	124	return 0;
	125	}
	126
	127	/* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
	128	static int pcie_phy_read(void __iomem dbi_base, int addr , int data)
	129	{
	130	u32 val, phy_ctl;
	131	int ret;
	132
	133	ret = pcie_phy_wait_ack(dbi_base, addr);
	134	if (ret)
	135	return ret;
	136
	137	/* assert Read signal */
	138	phy_ctl = 0x1 << PCIE_PHY_CTRL_RD_LOC;
139	writel(phy_ctl, dbi_base + PCIE_PHY_CTRL);
140
141	ret = pcie_phy_poll_ack(dbi_base, 1);
142	if (ret)
143	return ret;
144
145	val = readl(dbi_base + PCIE_PHY_STAT);
146	*data = val & 0xffff;
147
148	/* deassert Read signal */
149	writel(0x00, dbi_base + PCIE_PHY_CTRL);
150
151	ret = pcie_phy_poll_ack(dbi_base, 0);
152	if (ret)
153	return ret;
154
155	return 0;
156	}
157
158	static int pcie_phy_write(void __iomem *dbi_base, int addr, int data)
159	{
160	u32 var;
161	int ret;
162
163	/* write addr */
164	/* cap addr */
165	ret = pcie_phy_wait_ack(dbi_base, addr);
166	if (ret)
167	return ret;
168
169	var = data << PCIE_PHY_CTRL_DATA_LOC;
170	writel(var, dbi_base + PCIE_PHY_CTRL);
171
172	/* capture data */
173	var \|= (0x1 << PCIE_PHY_CTRL_CAP_DAT_LOC);
174	writel(var, dbi_base + PCIE_PHY_CTRL);
175
176	ret = pcie_phy_poll_ack(dbi_base, 1);
177	if (ret)
178	return ret;
179
180	/* deassert cap data */
181	var = data << PCIE_PHY_CTRL_DATA_LOC;
182	writel(var, dbi_base + PCIE_PHY_CTRL);
183
184	/* wait for ack de-assertion */
185	ret = pcie_phy_poll_ack(dbi_base, 0);
186	if (ret)
187	return ret;
188
189	/* assert wr signal */
190	var = 0x1 << PCIE_PHY_CTRL_WR_LOC;
191	writel(var, dbi_base + PCIE_PHY_CTRL);
192
193	/* wait for ack */
194	ret = pcie_phy_poll_ack(dbi_base, 1);
195	if (ret)
196	return ret;
197
198	/* deassert wr signal */
199	var = data << PCIE_PHY_CTRL_DATA_LOC;
200	writel(var, dbi_base + PCIE_PHY_CTRL);
201
202	/* wait for ack de-assertion */
203	ret = pcie_phy_poll_ack(dbi_base, 0);
204	if (ret)
205	return ret;
206
207	writel(0x0, dbi_base + PCIE_PHY_CTRL);
208
209	return 0;
210	}
211
212	/* Added for PCI abort handling */
213	static int imx6q_pcie_abort_handler(unsigned long addr,
214	unsigned int fsr, struct pt_regs *regs)
215	{
bb38919e SC	216	return 0;
	217	}
	218
	219	static int imx6_pcie_assert_core_reset(struct pcie_port *pp)
	220	{
	221	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
3e3e406e LS	222	u32 val, gpr1, gpr12;
	223
	224	/*
	225	* If the bootloader already enabled the link we need some special
	226	* handling to get the core back into a state where it is safe to
	227	* touch it for configuration. As there is no dedicated reset signal
	228	* wired up for MX6QDL, we need to manually force LTSSM into "detect"
	229	* state before completely disabling LTSSM, which is a prerequisite
	230	* for core configuration.
	231	*
	232	* If both LTSSM_ENABLE and REF_SSP_ENABLE are active we have a strong
	233	* indication that the bootloader activated the link.
	234	*/
	235	regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1, &gpr1);
	236	regmap_read(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, &gpr12);
	237
	238	if ((gpr1 & IMX6Q_GPR1_PCIE_REF_CLK_EN) &&
	239	(gpr12 & IMX6Q_GPR12_PCIE_CTL_2)) {
	240	val = readl(pp->dbi_base + PCIE_PL_PFLR);
	241	val &= ~PCIE_PL_PFLR_LINK_STATE_MASK;
	242	val \|= PCIE_PL_PFLR_FORCE_LINK;
	243	writel(val, pp->dbi_base + PCIE_PL_PFLR);
	244
	245	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
	246	IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
	247	}
bb38919e SC	248
	249	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
	250	IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);
bb38919e SC	251	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
	252	IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);
	253
bb38919e SC	254	return 0;
	255	}
	256
	257	static int imx6_pcie_deassert_core_reset(struct pcie_port *pp)
	258	{
	259	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
	260	int ret;
	261
57526136	262	ret = clk_prepare_enable(imx6_pcie->pcie_phy);
bb38919e	263	if (ret) {
57526136 LS	264	dev_err(pp->dev, "unable to enable pcie_phy clock\n");
57526136 LS	265	goto err_pcie_phy;
bb38919e SC	266	}
bb38919e SC	267
57526136	268	ret = clk_prepare_enable(imx6_pcie->pcie_bus);
bb38919e	269	if (ret) {
57526136 LS	270	dev_err(pp->dev, "unable to enable pcie_bus clock\n");
57526136 LS	271	goto err_pcie_bus;
bb38919e SC	272	}
bb38919e SC	273
57526136	274	ret = clk_prepare_enable(imx6_pcie->pcie);
bb38919e	275	if (ret) {
57526136 LS	276	dev_err(pp->dev, "unable to enable pcie clock\n");
57526136 LS	277	goto err_pcie;
bb38919e SC	278	}
bb38919e SC	279
3fce0e88 TH	280	/* power up core phy and enable ref clock */
	281	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
	282	IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18);
a2fa6f64 RZ	283	/*
	284	* the async reset input need ref clock to sync internally,
	285	* when the ref clock comes after reset, internal synced
	286	* reset time is too short, cannot meet the requirement.
	287	* add one ~10us delay here.
	288	*/
	289	udelay(10);
3fce0e88 TH	290	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
	291	IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16);
	292
a2fa6f64 RZ	293	/* allow the clocks to stabilize */
	294	usleep_range(200, 500);
	295
bc9ef770 RZ	296	/* Some boards don't have PCIe reset GPIO. */
	297	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
	298	gpio_set_value(imx6_pcie->reset_gpio, 0);
	299	msleep(100);
	300	gpio_set_value(imx6_pcie->reset_gpio, 1);
	301	}
bb38919e SC	302	return 0;
bb38919e SC	303
57526136 LS	304	err_pcie:
	305	clk_disable_unprepare(imx6_pcie->pcie_bus);
	306	err_pcie_bus:
	307	clk_disable_unprepare(imx6_pcie->pcie_phy);
	308	err_pcie_phy:
bb38919e SC	309	return ret;
	310
	311	}
	312
	313	static void imx6_pcie_init_phy(struct pcie_port *pp)
	314	{
	315	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
	316
	317	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
	318	IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
	319
	320	/* configure constant input signal to the pcie ctrl and phy */
	321	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
	322	IMX6Q_GPR12_DEVICE_TYPE, PCI_EXP_TYPE_ROOT_PORT << 12);
	323	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
	324	IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
	325
	326	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
	327	IMX6Q_GPR8_TX_DEEMPH_GEN1, 0 << 0);
	328	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
	329	IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB, 0 << 6);
	330	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
	331	IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB, 20 << 12);
	332	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
	333	IMX6Q_GPR8_TX_SWING_FULL, 127 << 18);
	334	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
	335	IMX6Q_GPR8_TX_SWING_LOW, 127 << 25);
	336	}
	337
66a60f93 MV	338	static int imx6_pcie_wait_for_link(struct pcie_port *pp)
66a60f93 MV	339	{
6cbb247e	340	unsigned int retries;
66a60f93	341
6cbb247e BH	342	for (retries = 0; retries < 200; retries++) {
	343	if (dw_pcie_link_up(pp))
	344	return 0;
66a60f93	345	usleep_range(100, 1000);
66a60f93 MV	346	}
66a60f93 MV	347
6cbb247e BH	348	dev_err(pp->dev, "phy link never came up\n");
	349	dev_dbg(pp->dev, "DEBUG_R0: 0x%08x, DEBUG_R1: 0x%08x\n",
	350	readl(pp->dbi_base + PCIE_PHY_DEBUG_R0),
	351	readl(pp->dbi_base + PCIE_PHY_DEBUG_R1));
	352	return -EINVAL;
66a60f93 MV	353	}
66a60f93 MV	354
a0427464 TK	355	static int imx6_pcie_wait_for_speed_change(struct pcie_port *pp)
a0427464 TK	356	{
1c7fae18	357	u32 tmp;
a0427464 TK	358	unsigned int retries;
	359
	360	for (retries = 0; retries < 200; retries++) {
	361	tmp = readl(pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);
	362	/* Test if the speed change finished. */
	363	if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
	364	return 0;
	365	usleep_range(100, 1000);
	366	}
	367
	368	dev_err(pp->dev, "Speed change timeout\n");
	369	return -EINVAL;
66a60f93 MV	370	}
66a60f93 MV	371
d1dc9749 LS	372	static irqreturn_t imx6_pcie_msi_handler(int irq, void *arg)
	373	{
	374	struct pcie_port *pp = arg;
	375
	376	return dw_handle_msi_irq(pp);
	377	}
	378
fd5da208	379	static int imx6_pcie_establish_link(struct pcie_port *pp)
bb38919e	380	{
bb38919e	381	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pp);
1c7fae18	382	u32 tmp;
a0427464	383	int ret;
fa33a6d8 MV	384
	385	/*
	386	* Force Gen1 operation when starting the link. In case the link is
	387	* started in Gen2 mode, there is a possibility the devices on the
	388	* bus will not be detected at all. This happens with PCIe switches.
	389	*/
	390	tmp = readl(pp->dbi_base + PCIE_RC_LCR);
	391	tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
	392	tmp \|= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN1;
	393	writel(tmp, pp->dbi_base + PCIE_RC_LCR);
	394
	395	/* Start LTSSM. */
	396	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
	397	IMX6Q_GPR12_PCIE_CTL_2, 1 << 10);
	398
	399	ret = imx6_pcie_wait_for_link(pp);
	400	if (ret)
	401	return ret;
	402
	403	/* Allow Gen2 mode after the link is up. */
	404	tmp = readl(pp->dbi_base + PCIE_RC_LCR);
	405	tmp &= ~PCIE_RC_LCR_MAX_LINK_SPEEDS_MASK;
	406	tmp \|= PCIE_RC_LCR_MAX_LINK_SPEEDS_GEN2;
	407	writel(tmp, pp->dbi_base + PCIE_RC_LCR);
	408
	409	/*
	410	* Start Directed Speed Change so the best possible speed both link
	411	* partners support can be negotiated.
	412	*/
	413	tmp = readl(pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);
	414	tmp \|= PORT_LOGIC_SPEED_CHANGE;
	415	writel(tmp, pp->dbi_base + PCIE_LINK_WIDTH_SPEED_CONTROL);
	416
a0427464 TK	417	ret = imx6_pcie_wait_for_speed_change(pp);
	418	if (ret) {
	419	dev_err(pp->dev, "Failed to bring link up!\n");
	420	return ret;
fa33a6d8 MV	421	}
	422
	423	/* Make sure link training is finished as well! */
a0427464	424	ret = imx6_pcie_wait_for_link(pp);
fa33a6d8 MV	425	if (ret) {
fa33a6d8 MV	426	dev_err(pp->dev, "Failed to bring link up!\n");
a0427464	427	return ret;
fa33a6d8 MV	428	}
fa33a6d8 MV	429
2393f79c	430	tmp = readl(pp->dbi_base + PCIE_RC_LCSR);
a0427464 TK	431	dev_dbg(pp->dev, "Link up, Gen=%i\n", (tmp >> 16) & 0xf);
a0427464 TK	432	return 0;
fa33a6d8 MV	433	}
	434
	435	static void imx6_pcie_host_init(struct pcie_port *pp)
	436	{
bb38919e SC	437	imx6_pcie_assert_core_reset(pp);
	438
	439	imx6_pcie_init_phy(pp);
	440
	441	imx6_pcie_deassert_core_reset(pp);
	442
	443	dw_pcie_setup_rc(pp);
	444
fd5da208	445	imx6_pcie_establish_link(pp);
d1dc9749 LS	446
	447	if (IS_ENABLED(CONFIG_PCI_MSI))
	448	dw_pcie_msi_init(pp);
bb38919e SC	449	}
bb38919e SC	450
982aa234 MV	451	static void imx6_pcie_reset_phy(struct pcie_port *pp)
982aa234 MV	452	{
1c7fae18	453	u32 tmp;
982aa234	454
1c7fae18 BH	455	pcie_phy_read(pp->dbi_base, PHY_RX_OVRD_IN_LO, &tmp);
	456	tmp \|= (PHY_RX_OVRD_IN_LO_RX_DATA_EN \|
	457	PHY_RX_OVRD_IN_LO_RX_PLL_EN);
	458	pcie_phy_write(pp->dbi_base, PHY_RX_OVRD_IN_LO, tmp);
982aa234 MV	459
	460	usleep_range(2000, 3000);
	461
1c7fae18 BH	462	pcie_phy_read(pp->dbi_base, PHY_RX_OVRD_IN_LO, &tmp);
1c7fae18 BH	463	tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN \|
982aa234	464	PHY_RX_OVRD_IN_LO_RX_PLL_EN);
1c7fae18	465	pcie_phy_write(pp->dbi_base, PHY_RX_OVRD_IN_LO, tmp);
982aa234 MV	466	}
982aa234 MV	467
bb38919e SC	468	static int imx6_pcie_link_up(struct pcie_port *pp)
bb38919e SC	469	{
f95d3ae7 MV	470	u32 rc, debug_r0, rx_valid;
f95d3ae7 MV	471	int count = 5;
bb38919e	472
7f9f40c0	473	/*
f95d3ae7 MV	474	* Test if the PHY reports that the link is up and also that the LTSSM
	475	* training finished. There are three possible states of the link when
	476	* this code is called:
	477	* 1) The link is DOWN (unlikely)
	478	* The link didn't come up yet for some reason. This usually means
	479	* we have a real problem somewhere. Reset the PHY and exit. This
	480	* state calls for inspection of the DEBUG registers.
	481	* 2) The link is UP, but still in LTSSM training
	482	* Wait for the training to finish, which should take a very short
	483	* time. If the training does not finish, we have a problem and we
	484	* need to inspect the DEBUG registers. If the training does finish,
	485	* the link is up and operating correctly.
	486	* 3) The link is UP and no longer in LTSSM training
	487	* The link is up and operating correctly.
7f9f40c0	488	*/
f95d3ae7 MV	489	while (1) {
	490	rc = readl(pp->dbi_base + PCIE_PHY_DEBUG_R1);
	491	if (!(rc & PCIE_PHY_DEBUG_R1_XMLH_LINK_UP))
	492	break;
	493	if (!(rc & PCIE_PHY_DEBUG_R1_XMLH_LINK_IN_TRAINING))
	494	return 1;
	495	if (!count--)
	496	break;
	497	dev_dbg(pp->dev, "Link is up, but still in training\n");
	498	/*
	499	* Wait a little bit, then re-check if the link finished
	500	* the training.
	501	*/
	502	usleep_range(1000, 2000);
	503	}
bb38919e SC	504	/*
	505	* From L0, initiate MAC entry to gen2 if EP/RC supports gen2.
	506	* Wait 2ms (LTSSM timeout is 24ms, PHY lock is ~5us in gen2).
	507	* If (MAC/LTSSM.state == Recovery.RcvrLock)
	508	* && (PHY/rx_valid==0) then pulse PHY/rx_reset. Transition
	509	* to gen2 is stuck
	510	*/
	511	pcie_phy_read(pp->dbi_base, PCIE_PHY_RX_ASIC_OUT, &rx_valid);
f95d3ae7	512	debug_r0 = readl(pp->dbi_base + PCIE_PHY_DEBUG_R0);
bb38919e SC	513
	514	if (rx_valid & 0x01)
	515	return 0;
	516
f95d3ae7	517	if ((debug_r0 & 0x3f) != 0x0d)
bb38919e SC	518	return 0;
	519
	520	dev_err(pp->dev, "transition to gen2 is stuck, reset PHY!\n");
f95d3ae7	521	dev_dbg(pp->dev, "debug_r0=%08x debug_r1=%08x\n", debug_r0, rc);
bb38919e	522
982aa234	523	imx6_pcie_reset_phy(pp);
bb38919e SC	524
	525	return 0;
	526	}
	527
	528	static struct pcie_host_ops imx6_pcie_host_ops = {
	529	.link_up = imx6_pcie_link_up,
	530	.host_init = imx6_pcie_host_init,
	531	};
	532
44cb5e94	533	static int __init imx6_add_pcie_port(struct pcie_port *pp,
bb38919e SC	534	struct platform_device *pdev)
	535	{
	536	int ret;
	537
d1dc9749 LS	538	if (IS_ENABLED(CONFIG_PCI_MSI)) {
	539	pp->msi_irq = platform_get_irq_byname(pdev, "msi");
	540	if (pp->msi_irq <= 0) {
	541	dev_err(&pdev->dev, "failed to get MSI irq\n");
	542	return -ENODEV;
	543	}
	544
	545	ret = devm_request_irq(&pdev->dev, pp->msi_irq,
d88a7ef9 JH	546	imx6_pcie_msi_handler,
d88a7ef9 JH	547	IRQF_SHARED, "mx6-pcie-msi", pp);
d1dc9749 LS	548	if (ret) {
	549	dev_err(&pdev->dev, "failed to request MSI irq\n");
	550	return -ENODEV;
	551	}
	552	}
	553
bb38919e SC	554	pp->root_bus_nr = -1;
	555	pp->ops = &imx6_pcie_host_ops;
	556
bb38919e SC	557	ret = dw_pcie_host_init(pp);
	558	if (ret) {
	559	dev_err(&pdev->dev, "failed to initialize host\n");
	560	return ret;
	561	}
	562
	563	return 0;
	564	}
	565
	566	static int __init imx6_pcie_probe(struct platform_device *pdev)
	567	{
	568	struct imx6_pcie *imx6_pcie;
	569	struct pcie_port *pp;
	570	struct device_node *np = pdev->dev.of_node;
	571	struct resource *dbi_base;
	572	int ret;
	573
	574	imx6_pcie = devm_kzalloc(&pdev->dev, sizeof(*imx6_pcie), GFP_KERNEL);
	575	if (!imx6_pcie)
	576	return -ENOMEM;
	577
	578	pp = &imx6_pcie->pp;
	579	pp->dev = &pdev->dev;
	580
	581	/* Added for PCI abort handling */
	582	hook_fault_code(16 + 6, imx6q_pcie_abort_handler, SIGBUS, 0,
	583	"imprecise external abort");
	584
	585	dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
bb38919e	586	pp->dbi_base = devm_ioremap_resource(&pdev->dev, dbi_base);
b391bf31 FE	587	if (IS_ERR(pp->dbi_base))
b391bf31 FE	588	return PTR_ERR(pp->dbi_base);
bb38919e SC	589
	590	/* Fetch GPIOs */
	591	imx6_pcie->reset_gpio = of_get_named_gpio(np, "reset-gpio", 0);
c28f8a1f MV	592	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
	593	ret = devm_gpio_request_one(&pdev->dev, imx6_pcie->reset_gpio,
	594	GPIOF_OUT_INIT_LOW, "PCIe reset");
	595	if (ret) {
	596	dev_err(&pdev->dev, "unable to get reset gpio\n");
	597	return ret;
	598	}
bb38919e SC	599	}
bb38919e SC	600
bb38919e	601	/* Fetch clocks */
57526136 LS	602	imx6_pcie->pcie_phy = devm_clk_get(&pdev->dev, "pcie_phy");
57526136 LS	603	if (IS_ERR(imx6_pcie->pcie_phy)) {
bb38919e	604	dev_err(&pdev->dev,
57526136 LS	605	"pcie_phy clock source missing or invalid\n");
57526136 LS	606	return PTR_ERR(imx6_pcie->pcie_phy);
bb38919e SC	607	}
bb38919e SC	608
57526136 LS	609	imx6_pcie->pcie_bus = devm_clk_get(&pdev->dev, "pcie_bus");
57526136 LS	610	if (IS_ERR(imx6_pcie->pcie_bus)) {
bb38919e	611	dev_err(&pdev->dev,
57526136 LS	612	"pcie_bus clock source missing or invalid\n");
57526136 LS	613	return PTR_ERR(imx6_pcie->pcie_bus);
bb38919e SC	614	}
bb38919e SC	615
57526136 LS	616	imx6_pcie->pcie = devm_clk_get(&pdev->dev, "pcie");
57526136 LS	617	if (IS_ERR(imx6_pcie->pcie)) {
bb38919e	618	dev_err(&pdev->dev,
57526136 LS	619	"pcie clock source missing or invalid\n");
57526136 LS	620	return PTR_ERR(imx6_pcie->pcie);
bb38919e SC	621	}
	622
	623	/* Grab GPR config register range */
	624	imx6_pcie->iomuxc_gpr =
	625	syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr");
	626	if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
	627	dev_err(&pdev->dev, "unable to find iomuxc registers\n");
b391bf31	628	return PTR_ERR(imx6_pcie->iomuxc_gpr);
bb38919e SC	629	}
	630
	631	ret = imx6_add_pcie_port(pp, pdev);
	632	if (ret < 0)
b391bf31	633	return ret;
bb38919e SC	634
	635	platform_set_drvdata(pdev, imx6_pcie);
	636	return 0;
bb38919e SC	637	}
bb38919e SC	638
3e3e406e LS	639	static void imx6_pcie_shutdown(struct platform_device *pdev)
	640	{
	641	struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev);
	642
	643	/* bring down link, so bootloader gets clean state in case of reboot */
	644	imx6_pcie_assert_core_reset(&imx6_pcie->pp);
	645	}
	646
bb38919e SC	647	static const struct of_device_id imx6_pcie_of_match[] = {
	648	{ .compatible = "fsl,imx6q-pcie", },
	649	{},
	650	};
	651	MODULE_DEVICE_TABLE(of, imx6_pcie_of_match);
	652
	653	static struct platform_driver imx6_pcie_driver = {
	654	.driver = {
	655	.name = "imx6q-pcie",
8bcadbe1	656	.of_match_table = imx6_pcie_of_match,
bb38919e	657	},
3e3e406e	658	.shutdown = imx6_pcie_shutdown,
bb38919e SC	659	};
	660
	661	/* Freescale PCIe driver does not allow module unload */
	662
	663	static int __init imx6_pcie_init(void)
	664	{
	665	return platform_driver_probe(&imx6_pcie_driver, imx6_pcie_probe);
	666	}
61da50da	667	module_init(imx6_pcie_init);
bb38919e SC	668
	669	MODULE_AUTHOR("Sean Cross <xobs@kosagi.com>");
	670	MODULE_DESCRIPTION("Freescale i.MX6 PCIe host controller driver");
	671	MODULE_LICENSE("GPL v2");