[mirror_ubuntu-disco-kernel.git] / drivers / memory / tegra / mc.c

/*
 * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/sort.h>

#include <soc/tegra/fuse.h>

#include "mc.h"

#define MC_INTSTATUS 0x000

#define MC_INTMASK 0x004

#define MC_ERR_STATUS 0x08
#define  MC_ERR_STATUS_TYPE_SHIFT 28
#define  MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE (6 << MC_ERR_STATUS_TYPE_SHIFT)
#define  MC_ERR_STATUS_TYPE_MASK (0x7 << MC_ERR_STATUS_TYPE_SHIFT)
#define  MC_ERR_STATUS_READABLE (1 << 27)
#define  MC_ERR_STATUS_WRITABLE (1 << 26)
#define  MC_ERR_STATUS_NONSECURE (1 << 25)
#define  MC_ERR_STATUS_ADR_HI_SHIFT 20
#define  MC_ERR_STATUS_ADR_HI_MASK 0x3
#define  MC_ERR_STATUS_SECURITY (1 << 17)
#define  MC_ERR_STATUS_RW (1 << 16)

#define MC_ERR_ADR 0x0c

#define MC_DECERR_EMEM_OTHERS_STATUS	0x58
#define MC_SECURITY_VIOLATION_STATUS	0x74

#define MC_EMEM_ARB_CFG 0x90
#define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x)	(((x) & 0x1ff) << 0)
#define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK	0x1ff
#define MC_EMEM_ARB_MISC0 0xd8

#define MC_EMEM_ADR_CFG 0x54
#define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0)

static const struct of_device_id tegra_mc_of_match[] = {
#ifdef CONFIG_ARCH_TEGRA_2x_SOC
	{ .compatible = "nvidia,tegra20-mc", .data = &tegra20_mc_soc },
#endif
#ifdef CONFIG_ARCH_TEGRA_3x_SOC
	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
#endif
#ifdef CONFIG_ARCH_TEGRA_114_SOC
	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
#endif
#ifdef CONFIG_ARCH_TEGRA_124_SOC
	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
#endif
#ifdef CONFIG_ARCH_TEGRA_132_SOC
	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
#endif
#ifdef CONFIG_ARCH_TEGRA_210_SOC
	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
#endif
	{ }
};
MODULE_DEVICE_TABLE(of, tegra_mc_of_match);

static int terga_mc_block_dma_common(struct tegra_mc *mc,
				     const struct tegra_mc_reset *rst)
{
	unsigned long flags;
	u32 value;

	spin_lock_irqsave(&mc->lock, flags);

	value = mc_readl(mc, rst->control) | BIT(rst->bit);
	mc_writel(mc, value, rst->control);

	spin_unlock_irqrestore(&mc->lock, flags);

	return 0;
}

static bool terga_mc_dma_idling_common(struct tegra_mc *mc,
				       const struct tegra_mc_reset *rst)
{
	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
}

static int terga_mc_unblock_dma_common(struct tegra_mc *mc,
				       const struct tegra_mc_reset *rst)
{
	unsigned long flags;
	u32 value;

	spin_lock_irqsave(&mc->lock, flags);

	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
	mc_writel(mc, value, rst->control);

	spin_unlock_irqrestore(&mc->lock, flags);

	return 0;
}

static int terga_mc_reset_status_common(struct tegra_mc *mc,
					const struct tegra_mc_reset *rst)
{
	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
}

const struct tegra_mc_reset_ops terga_mc_reset_ops_common = {
	.block_dma = terga_mc_block_dma_common,
	.dma_idling = terga_mc_dma_idling_common,
	.unblock_dma = terga_mc_unblock_dma_common,
	.reset_status = terga_mc_reset_status_common,
};

static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
{
	return container_of(rcdev, struct tegra_mc, reset);
}

static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
							unsigned long id)
{
	unsigned int i;

	for (i = 0; i < mc->soc->num_resets; i++)
		if (mc->soc->resets[i].id == id)
			return &mc->soc->resets[i];

	return NULL;
}

static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
				    unsigned long id)
{
	struct tegra_mc *mc = reset_to_mc(rcdev);
	const struct tegra_mc_reset_ops *rst_ops;
	const struct tegra_mc_reset *rst;
	int retries = 500;
	int err;

	rst = tegra_mc_reset_find(mc, id);
	if (!rst)
		return -ENODEV;

	rst_ops = mc->soc->reset_ops;
	if (!rst_ops)
		return -ENODEV;

	if (rst_ops->block_dma) {
		/* block clients DMA requests */
		err = rst_ops->block_dma(mc, rst);
		if (err) {
			dev_err(mc->dev, "Failed to block %s DMA: %d\n",
				rst->name, err);
			return err;
		}
	}

	if (rst_ops->dma_idling) {
		/* wait for completion of the outstanding DMA requests */
		while (!rst_ops->dma_idling(mc, rst)) {
			if (!retries--) {
				dev_err(mc->dev, "Failed to flush %s DMA\n",
					rst->name);
				return -EBUSY;
			}

			usleep_range(10, 100);
		}
	}

	if (rst_ops->hotreset_assert) {
		/* clear clients DMA requests sitting before arbitration */
		err = rst_ops->hotreset_assert(mc, rst);
		if (err) {
			dev_err(mc->dev, "Failed to hot reset %s: %d\n",
				rst->name, err);
			return err;
		}
	}

	return 0;
}

static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
				      unsigned long id)
{
	struct tegra_mc *mc = reset_to_mc(rcdev);
	const struct tegra_mc_reset_ops *rst_ops;
	const struct tegra_mc_reset *rst;
	int err;

	rst = tegra_mc_reset_find(mc, id);
	if (!rst)
		return -ENODEV;

	rst_ops = mc->soc->reset_ops;
	if (!rst_ops)
		return -ENODEV;

	if (rst_ops->hotreset_deassert) {
		/* take out client from hot reset */
		err = rst_ops->hotreset_deassert(mc, rst);
		if (err) {
			dev_err(mc->dev, "Failed to deassert hot reset %s: %d\n",
				rst->name, err);
			return err;
		}
	}

	if (rst_ops->unblock_dma) {
		/* allow new DMA requests to proceed to arbitration */
		err = rst_ops->unblock_dma(mc, rst);
		if (err) {
			dev_err(mc->dev, "Failed to unblock %s DMA : %d\n",
				rst->name, err);
			return err;
		}
	}

	return 0;
}

static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
				    unsigned long id)
{
	struct tegra_mc *mc = reset_to_mc(rcdev);
	const struct tegra_mc_reset_ops *rst_ops;
	const struct tegra_mc_reset *rst;

	rst = tegra_mc_reset_find(mc, id);
	if (!rst)
		return -ENODEV;

	rst_ops = mc->soc->reset_ops;
	if (!rst_ops)
		return -ENODEV;

	return rst_ops->reset_status(mc, rst);
}

static const struct reset_control_ops tegra_mc_reset_ops = {
	.assert = tegra_mc_hotreset_assert,
	.deassert = tegra_mc_hotreset_deassert,
	.status = tegra_mc_hotreset_status,
};

static int tegra_mc_reset_setup(struct tegra_mc *mc)
{
	int err;

	mc->reset.ops = &tegra_mc_reset_ops;
	mc->reset.owner = THIS_MODULE;
	mc->reset.of_node = mc->dev->of_node;
	mc->reset.of_reset_n_cells = 1;
	mc->reset.nr_resets = mc->soc->num_resets;

	err = reset_controller_register(&mc->reset);
	if (err < 0)
		return err;

	return 0;
}

static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
{
	unsigned long long tick;
	unsigned int i;
	u32 value;

	/* compute the number of MC clock cycles per tick */
	tick = mc->tick * clk_get_rate(mc->clk);
	do_div(tick, NSEC_PER_SEC);

	value = readl(mc->regs + MC_EMEM_ARB_CFG);
	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
	value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
	writel(value, mc->regs + MC_EMEM_ARB_CFG);

	/* write latency allowance defaults */
	for (i = 0; i < mc->soc->num_clients; i++) {
		const struct tegra_mc_la *la = &mc->soc->clients[i].la;
		u32 value;

		value = readl(mc->regs + la->reg);
		value &= ~(la->mask << la->shift);
		value |= (la->def & la->mask) << la->shift;
		writel(value, mc->regs + la->reg);
	}

	return 0;
}

void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
{
	unsigned int i;
	struct tegra_mc_timing *timing = NULL;

	for (i = 0; i < mc->num_timings; i++) {
		if (mc->timings[i].rate == rate) {
			timing = &mc->timings[i];
			break;
		}
	}

	if (!timing) {
		dev_err(mc->dev, "no memory timing registered for rate %lu\n",
			rate);
		return;
	}

	for (i = 0; i < mc->soc->num_emem_regs; ++i)
		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
}

unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
{
	u8 dram_count;

	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
	dram_count++;

	return dram_count;
}

static int load_one_timing(struct tegra_mc *mc,
			   struct tegra_mc_timing *timing,
			   struct device_node *node)
{
	int err;
	u32 tmp;

	err = of_property_read_u32(node, "clock-frequency", &tmp);
	if (err) {
		dev_err(mc->dev,
			"timing %s: failed to read rate\n", node->name);
		return err;
	}

	timing->rate = tmp;
	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
					 sizeof(u32), GFP_KERNEL);
	if (!timing->emem_data)
		return -ENOMEM;

	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
					 timing->emem_data,
					 mc->soc->num_emem_regs);
	if (err) {
		dev_err(mc->dev,
			"timing %s: failed to read EMEM configuration\n",
			node->name);
		return err;
	}

	return 0;
}

static int load_timings(struct tegra_mc *mc, struct device_node *node)
{
	struct device_node *child;
	struct tegra_mc_timing *timing;
	int child_count = of_get_child_count(node);
	int i = 0, err;

	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
				   GFP_KERNEL);
	if (!mc->timings)
		return -ENOMEM;

	mc->num_timings = child_count;

	for_each_child_of_node(node, child) {
		timing = &mc->timings[i++];

		err = load_one_timing(mc, timing, child);
		if (err) {
			of_node_put(child);
			return err;
		}
	}

	return 0;
}

static int tegra_mc_setup_timings(struct tegra_mc *mc)
{
	struct device_node *node;
	u32 ram_code, node_ram_code;
	int err;

	ram_code = tegra_read_ram_code();

	mc->num_timings = 0;

	for_each_child_of_node(mc->dev->of_node, node) {
		err = of_property_read_u32(node, "nvidia,ram-code",
					   &node_ram_code);
		if (err || (node_ram_code != ram_code))
			continue;

		err = load_timings(mc, node);
		of_node_put(node);
		if (err)
			return err;
		break;
	}

	if (mc->num_timings == 0)
		dev_warn(mc->dev,
			 "no memory timings for RAM code %u registered\n",
			 ram_code);

	return 0;
}

static const char *const status_names[32] = {
	[ 1] = "External interrupt",
	[ 6] = "EMEM address decode error",
	[ 7] = "GART page fault",
	[ 8] = "Security violation",
	[ 9] = "EMEM arbitration error",
	[10] = "Page fault",
	[11] = "Invalid APB ASID update",
	[12] = "VPR violation",
	[13] = "Secure carveout violation",
	[16] = "MTS carveout violation",
};

static const char *const error_names[8] = {
	[2] = "EMEM decode error",
	[3] = "TrustZone violation",
	[4] = "Carveout violation",
	[6] = "SMMU translation error",
};

static irqreturn_t tegra_mc_irq(int irq, void *data)
{
	struct tegra_mc *mc = data;
	unsigned long status;
	unsigned int bit;

	/* mask all interrupts to avoid flooding */
	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
	if (!status)
		return IRQ_NONE;

	for_each_set_bit(bit, &status, 32) {
		const char *error = status_names[bit] ?: "unknown";
		const char *client = "unknown", *desc;
		const char *direction, *secure;
		phys_addr_t addr = 0;
		unsigned int i;
		char perm[7];
		u8 id, type;
		u32 value;

		value = mc_readl(mc, MC_ERR_STATUS);

#ifdef CONFIG_PHYS_ADDR_T_64BIT
		if (mc->soc->num_address_bits > 32) {
			addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
				MC_ERR_STATUS_ADR_HI_MASK);
			addr <<= 32;
		}
#endif

		if (value & MC_ERR_STATUS_RW)
			direction = "write";
		else
			direction = "read";

		if (value & MC_ERR_STATUS_SECURITY)
			secure = "secure ";
		else
			secure = "";

		id = value & mc->soc->client_id_mask;

		for (i = 0; i < mc->soc->num_clients; i++) {
			if (mc->soc->clients[i].id == id) {
				client = mc->soc->clients[i].name;
				break;
			}
		}

		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
		       MC_ERR_STATUS_TYPE_SHIFT;
		desc = error_names[type];

		switch (value & MC_ERR_STATUS_TYPE_MASK) {
		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
			perm[0] = ' ';
			perm[1] = '[';

			if (value & MC_ERR_STATUS_READABLE)
				perm[2] = 'R';
			else
				perm[2] = '-';

			if (value & MC_ERR_STATUS_WRITABLE)
				perm[3] = 'W';
			else
				perm[3] = '-';

			if (value & MC_ERR_STATUS_NONSECURE)
				perm[4] = '-';
			else
				perm[4] = 'S';

			perm[5] = ']';
			perm[6] = '\0';
			break;

		default:
			perm[0] = '\0';
			break;
		}

		value = mc_readl(mc, MC_ERR_ADR);
		addr |= value;

		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
				    client, secure, direction, &addr, error,
				    desc, perm);
	}

	/* clear interrupts */
	mc_writel(mc, status, MC_INTSTATUS);

	return IRQ_HANDLED;
}

static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
{
	struct tegra_mc *mc = data;
	unsigned long status;
	unsigned int bit;

	/* mask all interrupts to avoid flooding */
	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
	if (!status)
		return IRQ_NONE;

	for_each_set_bit(bit, &status, 32) {
		const char *direction = "read", *secure = "";
		const char *error = status_names[bit];
		const char *client, *desc;
		phys_addr_t addr;
		u32 value, reg;
		u8 id, type;

		switch (BIT(bit)) {
		case MC_INT_DECERR_EMEM:
			reg = MC_DECERR_EMEM_OTHERS_STATUS;
			value = mc_readl(mc, reg);

			id = value & mc->soc->client_id_mask;
			desc = error_names[2];

			if (value & BIT(31))
				direction = "write";
			break;

		case MC_INT_INVALID_GART_PAGE:
			dev_err_ratelimited(mc->dev, "%s\n", error);
			continue;

		case MC_INT_SECURITY_VIOLATION:
			reg = MC_SECURITY_VIOLATION_STATUS;
			value = mc_readl(mc, reg);

			id = value & mc->soc->client_id_mask;
			type = (value & BIT(30)) ? 4 : 3;
			desc = error_names[type];
			secure = "secure ";

			if (value & BIT(31))
				direction = "write";
			break;

		default:
			continue;
		}

		client = mc->soc->clients[id].name;
		addr = mc_readl(mc, reg + sizeof(u32));

		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
				    client, secure, direction, &addr, error,
				    desc);
	}

	/* clear interrupts */
	mc_writel(mc, status, MC_INTSTATUS);

	return IRQ_HANDLED;
}

static int tegra_mc_probe(struct platform_device *pdev)
{
	const struct of_device_id *match;
	struct resource *res;
	struct tegra_mc *mc;
	void *isr;
	int err;

	match = of_match_node(tegra_mc_of_match, pdev->dev.of_node);
	if (!match)
		return -ENODEV;

	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
	if (!mc)
		return -ENOMEM;

	platform_set_drvdata(pdev, mc);
	spin_lock_init(&mc->lock);
	mc->soc = match->data;
	mc->dev = &pdev->dev;

	/* length of MC tick in nanoseconds */
	mc->tick = 30;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	mc->regs = devm_ioremap_resource(&pdev->dev, res);
	if (IS_ERR(mc->regs))
		return PTR_ERR(mc->regs);

#ifdef CONFIG_ARCH_TEGRA_2x_SOC
	if (mc->soc == &tegra20_mc_soc) {
		res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
		mc->regs2 = devm_ioremap_resource(&pdev->dev, res);
		if (IS_ERR(mc->regs2))
			return PTR_ERR(mc->regs2);

		isr = tegra20_mc_irq;
	} else
#endif
	{
		mc->clk = devm_clk_get(&pdev->dev, "mc");
		if (IS_ERR(mc->clk)) {
			dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
				PTR_ERR(mc->clk));
			return PTR_ERR(mc->clk);
		}

		err = tegra_mc_setup_latency_allowance(mc);
		if (err < 0) {
			dev_err(&pdev->dev, "failed to setup latency allowance: %d\n",
				err);
			return err;
		}

		isr = tegra_mc_irq;
	}

	err = tegra_mc_setup_timings(mc);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to setup timings: %d\n", err);
		return err;
	}

	mc->irq = platform_get_irq(pdev, 0);
	if (mc->irq < 0) {
		dev_err(&pdev->dev, "interrupt not specified\n");
		return mc->irq;
	}

	WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n");

	mc_writel(mc, mc->soc->intmask, MC_INTMASK);

	err = devm_request_irq(&pdev->dev, mc->irq, isr, IRQF_SHARED,
			       dev_name(&pdev->dev), mc);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
			err);
		return err;
	}

	err = tegra_mc_reset_setup(mc);
	if (err < 0)
		dev_err(&pdev->dev, "failed to register reset controller: %d\n",
			err);

	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) {
		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
		if (IS_ERR(mc->smmu))
			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
				PTR_ERR(mc->smmu));
	}

	return 0;
}

static struct platform_driver tegra_mc_driver = {
	.driver = {
		.name = "tegra-mc",
		.of_match_table = tegra_mc_of_match,
		.suppress_bind_attrs = true,
	},
	.prevent_deferred_probe = true,
	.probe = tegra_mc_probe,
};

static int tegra_mc_init(void)
{
	return platform_driver_register(&tegra_mc_driver);
}
arch_initcall(tegra_mc_init);

MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
89184651 TR	1	/*
	2	* Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License version 2 as
	6	* published by the Free Software Foundation.
	7	*/
	8
	9	#include <linux/clk.h>
20e92462	10	#include <linux/delay.h>
89184651 TR	11	#include <linux/interrupt.h>
	12	#include <linux/kernel.h>
	13	#include <linux/module.h>
	14	#include <linux/of.h>
	15	#include <linux/platform_device.h>
	16	#include <linux/slab.h>
3d9dd6fd MP	17	#include <linux/sort.h>
	18
	19	#include <soc/tegra/fuse.h>
89184651 TR	20
	21	#include "mc.h"
	22
	23	#define MC_INTSTATUS 0x000
89184651 TR	24
	25	#define MC_INTMASK 0x004
	26
	27	#define MC_ERR_STATUS 0x08
	28	#define MC_ERR_STATUS_TYPE_SHIFT 28
	29	#define MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE (6 << MC_ERR_STATUS_TYPE_SHIFT)
	30	#define MC_ERR_STATUS_TYPE_MASK (0x7 << MC_ERR_STATUS_TYPE_SHIFT)
	31	#define MC_ERR_STATUS_READABLE (1 << 27)
	32	#define MC_ERR_STATUS_WRITABLE (1 << 26)
	33	#define MC_ERR_STATUS_NONSECURE (1 << 25)
	34	#define MC_ERR_STATUS_ADR_HI_SHIFT 20
	35	#define MC_ERR_STATUS_ADR_HI_MASK 0x3
	36	#define MC_ERR_STATUS_SECURITY (1 << 17)
	37	#define MC_ERR_STATUS_RW (1 << 16)
89184651 TR	38
	39	#define MC_ERR_ADR 0x0c
	40
a8d502fd DO	41	#define MC_DECERR_EMEM_OTHERS_STATUS 0x58
	42	#define MC_SECURITY_VIOLATION_STATUS 0x74
	43
89184651 TR	44	#define MC_EMEM_ARB_CFG 0x90
	45	#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x) (((x) & 0x1ff) << 0)
	46	#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK 0x1ff
	47	#define MC_EMEM_ARB_MISC0 0xd8
	48
3d9dd6fd MP	49	#define MC_EMEM_ADR_CFG 0x54
	50	#define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0)
	51
89184651	52	static const struct of_device_id tegra_mc_of_match[] = {
a8d502fd DO	53	#ifdef CONFIG_ARCH_TEGRA_2x_SOC
	54	{ .compatible = "nvidia,tegra20-mc", .data = &tegra20_mc_soc },
	55	#endif
89184651 TR	56	#ifdef CONFIG_ARCH_TEGRA_3x_SOC
	57	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
	58	#endif
	59	#ifdef CONFIG_ARCH_TEGRA_114_SOC
	60	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
	61	#endif
	62	#ifdef CONFIG_ARCH_TEGRA_124_SOC
	63	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
242b1d71 TR	64	#endif
	65	#ifdef CONFIG_ARCH_TEGRA_132_SOC
	66	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
588c43a7 TR	67	#endif
	68	#ifdef CONFIG_ARCH_TEGRA_210_SOC
	69	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
89184651 TR	70	#endif
	71	{ }
	72	};
	73	MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
	74
20e92462 DO	75	static int terga_mc_block_dma_common(struct tegra_mc *mc,
	76	const struct tegra_mc_reset *rst)
	77	{
	78	unsigned long flags;
	79	u32 value;
	80
	81	spin_lock_irqsave(&mc->lock, flags);
	82
	83	value = mc_readl(mc, rst->control) \| BIT(rst->bit);
	84	mc_writel(mc, value, rst->control);
	85
	86	spin_unlock_irqrestore(&mc->lock, flags);
	87
	88	return 0;
	89	}
	90
	91	static bool terga_mc_dma_idling_common(struct tegra_mc *mc,
	92	const struct tegra_mc_reset *rst)
	93	{
	94	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
	95	}
	96
	97	static int terga_mc_unblock_dma_common(struct tegra_mc *mc,
	98	const struct tegra_mc_reset *rst)
	99	{
	100	unsigned long flags;
	101	u32 value;
	102
	103	spin_lock_irqsave(&mc->lock, flags);
	104
	105	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
	106	mc_writel(mc, value, rst->control);
	107
	108	spin_unlock_irqrestore(&mc->lock, flags);
	109
	110	return 0;
	111	}
	112
	113	static int terga_mc_reset_status_common(struct tegra_mc *mc,
	114	const struct tegra_mc_reset *rst)
	115	{
	116	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
	117	}
	118
	119	const struct tegra_mc_reset_ops terga_mc_reset_ops_common = {
	120	.block_dma = terga_mc_block_dma_common,
	121	.dma_idling = terga_mc_dma_idling_common,
	122	.unblock_dma = terga_mc_unblock_dma_common,
	123	.reset_status = terga_mc_reset_status_common,
	124	};
	125
	126	static inline struct tegra_mc reset_to_mc(struct reset_controller_dev rcdev)
	127	{
	128	return container_of(rcdev, struct tegra_mc, reset);
	129	}
	130
	131	static const struct tegra_mc_reset tegra_mc_reset_find(struct tegra_mc mc,
	132	unsigned long id)
	133	{
	134	unsigned int i;
	135
	136	for (i = 0; i < mc->soc->num_resets; i++)
	137	if (mc->soc->resets[i].id == id)
	138	return &mc->soc->resets[i];
139
140	return NULL;
141	}
142
143	static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
144	unsigned long id)
145	{
146	struct tegra_mc *mc = reset_to_mc(rcdev);
147	const struct tegra_mc_reset_ops *rst_ops;
148	const struct tegra_mc_reset *rst;
149	int retries = 500;
150	int err;
151
152	rst = tegra_mc_reset_find(mc, id);
153	if (!rst)
154	return -ENODEV;
155
156	rst_ops = mc->soc->reset_ops;
157	if (!rst_ops)
158	return -ENODEV;
159
160	if (rst_ops->block_dma) {
161	/* block clients DMA requests */
162	err = rst_ops->block_dma(mc, rst);
163	if (err) {
164	dev_err(mc->dev, "Failed to block %s DMA: %d\n",
165	rst->name, err);
166	return err;
167	}
168	}
169
170	if (rst_ops->dma_idling) {
171	/* wait for completion of the outstanding DMA requests */
172	while (!rst_ops->dma_idling(mc, rst)) {
173	if (!retries--) {
174	dev_err(mc->dev, "Failed to flush %s DMA\n",
175	rst->name);
176	return -EBUSY;
177	}
178
179	usleep_range(10, 100);
180	}
181	}
182
183	if (rst_ops->hotreset_assert) {
184	/* clear clients DMA requests sitting before arbitration */
185	err = rst_ops->hotreset_assert(mc, rst);
186	if (err) {
187	dev_err(mc->dev, "Failed to hot reset %s: %d\n",
188	rst->name, err);
189	return err;
190	}
191	}
192
193	return 0;
194	}
195
196	static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
197	unsigned long id)
198	{
199	struct tegra_mc *mc = reset_to_mc(rcdev);
200	const struct tegra_mc_reset_ops *rst_ops;
201	const struct tegra_mc_reset *rst;
202	int err;
203
204	rst = tegra_mc_reset_find(mc, id);
205	if (!rst)
206	return -ENODEV;
207
208	rst_ops = mc->soc->reset_ops;
209	if (!rst_ops)
210	return -ENODEV;
211
212	if (rst_ops->hotreset_deassert) {
213	/* take out client from hot reset */
214	err = rst_ops->hotreset_deassert(mc, rst);
215	if (err) {
216	dev_err(mc->dev, "Failed to deassert hot reset %s: %d\n",
217	rst->name, err);
218	return err;
219	}
220	}
221
222	if (rst_ops->unblock_dma) {
223	/* allow new DMA requests to proceed to arbitration */
224	err = rst_ops->unblock_dma(mc, rst);
225	if (err) {
226	dev_err(mc->dev, "Failed to unblock %s DMA : %d\n",
227	rst->name, err);
228	return err;
229	}
230	}
231
232	return 0;
233	}
234
235	static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
236	unsigned long id)
237	{
238	struct tegra_mc *mc = reset_to_mc(rcdev);
239	const struct tegra_mc_reset_ops *rst_ops;
240	const struct tegra_mc_reset *rst;
241
242	rst = tegra_mc_reset_find(mc, id);
243	if (!rst)
244	return -ENODEV;
245
246	rst_ops = mc->soc->reset_ops;
247	if (!rst_ops)
248	return -ENODEV;
249
250	return rst_ops->reset_status(mc, rst);
251	}
252
253	static const struct reset_control_ops tegra_mc_reset_ops = {
254	.assert = tegra_mc_hotreset_assert,
255	.deassert = tegra_mc_hotreset_deassert,
256	.status = tegra_mc_hotreset_status,
257	};
258
259	static int tegra_mc_reset_setup(struct tegra_mc *mc)
260	{
261	int err;
262
263	mc->reset.ops = &tegra_mc_reset_ops;
264	mc->reset.owner = THIS_MODULE;
265	mc->reset.of_node = mc->dev->of_node;
266	mc->reset.of_reset_n_cells = 1;
267	mc->reset.nr_resets = mc->soc->num_resets;
268
269	err = reset_controller_register(&mc->reset);
270	if (err < 0)
271	return err;
272
273	return 0;
274	}
275
89184651 TR	276	static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
	277	{
	278	unsigned long long tick;
	279	unsigned int i;
	280	u32 value;
	281
	282	/* compute the number of MC clock cycles per tick */
	283	tick = mc->tick * clk_get_rate(mc->clk);
	284	do_div(tick, NSEC_PER_SEC);
	285
	286	value = readl(mc->regs + MC_EMEM_ARB_CFG);
	287	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
	288	value \|= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
	289	writel(value, mc->regs + MC_EMEM_ARB_CFG);
	290
	291	/* write latency allowance defaults */
	292	for (i = 0; i < mc->soc->num_clients; i++) {
	293	const struct tegra_mc_la *la = &mc->soc->clients[i].la;
	294	u32 value;
	295
	296	value = readl(mc->regs + la->reg);
	297	value &= ~(la->mask << la->shift);
	298	value \|= (la->def & la->mask) << la->shift;
	299	writel(value, mc->regs + la->reg);
	300	}
	301
	302	return 0;
	303	}
	304
3d9dd6fd MP	305	void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
	306	{
	307	unsigned int i;
	308	struct tegra_mc_timing *timing = NULL;
	309
	310	for (i = 0; i < mc->num_timings; i++) {
	311	if (mc->timings[i].rate == rate) {
	312	timing = &mc->timings[i];
	313	break;
	314	}
	315	}
	316
	317	if (!timing) {
	318	dev_err(mc->dev, "no memory timing registered for rate %lu\n",
	319	rate);
	320	return;
	321	}
	322
	323	for (i = 0; i < mc->soc->num_emem_regs; ++i)
	324	mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
	325	}
	326
	327	unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
	328	{
	329	u8 dram_count;
	330
	331	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
	332	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
	333	dram_count++;
	334
	335	return dram_count;
	336	}
	337
	338	static int load_one_timing(struct tegra_mc *mc,
	339	struct tegra_mc_timing *timing,
	340	struct device_node *node)
	341	{
	342	int err;
	343	u32 tmp;
	344
	345	err = of_property_read_u32(node, "clock-frequency", &tmp);
	346	if (err) {
	347	dev_err(mc->dev,
	348	"timing %s: failed to read rate\n", node->name);
	349	return err;
	350	}
	351
	352	timing->rate = tmp;
	353	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
	354	sizeof(u32), GFP_KERNEL);
	355	if (!timing->emem_data)
	356	return -ENOMEM;
	357
	358	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
	359	timing->emem_data,
	360	mc->soc->num_emem_regs);
	361	if (err) {
	362	dev_err(mc->dev,
	363	"timing %s: failed to read EMEM configuration\n",
	364	node->name);
	365	return err;
	366	}
	367
	368	return 0;
369	}
370
371	static int load_timings(struct tegra_mc mc, struct device_node node)
372	{
373	struct device_node *child;
374	struct tegra_mc_timing *timing;
375	int child_count = of_get_child_count(node);
376	int i = 0, err;
377
378	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
379	GFP_KERNEL);
380	if (!mc->timings)
381	return -ENOMEM;
382
383	mc->num_timings = child_count;
384
385	for_each_child_of_node(node, child) {
386	timing = &mc->timings[i++];
387
388	err = load_one_timing(mc, timing, child);
55bb1d83 AKC	389	if (err) {
55bb1d83 AKC	390	of_node_put(child);
3d9dd6fd	391	return err;
55bb1d83	392	}
3d9dd6fd MP	393	}
	394
	395	return 0;
	396	}
	397
	398	static int tegra_mc_setup_timings(struct tegra_mc *mc)
	399	{
	400	struct device_node *node;
	401	u32 ram_code, node_ram_code;
	402	int err;
	403
	404	ram_code = tegra_read_ram_code();
	405
	406	mc->num_timings = 0;
	407
	408	for_each_child_of_node(mc->dev->of_node, node) {
	409	err = of_property_read_u32(node, "nvidia,ram-code",
	410	&node_ram_code);
d1122e4b	411	if (err \|\| (node_ram_code != ram_code))
3d9dd6fd	412	continue;
3d9dd6fd MP	413
3d9dd6fd MP	414	err = load_timings(mc, node);
55bb1d83	415	of_node_put(node);
3d9dd6fd MP	416	if (err)
3d9dd6fd MP	417	return err;
3d9dd6fd MP	418	break;
	419	}
	420
	421	if (mc->num_timings == 0)
	422	dev_warn(mc->dev,
	423	"no memory timings for RAM code %u registered\n",
	424	ram_code);
	425
	426	return 0;
	427	}
	428
89184651 TR	429	static const char *const status_names[32] = {
	430	[ 1] = "External interrupt",
	431	[ 6] = "EMEM address decode error",
a8d502fd	432	[ 7] = "GART page fault",
89184651 TR	433	[ 8] = "Security violation",
	434	[ 9] = "EMEM arbitration error",
	435	[10] = "Page fault",
	436	[11] = "Invalid APB ASID update",
	437	[12] = "VPR violation",
	438	[13] = "Secure carveout violation",
	439	[16] = "MTS carveout violation",
	440	};
	441
	442	static const char *const error_names[8] = {
	443	[2] = "EMEM decode error",
	444	[3] = "TrustZone violation",
	445	[4] = "Carveout violation",
	446	[6] = "SMMU translation error",
	447	};
	448
	449	static irqreturn_t tegra_mc_irq(int irq, void *data)
	450	{
	451	struct tegra_mc *mc = data;
1c74d5c0	452	unsigned long status;
89184651 TR	453	unsigned int bit;
	454
	455	/* mask all interrupts to avoid flooding */
1c74d5c0	456	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
bf3fbdfb DO	457	if (!status)
bf3fbdfb DO	458	return IRQ_NONE;
89184651 TR	459
	460	for_each_set_bit(bit, &status, 32) {
	461	const char *error = status_names[bit] ?: "unknown";
	462	const char client = "unknown", desc;
	463	const char direction, secure;
	464	phys_addr_t addr = 0;
	465	unsigned int i;
	466	char perm[7];
	467	u8 id, type;
	468	u32 value;
	469
	470	value = mc_readl(mc, MC_ERR_STATUS);
	471
	472	#ifdef CONFIG_PHYS_ADDR_T_64BIT
	473	if (mc->soc->num_address_bits > 32) {
	474	addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
	475	MC_ERR_STATUS_ADR_HI_MASK);
	476	addr <<= 32;
	477	}
	478	#endif
	479
	480	if (value & MC_ERR_STATUS_RW)
	481	direction = "write";
	482	else
	483	direction = "read";
	484
	485	if (value & MC_ERR_STATUS_SECURITY)
	486	secure = "secure ";
	487	else
	488	secure = "";
	489
3c01cf3b	490	id = value & mc->soc->client_id_mask;
89184651 TR	491
	492	for (i = 0; i < mc->soc->num_clients; i++) {
	493	if (mc->soc->clients[i].id == id) {
	494	client = mc->soc->clients[i].name;
	495	break;
	496	}
	497	}
	498
	499	type = (value & MC_ERR_STATUS_TYPE_MASK) >>
	500	MC_ERR_STATUS_TYPE_SHIFT;
	501	desc = error_names[type];
	502
	503	switch (value & MC_ERR_STATUS_TYPE_MASK) {
	504	case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
	505	perm[0] = ' ';
	506	perm[1] = '[';
	507
	508	if (value & MC_ERR_STATUS_READABLE)
	509	perm[2] = 'R';
	510	else
	511	perm[2] = '-';
	512
	513	if (value & MC_ERR_STATUS_WRITABLE)
	514	perm[3] = 'W';
	515	else
	516	perm[3] = '-';
	517
	518	if (value & MC_ERR_STATUS_NONSECURE)
	519	perm[4] = '-';
	520	else
	521	perm[4] = 'S';
	522
	523	perm[5] = ']';
	524	perm[6] = '\0';
	525	break;
	526
	527	default:
	528	perm[0] = '\0';
	529	break;
	530	}
	531
	532	value = mc_readl(mc, MC_ERR_ADR);
	533	addr \|= value;
	534
	535	dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
	536	client, secure, direction, &addr, error,
	537	desc, perm);
	538	}
	539
	540	/* clear interrupts */
	541	mc_writel(mc, status, MC_INTSTATUS);
	542
	543	return IRQ_HANDLED;
	544	}
	545
a8d502fd DO	546	static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
	547	{
	548	struct tegra_mc *mc = data;
	549	unsigned long status;
	550	unsigned int bit;
	551
	552	/* mask all interrupts to avoid flooding */
	553	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
	554	if (!status)
	555	return IRQ_NONE;
	556
	557	for_each_set_bit(bit, &status, 32) {
	558	const char direction = "read", secure = "";
	559	const char *error = status_names[bit];
	560	const char client, desc;
	561	phys_addr_t addr;
	562	u32 value, reg;
	563	u8 id, type;
	564
	565	switch (BIT(bit)) {
	566	case MC_INT_DECERR_EMEM:
	567	reg = MC_DECERR_EMEM_OTHERS_STATUS;
	568	value = mc_readl(mc, reg);
	569
	570	id = value & mc->soc->client_id_mask;
	571	desc = error_names[2];
	572
	573	if (value & BIT(31))
	574	direction = "write";
	575	break;
	576
	577	case MC_INT_INVALID_GART_PAGE:
	578	dev_err_ratelimited(mc->dev, "%s\n", error);
	579	continue;
	580
	581	case MC_INT_SECURITY_VIOLATION:
	582	reg = MC_SECURITY_VIOLATION_STATUS;
	583	value = mc_readl(mc, reg);
	584
	585	id = value & mc->soc->client_id_mask;
	586	type = (value & BIT(30)) ? 4 : 3;
	587	desc = error_names[type];
	588	secure = "secure ";
	589
	590	if (value & BIT(31))
	591	direction = "write";
	592	break;
	593
	594	default:
	595	continue;
	596	}
	597
	598	client = mc->soc->clients[id].name;
	599	addr = mc_readl(mc, reg + sizeof(u32));
	600
	601	dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
	602	client, secure, direction, &addr, error,
	603	desc);
	604	}
	605
	606	/* clear interrupts */
	607	mc_writel(mc, status, MC_INTSTATUS);
	608
	609	return IRQ_HANDLED;
610	}
611
89184651 TR	612	static int tegra_mc_probe(struct platform_device *pdev)
	613	{
	614	const struct of_device_id *match;
	615	struct resource *res;
	616	struct tegra_mc *mc;
a8d502fd	617	void *isr;
89184651 TR	618	int err;
	619
	620	match = of_match_node(tegra_mc_of_match, pdev->dev.of_node);
	621	if (!match)
	622	return -ENODEV;
	623
	624	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
	625	if (!mc)
	626	return -ENOMEM;
	627
	628	platform_set_drvdata(pdev, mc);
20e92462	629	spin_lock_init(&mc->lock);
89184651 TR	630	mc->soc = match->data;
	631	mc->dev = &pdev->dev;
	632
	633	/* length of MC tick in nanoseconds */
	634	mc->tick = 30;
	635
	636	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	637	mc->regs = devm_ioremap_resource(&pdev->dev, res);
	638	if (IS_ERR(mc->regs))
	639	return PTR_ERR(mc->regs);
	640
a8d502fd DO	641	#ifdef CONFIG_ARCH_TEGRA_2x_SOC
	642	if (mc->soc == &tegra20_mc_soc) {
	643	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
	644	mc->regs2 = devm_ioremap_resource(&pdev->dev, res);
	645	if (IS_ERR(mc->regs2))
	646	return PTR_ERR(mc->regs2);
89184651	647
a8d502fd DO	648	isr = tegra20_mc_irq;
	649	} else
	650	#endif
	651	{
	652	mc->clk = devm_clk_get(&pdev->dev, "mc");
	653	if (IS_ERR(mc->clk)) {
	654	dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
	655	PTR_ERR(mc->clk));
	656	return PTR_ERR(mc->clk);
	657	}
	658
	659	err = tegra_mc_setup_latency_allowance(mc);
	660	if (err < 0) {
	661	dev_err(&pdev->dev, "failed to setup latency allowance: %d\n",
	662	err);
	663	return err;
	664	}
	665
	666	isr = tegra_mc_irq;
89184651 TR	667	}
89184651 TR	668
3d9dd6fd MP	669	err = tegra_mc_setup_timings(mc);
	670	if (err < 0) {
	671	dev_err(&pdev->dev, "failed to setup timings: %d\n", err);
	672	return err;
	673	}
	674
89184651 TR	675	mc->irq = platform_get_irq(pdev, 0);
	676	if (mc->irq < 0) {
	677	dev_err(&pdev->dev, "interrupt not specified\n");
	678	return mc->irq;
	679	}
	680
3c01cf3b PW	681	WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n");
3c01cf3b PW	682
1c74d5c0	683	mc_writel(mc, mc->soc->intmask, MC_INTMASK);
89184651	684
a8d502fd	685	err = devm_request_irq(&pdev->dev, mc->irq, isr, IRQF_SHARED,
db4a9c19 DO	686	dev_name(&pdev->dev), mc);
	687	if (err < 0) {
	688	dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
	689	err);
	690	return err;
	691	}
	692
1662dd64 DO	693	err = tegra_mc_reset_setup(mc);
	694	if (err < 0)
	695	dev_err(&pdev->dev, "failed to register reset controller: %d\n",
	696	err);
	697
45a81df0 DO	698	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) {
45a81df0 DO	699	mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
1662dd64	700	if (IS_ERR(mc->smmu))
45a81df0 DO	701	dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
45a81df0 DO	702	PTR_ERR(mc->smmu));
45a81df0 DO	703	}
45a81df0 DO	704
89184651 TR	705	return 0;
	706	}
	707
	708	static struct platform_driver tegra_mc_driver = {
	709	.driver = {
	710	.name = "tegra-mc",
	711	.of_match_table = tegra_mc_of_match,
	712	.suppress_bind_attrs = true,
	713	},
	714	.prevent_deferred_probe = true,
	715	.probe = tegra_mc_probe,
	716	};
	717
	718	static int tegra_mc_init(void)
	719	{
	720	return platform_driver_register(&tegra_mc_driver);
	721	}
	722	arch_initcall(tegra_mc_init);
	723
	724	MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
	725	MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
	726	MODULE_LICENSE("GPL v2");