[mirror_ubuntu-jammy-kernel.git] / drivers / iommu / arm-smmu.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * IOMMU API for ARM architected SMMU implementations.
 *
 * Copyright (C) 2013 ARM Limited
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

#ifndef _ARM_SMMU_H
#define _ARM_SMMU_H

#include <linux/atomic.h>
#include <linux/bitfield.h>
#include <linux/bits.h>
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/io-pgtable.h>
#include <linux/iommu.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/types.h>

/* Configuration registers */
#define ARM_SMMU_GR0_sCR0		0x0
#define ARM_SMMU_sCR0_VMID16EN		BIT(31)
#define ARM_SMMU_sCR0_BSU		GENMASK(15, 14)
#define ARM_SMMU_sCR0_FB		BIT(13)
#define ARM_SMMU_sCR0_PTM		BIT(12)
#define ARM_SMMU_sCR0_VMIDPNE		BIT(11)
#define ARM_SMMU_sCR0_USFCFG		BIT(10)
#define ARM_SMMU_sCR0_GCFGFIE		BIT(5)
#define ARM_SMMU_sCR0_GCFGFRE		BIT(4)
#define ARM_SMMU_sCR0_EXIDENABLE	BIT(3)
#define ARM_SMMU_sCR0_GFIE		BIT(2)
#define ARM_SMMU_sCR0_GFRE		BIT(1)
#define ARM_SMMU_sCR0_CLIENTPD		BIT(0)

/* Auxiliary Configuration register */
#define ARM_SMMU_GR0_sACR		0x10

/* Identification registers */
#define ARM_SMMU_GR0_ID0		0x20
#define ARM_SMMU_ID0_S1TS		BIT(30)
#define ARM_SMMU_ID0_S2TS		BIT(29)
#define ARM_SMMU_ID0_NTS		BIT(28)
#define ARM_SMMU_ID0_SMS		BIT(27)
#define ARM_SMMU_ID0_ATOSNS		BIT(26)
#define ARM_SMMU_ID0_PTFS_NO_AARCH32	BIT(25)
#define ARM_SMMU_ID0_PTFS_NO_AARCH32S	BIT(24)
#define ARM_SMMU_ID0_NUMIRPT		GENMASK(23, 16)
#define ARM_SMMU_ID0_CTTW		BIT(14)
#define ARM_SMMU_ID0_NUMSIDB		GENMASK(12, 9)
#define ARM_SMMU_ID0_EXIDS		BIT(8)
#define ARM_SMMU_ID0_NUMSMRG		GENMASK(7, 0)

#define ARM_SMMU_GR0_ID1		0x24
#define ARM_SMMU_ID1_PAGESIZE		BIT(31)
#define ARM_SMMU_ID1_NUMPAGENDXB	GENMASK(30, 28)
#define ARM_SMMU_ID1_NUMS2CB		GENMASK(23, 16)
#define ARM_SMMU_ID1_NUMCB		GENMASK(7, 0)

#define ARM_SMMU_GR0_ID2		0x28
#define ARM_SMMU_ID2_VMID16		BIT(15)
#define ARM_SMMU_ID2_PTFS_64K		BIT(14)
#define ARM_SMMU_ID2_PTFS_16K		BIT(13)
#define ARM_SMMU_ID2_PTFS_4K		BIT(12)
#define ARM_SMMU_ID2_UBS		GENMASK(11, 8)
#define ARM_SMMU_ID2_OAS		GENMASK(7, 4)
#define ARM_SMMU_ID2_IAS		GENMASK(3, 0)

#define ARM_SMMU_GR0_ID3		0x2c
#define ARM_SMMU_GR0_ID4		0x30
#define ARM_SMMU_GR0_ID5		0x34
#define ARM_SMMU_GR0_ID6		0x38

#define ARM_SMMU_GR0_ID7		0x3c
#define ARM_SMMU_ID7_MAJOR		GENMASK(7, 4)
#define ARM_SMMU_ID7_MINOR		GENMASK(3, 0)

#define ARM_SMMU_GR0_sGFSR		0x48
#define ARM_SMMU_sGFSR_USF		BIT(1)

#define ARM_SMMU_GR0_sGFSYNR0		0x50
#define ARM_SMMU_GR0_sGFSYNR1		0x54
#define ARM_SMMU_GR0_sGFSYNR2		0x58

/* Global TLB invalidation */
#define ARM_SMMU_GR0_TLBIVMID		0x64
#define ARM_SMMU_GR0_TLBIALLNSNH	0x68
#define ARM_SMMU_GR0_TLBIALLH		0x6c
#define ARM_SMMU_GR0_sTLBGSYNC		0x70

#define ARM_SMMU_GR0_sTLBGSTATUS	0x74
#define ARM_SMMU_sTLBGSTATUS_GSACTIVE	BIT(0)

/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
#define ARM_SMMU_SMR_VALID		BIT(31)
#define ARM_SMMU_SMR_MASK		GENMASK(31, 16)
#define ARM_SMMU_SMR_ID			GENMASK(15, 0)

#define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
#define ARM_SMMU_S2CR_PRIVCFG		GENMASK(25, 24)
enum arm_smmu_s2cr_privcfg {
	S2CR_PRIVCFG_DEFAULT,
	S2CR_PRIVCFG_DIPAN,
	S2CR_PRIVCFG_UNPRIV,
	S2CR_PRIVCFG_PRIV,
};
#define ARM_SMMU_S2CR_TYPE		GENMASK(17, 16)
enum arm_smmu_s2cr_type {
	S2CR_TYPE_TRANS,
	S2CR_TYPE_BYPASS,
	S2CR_TYPE_FAULT,
};
#define ARM_SMMU_S2CR_EXIDVALID		BIT(10)
#define ARM_SMMU_S2CR_CBNDX		GENMASK(7, 0)

/* Context bank attribute registers */
#define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
#define ARM_SMMU_CBAR_IRPTNDX		GENMASK(31, 24)
#define ARM_SMMU_CBAR_TYPE		GENMASK(17, 16)
enum arm_smmu_cbar_type {
	CBAR_TYPE_S2_TRANS,
	CBAR_TYPE_S1_TRANS_S2_BYPASS,
	CBAR_TYPE_S1_TRANS_S2_FAULT,
	CBAR_TYPE_S1_TRANS_S2_TRANS,
};
#define ARM_SMMU_CBAR_S1_MEMATTR	GENMASK(15, 12)
#define ARM_SMMU_CBAR_S1_MEMATTR_WB	0xf
#define ARM_SMMU_CBAR_S1_BPSHCFG	GENMASK(9, 8)
#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH	3
#define ARM_SMMU_CBAR_VMID		GENMASK(7, 0)

#define ARM_SMMU_GR1_CBFRSYNRA(n)	(0x400 + ((n) << 2))

#define ARM_SMMU_GR1_CBA2R(n)		(0x800 + ((n) << 2))
#define ARM_SMMU_CBA2R_VMID16		GENMASK(31, 16)
#define ARM_SMMU_CBA2R_VA64		BIT(0)

#define ARM_SMMU_CB_SCTLR		0x0
#define ARM_SMMU_SCTLR_S1_ASIDPNE	BIT(12)
#define ARM_SMMU_SCTLR_CFCFG		BIT(7)
#define ARM_SMMU_SCTLR_CFIE		BIT(6)
#define ARM_SMMU_SCTLR_CFRE		BIT(5)
#define ARM_SMMU_SCTLR_E		BIT(4)
#define ARM_SMMU_SCTLR_AFE		BIT(2)
#define ARM_SMMU_SCTLR_TRE		BIT(1)
#define ARM_SMMU_SCTLR_M		BIT(0)

#define ARM_SMMU_CB_ACTLR		0x4

#define ARM_SMMU_CB_RESUME		0x8
#define ARM_SMMU_RESUME_TERMINATE	BIT(0)

#define ARM_SMMU_CB_TCR2		0x10
#define ARM_SMMU_TCR2_SEP		GENMASK(17, 15)
#define ARM_SMMU_TCR2_SEP_UPSTREAM	0x7
#define ARM_SMMU_TCR2_AS		BIT(4)
#define ARM_SMMU_TCR2_PASIZE		GENMASK(3, 0)

#define ARM_SMMU_CB_TTBR0		0x20
#define ARM_SMMU_CB_TTBR1		0x28
#define ARM_SMMU_TTBRn_ASID		GENMASK_ULL(63, 48)

#define ARM_SMMU_CB_TCR			0x30
#define ARM_SMMU_TCR_EAE		BIT(31)
#define ARM_SMMU_TCR_EPD1		BIT(23)
#define ARM_SMMU_TCR_TG0		GENMASK(15, 14)
#define ARM_SMMU_TCR_SH0		GENMASK(13, 12)
#define ARM_SMMU_TCR_ORGN0		GENMASK(11, 10)
#define ARM_SMMU_TCR_IRGN0		GENMASK(9, 8)
#define ARM_SMMU_TCR_T0SZ		GENMASK(5, 0)

#define ARM_SMMU_VTCR_RES1		BIT(31)
#define ARM_SMMU_VTCR_PS		GENMASK(18, 16)
#define ARM_SMMU_VTCR_TG0		ARM_SMMU_TCR_TG0
#define ARM_SMMU_VTCR_SH0		ARM_SMMU_TCR_SH0
#define ARM_SMMU_VTCR_ORGN0		ARM_SMMU_TCR_ORGN0
#define ARM_SMMU_VTCR_IRGN0		ARM_SMMU_TCR_IRGN0
#define ARM_SMMU_VTCR_SL0		GENMASK(7, 6)
#define ARM_SMMU_VTCR_T0SZ		ARM_SMMU_TCR_T0SZ

#define ARM_SMMU_CB_CONTEXTIDR		0x34
#define ARM_SMMU_CB_S1_MAIR0		0x38
#define ARM_SMMU_CB_S1_MAIR1		0x3c

#define ARM_SMMU_CB_PAR			0x50
#define ARM_SMMU_CB_PAR_F		BIT(0)

#define ARM_SMMU_CB_FSR			0x58
#define ARM_SMMU_FSR_MULTI		BIT(31)
#define ARM_SMMU_FSR_SS			BIT(30)
#define ARM_SMMU_FSR_UUT		BIT(8)
#define ARM_SMMU_FSR_ASF		BIT(7)
#define ARM_SMMU_FSR_TLBLKF		BIT(6)
#define ARM_SMMU_FSR_TLBMCF		BIT(5)
#define ARM_SMMU_FSR_EF			BIT(4)
#define ARM_SMMU_FSR_PF			BIT(3)
#define ARM_SMMU_FSR_AFF		BIT(2)
#define ARM_SMMU_FSR_TF			BIT(1)

#define ARM_SMMU_FSR_IGN		(ARM_SMMU_FSR_AFF |		\
					 ARM_SMMU_FSR_ASF |		\
					 ARM_SMMU_FSR_TLBMCF |		\
					 ARM_SMMU_FSR_TLBLKF)

#define ARM_SMMU_FSR_FAULT		(ARM_SMMU_FSR_MULTI |		\
					 ARM_SMMU_FSR_SS |		\
					 ARM_SMMU_FSR_UUT |		\
					 ARM_SMMU_FSR_EF |		\
					 ARM_SMMU_FSR_PF |		\
					 ARM_SMMU_FSR_TF |		\
					 ARM_SMMU_FSR_IGN)

#define ARM_SMMU_CB_FAR			0x60

#define ARM_SMMU_CB_FSYNR0		0x68
#define ARM_SMMU_FSYNR0_WNR		BIT(4)

#define ARM_SMMU_CB_S1_TLBIVA		0x600
#define ARM_SMMU_CB_S1_TLBIASID		0x610
#define ARM_SMMU_CB_S1_TLBIVAL		0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2	0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L	0x638
#define ARM_SMMU_CB_TLBSYNC		0x7f0
#define ARM_SMMU_CB_TLBSTATUS		0x7f4
#define ARM_SMMU_CB_ATS1PR		0x800

#define ARM_SMMU_CB_ATSR		0x8f0
#define ARM_SMMU_ATSR_ACTIVE		BIT(0)


/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS		128


/* Shared driver definitions */
enum arm_smmu_arch_version {
	ARM_SMMU_V1,
	ARM_SMMU_V1_64K,
	ARM_SMMU_V2,
};

enum arm_smmu_implementation {
	GENERIC_SMMU,
	ARM_MMU500,
	CAVIUM_SMMUV2,
	QCOM_SMMUV2,
};

struct arm_smmu_device {
	struct device			*dev;

	void __iomem			*base;
	unsigned int			numpage;
	unsigned int			pgshift;

#define ARM_SMMU_FEAT_COHERENT_WALK	(1 << 0)
#define ARM_SMMU_FEAT_STREAM_MATCH	(1 << 1)
#define ARM_SMMU_FEAT_TRANS_S1		(1 << 2)
#define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
#define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
#define ARM_SMMU_FEAT_TRANS_OPS		(1 << 5)
#define ARM_SMMU_FEAT_VMID16		(1 << 6)
#define ARM_SMMU_FEAT_FMT_AARCH64_4K	(1 << 7)
#define ARM_SMMU_FEAT_FMT_AARCH64_16K	(1 << 8)
#define ARM_SMMU_FEAT_FMT_AARCH64_64K	(1 << 9)
#define ARM_SMMU_FEAT_FMT_AARCH32_L	(1 << 10)
#define ARM_SMMU_FEAT_FMT_AARCH32_S	(1 << 11)
#define ARM_SMMU_FEAT_EXIDS		(1 << 12)
	u32				features;

	enum arm_smmu_arch_version	version;
	enum arm_smmu_implementation	model;
	const struct arm_smmu_impl	*impl;

	u32				num_context_banks;
	u32				num_s2_context_banks;
	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
	struct arm_smmu_cb		*cbs;
	atomic_t			irptndx;

	u32				num_mapping_groups;
	u16				streamid_mask;
	u16				smr_mask_mask;
	struct arm_smmu_smr		*smrs;
	struct arm_smmu_s2cr		*s2crs;
	struct mutex			stream_map_mutex;

	unsigned long			va_size;
	unsigned long			ipa_size;
	unsigned long			pa_size;
	unsigned long			pgsize_bitmap;

	u32				num_global_irqs;
	u32				num_context_irqs;
	unsigned int			*irqs;
	struct clk_bulk_data		*clks;
	int				num_clks;

	spinlock_t			global_sync_lock;

	/* IOMMU core code handle */
	struct iommu_device		iommu;
};

enum arm_smmu_context_fmt {
	ARM_SMMU_CTX_FMT_NONE,
	ARM_SMMU_CTX_FMT_AARCH64,
	ARM_SMMU_CTX_FMT_AARCH32_L,
	ARM_SMMU_CTX_FMT_AARCH32_S,
};

struct arm_smmu_cfg {
	u8				cbndx;
	u8				irptndx;
	union {
		u16			asid;
		u16			vmid;
	};
	enum arm_smmu_cbar_type		cbar;
	enum arm_smmu_context_fmt	fmt;
};
#define ARM_SMMU_INVALID_IRPTNDX	0xff

enum arm_smmu_domain_stage {
	ARM_SMMU_DOMAIN_S1 = 0,
	ARM_SMMU_DOMAIN_S2,
	ARM_SMMU_DOMAIN_NESTED,
	ARM_SMMU_DOMAIN_BYPASS,
};

struct arm_smmu_domain {
	struct arm_smmu_device		*smmu;
	struct io_pgtable_ops		*pgtbl_ops;
	const struct iommu_flush_ops	*flush_ops;
	struct arm_smmu_cfg		cfg;
	enum arm_smmu_domain_stage	stage;
	bool				non_strict;
	struct mutex			init_mutex; /* Protects smmu pointer */
	spinlock_t			cb_lock; /* Serialises ATS1* ops and TLB syncs */
	struct iommu_domain		domain;
};

static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg)
{
	return ARM_SMMU_TCR_EPD1 |
	       FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
	       FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
	       FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
	       FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
	       FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
}

static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg)
{
	return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
	       FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
}

static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg)
{
	return ARM_SMMU_VTCR_RES1 |
	       FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
	       FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) |
	       FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) |
	       FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) |
	       FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) |
	       FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) |
	       FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
}

/* Implementation details, yay! */
struct arm_smmu_impl {
	u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
	void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
			  u32 val);
	u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
	void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
			    u64 val);
	int (*cfg_probe)(struct arm_smmu_device *smmu);
	int (*reset)(struct arm_smmu_device *smmu);
	int (*init_context)(struct arm_smmu_domain *smmu_domain);
	void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
			 int status);
};

static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
{
	return smmu->base + (n << smmu->pgshift);
}

static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
{
	if (smmu->impl && unlikely(smmu->impl->read_reg))
		return smmu->impl->read_reg(smmu, page, offset);
	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
}

static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
				   int offset, u32 val)
{
	if (smmu->impl && unlikely(smmu->impl->write_reg))
		smmu->impl->write_reg(smmu, page, offset, val);
	else
		writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
}

static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
{
	if (smmu->impl && unlikely(smmu->impl->read_reg64))
		return smmu->impl->read_reg64(smmu, page, offset);
	return readq_relaxed(arm_smmu_page(smmu, page) + offset);
}

static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
				   int offset, u64 val)
{
	if (smmu->impl && unlikely(smmu->impl->write_reg64))
		smmu->impl->write_reg64(smmu, page, offset, val);
	else
		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
}

#define ARM_SMMU_GR0		0
#define ARM_SMMU_GR1		1
#define ARM_SMMU_CB(s, n)	((s)->numpage + (n))

#define arm_smmu_gr0_read(s, o)		\
	arm_smmu_readl((s), ARM_SMMU_GR0, (o))
#define arm_smmu_gr0_write(s, o, v)	\
	arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))

#define arm_smmu_gr1_read(s, o)		\
	arm_smmu_readl((s), ARM_SMMU_GR1, (o))
#define arm_smmu_gr1_write(s, o, v)	\
	arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))

#define arm_smmu_cb_read(s, n, o)	\
	arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_write(s, n, o, v)	\
	arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
#define arm_smmu_cb_readq(s, n, o)	\
	arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_writeq(s, n, o, v)	\
	arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))

struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);

int arm_mmu500_reset(struct arm_smmu_device *smmu);

#endif /* _ARM_SMMU_H */
Commit	Line	Data
45051539	1	/* SPDX-License-Identifier: GPL-2.0-only */
2b03774b RC	2	/*
	3	* IOMMU API for ARM architected SMMU implementations.
	4	*
2b03774b RC	5	* Copyright (C) 2013 ARM Limited
	6	*
	7	* Author: Will Deacon <will.deacon@arm.com>
	8	*/
	9
c5fc6488 RM	10	#ifndef _ARM_SMMU_H
c5fc6488 RM	11	#define _ARM_SMMU_H
2b03774b	12
fc058d37	13	#include <linux/atomic.h>
fb485eb1	14	#include <linux/bitfield.h>
0caf5f4e	15	#include <linux/bits.h>
fc058d37 RM	16	#include <linux/clk.h>
fc058d37 RM	17	#include <linux/device.h>
d720e641	18	#include <linux/io-64-nonatomic-hi-lo.h>
ba7e4a08	19	#include <linux/io-pgtable.h>
fc058d37 RM	20	#include <linux/iommu.h>
	21	#include <linux/mutex.h>
	22	#include <linux/spinlock.h>
	23	#include <linux/types.h>
0caf5f4e	24
2b03774b RC	25	/* Configuration registers */
2b03774b RC	26	#define ARM_SMMU_GR0_sCR0 0x0
fba6e960 WD	27	#define ARM_SMMU_sCR0_VMID16EN BIT(31)
	28	#define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
	29	#define ARM_SMMU_sCR0_FB BIT(13)
	30	#define ARM_SMMU_sCR0_PTM BIT(12)
	31	#define ARM_SMMU_sCR0_VMIDPNE BIT(11)
	32	#define ARM_SMMU_sCR0_USFCFG BIT(10)
	33	#define ARM_SMMU_sCR0_GCFGFIE BIT(5)
	34	#define ARM_SMMU_sCR0_GCFGFRE BIT(4)
	35	#define ARM_SMMU_sCR0_EXIDENABLE BIT(3)
	36	#define ARM_SMMU_sCR0_GFIE BIT(2)
	37	#define ARM_SMMU_sCR0_GFRE BIT(1)
	38	#define ARM_SMMU_sCR0_CLIENTPD BIT(0)
2b03774b RC	39
	40	/* Auxiliary Configuration register */
	41	#define ARM_SMMU_GR0_sACR 0x10
	42
	43	/* Identification registers */
	44	#define ARM_SMMU_GR0_ID0 0x20
fba6e960 WD	45	#define ARM_SMMU_ID0_S1TS BIT(30)
	46	#define ARM_SMMU_ID0_S2TS BIT(29)
	47	#define ARM_SMMU_ID0_NTS BIT(28)
	48	#define ARM_SMMU_ID0_SMS BIT(27)
	49	#define ARM_SMMU_ID0_ATOSNS BIT(26)
	50	#define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25)
	51	#define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24)
	52	#define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16)
	53	#define ARM_SMMU_ID0_CTTW BIT(14)
	54	#define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9)
	55	#define ARM_SMMU_ID0_EXIDS BIT(8)
	56	#define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0)
0caf5f4e	57
2b03774b	58	#define ARM_SMMU_GR0_ID1 0x24
fba6e960 WD	59	#define ARM_SMMU_ID1_PAGESIZE BIT(31)
	60	#define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28)
	61	#define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16)
	62	#define ARM_SMMU_ID1_NUMCB GENMASK(7, 0)
0caf5f4e	63
2b03774b	64	#define ARM_SMMU_GR0_ID2 0x28
fba6e960 WD	65	#define ARM_SMMU_ID2_VMID16 BIT(15)
	66	#define ARM_SMMU_ID2_PTFS_64K BIT(14)
	67	#define ARM_SMMU_ID2_PTFS_16K BIT(13)
	68	#define ARM_SMMU_ID2_PTFS_4K BIT(12)
	69	#define ARM_SMMU_ID2_UBS GENMASK(11, 8)
	70	#define ARM_SMMU_ID2_OAS GENMASK(7, 4)
	71	#define ARM_SMMU_ID2_IAS GENMASK(3, 0)
0caf5f4e	72
2b03774b RC	73	#define ARM_SMMU_GR0_ID3 0x2c
	74	#define ARM_SMMU_GR0_ID4 0x30
	75	#define ARM_SMMU_GR0_ID5 0x34
	76	#define ARM_SMMU_GR0_ID6 0x38
0caf5f4e	77
2b03774b	78	#define ARM_SMMU_GR0_ID7 0x3c
fba6e960 WD	79	#define ARM_SMMU_ID7_MAJOR GENMASK(7, 4)
fba6e960 WD	80	#define ARM_SMMU_ID7_MINOR GENMASK(3, 0)
0caf5f4e	81
2b03774b	82	#define ARM_SMMU_GR0_sGFSR 0x48
fba6e960	83	#define ARM_SMMU_sGFSR_USF BIT(1)
931a0ba6	84
2b03774b RC	85	#define ARM_SMMU_GR0_sGFSYNR0 0x50
	86	#define ARM_SMMU_GR0_sGFSYNR1 0x54
	87	#define ARM_SMMU_GR0_sGFSYNR2 0x58
	88
2b03774b RC	89	/* Global TLB invalidation */
	90	#define ARM_SMMU_GR0_TLBIVMID 0x64
	91	#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
	92	#define ARM_SMMU_GR0_TLBIALLH 0x6c
	93	#define ARM_SMMU_GR0_sTLBGSYNC 0x70
0caf5f4e	94
2b03774b	95	#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
fba6e960	96	#define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0)
2b03774b RC	97
	98	/* Stream mapping registers */
	99	#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
fba6e960 WD	100	#define ARM_SMMU_SMR_VALID BIT(31)
	101	#define ARM_SMMU_SMR_MASK GENMASK(31, 16)
	102	#define ARM_SMMU_SMR_ID GENMASK(15, 0)
2b03774b RC	103
2b03774b RC	104	#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
fba6e960	105	#define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24)
2b03774b RC	106	enum arm_smmu_s2cr_privcfg {
	107	S2CR_PRIVCFG_DEFAULT,
	108	S2CR_PRIVCFG_DIPAN,
	109	S2CR_PRIVCFG_UNPRIV,
	110	S2CR_PRIVCFG_PRIV,
	111	};
fba6e960	112	#define ARM_SMMU_S2CR_TYPE GENMASK(17, 16)
0caf5f4e RM	113	enum arm_smmu_s2cr_type {
	114	S2CR_TYPE_TRANS,
	115	S2CR_TYPE_BYPASS,
	116	S2CR_TYPE_FAULT,
	117	};
fba6e960 WD	118	#define ARM_SMMU_S2CR_EXIDVALID BIT(10)
fba6e960 WD	119	#define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
2b03774b RC	120
	121	/* Context bank attribute registers */
	122	#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
fba6e960 WD	123	#define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24)
fba6e960 WD	124	#define ARM_SMMU_CBAR_TYPE GENMASK(17, 16)
5114e96c RM	125	enum arm_smmu_cbar_type {
	126	CBAR_TYPE_S2_TRANS,
	127	CBAR_TYPE_S1_TRANS_S2_BYPASS,
	128	CBAR_TYPE_S1_TRANS_S2_FAULT,
	129	CBAR_TYPE_S1_TRANS_S2_TRANS,
	130	};
fba6e960 WD	131	#define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12)
	132	#define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf
	133	#define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8)
	134	#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3
	135	#define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
2b03774b	136
bc580b56 VG	137	#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
bc580b56 VG	138
2b03774b	139	#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
fba6e960 WD	140	#define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
fba6e960 WD	141	#define ARM_SMMU_CBA2R_VA64 BIT(0)
2b03774b RC	142
2b03774b RC	143	#define ARM_SMMU_CB_SCTLR 0x0
fba6e960 WD	144	#define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
	145	#define ARM_SMMU_SCTLR_CFCFG BIT(7)
	146	#define ARM_SMMU_SCTLR_CFIE BIT(6)
	147	#define ARM_SMMU_SCTLR_CFRE BIT(5)
	148	#define ARM_SMMU_SCTLR_E BIT(4)
	149	#define ARM_SMMU_SCTLR_AFE BIT(2)
	150	#define ARM_SMMU_SCTLR_TRE BIT(1)
	151	#define ARM_SMMU_SCTLR_M BIT(0)
620565a7	152
2b03774b	153	#define ARM_SMMU_CB_ACTLR 0x4
620565a7	154
2b03774b	155	#define ARM_SMMU_CB_RESUME 0x8
fba6e960	156	#define ARM_SMMU_RESUME_TERMINATE BIT(0)
620565a7 RM	157
620565a7 RM	158	#define ARM_SMMU_CB_TCR2 0x10
fba6e960 WD	159	#define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
	160	#define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7
	161	#define ARM_SMMU_TCR2_AS BIT(4)
	162	#define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0)
620565a7	163
2b03774b RC	164	#define ARM_SMMU_CB_TTBR0 0x20
2b03774b RC	165	#define ARM_SMMU_CB_TTBR1 0x28
fba6e960	166	#define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48)
fb485eb1	167
620565a7	168	#define ARM_SMMU_CB_TCR 0x30
fba6e960 WD	169	#define ARM_SMMU_TCR_EAE BIT(31)
	170	#define ARM_SMMU_TCR_EPD1 BIT(23)
	171	#define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
	172	#define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
	173	#define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
	174	#define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
	175	#define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
fb485eb1	176
ac4b80e5 WD	177	#define ARM_SMMU_VTCR_RES1 BIT(31)
	178	#define ARM_SMMU_VTCR_PS GENMASK(18, 16)
	179	#define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0
	180	#define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0
	181	#define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0
	182	#define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0
	183	#define ARM_SMMU_VTCR_SL0 GENMASK(7, 6)
	184	#define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ
	185
2b03774b RC	186	#define ARM_SMMU_CB_CONTEXTIDR 0x34
	187	#define ARM_SMMU_CB_S1_MAIR0 0x38
	188	#define ARM_SMMU_CB_S1_MAIR1 0x3c
620565a7	189
2b03774b	190	#define ARM_SMMU_CB_PAR 0x50
fba6e960	191	#define ARM_SMMU_CB_PAR_F BIT(0)
620565a7	192
2b03774b	193	#define ARM_SMMU_CB_FSR 0x58
fba6e960 WD	194	#define ARM_SMMU_FSR_MULTI BIT(31)
	195	#define ARM_SMMU_FSR_SS BIT(30)
	196	#define ARM_SMMU_FSR_UUT BIT(8)
	197	#define ARM_SMMU_FSR_ASF BIT(7)
	198	#define ARM_SMMU_FSR_TLBLKF BIT(6)
	199	#define ARM_SMMU_FSR_TLBMCF BIT(5)
	200	#define ARM_SMMU_FSR_EF BIT(4)
	201	#define ARM_SMMU_FSR_PF BIT(3)
	202	#define ARM_SMMU_FSR_AFF BIT(2)
	203	#define ARM_SMMU_FSR_TF BIT(1)
	204
	205	#define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF \| \
	206	ARM_SMMU_FSR_ASF \| \
	207	ARM_SMMU_FSR_TLBMCF \| \
	208	ARM_SMMU_FSR_TLBLKF)
	209
	210	#define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI \| \
	211	ARM_SMMU_FSR_SS \| \
	212	ARM_SMMU_FSR_UUT \| \
	213	ARM_SMMU_FSR_EF \| \
	214	ARM_SMMU_FSR_PF \| \
	215	ARM_SMMU_FSR_TF \| \
	216	ARM_SMMU_FSR_IGN)
620565a7	217
2b03774b	218	#define ARM_SMMU_CB_FAR 0x60
620565a7	219
2b03774b	220	#define ARM_SMMU_CB_FSYNR0 0x68
fba6e960	221	#define ARM_SMMU_FSYNR0_WNR BIT(4)
620565a7	222
2b03774b RC	223	#define ARM_SMMU_CB_S1_TLBIVA 0x600
	224	#define ARM_SMMU_CB_S1_TLBIASID 0x610
	225	#define ARM_SMMU_CB_S1_TLBIVAL 0x620
	226	#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
	227	#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
	228	#define ARM_SMMU_CB_TLBSYNC 0x7f0
	229	#define ARM_SMMU_CB_TLBSTATUS 0x7f4
	230	#define ARM_SMMU_CB_ATS1PR 0x800
2b03774b	231
620565a7	232	#define ARM_SMMU_CB_ATSR 0x8f0
fba6e960	233	#define ARM_SMMU_ATSR_ACTIVE BIT(0)
2b03774b	234
fc058d37 RM	235
	236	/* Maximum number of context banks per SMMU */
	237	#define ARM_SMMU_MAX_CBS 128
	238
	239
	240	/* Shared driver definitions */
	241	enum arm_smmu_arch_version {
	242	ARM_SMMU_V1,
	243	ARM_SMMU_V1_64K,
	244	ARM_SMMU_V2,
	245	};
	246
	247	enum arm_smmu_implementation {
	248	GENERIC_SMMU,
	249	ARM_MMU500,
	250	CAVIUM_SMMUV2,
	251	QCOM_SMMUV2,
	252	};
	253
	254	struct arm_smmu_device {
	255	struct device *dev;
	256
	257	void __iomem *base;
	258	unsigned int numpage;
	259	unsigned int pgshift;
	260
	261	#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
	262	#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
	263	#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
	264	#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
	265	#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
	266	#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
	267	#define ARM_SMMU_FEAT_VMID16 (1 << 6)
	268	#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
	269	#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
	270	#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
	271	#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
	272	#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
	273	#define ARM_SMMU_FEAT_EXIDS (1 << 12)
	274	u32 features;
	275
fc058d37 RM	276	enum arm_smmu_arch_version version;
fc058d37 RM	277	enum arm_smmu_implementation model;
6d7dff62	278	const struct arm_smmu_impl *impl;
fc058d37 RM	279
	280	u32 num_context_banks;
	281	u32 num_s2_context_banks;
	282	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
	283	struct arm_smmu_cb *cbs;
	284	atomic_t irptndx;
	285
	286	u32 num_mapping_groups;
	287	u16 streamid_mask;
	288	u16 smr_mask_mask;
	289	struct arm_smmu_smr *smrs;
	290	struct arm_smmu_s2cr *s2crs;
	291	struct mutex stream_map_mutex;
	292
	293	unsigned long va_size;
	294	unsigned long ipa_size;
	295	unsigned long pa_size;
	296	unsigned long pgsize_bitmap;
	297
	298	u32 num_global_irqs;
	299	u32 num_context_irqs;
	300	unsigned int *irqs;
	301	struct clk_bulk_data *clks;
	302	int num_clks;
	303
fc058d37 RM	304	spinlock_t global_sync_lock;
	305
	306	/* IOMMU core code handle */
	307	struct iommu_device iommu;
	308	};
	309
ba7e4a08 RM	310	enum arm_smmu_context_fmt {
	311	ARM_SMMU_CTX_FMT_NONE,
	312	ARM_SMMU_CTX_FMT_AARCH64,
	313	ARM_SMMU_CTX_FMT_AARCH32_L,
	314	ARM_SMMU_CTX_FMT_AARCH32_S,
	315	};
	316
	317	struct arm_smmu_cfg {
	318	u8 cbndx;
	319	u8 irptndx;
	320	union {
	321	u16 asid;
	322	u16 vmid;
	323	};
	324	enum arm_smmu_cbar_type cbar;
	325	enum arm_smmu_context_fmt fmt;
	326	};
fba6e960	327	#define ARM_SMMU_INVALID_IRPTNDX 0xff
ba7e4a08 RM	328
	329	enum arm_smmu_domain_stage {
	330	ARM_SMMU_DOMAIN_S1 = 0,
	331	ARM_SMMU_DOMAIN_S2,
	332	ARM_SMMU_DOMAIN_NESTED,
	333	ARM_SMMU_DOMAIN_BYPASS,
	334	};
	335
	336	struct arm_smmu_domain {
	337	struct arm_smmu_device *smmu;
	338	struct io_pgtable_ops *pgtbl_ops;
696bcfb7	339	const struct iommu_flush_ops *flush_ops;
ba7e4a08 RM	340	struct arm_smmu_cfg cfg;
	341	enum arm_smmu_domain_stage stage;
	342	bool non_strict;
	343	struct mutex init_mutex; /* Protects smmu pointer */
	344	spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
	345	struct iommu_domain domain;
	346	};
	347
fb485eb1 RM	348	static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg)
fb485eb1 RM	349	{
fba6e960 WD	350	return ARM_SMMU_TCR_EPD1 \|
	351	FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) \|
	352	FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) \|
	353	FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) \|
	354	FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) \|
	355	FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
fb485eb1 RM	356	}
	357
	358	static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg)
	359	{
fba6e960 WD	360	return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) \|
fba6e960 WD	361	FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
fb485eb1	362	}
fc058d37	363
ac4b80e5 WD	364	static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg)
	365	{
	366	return ARM_SMMU_VTCR_RES1 \|
	367	FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) \|
	368	FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) \|
	369	FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) \|
	370	FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) \|
	371	FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) \|
	372	FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) \|
	373	FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
	374	}
	375
fc058d37	376	/* Implementation details, yay! */
6d7dff62 RM	377	struct arm_smmu_impl {
	378	u32 (read_reg)(struct arm_smmu_device smmu, int page, int offset);
	379	void (write_reg)(struct arm_smmu_device smmu, int page, int offset,
	380	u32 val);
	381	u64 (read_reg64)(struct arm_smmu_device smmu, int page, int offset);
	382	void (write_reg64)(struct arm_smmu_device smmu, int page, int offset,
	383	u64 val);
3995e186	384	int (cfg_probe)(struct arm_smmu_device smmu);
62b993a3	385	int (reset)(struct arm_smmu_device smmu);
ba7e4a08	386	int (init_context)(struct arm_smmu_domain smmu_domain);
ae2b60f3 RM	387	void (tlb_sync)(struct arm_smmu_device smmu, int page, int sync,
ae2b60f3 RM	388	int status);
6d7dff62 RM	389	};
	390
	391	static inline void __iomem arm_smmu_page(struct arm_smmu_device smmu, int n)
	392	{
	393	return smmu->base + (n << smmu->pgshift);
	394	}
	395
	396	static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
	397	{
	398	if (smmu->impl && unlikely(smmu->impl->read_reg))
	399	return smmu->impl->read_reg(smmu, page, offset);
	400	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
	401	}
	402
	403	static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
	404	int offset, u32 val)
	405	{
	406	if (smmu->impl && unlikely(smmu->impl->write_reg))
	407	smmu->impl->write_reg(smmu, page, offset, val);
	408	else
	409	writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
	410	}
	411
	412	static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
	413	{
	414	if (smmu->impl && unlikely(smmu->impl->read_reg64))
	415	return smmu->impl->read_reg64(smmu, page, offset);
	416	return readq_relaxed(arm_smmu_page(smmu, page) + offset);
	417	}
	418
	419	static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
	420	int offset, u64 val)
	421	{
	422	if (smmu->impl && unlikely(smmu->impl->write_reg64))
	423	smmu->impl->write_reg64(smmu, page, offset, val);
	424	else
	425	writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
	426	}
	427
	428	#define ARM_SMMU_GR0 0
	429	#define ARM_SMMU_GR1 1
	430	#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
	431
	432	#define arm_smmu_gr0_read(s, o) \
	433	arm_smmu_readl((s), ARM_SMMU_GR0, (o))
	434	#define arm_smmu_gr0_write(s, o, v) \
	435	arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
	436
	437	#define arm_smmu_gr1_read(s, o) \
	438	arm_smmu_readl((s), ARM_SMMU_GR1, (o))
	439	#define arm_smmu_gr1_write(s, o, v) \
	440	arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
	441
	442	#define arm_smmu_cb_read(s, n, o) \
	443	arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
	444	#define arm_smmu_cb_write(s, n, o, v) \
	445	arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
	446	#define arm_smmu_cb_readq(s, n, o) \
	447	arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
	448	#define arm_smmu_cb_writeq(s, n, o, v) \
	449	arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
	450
fc058d37	451	struct arm_smmu_device arm_smmu_impl_init(struct arm_smmu_device smmu);
759aaa10 VG	452	struct arm_smmu_device qcom_smmu_impl_init(struct arm_smmu_device smmu);
	453
	454	int arm_mmu500_reset(struct arm_smmu_device *smmu);
fc058d37	455
c5fc6488	456	#endif /* _ARM_SMMU_H */