[mirror_ubuntu-zesty-kernel.git] / drivers / clocksource / arm_arch_timer.c

/*
 *  linux/drivers/clocksource/arm_arch_timer.c
 *
 *  Copyright (C) 2011 ARM Ltd.
 *  All Rights Reserved
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/clockchips.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/of_address.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/sched_clock.h>

#include <asm/arch_timer.h>
#include <asm/virt.h>

#include <clocksource/arm_arch_timer.h>

#define CNTTIDR		0x08
#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))

#define CNTVCT_LO	0x08
#define CNTVCT_HI	0x0c
#define CNTFRQ		0x10
#define CNTP_TVAL	0x28
#define CNTP_CTL	0x2c
#define CNTV_TVAL	0x38
#define CNTV_CTL	0x3c

#define ARCH_CP15_TIMER	BIT(0)
#define ARCH_MEM_TIMER	BIT(1)
static unsigned arch_timers_present __initdata;

static void __iomem *arch_counter_base;

struct arch_timer {
	void __iomem *base;
	struct clock_event_device evt;
};

#define to_arch_timer(e) container_of(e, struct arch_timer, evt)

static u32 arch_timer_rate;

enum ppi_nr {
	PHYS_SECURE_PPI,
	PHYS_NONSECURE_PPI,
	VIRT_PPI,
	HYP_PPI,
	MAX_TIMER_PPI
};

static int arch_timer_ppi[MAX_TIMER_PPI];

static struct clock_event_device __percpu *arch_timer_evt;

static bool arch_timer_use_virtual = true;
static bool arch_timer_mem_use_virtual;

/*
 * Architected system timer support.
 */

static __always_inline
void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
			  struct clock_event_device *clk)
{
	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			writel_relaxed(val, timer->base + CNTP_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			writel_relaxed(val, timer->base + CNTP_TVAL);
			break;
		}
	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			writel_relaxed(val, timer->base + CNTV_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			writel_relaxed(val, timer->base + CNTV_TVAL);
			break;
		}
	} else {
		arch_timer_reg_write_cp15(access, reg, val);
	}
}

static __always_inline
u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
			struct clock_event_device *clk)
{
	u32 val;

	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			val = readl_relaxed(timer->base + CNTP_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			val = readl_relaxed(timer->base + CNTP_TVAL);
			break;
		}
	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			val = readl_relaxed(timer->base + CNTV_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			val = readl_relaxed(timer->base + CNTV_TVAL);
			break;
		}
	} else {
		val = arch_timer_reg_read_cp15(access, reg);
	}

	return val;
}

static __always_inline irqreturn_t timer_handler(const int access,
					struct clock_event_device *evt)
{
	unsigned long ctrl;

	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
		evt->event_handler(evt);
		return IRQ_HANDLED;
	}

	return IRQ_NONE;
}

static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
}

static __always_inline void timer_set_mode(const int access, int mode,
				  struct clock_event_device *clk)
{
	unsigned long ctrl;
	switch (mode) {
	case CLOCK_EVT_MODE_UNUSED:
	case CLOCK_EVT_MODE_SHUTDOWN:
		ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
		ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
		break;
	default:
		break;
	}
}

static void arch_timer_set_mode_virt(enum clock_event_mode mode,
				     struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk);
}

static void arch_timer_set_mode_phys(enum clock_event_mode mode,
				     struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk);
}

static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode,
					 struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk);
}

static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode,
					 struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk);
}

static __always_inline void set_next_event(const int access, unsigned long evt,
					   struct clock_event_device *clk)
{
	unsigned long ctrl;
	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
	ctrl |= ARCH_TIMER_CTRL_ENABLE;
	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
}

static int arch_timer_set_next_event_virt(unsigned long evt,
					  struct clock_event_device *clk)
{
	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
	return 0;
}

static int arch_timer_set_next_event_phys(unsigned long evt,
					  struct clock_event_device *clk)
{
	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
	return 0;
}

static int arch_timer_set_next_event_virt_mem(unsigned long evt,
					      struct clock_event_device *clk)
{
	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
	return 0;
}

static int arch_timer_set_next_event_phys_mem(unsigned long evt,
					      struct clock_event_device *clk)
{
	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
	return 0;
}

static void __arch_timer_setup(unsigned type,
			       struct clock_event_device *clk)
{
	clk->features = CLOCK_EVT_FEAT_ONESHOT;

	if (type == ARCH_CP15_TIMER) {
		clk->features |= CLOCK_EVT_FEAT_C3STOP;
		clk->name = "arch_sys_timer";
		clk->rating = 450;
		clk->cpumask = cpumask_of(smp_processor_id());
		if (arch_timer_use_virtual) {
			clk->irq = arch_timer_ppi[VIRT_PPI];
			clk->set_mode = arch_timer_set_mode_virt;
			clk->set_next_event = arch_timer_set_next_event_virt;
		} else {
			clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
			clk->set_mode = arch_timer_set_mode_phys;
			clk->set_next_event = arch_timer_set_next_event_phys;
		}
	} else {
		clk->name = "arch_mem_timer";
		clk->rating = 400;
		clk->cpumask = cpu_all_mask;
		if (arch_timer_mem_use_virtual) {
			clk->set_mode = arch_timer_set_mode_virt_mem;
			clk->set_next_event =
				arch_timer_set_next_event_virt_mem;
		} else {
			clk->set_mode = arch_timer_set_mode_phys_mem;
			clk->set_next_event =
				arch_timer_set_next_event_phys_mem;
		}
	}

	clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk);

	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
}

static int arch_timer_setup(struct clock_event_device *clk)
{
	__arch_timer_setup(ARCH_CP15_TIMER, clk);

	if (arch_timer_use_virtual)
		enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
	else {
		enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
	}

	arch_counter_set_user_access();

	return 0;
}

static void
arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
{
	/* Who has more than one independent system counter? */
	if (arch_timer_rate)
		return;

	/* Try to determine the frequency from the device tree or CNTFRQ */
	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
		if (cntbase)
			arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
		else
			arch_timer_rate = arch_timer_get_cntfrq();
	}

	/* Check the timer frequency. */
	if (arch_timer_rate == 0)
		pr_warn("Architected timer frequency not available\n");
}

static void arch_timer_banner(unsigned type)
{
	pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
		     type & ARCH_CP15_TIMER ? "cp15" : "",
		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
		     type & ARCH_MEM_TIMER ? "mmio" : "",
		     (unsigned long)arch_timer_rate / 1000000,
		     (unsigned long)(arch_timer_rate / 10000) % 100,
		     type & ARCH_CP15_TIMER ?
			arch_timer_use_virtual ? "virt" : "phys" :
			"",
		     type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
		     type & ARCH_MEM_TIMER ?
			arch_timer_mem_use_virtual ? "virt" : "phys" :
			"");
}

u32 arch_timer_get_rate(void)
{
	return arch_timer_rate;
}

static u64 arch_counter_get_cntvct_mem(void)
{
	u32 vct_lo, vct_hi, tmp_hi;

	do {
		vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
		vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
		tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
	} while (vct_hi != tmp_hi);

	return ((u64) vct_hi << 32) | vct_lo;
}

/*
 * Default to cp15 based access because arm64 uses this function for
 * sched_clock() before DT is probed and the cp15 method is guaranteed
 * to exist on arm64. arm doesn't use this before DT is probed so even
 * if we don't have the cp15 accessors we won't have a problem.
 */
u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;

static cycle_t arch_counter_read(struct clocksource *cs)
{
	return arch_timer_read_counter();
}

static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
{
	return arch_timer_read_counter();
}

static struct clocksource clocksource_counter = {
	.name	= "arch_sys_counter",
	.rating	= 400,
	.read	= arch_counter_read,
	.mask	= CLOCKSOURCE_MASK(56),
	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
};

static struct cyclecounter cyclecounter = {
	.read	= arch_counter_read_cc,
	.mask	= CLOCKSOURCE_MASK(56),
};

static struct timecounter timecounter;

struct timecounter *arch_timer_get_timecounter(void)
{
	return &timecounter;
}

static void __init arch_counter_register(unsigned type)
{
	u64 start_count;

	/* Register the CP15 based counter if we have one */
	if (type & ARCH_CP15_TIMER)
		arch_timer_read_counter = arch_counter_get_cntvct;
	else
		arch_timer_read_counter = arch_counter_get_cntvct_mem;

	start_count = arch_timer_read_counter();
	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	cyclecounter.mult = clocksource_counter.mult;
	cyclecounter.shift = clocksource_counter.shift;
	timecounter_init(&timecounter, &cyclecounter, start_count);
}

static void arch_timer_stop(struct clock_event_device *clk)
{
	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
		 clk->irq, smp_processor_id());

	if (arch_timer_use_virtual)
		disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
	else {
		disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
	}

	clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
}

static int arch_timer_cpu_notify(struct notifier_block *self,
					   unsigned long action, void *hcpu)
{
	/*
	 * Grab cpu pointer in each case to avoid spurious
	 * preemptible warnings
	 */
	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_STARTING:
		arch_timer_setup(this_cpu_ptr(arch_timer_evt));
		break;
	case CPU_DYING:
		arch_timer_stop(this_cpu_ptr(arch_timer_evt));
		break;
	}

	return NOTIFY_OK;
}

static struct notifier_block arch_timer_cpu_nb = {
	.notifier_call = arch_timer_cpu_notify,
};

static int __init arch_timer_register(void)
{
	int err;
	int ppi;

	arch_timer_evt = alloc_percpu(struct clock_event_device);
	if (!arch_timer_evt) {
		err = -ENOMEM;
		goto out;
	}

	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	cyclecounter.mult = clocksource_counter.mult;
	cyclecounter.shift = clocksource_counter.shift;
	timecounter_init(&timecounter, &cyclecounter,
			 arch_counter_get_cntvct());

	/* 56 bits minimum, so we assume worst case rollover */
	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);

	if (arch_timer_use_virtual) {
		ppi = arch_timer_ppi[VIRT_PPI];
		err = request_percpu_irq(ppi, arch_timer_handler_virt,
					 "arch_timer", arch_timer_evt);
	} else {
		ppi = arch_timer_ppi[PHYS_SECURE_PPI];
		err = request_percpu_irq(ppi, arch_timer_handler_phys,
					 "arch_timer", arch_timer_evt);
		if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
			ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
			err = request_percpu_irq(ppi, arch_timer_handler_phys,
						 "arch_timer", arch_timer_evt);
			if (err)
				free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
						arch_timer_evt);
		}
	}

	if (err) {
		pr_err("arch_timer: can't register interrupt %d (%d)\n",
		       ppi, err);
		goto out_free;
	}

	err = register_cpu_notifier(&arch_timer_cpu_nb);
	if (err)
		goto out_free_irq;

	/* Immediately configure the timer on the boot CPU */
	arch_timer_setup(this_cpu_ptr(arch_timer_evt));

	return 0;

out_free_irq:
	if (arch_timer_use_virtual)
		free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
	else {
		free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
				arch_timer_evt);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
					arch_timer_evt);
	}

out_free:
	free_percpu(arch_timer_evt);
out:
	return err;
}

static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
{
	int ret;
	irq_handler_t func;
	struct arch_timer *t;

	t = kzalloc(sizeof(*t), GFP_KERNEL);
	if (!t)
		return -ENOMEM;

	t->base = base;
	t->evt.irq = irq;
	__arch_timer_setup(ARCH_MEM_TIMER, &t->evt);

	if (arch_timer_mem_use_virtual)
		func = arch_timer_handler_virt_mem;
	else
		func = arch_timer_handler_phys_mem;

	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
	if (ret) {
		pr_err("arch_timer: Failed to request mem timer irq\n");
		kfree(t);
	}

	return ret;
}

static const struct of_device_id arch_timer_of_match[] __initconst = {
	{ .compatible   = "arm,armv7-timer",    },
	{ .compatible   = "arm,armv8-timer",    },
	{},
};

static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
	{ .compatible   = "arm,armv7-timer-mem", },
	{},
};

static void __init arch_timer_common_init(void)
{
	unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;

	/* Wait until both nodes are probed if we have two timers */
	if ((arch_timers_present & mask) != mask) {
		if (of_find_matching_node(NULL, arch_timer_mem_of_match) &&
				!(arch_timers_present & ARCH_MEM_TIMER))
			return;
		if (of_find_matching_node(NULL, arch_timer_of_match) &&
				!(arch_timers_present & ARCH_CP15_TIMER))
			return;
	}

	arch_timer_banner(arch_timers_present);
	arch_counter_register(arch_timers_present);
	arch_timer_arch_init();
}

static void __init arch_timer_init(struct device_node *np)
{
	int i;

	if (arch_timers_present & ARCH_CP15_TIMER) {
		pr_warn("arch_timer: multiple nodes in dt, skipping\n");
		return;
	}

	arch_timers_present |= ARCH_CP15_TIMER;
	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
	arch_timer_detect_rate(NULL, np);

	/*
	 * If HYP mode is available, we know that the physical timer
	 * has been configured to be accessible from PL1. Use it, so
	 * that a guest can use the virtual timer instead.
	 *
	 * If no interrupt provided for virtual timer, we'll have to
	 * stick to the physical timer. It'd better be accessible...
	 */
	if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
		arch_timer_use_virtual = false;

		if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
		    !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
			pr_warn("arch_timer: No interrupt available, giving up\n");
			return;
		}
	}

	arch_timer_register();
	arch_timer_common_init();
}
CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_init);
CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_init);

static void __init arch_timer_mem_init(struct device_node *np)
{
	struct device_node *frame, *best_frame = NULL;
	void __iomem *cntctlbase, *base;
	unsigned int irq;
	u32 cnttidr;

	arch_timers_present |= ARCH_MEM_TIMER;
	cntctlbase = of_iomap(np, 0);
	if (!cntctlbase) {
		pr_err("arch_timer: Can't find CNTCTLBase\n");
		return;
	}

	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
	iounmap(cntctlbase);

	/*
	 * Try to find a virtual capable frame. Otherwise fall back to a
	 * physical capable frame.
	 */
	for_each_available_child_of_node(np, frame) {
		int n;

		if (of_property_read_u32(frame, "frame-number", &n)) {
			pr_err("arch_timer: Missing frame-number\n");
			of_node_put(best_frame);
			of_node_put(frame);
			return;
		}

		if (cnttidr & CNTTIDR_VIRT(n)) {
			of_node_put(best_frame);
			best_frame = frame;
			arch_timer_mem_use_virtual = true;
			break;
		}
		of_node_put(best_frame);
		best_frame = of_node_get(frame);
	}

	base = arch_counter_base = of_iomap(best_frame, 0);
	if (!base) {
		pr_err("arch_timer: Can't map frame's registers\n");
		of_node_put(best_frame);
		return;
	}

	if (arch_timer_mem_use_virtual)
		irq = irq_of_parse_and_map(best_frame, 1);
	else
		irq = irq_of_parse_and_map(best_frame, 0);
	of_node_put(best_frame);
	if (!irq) {
		pr_err("arch_timer: Frame missing %s irq",
		       arch_timer_mem_use_virtual ? "virt" : "phys");
		return;
	}

	arch_timer_detect_rate(base, np);
	arch_timer_mem_register(base, irq);
	arch_timer_common_init();
}
CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
		       arch_timer_mem_init);
Commit	Line	Data
8a4da6e3 MR	1	/*
	2	* linux/drivers/clocksource/arm_arch_timer.c
	3	*
	4	* Copyright (C) 2011 ARM Ltd.
	5	* All Rights Reserved
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*/
	11	#include <linux/init.h>
	12	#include <linux/kernel.h>
	13	#include <linux/device.h>
	14	#include <linux/smp.h>
	15	#include <linux/cpu.h>
	16	#include <linux/clockchips.h>
	17	#include <linux/interrupt.h>
	18	#include <linux/of_irq.h>
22006994	19	#include <linux/of_address.h>
8a4da6e3	20	#include <linux/io.h>
22006994	21	#include <linux/slab.h>
65cd4f6c	22	#include <linux/sched_clock.h>
8a4da6e3 MR	23
8a4da6e3 MR	24	#include <asm/arch_timer.h>
8266891e	25	#include <asm/virt.h>
8a4da6e3 MR	26
	27	#include <clocksource/arm_arch_timer.h>
	28
22006994 SB	29	#define CNTTIDR 0x08
	30	#define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
	31
	32	#define CNTVCT_LO 0x08
	33	#define CNTVCT_HI 0x0c
	34	#define CNTFRQ 0x10
	35	#define CNTP_TVAL 0x28
	36	#define CNTP_CTL 0x2c
	37	#define CNTV_TVAL 0x38
	38	#define CNTV_CTL 0x3c
	39
	40	#define ARCH_CP15_TIMER BIT(0)
	41	#define ARCH_MEM_TIMER BIT(1)
	42	static unsigned arch_timers_present __initdata;
	43
	44	static void __iomem *arch_counter_base;
	45
	46	struct arch_timer {
	47	void __iomem *base;
	48	struct clock_event_device evt;
	49	};
	50
	51	#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
	52
8a4da6e3 MR	53	static u32 arch_timer_rate;
	54
	55	enum ppi_nr {
	56	PHYS_SECURE_PPI,
	57	PHYS_NONSECURE_PPI,
	58	VIRT_PPI,
	59	HYP_PPI,
	60	MAX_TIMER_PPI
	61	};
	62
	63	static int arch_timer_ppi[MAX_TIMER_PPI];
	64
	65	static struct clock_event_device __percpu *arch_timer_evt;
	66
	67	static bool arch_timer_use_virtual = true;
22006994	68	static bool arch_timer_mem_use_virtual;
8a4da6e3 MR	69
	70	/*
	71	* Architected system timer support.
	72	*/
	73
60faddf6 SB	74	static __always_inline
60faddf6 SB	75	void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
cfb6d656	76	struct clock_event_device *clk)
60faddf6	77	{
22006994 SB	78	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
	79	struct arch_timer *timer = to_arch_timer(clk);
	80	switch (reg) {
	81	case ARCH_TIMER_REG_CTRL:
	82	writel_relaxed(val, timer->base + CNTP_CTL);
	83	break;
	84	case ARCH_TIMER_REG_TVAL:
	85	writel_relaxed(val, timer->base + CNTP_TVAL);
	86	break;
	87	}
	88	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
	89	struct arch_timer *timer = to_arch_timer(clk);
	90	switch (reg) {
	91	case ARCH_TIMER_REG_CTRL:
	92	writel_relaxed(val, timer->base + CNTV_CTL);
	93	break;
	94	case ARCH_TIMER_REG_TVAL:
	95	writel_relaxed(val, timer->base + CNTV_TVAL);
	96	break;
	97	}
	98	} else {
	99	arch_timer_reg_write_cp15(access, reg, val);
	100	}
60faddf6 SB	101	}
	102
	103	static __always_inline
	104	u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
cfb6d656	105	struct clock_event_device *clk)
60faddf6	106	{
22006994 SB	107	u32 val;
	108
	109	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
	110	struct arch_timer *timer = to_arch_timer(clk);
	111	switch (reg) {
	112	case ARCH_TIMER_REG_CTRL:
	113	val = readl_relaxed(timer->base + CNTP_CTL);
	114	break;
	115	case ARCH_TIMER_REG_TVAL:
	116	val = readl_relaxed(timer->base + CNTP_TVAL);
	117	break;
	118	}
	119	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
	120	struct arch_timer *timer = to_arch_timer(clk);
	121	switch (reg) {
	122	case ARCH_TIMER_REG_CTRL:
	123	val = readl_relaxed(timer->base + CNTV_CTL);
	124	break;
	125	case ARCH_TIMER_REG_TVAL:
	126	val = readl_relaxed(timer->base + CNTV_TVAL);
	127	break;
	128	}
	129	} else {
	130	val = arch_timer_reg_read_cp15(access, reg);
	131	}
	132
	133	return val;
60faddf6 SB	134	}
60faddf6 SB	135
e09f3cc0	136	static __always_inline irqreturn_t timer_handler(const int access,
8a4da6e3 MR	137	struct clock_event_device *evt)
	138	{
	139	unsigned long ctrl;
cfb6d656	140
60faddf6	141	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
8a4da6e3 MR	142	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
8a4da6e3 MR	143	ctrl \|= ARCH_TIMER_CTRL_IT_MASK;
60faddf6	144	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
8a4da6e3 MR	145	evt->event_handler(evt);
	146	return IRQ_HANDLED;
	147	}
	148
	149	return IRQ_NONE;
	150	}
	151
	152	static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
	153	{
	154	struct clock_event_device *evt = dev_id;
	155
	156	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
	157	}
	158
	159	static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
	160	{
	161	struct clock_event_device *evt = dev_id;
	162
	163	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
	164	}
	165
22006994 SB	166	static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
	167	{
	168	struct clock_event_device *evt = dev_id;
	169
	170	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
	171	}
	172
	173	static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
	174	{
	175	struct clock_event_device *evt = dev_id;
	176
	177	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
	178	}
	179
60faddf6 SB	180	static __always_inline void timer_set_mode(const int access, int mode,
60faddf6 SB	181	struct clock_event_device *clk)
8a4da6e3 MR	182	{
	183	unsigned long ctrl;
	184	switch (mode) {
	185	case CLOCK_EVT_MODE_UNUSED:
	186	case CLOCK_EVT_MODE_SHUTDOWN:
60faddf6	187	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
8a4da6e3	188	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
60faddf6	189	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
8a4da6e3 MR	190	break;
	191	default:
	192	break;
	193	}
	194	}
	195
	196	static void arch_timer_set_mode_virt(enum clock_event_mode mode,
	197	struct clock_event_device *clk)
	198	{
60faddf6	199	timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk);
8a4da6e3 MR	200	}
	201
	202	static void arch_timer_set_mode_phys(enum clock_event_mode mode,
	203	struct clock_event_device *clk)
	204	{
60faddf6	205	timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk);
8a4da6e3 MR	206	}
8a4da6e3 MR	207
22006994 SB	208	static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode,
	209	struct clock_event_device *clk)
	210	{
	211	timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk);
8a4da6e3 MR	212	}
8a4da6e3 MR	213
22006994 SB	214	static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode,
	215	struct clock_event_device *clk)
	216	{
	217	timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk);
	218	}
	219
60faddf6	220	static __always_inline void set_next_event(const int access, unsigned long evt,
cfb6d656	221	struct clock_event_device *clk)
8a4da6e3 MR	222	{
8a4da6e3 MR	223	unsigned long ctrl;
60faddf6	224	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
8a4da6e3 MR	225	ctrl \|= ARCH_TIMER_CTRL_ENABLE;
8a4da6e3 MR	226	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
60faddf6 SB	227	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
60faddf6 SB	228	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
8a4da6e3 MR	229	}
	230
	231	static int arch_timer_set_next_event_virt(unsigned long evt,
60faddf6	232	struct clock_event_device *clk)
8a4da6e3	233	{
60faddf6	234	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
8a4da6e3 MR	235	return 0;
	236	}
	237
	238	static int arch_timer_set_next_event_phys(unsigned long evt,
60faddf6	239	struct clock_event_device *clk)
8a4da6e3	240	{
60faddf6	241	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
8a4da6e3 MR	242	return 0;
	243	}
	244
22006994 SB	245	static int arch_timer_set_next_event_virt_mem(unsigned long evt,
22006994 SB	246	struct clock_event_device *clk)
8a4da6e3	247	{
22006994 SB	248	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
	249	return 0;
	250	}
	251
	252	static int arch_timer_set_next_event_phys_mem(unsigned long evt,
	253	struct clock_event_device *clk)
	254	{
	255	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
	256	return 0;
	257	}
	258
cfb6d656 TG	259	static void __arch_timer_setup(unsigned type,
cfb6d656 TG	260	struct clock_event_device *clk)
22006994 SB	261	{
	262	clk->features = CLOCK_EVT_FEAT_ONESHOT;
	263
	264	if (type == ARCH_CP15_TIMER) {
	265	clk->features \|= CLOCK_EVT_FEAT_C3STOP;
	266	clk->name = "arch_sys_timer";
	267	clk->rating = 450;
	268	clk->cpumask = cpumask_of(smp_processor_id());
	269	if (arch_timer_use_virtual) {
	270	clk->irq = arch_timer_ppi[VIRT_PPI];
	271	clk->set_mode = arch_timer_set_mode_virt;
	272	clk->set_next_event = arch_timer_set_next_event_virt;
	273	} else {
	274	clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
	275	clk->set_mode = arch_timer_set_mode_phys;
	276	clk->set_next_event = arch_timer_set_next_event_phys;
	277	}
8a4da6e3	278	} else {
22006994 SB	279	clk->name = "arch_mem_timer";
	280	clk->rating = 400;
	281	clk->cpumask = cpu_all_mask;
	282	if (arch_timer_mem_use_virtual) {
	283	clk->set_mode = arch_timer_set_mode_virt_mem;
	284	clk->set_next_event =
	285	arch_timer_set_next_event_virt_mem;
	286	} else {
	287	clk->set_mode = arch_timer_set_mode_phys_mem;
	288	clk->set_next_event =
	289	arch_timer_set_next_event_phys_mem;
	290	}
8a4da6e3 MR	291	}
8a4da6e3 MR	292
1ff99ea6	293	clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk);
8a4da6e3	294
22006994 SB	295	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
22006994 SB	296	}
8a4da6e3	297
cfb6d656	298	static int arch_timer_setup(struct clock_event_device *clk)
22006994 SB	299	{
22006994 SB	300	__arch_timer_setup(ARCH_CP15_TIMER, clk);
8a4da6e3 MR	301
	302	if (arch_timer_use_virtual)
	303	enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
	304	else {
	305	enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
	306	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	307	enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
	308	}
	309
	310	arch_counter_set_user_access();
	311
	312	return 0;
	313	}
	314
22006994 SB	315	static void
22006994 SB	316	arch_timer_detect_rate(void __iomem cntbase, struct device_node np)
8a4da6e3	317	{
22006994 SB	318	/* Who has more than one independent system counter? */
	319	if (arch_timer_rate)
	320	return;
8a4da6e3	321
22006994 SB	322	/* Try to determine the frequency from the device tree or CNTFRQ */
	323	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
	324	if (cntbase)
	325	arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
	326	else
	327	arch_timer_rate = arch_timer_get_cntfrq();
8a4da6e3 MR	328	}
8a4da6e3 MR	329
22006994 SB	330	/* Check the timer frequency. */
	331	if (arch_timer_rate == 0)
	332	pr_warn("Architected timer frequency not available\n");
	333	}
	334
	335	static void arch_timer_banner(unsigned type)
	336	{
	337	pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
	338	type & ARCH_CP15_TIMER ? "cp15" : "",
	339	type == (ARCH_CP15_TIMER \| ARCH_MEM_TIMER) ? " and " : "",
	340	type & ARCH_MEM_TIMER ? "mmio" : "",
8a4da6e3 MR	341	(unsigned long)arch_timer_rate / 1000000,
8a4da6e3 MR	342	(unsigned long)(arch_timer_rate / 10000) % 100,
22006994 SB	343	type & ARCH_CP15_TIMER ?
	344	arch_timer_use_virtual ? "virt" : "phys" :
	345	"",
	346	type == (ARCH_CP15_TIMER \| ARCH_MEM_TIMER) ? "/" : "",
	347	type & ARCH_MEM_TIMER ?
	348	arch_timer_mem_use_virtual ? "virt" : "phys" :
	349	"");
8a4da6e3 MR	350	}
	351
	352	u32 arch_timer_get_rate(void)
	353	{
	354	return arch_timer_rate;
	355	}
	356
22006994	357	static u64 arch_counter_get_cntvct_mem(void)
8a4da6e3	358	{
22006994 SB	359	u32 vct_lo, vct_hi, tmp_hi;
	360
	361	do {
	362	vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
	363	vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
	364	tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
	365	} while (vct_hi != tmp_hi);
	366
	367	return ((u64) vct_hi << 32) \| vct_lo;
8a4da6e3 MR	368	}
8a4da6e3 MR	369
22006994 SB	370	/*
	371	* Default to cp15 based access because arm64 uses this function for
	372	* sched_clock() before DT is probed and the cp15 method is guaranteed
	373	* to exist on arm64. arm doesn't use this before DT is probed so even
	374	* if we don't have the cp15 accessors we won't have a problem.
	375	*/
	376	u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
	377
8a4da6e3 MR	378	static cycle_t arch_counter_read(struct clocksource *cs)
8a4da6e3 MR	379	{
22006994	380	return arch_timer_read_counter();
8a4da6e3 MR	381	}
	382
	383	static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
	384	{
22006994	385	return arch_timer_read_counter();
8a4da6e3 MR	386	}
	387
	388	static struct clocksource clocksource_counter = {
	389	.name = "arch_sys_counter",
	390	.rating = 400,
	391	.read = arch_counter_read,
	392	.mask = CLOCKSOURCE_MASK(56),
	393	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
	394	};
	395
	396	static struct cyclecounter cyclecounter = {
	397	.read = arch_counter_read_cc,
	398	.mask = CLOCKSOURCE_MASK(56),
	399	};
	400
	401	static struct timecounter timecounter;
	402
	403	struct timecounter *arch_timer_get_timecounter(void)
	404	{
	405	return &timecounter;
	406	}
	407
22006994 SB	408	static void __init arch_counter_register(unsigned type)
	409	{
	410	u64 start_count;
	411
	412	/* Register the CP15 based counter if we have one */
	413	if (type & ARCH_CP15_TIMER)
	414	arch_timer_read_counter = arch_counter_get_cntvct;
	415	else
	416	arch_timer_read_counter = arch_counter_get_cntvct_mem;
	417
	418	start_count = arch_timer_read_counter();
	419	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	420	cyclecounter.mult = clocksource_counter.mult;
	421	cyclecounter.shift = clocksource_counter.shift;
	422	timecounter_init(&timecounter, &cyclecounter, start_count);
	423	}
	424
8c37bb3a	425	static void arch_timer_stop(struct clock_event_device *clk)
8a4da6e3 MR	426	{
	427	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
	428	clk->irq, smp_processor_id());
	429
	430	if (arch_timer_use_virtual)
	431	disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
	432	else {
	433	disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
	434	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	435	disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
	436	}
	437
	438	clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
	439	}
	440
8c37bb3a	441	static int arch_timer_cpu_notify(struct notifier_block *self,
8a4da6e3 MR	442	unsigned long action, void *hcpu)
8a4da6e3 MR	443	{
f31c2f1c SB	444	/*
	445	* Grab cpu pointer in each case to avoid spurious
	446	* preemptible warnings
	447	*/
8a4da6e3 MR	448	switch (action & ~CPU_TASKS_FROZEN) {
8a4da6e3 MR	449	case CPU_STARTING:
f31c2f1c	450	arch_timer_setup(this_cpu_ptr(arch_timer_evt));
8a4da6e3 MR	451	break;
8a4da6e3 MR	452	case CPU_DYING:
f31c2f1c	453	arch_timer_stop(this_cpu_ptr(arch_timer_evt));
8a4da6e3 MR	454	break;
	455	}
	456
	457	return NOTIFY_OK;
	458	}
	459
8c37bb3a	460	static struct notifier_block arch_timer_cpu_nb = {
8a4da6e3 MR	461	.notifier_call = arch_timer_cpu_notify,
	462	};
	463
	464	static int __init arch_timer_register(void)
	465	{
	466	int err;
	467	int ppi;
	468
8a4da6e3 MR	469	arch_timer_evt = alloc_percpu(struct clock_event_device);
	470	if (!arch_timer_evt) {
	471	err = -ENOMEM;
	472	goto out;
	473	}
	474
65cd4f6c SB	475	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	476	cyclecounter.mult = clocksource_counter.mult;
	477	cyclecounter.shift = clocksource_counter.shift;
	478	timecounter_init(&timecounter, &cyclecounter,
	479	arch_counter_get_cntvct());
	480
	481	/* 56 bits minimum, so we assume worst case rollover */
	482	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
	483
8a4da6e3 MR	484	if (arch_timer_use_virtual) {
	485	ppi = arch_timer_ppi[VIRT_PPI];
	486	err = request_percpu_irq(ppi, arch_timer_handler_virt,
	487	"arch_timer", arch_timer_evt);
	488	} else {
	489	ppi = arch_timer_ppi[PHYS_SECURE_PPI];
	490	err = request_percpu_irq(ppi, arch_timer_handler_phys,
	491	"arch_timer", arch_timer_evt);
	492	if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
	493	ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
	494	err = request_percpu_irq(ppi, arch_timer_handler_phys,
	495	"arch_timer", arch_timer_evt);
	496	if (err)
	497	free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
	498	arch_timer_evt);
	499	}
	500	}
	501
	502	if (err) {
	503	pr_err("arch_timer: can't register interrupt %d (%d)\n",
	504	ppi, err);
	505	goto out_free;
	506	}
	507
	508	err = register_cpu_notifier(&arch_timer_cpu_nb);
	509	if (err)
	510	goto out_free_irq;
	511
	512	/* Immediately configure the timer on the boot CPU */
	513	arch_timer_setup(this_cpu_ptr(arch_timer_evt));
	514
	515	return 0;
	516
	517	out_free_irq:
	518	if (arch_timer_use_virtual)
	519	free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
	520	else {
	521	free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
	522	arch_timer_evt);
	523	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	524	free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
	525	arch_timer_evt);
	526	}
	527
	528	out_free:
	529	free_percpu(arch_timer_evt);
	530	out:
	531	return err;
	532	}
	533
22006994 SB	534	static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
	535	{
	536	int ret;
	537	irq_handler_t func;
	538	struct arch_timer *t;
	539
	540	t = kzalloc(sizeof(*t), GFP_KERNEL);
	541	if (!t)
	542	return -ENOMEM;
	543
	544	t->base = base;
	545	t->evt.irq = irq;
	546	__arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
	547
	548	if (arch_timer_mem_use_virtual)
	549	func = arch_timer_handler_virt_mem;
	550	else
	551	func = arch_timer_handler_phys_mem;
	552
	553	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
	554	if (ret) {
	555	pr_err("arch_timer: Failed to request mem timer irq\n");
	556	kfree(t);
	557	}
	558
	559	return ret;
	560	}
	561
	562	static const struct of_device_id arch_timer_of_match[] __initconst = {
	563	{ .compatible = "arm,armv7-timer", },
	564	{ .compatible = "arm,armv8-timer", },
	565	{},
	566	};
	567
	568	static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
	569	{ .compatible = "arm,armv7-timer-mem", },
	570	{},
	571	};
	572
	573	static void __init arch_timer_common_init(void)
	574	{
	575	unsigned mask = ARCH_CP15_TIMER \| ARCH_MEM_TIMER;
	576
	577	/* Wait until both nodes are probed if we have two timers */
	578	if ((arch_timers_present & mask) != mask) {
	579	if (of_find_matching_node(NULL, arch_timer_mem_of_match) &&
	580	!(arch_timers_present & ARCH_MEM_TIMER))
	581	return;
	582	if (of_find_matching_node(NULL, arch_timer_of_match) &&
	583	!(arch_timers_present & ARCH_CP15_TIMER))
	584	return;
	585	}
	586
	587	arch_timer_banner(arch_timers_present);
	588	arch_counter_register(arch_timers_present);
	589	arch_timer_arch_init();
	590	}
	591
0583fe47	592	static void __init arch_timer_init(struct device_node *np)
8a4da6e3	593	{
8a4da6e3 MR	594	int i;
8a4da6e3 MR	595
22006994	596	if (arch_timers_present & ARCH_CP15_TIMER) {
0583fe47 RH	597	pr_warn("arch_timer: multiple nodes in dt, skipping\n");
0583fe47 RH	598	return;
8a4da6e3 MR	599	}
8a4da6e3 MR	600
22006994	601	arch_timers_present \|= ARCH_CP15_TIMER;
8a4da6e3 MR	602	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
8a4da6e3 MR	603	arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
22006994	604	arch_timer_detect_rate(NULL, np);
8a4da6e3 MR	605
8a4da6e3 MR	606	/*
8266891e MZ	607	* If HYP mode is available, we know that the physical timer
	608	* has been configured to be accessible from PL1. Use it, so
	609	* that a guest can use the virtual timer instead.
	610	*
8a4da6e3 MR	611	* If no interrupt provided for virtual timer, we'll have to
	612	* stick to the physical timer. It'd better be accessible...
	613	*/
8266891e	614	if (is_hyp_mode_available() \|\| !arch_timer_ppi[VIRT_PPI]) {
8a4da6e3 MR	615	arch_timer_use_virtual = false;
	616
	617	if (!arch_timer_ppi[PHYS_SECURE_PPI] \|\|
	618	!arch_timer_ppi[PHYS_NONSECURE_PPI]) {
	619	pr_warn("arch_timer: No interrupt available, giving up\n");
0583fe47	620	return;
8a4da6e3 MR	621	}
	622	}
	623
0583fe47	624	arch_timer_register();
22006994	625	arch_timer_common_init();
8a4da6e3	626	}
0583fe47 RH	627	CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_init);
0583fe47 RH	628	CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_init);
22006994 SB	629
	630	static void __init arch_timer_mem_init(struct device_node *np)
	631	{
	632	struct device_node frame, best_frame = NULL;
	633	void __iomem cntctlbase, base;
	634	unsigned int irq;
	635	u32 cnttidr;
	636
	637	arch_timers_present \|= ARCH_MEM_TIMER;
	638	cntctlbase = of_iomap(np, 0);
	639	if (!cntctlbase) {
	640	pr_err("arch_timer: Can't find CNTCTLBase\n");
	641	return;
	642	}
	643
	644	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
	645	iounmap(cntctlbase);
	646
	647	/*
	648	* Try to find a virtual capable frame. Otherwise fall back to a
	649	* physical capable frame.
	650	*/
	651	for_each_available_child_of_node(np, frame) {
	652	int n;
	653
	654	if (of_property_read_u32(frame, "frame-number", &n)) {
	655	pr_err("arch_timer: Missing frame-number\n");
	656	of_node_put(best_frame);
	657	of_node_put(frame);
	658	return;
	659	}
	660
	661	if (cnttidr & CNTTIDR_VIRT(n)) {
	662	of_node_put(best_frame);
	663	best_frame = frame;
	664	arch_timer_mem_use_virtual = true;
	665	break;
	666	}
	667	of_node_put(best_frame);
	668	best_frame = of_node_get(frame);
	669	}
	670
	671	base = arch_counter_base = of_iomap(best_frame, 0);
	672	if (!base) {
	673	pr_err("arch_timer: Can't map frame's registers\n");
	674	of_node_put(best_frame);
	675	return;
	676	}
	677
	678	if (arch_timer_mem_use_virtual)
	679	irq = irq_of_parse_and_map(best_frame, 1);
	680	else
	681	irq = irq_of_parse_and_map(best_frame, 0);
	682	of_node_put(best_frame);
	683	if (!irq) {
	684	pr_err("arch_timer: Frame missing %s irq",
cfb6d656	685	arch_timer_mem_use_virtual ? "virt" : "phys");
22006994 SB	686	return;
	687	}
	688
	689	arch_timer_detect_rate(base, np);
	690	arch_timer_mem_register(base, irq);
	691	arch_timer_common_init();
	692	}
	693	CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
	694	arch_timer_mem_init);