[mirror_ubuntu-jammy-kernel.git] / drivers / clocksource / timer-clint.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
 *
 * Most of the M-mode (i.e. NoMMU) RISC-V systems usually have a
 * CLINT MMIO timer device.
 */

#define pr_fmt(fmt) "clint: " fmt
#include <linux/bitops.h>
#include <linux/clocksource.h>
#include <linux/clockchips.h>
#include <linux/cpu.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/of_address.h>
#include <linux/sched_clock.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/smp.h>
#include <linux/timex.h>

#ifndef CONFIG_RISCV_M_MODE
#include <asm/clint.h>
#endif

#define CLINT_IPI_OFF		0
#define CLINT_TIMER_CMP_OFF	0x4000
#define CLINT_TIMER_VAL_OFF	0xbff8

/* CLINT manages IPI and Timer for RISC-V M-mode  */
static u32 __iomem *clint_ipi_base;
static u64 __iomem *clint_timer_cmp;
static u64 __iomem *clint_timer_val;
static unsigned long clint_timer_freq;
static unsigned int clint_timer_irq;

#ifdef CONFIG_RISCV_M_MODE
u64 __iomem *clint_time_val;
EXPORT_SYMBOL(clint_time_val);
#endif

static void clint_send_ipi(const struct cpumask *target)
{
	unsigned int cpu;

	for_each_cpu(cpu, target)
		writel(1, clint_ipi_base + cpuid_to_hartid_map(cpu));
}

static void clint_clear_ipi(void)
{
	writel(0, clint_ipi_base + cpuid_to_hartid_map(smp_processor_id()));
}

static struct riscv_ipi_ops clint_ipi_ops = {
	.ipi_inject = clint_send_ipi,
	.ipi_clear = clint_clear_ipi,
};

#ifdef CONFIG_64BIT
#define clint_get_cycles()	readq_relaxed(clint_timer_val)
#else
#define clint_get_cycles()	readl_relaxed(clint_timer_val)
#define clint_get_cycles_hi()	readl_relaxed(((u32 *)clint_timer_val) + 1)
#endif

#ifdef CONFIG_64BIT
static u64 notrace clint_get_cycles64(void)
{
	return clint_get_cycles();
}
#else /* CONFIG_64BIT */
static u64 notrace clint_get_cycles64(void)
{
	u32 hi, lo;

	do {
		hi = clint_get_cycles_hi();
		lo = clint_get_cycles();
	} while (hi != clint_get_cycles_hi());

	return ((u64)hi << 32) | lo;
}
#endif /* CONFIG_64BIT */

static u64 clint_rdtime(struct clocksource *cs)
{
	return clint_get_cycles64();
}

static struct clocksource clint_clocksource = {
	.name		= "clint_clocksource",
	.rating		= 300,
	.mask		= CLOCKSOURCE_MASK(64),
	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
	.read		= clint_rdtime,
};

static int clint_clock_next_event(unsigned long delta,
				   struct clock_event_device *ce)
{
	void __iomem *r = clint_timer_cmp +
			  cpuid_to_hartid_map(smp_processor_id());

	csr_set(CSR_IE, IE_TIE);
	writeq_relaxed(clint_get_cycles64() + delta, r);
	return 0;
}

static DEFINE_PER_CPU(struct clock_event_device, clint_clock_event) = {
	.name		= "clint_clockevent",
	.features	= CLOCK_EVT_FEAT_ONESHOT,
	.rating		= 100,
	.set_next_event	= clint_clock_next_event,
};

static int clint_timer_starting_cpu(unsigned int cpu)
{
	struct clock_event_device *ce = per_cpu_ptr(&clint_clock_event, cpu);

	ce->cpumask = cpumask_of(cpu);
	clockevents_config_and_register(ce, clint_timer_freq, 100, 0x7fffffff);

	enable_percpu_irq(clint_timer_irq,
			  irq_get_trigger_type(clint_timer_irq));
	return 0;
}

static int clint_timer_dying_cpu(unsigned int cpu)
{
	disable_percpu_irq(clint_timer_irq);
	return 0;
}

static irqreturn_t clint_timer_interrupt(int irq, void *dev_id)
{
	struct clock_event_device *evdev = this_cpu_ptr(&clint_clock_event);

	csr_clear(CSR_IE, IE_TIE);
	evdev->event_handler(evdev);

	return IRQ_HANDLED;
}

static int __init clint_timer_init_dt(struct device_node *np)
{
	int rc;
	u32 i, nr_irqs;
	void __iomem *base;
	struct of_phandle_args oirq;

	/*
	 * Ensure that CLINT device interrupts are either RV_IRQ_TIMER or
	 * RV_IRQ_SOFT. If it's anything else then we ignore the device.
	 */
	nr_irqs = of_irq_count(np);
	for (i = 0; i < nr_irqs; i++) {
		if (of_irq_parse_one(np, i, &oirq)) {
			pr_err("%pOFP: failed to parse irq %d.\n", np, i);
			continue;
		}

		if ((oirq.args_count != 1) ||
		    (oirq.args[0] != RV_IRQ_TIMER &&
		     oirq.args[0] != RV_IRQ_SOFT)) {
			pr_err("%pOFP: invalid irq %d (hwirq %d)\n",
			       np, i, oirq.args[0]);
			return -ENODEV;
		}

		/* Find parent irq domain and map timer irq */
		if (!clint_timer_irq &&
		    oirq.args[0] == RV_IRQ_TIMER &&
		    irq_find_host(oirq.np))
			clint_timer_irq = irq_of_parse_and_map(np, i);
	}

	/* If CLINT timer irq not found then fail */
	if (!clint_timer_irq) {
		pr_err("%pOFP: timer irq not found\n", np);
		return -ENODEV;
	}

	base = of_iomap(np, 0);
	if (!base) {
		pr_err("%pOFP: could not map registers\n", np);
		return -ENODEV;
	}

	clint_ipi_base = base + CLINT_IPI_OFF;
	clint_timer_cmp = base + CLINT_TIMER_CMP_OFF;
	clint_timer_val = base + CLINT_TIMER_VAL_OFF;
	clint_timer_freq = riscv_timebase;

#ifdef CONFIG_RISCV_M_MODE
	/*
	 * Yes, that's an odd naming scheme.  time_val is public, but hopefully
	 * will die in favor of something cleaner.
	 */
	clint_time_val = clint_timer_val;
#endif

	pr_info("%pOFP: timer running at %ld Hz\n", np, clint_timer_freq);

	rc = clocksource_register_hz(&clint_clocksource, clint_timer_freq);
	if (rc) {
		pr_err("%pOFP: clocksource register failed [%d]\n", np, rc);
		goto fail_iounmap;
	}

	sched_clock_register(clint_get_cycles64, 64, clint_timer_freq);

	rc = request_percpu_irq(clint_timer_irq, clint_timer_interrupt,
				 "clint-timer", &clint_clock_event);
	if (rc) {
		pr_err("registering percpu irq failed [%d]\n", rc);
		goto fail_iounmap;
	}

	rc = cpuhp_setup_state(CPUHP_AP_CLINT_TIMER_STARTING,
				"clockevents/clint/timer:starting",
				clint_timer_starting_cpu,
				clint_timer_dying_cpu);
	if (rc) {
		pr_err("%pOFP: cpuhp setup state failed [%d]\n", np, rc);
		goto fail_free_irq;
	}

	riscv_set_ipi_ops(&clint_ipi_ops);
	clint_clear_ipi();

	return 0;

fail_free_irq:
	free_irq(clint_timer_irq, &clint_clock_event);
fail_iounmap:
	iounmap(base);
	return rc;
}

TIMER_OF_DECLARE(clint_timer, "riscv,clint0", clint_timer_init_dt);
TIMER_OF_DECLARE(clint_timer1, "sifive,clint0", clint_timer_init_dt);
Commit	Line	Data
2ac6795f AP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2020 Western Digital Corporation or its affiliates.
	4	*
	5	* Most of the M-mode (i.e. NoMMU) RISC-V systems usually have a
	6	* CLINT MMIO timer device.
	7	*/
	8
	9	#define pr_fmt(fmt) "clint: " fmt
	10	#include <linux/bitops.h>
	11	#include <linux/clocksource.h>
	12	#include <linux/clockchips.h>
	13	#include <linux/cpu.h>
	14	#include <linux/delay.h>
	15	#include <linux/module.h>
	16	#include <linux/of_address.h>
	17	#include <linux/sched_clock.h>
	18	#include <linux/io-64-nonatomic-lo-hi.h>
	19	#include <linux/interrupt.h>
	20	#include <linux/of_irq.h>
	21	#include <linux/smp.h>
d5be89a8 PD	22	#include <linux/timex.h>
	23
	24	#ifndef CONFIG_RISCV_M_MODE
	25	#include <asm/clint.h>
	26	#endif
2ac6795f AP	27
	28	#define CLINT_IPI_OFF 0
	29	#define CLINT_TIMER_CMP_OFF 0x4000
	30	#define CLINT_TIMER_VAL_OFF 0xbff8
	31
	32	/* CLINT manages IPI and Timer for RISC-V M-mode */
	33	static u32 __iomem *clint_ipi_base;
	34	static u64 __iomem *clint_timer_cmp;
	35	static u64 __iomem *clint_timer_val;
	36	static unsigned long clint_timer_freq;
	37	static unsigned int clint_timer_irq;
	38
d5be89a8 PD	39	#ifdef CONFIG_RISCV_M_MODE
d5be89a8 PD	40	u64 __iomem *clint_time_val;
c14decfc	41	EXPORT_SYMBOL(clint_time_val);
d5be89a8 PD	42	#endif
d5be89a8 PD	43
2ac6795f AP	44	static void clint_send_ipi(const struct cpumask *target)
	45	{
	46	unsigned int cpu;
	47
	48	for_each_cpu(cpu, target)
	49	writel(1, clint_ipi_base + cpuid_to_hartid_map(cpu));
	50	}
	51
	52	static void clint_clear_ipi(void)
	53	{
	54	writel(0, clint_ipi_base + cpuid_to_hartid_map(smp_processor_id()));
	55	}
	56
	57	static struct riscv_ipi_ops clint_ipi_ops = {
	58	.ipi_inject = clint_send_ipi,
	59	.ipi_clear = clint_clear_ipi,
	60	};
	61
	62	#ifdef CONFIG_64BIT
	63	#define clint_get_cycles() readq_relaxed(clint_timer_val)
	64	#else
	65	#define clint_get_cycles() readl_relaxed(clint_timer_val)
	66	#define clint_get_cycles_hi() readl_relaxed(((u32 *)clint_timer_val) + 1)
	67	#endif
	68
	69	#ifdef CONFIG_64BIT
	70	static u64 notrace clint_get_cycles64(void)
	71	{
	72	return clint_get_cycles();
	73	}
	74	#else /* CONFIG_64BIT */
	75	static u64 notrace clint_get_cycles64(void)
	76	{
	77	u32 hi, lo;
	78
	79	do {
	80	hi = clint_get_cycles_hi();
	81	lo = clint_get_cycles();
	82	} while (hi != clint_get_cycles_hi());
	83
	84	return ((u64)hi << 32) \| lo;
	85	}
	86	#endif /* CONFIG_64BIT */
	87
	88	static u64 clint_rdtime(struct clocksource *cs)
	89	{
	90	return clint_get_cycles64();
	91	}
	92
	93	static struct clocksource clint_clocksource = {
	94	.name = "clint_clocksource",
	95	.rating = 300,
	96	.mask = CLOCKSOURCE_MASK(64),
	97	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
	98	.read = clint_rdtime,
	99	};
	100
	101	static int clint_clock_next_event(unsigned long delta,
	102	struct clock_event_device *ce)
	103	{
	104	void __iomem *r = clint_timer_cmp +
	105	cpuid_to_hartid_map(smp_processor_id());
	106
	107	csr_set(CSR_IE, IE_TIE);
108	writeq_relaxed(clint_get_cycles64() + delta, r);
109	return 0;
110	}
111
112	static DEFINE_PER_CPU(struct clock_event_device, clint_clock_event) = {
113	.name = "clint_clockevent",
114	.features = CLOCK_EVT_FEAT_ONESHOT,
115	.rating = 100,
116	.set_next_event = clint_clock_next_event,
117	};
118
119	static int clint_timer_starting_cpu(unsigned int cpu)
120	{
121	struct clock_event_device *ce = per_cpu_ptr(&clint_clock_event, cpu);
122
123	ce->cpumask = cpumask_of(cpu);
124	clockevents_config_and_register(ce, clint_timer_freq, 100, 0x7fffffff);
125
126	enable_percpu_irq(clint_timer_irq,
127	irq_get_trigger_type(clint_timer_irq));
128	return 0;
129	}
130
131	static int clint_timer_dying_cpu(unsigned int cpu)
132	{
133	disable_percpu_irq(clint_timer_irq);
134	return 0;
135	}
136
137	static irqreturn_t clint_timer_interrupt(int irq, void *dev_id)
138	{
139	struct clock_event_device *evdev = this_cpu_ptr(&clint_clock_event);
140
141	csr_clear(CSR_IE, IE_TIE);
142	evdev->event_handler(evdev);
143
144	return IRQ_HANDLED;
145	}
146
147	static int __init clint_timer_init_dt(struct device_node *np)
148	{
149	int rc;
150	u32 i, nr_irqs;
151	void __iomem *base;
152	struct of_phandle_args oirq;
153
154	/*
155	* Ensure that CLINT device interrupts are either RV_IRQ_TIMER or
156	* RV_IRQ_SOFT. If it's anything else then we ignore the device.
157	*/
158	nr_irqs = of_irq_count(np);
159	for (i = 0; i < nr_irqs; i++) {
160	if (of_irq_parse_one(np, i, &oirq)) {
161	pr_err("%pOFP: failed to parse irq %d.\n", np, i);
162	continue;
163	}
164
165	if ((oirq.args_count != 1) \|\|
166	(oirq.args[0] != RV_IRQ_TIMER &&
167	oirq.args[0] != RV_IRQ_SOFT)) {
168	pr_err("%pOFP: invalid irq %d (hwirq %d)\n",
169	np, i, oirq.args[0]);
170	return -ENODEV;
171	}
172
173	/* Find parent irq domain and map timer irq */
174	if (!clint_timer_irq &&
175	oirq.args[0] == RV_IRQ_TIMER &&
176	irq_find_host(oirq.np))
177	clint_timer_irq = irq_of_parse_and_map(np, i);
178	}
179
180	/* If CLINT timer irq not found then fail */
181	if (!clint_timer_irq) {
182	pr_err("%pOFP: timer irq not found\n", np);
183	return -ENODEV;
184	}
185
186	base = of_iomap(np, 0);
187	if (!base) {
188	pr_err("%pOFP: could not map registers\n", np);
189	return -ENODEV;
190	}
191
192	clint_ipi_base = base + CLINT_IPI_OFF;
193	clint_timer_cmp = base + CLINT_TIMER_CMP_OFF;
194	clint_timer_val = base + CLINT_TIMER_VAL_OFF;
195	clint_timer_freq = riscv_timebase;
196
d5be89a8 PD	197	#ifdef CONFIG_RISCV_M_MODE
	198	/*
	199	* Yes, that's an odd naming scheme. time_val is public, but hopefully
	200	* will die in favor of something cleaner.
	201	*/
	202	clint_time_val = clint_timer_val;
	203	#endif
	204
2ac6795f AP	205	pr_info("%pOFP: timer running at %ld Hz\n", np, clint_timer_freq);
	206
	207	rc = clocksource_register_hz(&clint_clocksource, clint_timer_freq);
	208	if (rc) {
	209	pr_err("%pOFP: clocksource register failed [%d]\n", np, rc);
	210	goto fail_iounmap;
	211	}
	212
	213	sched_clock_register(clint_get_cycles64, 64, clint_timer_freq);
	214
	215	rc = request_percpu_irq(clint_timer_irq, clint_timer_interrupt,
	216	"clint-timer", &clint_clock_event);
	217	if (rc) {
	218	pr_err("registering percpu irq failed [%d]\n", rc);
	219	goto fail_iounmap;
	220	}
	221
	222	rc = cpuhp_setup_state(CPUHP_AP_CLINT_TIMER_STARTING,
	223	"clockevents/clint/timer:starting",
	224	clint_timer_starting_cpu,
	225	clint_timer_dying_cpu);
	226	if (rc) {
	227	pr_err("%pOFP: cpuhp setup state failed [%d]\n", np, rc);
	228	goto fail_free_irq;
	229	}
	230
	231	riscv_set_ipi_ops(&clint_ipi_ops);
	232	clint_clear_ipi();
	233
	234	return 0;
	235
	236	fail_free_irq:
	237	free_irq(clint_timer_irq, &clint_clock_event);
	238	fail_iounmap:
	239	iounmap(base);
	240	return rc;
	241	}
	242
	243	TIMER_OF_DECLARE(clint_timer, "riscv,clint0", clint_timer_init_dt);
	244	TIMER_OF_DECLARE(clint_timer1, "sifive,clint0", clint_timer_init_dt);