[mirror_ubuntu-focal-kernel.git] / arch / s390 / kernel / vtime.c

// SPDX-License-Identifier: GPL-2.0
/*
 *    Virtual cpu timer based timer functions.
 *
 *    Copyright IBM Corp. 2004, 2012
 *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
 */

#include <linux/kernel_stat.h>
#include <linux/sched/cputime.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/timex.h>
#include <linux/types.h>
#include <linux/time.h>

#include <asm/vtimer.h>
#include <asm/vtime.h>
#include <asm/cpu_mf.h>
#include <asm/smp.h>

#include "entry.h"

static void virt_timer_expire(void);

static LIST_HEAD(virt_timer_list);
static DEFINE_SPINLOCK(virt_timer_lock);
static atomic64_t virt_timer_current;
static atomic64_t virt_timer_elapsed;

DEFINE_PER_CPU(u64, mt_cycles[8]);
static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_jiffies);

static inline u64 get_vtimer(void)
{
	u64 timer;

	asm volatile("stpt %0" : "=m" (timer));
	return timer;
}

static inline void set_vtimer(u64 expires)
{
	u64 timer;

	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
		"	spt	%1"	/* Set new value imm. afterwards */
		: "=m" (timer) : "m" (expires));
	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	S390_lowcore.last_update_timer = expires;
}

static inline int virt_timer_forward(u64 elapsed)
{
	BUG_ON(!irqs_disabled());

	if (list_empty(&virt_timer_list))
		return 0;
	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	return elapsed >= atomic64_read(&virt_timer_current);
}

static void update_mt_scaling(void)
{
	u64 cycles_new[8], *cycles_old;
	u64 delta, fac, mult, div;
	int i;

	stcctm5(smp_cpu_mtid + 1, cycles_new);
	cycles_old = this_cpu_ptr(mt_cycles);
	fac = 1;
	mult = div = 0;
	for (i = 0; i <= smp_cpu_mtid; i++) {
		delta = cycles_new[i] - cycles_old[i];
		div += delta;
		mult *= i + 1;
		mult += delta * fac;
		fac *= i + 1;
	}
	div *= fac;
	if (div > 0) {
		/* Update scaling factor */
		__this_cpu_write(mt_scaling_mult, mult);
		__this_cpu_write(mt_scaling_div, div);
		memcpy(cycles_old, cycles_new,
		       sizeof(u64) * (smp_cpu_mtid + 1));
	}
	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
}

static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
{
	u64 delta;

	delta = new - *tsk_vtime;
	*tsk_vtime = new;
	return delta;
}


static inline u64 scale_vtime(u64 vtime)
{
	u64 mult = __this_cpu_read(mt_scaling_mult);
	u64 div = __this_cpu_read(mt_scaling_div);

	if (smp_cpu_mtid)
		return vtime * mult / div;
	return vtime;
}

static void account_system_index_scaled(struct task_struct *p, u64 cputime,
					enum cpu_usage_stat index)
{
	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
	account_system_index_time(p, cputime_to_nsecs(cputime), index);
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
static int do_account_vtime(struct task_struct *tsk)
{
	u64 timer, clock, user, guest, system, hardirq, softirq, steal;

	timer = S390_lowcore.last_update_timer;
	clock = S390_lowcore.last_update_clock;
	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
		"	stckf	%1"	/* Store current tod clock value */
#else
		"	stck	%1"	/* Store current tod clock value */
#endif
		: "=m" (S390_lowcore.last_update_timer),
		  "=m" (S390_lowcore.last_update_clock));
	clock = S390_lowcore.last_update_clock - clock;
	timer -= S390_lowcore.last_update_timer;

	if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	/* Update MT utilization calculation */
	if (smp_cpu_mtid &&
	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
		update_mt_scaling();

	/* Calculate cputime delta */
	user = update_tsk_timer(&tsk->thread.user_timer,
				READ_ONCE(S390_lowcore.user_timer));
	guest = update_tsk_timer(&tsk->thread.guest_timer,
				 READ_ONCE(S390_lowcore.guest_timer));
	system = update_tsk_timer(&tsk->thread.system_timer,
				  READ_ONCE(S390_lowcore.system_timer));
	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
				   READ_ONCE(S390_lowcore.hardirq_timer));
	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
				   READ_ONCE(S390_lowcore.softirq_timer));
	S390_lowcore.steal_timer +=
		clock - user - guest - system - hardirq - softirq;

	/* Push account value */
	if (user) {
		account_user_time(tsk, cputime_to_nsecs(user));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
	}

	if (guest) {
		account_guest_time(tsk, cputime_to_nsecs(guest));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
	}

	if (system)
		account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
	if (hardirq)
		account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
	if (softirq)
		account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);

	steal = S390_lowcore.steal_timer;
	if ((s64) steal > 0) {
		S390_lowcore.steal_timer = 0;
		account_steal_time(cputime_to_nsecs(steal));
	}

	return virt_timer_forward(user + guest + system + hardirq + softirq);
}

void vtime_task_switch(struct task_struct *prev)
{
	do_account_vtime(prev);
	prev->thread.user_timer = S390_lowcore.user_timer;
	prev->thread.guest_timer = S390_lowcore.guest_timer;
	prev->thread.system_timer = S390_lowcore.system_timer;
	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
	S390_lowcore.user_timer = current->thread.user_timer;
	S390_lowcore.guest_timer = current->thread.guest_timer;
	S390_lowcore.system_timer = current->thread.system_timer;
	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
	S390_lowcore.softirq_timer = current->thread.softirq_timer;
}

/*
 * In s390, accounting pending user time also implies
 * accounting system time in order to correctly compute
 * the stolen time accounting.
 */
void vtime_flush(struct task_struct *tsk)
{
	if (do_account_vtime(tsk))
		virt_timer_expire();
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
void vtime_account_irq_enter(struct task_struct *tsk)
{
	u64 timer;

	timer = S390_lowcore.last_update_timer;
	S390_lowcore.last_update_timer = get_vtimer();
	timer -= S390_lowcore.last_update_timer;

	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
		S390_lowcore.guest_timer += timer;
	else if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else if (in_serving_softirq())
		S390_lowcore.softirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	virt_timer_forward(timer);
}
EXPORT_SYMBOL_GPL(vtime_account_irq_enter);

void vtime_account_system(struct task_struct *tsk)
__attribute__((alias("vtime_account_irq_enter")));
EXPORT_SYMBOL_GPL(vtime_account_system);

/*
 * Sorted add to a list. List is linear searched until first bigger
 * element is found.
 */
static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
{
	struct vtimer_list *tmp;

	list_for_each_entry(tmp, head, entry) {
		if (tmp->expires > timer->expires) {
			list_add_tail(&timer->entry, &tmp->entry);
			return;
		}
	}
	list_add_tail(&timer->entry, head);
}

/*
 * Handler for expired virtual CPU timer.
 */
static void virt_timer_expire(void)
{
	struct vtimer_list *timer, *tmp;
	unsigned long elapsed;
	LIST_HEAD(cb_list);

	/* walk timer list, fire all expired timers */
	spin_lock(&virt_timer_lock);
	elapsed = atomic64_read(&virt_timer_elapsed);
	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
		if (timer->expires < elapsed)
			/* move expired timer to the callback queue */
			list_move_tail(&timer->entry, &cb_list);
		else
			timer->expires -= elapsed;
	}
	if (!list_empty(&virt_timer_list)) {
		timer = list_first_entry(&virt_timer_list,
					 struct vtimer_list, entry);
		atomic64_set(&virt_timer_current, timer->expires);
	}
	atomic64_sub(elapsed, &virt_timer_elapsed);
	spin_unlock(&virt_timer_lock);

	/* Do callbacks and recharge periodic timers */
	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
		list_del_init(&timer->entry);
		timer->function(timer->data);
		if (timer->interval) {
			/* Recharge interval timer */
			timer->expires = timer->interval +
				atomic64_read(&virt_timer_elapsed);
			spin_lock(&virt_timer_lock);
			list_add_sorted(timer, &virt_timer_list);
			spin_unlock(&virt_timer_lock);
		}
	}
}

void init_virt_timer(struct vtimer_list *timer)
{
	timer->function = NULL;
	INIT_LIST_HEAD(&timer->entry);
}
EXPORT_SYMBOL(init_virt_timer);

static inline int vtimer_pending(struct vtimer_list *timer)
{
	return !list_empty(&timer->entry);
}

static void internal_add_vtimer(struct vtimer_list *timer)
{
	if (list_empty(&virt_timer_list)) {
		/* First timer, just program it. */
		atomic64_set(&virt_timer_current, timer->expires);
		atomic64_set(&virt_timer_elapsed, 0);
		list_add(&timer->entry, &virt_timer_list);
	} else {
		/* Update timer against current base. */
		timer->expires += atomic64_read(&virt_timer_elapsed);
		if (likely((s64) timer->expires <
			   (s64) atomic64_read(&virt_timer_current)))
			/* The new timer expires before the current timer. */
			atomic64_set(&virt_timer_current, timer->expires);
		/* Insert new timer into the list. */
		list_add_sorted(timer, &virt_timer_list);
	}
}

static void __add_vtimer(struct vtimer_list *timer, int periodic)
{
	unsigned long flags;

	timer->interval = periodic ? timer->expires : 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
}

/*
 * add_virt_timer - add a oneshot virtual CPU timer
 */
void add_virt_timer(struct vtimer_list *timer)
{
	__add_vtimer(timer, 0);
}
EXPORT_SYMBOL(add_virt_timer);

/*
 * add_virt_timer_int - add an interval virtual CPU timer
 */
void add_virt_timer_periodic(struct vtimer_list *timer)
{
	__add_vtimer(timer, 1);
}
EXPORT_SYMBOL(add_virt_timer_periodic);

static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
{
	unsigned long flags;
	int rc;

	BUG_ON(!timer->function);

	if (timer->expires == expires && vtimer_pending(timer))
		return 1;
	spin_lock_irqsave(&virt_timer_lock, flags);
	rc = vtimer_pending(timer);
	if (rc)
		list_del_init(&timer->entry);
	timer->interval = periodic ? expires : 0;
	timer->expires = expires;
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return rc;
}

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 0);
}
EXPORT_SYMBOL(mod_virt_timer);

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 1);
}
EXPORT_SYMBOL(mod_virt_timer_periodic);

/*
 * Delete a virtual timer.
 *
 * returns whether the deleted timer was pending (1) or not (0)
 */
int del_virt_timer(struct vtimer_list *timer)
{
	unsigned long flags;

	if (!vtimer_pending(timer))
		return 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	list_del_init(&timer->entry);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return 1;
}
EXPORT_SYMBOL(del_virt_timer);

/*
 * Start the virtual CPU timer on the current CPU.
 */
void vtime_init(void)
{
	/* set initial cpu timer */
	set_vtimer(VTIMER_MAX_SLICE);
	/* Setup initial MT scaling values */
	if (smp_cpu_mtid) {
		__this_cpu_write(mt_scaling_jiffies, jiffies);
		__this_cpu_write(mt_scaling_mult, 1);
		__this_cpu_write(mt_scaling_div, 1);
		stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
	}
}
Commit	Line	Data
a17ae4c3	1	// SPDX-License-Identifier: GPL-2.0
1da177e4	2	/*
1da177e4 LT	3	* Virtual cpu timer based timer functions.
1da177e4 LT	4	*
27f6b416	5	* Copyright IBM Corp. 2004, 2012
1da177e4 LT	6	* Author(s): Jan Glauber <jan.glauber@de.ibm.com>
	7	*/
	8
27f6b416	9	#include <linux/kernel_stat.h>
32ef5517	10	#include <linux/sched/cputime.h>
27f6b416	11	#include <linux/export.h>
1da177e4	12	#include <linux/kernel.h>
1da177e4	13	#include <linux/timex.h>
27f6b416 MS	14	#include <linux/types.h>
27f6b416 MS	15	#include <linux/time.h>
1da177e4	16
27f6b416	17	#include <asm/vtimer.h>
a5725ac2	18	#include <asm/vtime.h>
10ad34bc MS	19	#include <asm/cpu_mf.h>
10ad34bc MS	20	#include <asm/smp.h>
1da177e4	21
521b00cd HC	22	#include "entry.h"
521b00cd HC	23
27f6b416	24	static void virt_timer_expire(void);
1da177e4	25
27f6b416 MS	26	static LIST_HEAD(virt_timer_list);
	27	static DEFINE_SPINLOCK(virt_timer_lock);
	28	static atomic64_t virt_timer_current;
	29	static atomic64_t virt_timer_elapsed;
	30
72d38b19	31	DEFINE_PER_CPU(u64, mt_cycles[8]);
10ad34bc MS	32	static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
10ad34bc MS	33	static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
f341b8df	34	static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
10ad34bc	35
27f6b416	36	static inline u64 get_vtimer(void)
9cfb9b3c	37	{
27f6b416	38	u64 timer;
9cfb9b3c	39
27f6b416	40	asm volatile("stpt %0" : "=m" (timer));
9cfb9b3c MS	41	return timer;
	42	}
	43
27f6b416	44	static inline void set_vtimer(u64 expires)
9cfb9b3c	45	{
27f6b416	46	u64 timer;
9cfb9b3c	47
27f6b416 MS	48	asm volatile(
	49	" stpt %0\n" /* Store current cpu timer value */
	50	" spt %1" /* Set new value imm. afterwards */
	51	: "=m" (timer) : "m" (expires));
9cfb9b3c MS	52	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	53	S390_lowcore.last_update_timer = expires;
	54	}
	55
27f6b416 MS	56	static inline int virt_timer_forward(u64 elapsed)
	57	{
	58	BUG_ON(!irqs_disabled());
	59
	60	if (list_empty(&virt_timer_list))
	61	return 0;
	62	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	63	return elapsed >= atomic64_read(&virt_timer_current);
	64	}
	65
72d38b19 MS	66	static void update_mt_scaling(void)
	67	{
	68	u64 cycles_new[8], *cycles_old;
	69	u64 delta, fac, mult, div;
	70	int i;
	71
	72	stcctm5(smp_cpu_mtid + 1, cycles_new);
	73	cycles_old = this_cpu_ptr(mt_cycles);
	74	fac = 1;
	75	mult = div = 0;
	76	for (i = 0; i <= smp_cpu_mtid; i++) {
	77	delta = cycles_new[i] - cycles_old[i];
	78	div += delta;
	79	mult *= i + 1;
	80	mult += delta * fac;
	81	fac *= i + 1;
	82	}
	83	div *= fac;
	84	if (div > 0) {
	85	/* Update scaling factor */
	86	__this_cpu_write(mt_scaling_mult, mult);
	87	__this_cpu_write(mt_scaling_div, div);
	88	memcpy(cycles_old, cycles_new,
	89	sizeof(u64) * (smp_cpu_mtid + 1));
	90	}
	91	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
	92	}
	93
b7394a5f MS	94	static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
	95	{
	96	u64 delta;
	97
	98	delta = new - *tsk_vtime;
	99	*tsk_vtime = new;
	100	return delta;
	101	}
	102
	103
	104	static inline u64 scale_vtime(u64 vtime)
	105	{
	106	u64 mult = __this_cpu_read(mt_scaling_mult);
	107	u64 div = __this_cpu_read(mt_scaling_div);
	108
	109	if (smp_cpu_mtid)
	110	return vtime * mult / div;
	111	return vtime;
	112	}
	113
b29e061b	114	static void account_system_index_scaled(struct task_struct *p, u64 cputime,
b7394a5f MS	115	enum cpu_usage_stat index)
b7394a5f MS	116	{
b29e061b	117	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
fb8b049c	118	account_system_index_time(p, cputime_to_nsecs(cputime), index);
b7394a5f MS	119	}
b7394a5f MS	120
1da177e4 LT	121	/*
	122	* Update process times based on virtual cpu times stored by entry.S
	123	* to the lowcore fields user_timer, system_timer & steal_clock.
	124	*/
8f2b468a	125	static int do_account_vtime(struct task_struct *tsk)
1da177e4	126	{
b7394a5f	127	u64 timer, clock, user, guest, system, hardirq, softirq, steal;
1da177e4 LT	128
	129	timer = S390_lowcore.last_update_timer;
	130	clock = S390_lowcore.last_update_clock;
27f6b416 MS	131	asm volatile(
27f6b416 MS	132	" stpt %0\n" /* Store current cpu timer value */
1f759bb3 MS	133	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	134	" stckf %1" /* Store current tod clock value */
	135	#else
27f6b416	136	" stck %1" /* Store current tod clock value */
1f759bb3	137	#endif
27f6b416 MS	138	: "=m" (S390_lowcore.last_update_timer),
27f6b416 MS	139	"=m" (S390_lowcore.last_update_clock));
b7394a5f MS	140	clock = S390_lowcore.last_update_clock - clock;
	141	timer -= S390_lowcore.last_update_timer;
	142
	143	if (hardirq_count())
	144	S390_lowcore.hardirq_timer += timer;
	145	else
	146	S390_lowcore.system_timer += timer;
1da177e4	147
72d38b19	148	/* Update MT utilization calculation */
f341b8df	149	if (smp_cpu_mtid &&
72d38b19 MS	150	time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
72d38b19 MS	151	update_mt_scaling();
10ad34bc	152
b7394a5f MS	153	/* Calculate cputime delta */
	154	user = update_tsk_timer(&tsk->thread.user_timer,
	155	READ_ONCE(S390_lowcore.user_timer));
	156	guest = update_tsk_timer(&tsk->thread.guest_timer,
	157	READ_ONCE(S390_lowcore.guest_timer));
	158	system = update_tsk_timer(&tsk->thread.system_timer,
	159	READ_ONCE(S390_lowcore.system_timer));
	160	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
	161	READ_ONCE(S390_lowcore.hardirq_timer));
	162	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
	163	READ_ONCE(S390_lowcore.softirq_timer));
	164	S390_lowcore.steal_timer +=
	165	clock - user - guest - system - hardirq - softirq;
	166
	167	/* Push account value */
	168	if (user) {
23244a5c	169	account_user_time(tsk, cputime_to_nsecs(user));
5613fda9	170	tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
b7394a5f	171	}
10ad34bc	172
b7394a5f	173	if (guest) {
fb8b049c	174	account_guest_time(tsk, cputime_to_nsecs(guest));
5613fda9	175	tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
10ad34bc	176	}
b7394a5f MS	177
b7394a5f MS	178	if (system)
b29e061b	179	account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
b7394a5f	180	if (hardirq)
b29e061b	181	account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
b7394a5f	182	if (softirq)
b29e061b	183	account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
1da177e4	184
aa5e97ce MS	185	steal = S390_lowcore.steal_timer;
	186	if ((s64) steal > 0) {
	187	S390_lowcore.steal_timer = 0;
be9095ed	188	account_steal_time(cputime_to_nsecs(steal));
1da177e4	189	}
27f6b416	190
b7394a5f	191	return virt_timer_forward(user + guest + system + hardirq + softirq);
1da177e4 LT	192	}
1da177e4 LT	193
bf9fae9f	194	void vtime_task_switch(struct task_struct *prev)
1f1c12af	195	{
8f2b468a	196	do_account_vtime(prev);
90c53e65	197	prev->thread.user_timer = S390_lowcore.user_timer;
b7394a5f	198	prev->thread.guest_timer = S390_lowcore.guest_timer;
90c53e65	199	prev->thread.system_timer = S390_lowcore.system_timer;
b7394a5f MS	200	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
b7394a5f MS	201	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
90c53e65	202	S390_lowcore.user_timer = current->thread.user_timer;
b7394a5f	203	S390_lowcore.guest_timer = current->thread.guest_timer;
90c53e65	204	S390_lowcore.system_timer = current->thread.system_timer;
b7394a5f MS	205	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
b7394a5f MS	206	S390_lowcore.softirq_timer = current->thread.softirq_timer;
aa5e97ce	207	}
1f1c12af	208
bcebdf84 FW	209	/*
	210	* In s390, accounting pending user time also implies
	211	* accounting system time in order to correctly compute
	212	* the stolen time accounting.
	213	*/
c8d7dabf	214	void vtime_flush(struct task_struct *tsk)
aa5e97ce	215	{
8f2b468a	216	if (do_account_vtime(tsk))
27f6b416	217	virt_timer_expire();
1f1c12af MS	218	}
1f1c12af MS	219
1da177e4 LT	220	/*
	221	* Update process times based on virtual cpu times stored by entry.S
	222	* to the lowcore fields user_timer, system_timer & steal_clock.
	223	*/
6a61671b	224	void vtime_account_irq_enter(struct task_struct *tsk)
1da177e4	225	{
b7394a5f	226	u64 timer;
1da177e4 LT	227
1da177e4 LT	228	timer = S390_lowcore.last_update_timer;
9cfb9b3c	229	S390_lowcore.last_update_timer = get_vtimer();
b7394a5f MS	230	timer -= S390_lowcore.last_update_timer;
	231
	232	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
	233	S390_lowcore.guest_timer += timer;
	234	else if (hardirq_count())
	235	S390_lowcore.hardirq_timer += timer;
	236	else if (in_serving_softirq())
	237	S390_lowcore.softirq_timer += timer;
	238	else
	239	S390_lowcore.system_timer += timer;
	240
	241	virt_timer_forward(timer);
1da177e4	242	}
6a61671b	243	EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
1da177e4	244
fd25b4c2	245	void vtime_account_system(struct task_struct *tsk)
6a61671b	246	__attribute__((alias("vtime_account_irq_enter")));
fd25b4c2	247	EXPORT_SYMBOL_GPL(vtime_account_system);
11113334	248
1da177e4 LT	249	/*
	250	* Sorted add to a list. List is linear searched until first bigger
	251	* element is found.
	252	*/
	253	static void list_add_sorted(struct vtimer_list timer, struct list_head head)
	254	{
27f6b416	255	struct vtimer_list *tmp;
1da177e4	256
27f6b416 MS	257	list_for_each_entry(tmp, head, entry) {
	258	if (tmp->expires > timer->expires) {
	259	list_add_tail(&timer->entry, &tmp->entry);
1da177e4 LT	260	return;
	261	}
	262	}
	263	list_add_tail(&timer->entry, head);
	264	}
	265
	266	/*
27f6b416	267	* Handler for expired virtual CPU timer.
1da177e4	268	*/
27f6b416	269	static void virt_timer_expire(void)
1da177e4	270	{
27f6b416 MS	271	struct vtimer_list timer, tmp;
	272	unsigned long elapsed;
	273	LIST_HEAD(cb_list);
	274
	275	/* walk timer list, fire all expired timers */
	276	spin_lock(&virt_timer_lock);
	277	elapsed = atomic64_read(&virt_timer_elapsed);
	278	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
	279	if (timer->expires < elapsed)
9cfb9b3c	280	/* move expired timer to the callback queue */
27f6b416	281	list_move_tail(&timer->entry, &cb_list);
9cfb9b3c	282	else
27f6b416	283	timer->expires -= elapsed;
1da177e4	284	}
27f6b416 MS	285	if (!list_empty(&virt_timer_list)) {
	286	timer = list_first_entry(&virt_timer_list,
	287	struct vtimer_list, entry);
	288	atomic64_set(&virt_timer_current, timer->expires);
	289	}
	290	atomic64_sub(elapsed, &virt_timer_elapsed);
	291	spin_unlock(&virt_timer_lock);
	292
	293	/* Do callbacks and recharge periodic timers */
	294	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
	295	list_del_init(&timer->entry);
	296	timer->function(timer->data);
	297	if (timer->interval) {
	298	/* Recharge interval timer */
	299	timer->expires = timer->interval +
	300	atomic64_read(&virt_timer_elapsed);
	301	spin_lock(&virt_timer_lock);
	302	list_add_sorted(timer, &virt_timer_list);
	303	spin_unlock(&virt_timer_lock);
	304	}
4c1051e3	305	}
1da177e4 LT	306	}
	307
	308	void init_virt_timer(struct vtimer_list *timer)
	309	{
1da177e4 LT	310	timer->function = NULL;
1da177e4 LT	311	INIT_LIST_HEAD(&timer->entry);
1da177e4 LT	312	}
	313	EXPORT_SYMBOL(init_virt_timer);
	314
1da177e4 LT	315	static inline int vtimer_pending(struct vtimer_list *timer)
1da177e4 LT	316	{
27f6b416	317	return !list_empty(&timer->entry);
1da177e4 LT	318	}
1da177e4 LT	319
1da177e4 LT	320	static void internal_add_vtimer(struct vtimer_list *timer)
1da177e4 LT	321	{
27f6b416 MS	322	if (list_empty(&virt_timer_list)) {
	323	/* First timer, just program it. */
	324	atomic64_set(&virt_timer_current, timer->expires);
	325	atomic64_set(&virt_timer_elapsed, 0);
	326	list_add(&timer->entry, &virt_timer_list);
9cfb9b3c	327	} else {
27f6b416 MS	328	/* Update timer against current base. */
	329	timer->expires += atomic64_read(&virt_timer_elapsed);
	330	if (likely((s64) timer->expires <
	331	(s64) atomic64_read(&virt_timer_current)))
9cfb9b3c	332	/* The new timer expires before the current timer. */
27f6b416 MS	333	atomic64_set(&virt_timer_current, timer->expires);
	334	/* Insert new timer into the list. */
	335	list_add_sorted(timer, &virt_timer_list);
1da177e4	336	}
1da177e4 LT	337	}
1da177e4 LT	338
27f6b416	339	static void __add_vtimer(struct vtimer_list *timer, int periodic)
1da177e4	340	{
27f6b416 MS	341	unsigned long flags;
	342
	343	timer->interval = periodic ? timer->expires : 0;
	344	spin_lock_irqsave(&virt_timer_lock, flags);
	345	internal_add_vtimer(timer);
	346	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	347	}
	348
	349	/*
0f5e1558	350	* add_virt_timer - add a oneshot virtual CPU timer
1da177e4	351	*/
27f6b416	352	void add_virt_timer(struct vtimer_list *timer)
1da177e4	353	{
27f6b416	354	__add_vtimer(timer, 0);
1da177e4 LT	355	}
	356	EXPORT_SYMBOL(add_virt_timer);
	357
	358	/*
	359	* add_virt_timer_int - add an interval virtual CPU timer
	360	*/
27f6b416	361	void add_virt_timer_periodic(struct vtimer_list *timer)
1da177e4	362	{
27f6b416	363	__add_vtimer(timer, 1);
1da177e4 LT	364	}
	365	EXPORT_SYMBOL(add_virt_timer_periodic);
	366
27f6b416	367	static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
1da177e4	368	{
1da177e4	369	unsigned long flags;
27f6b416	370	int rc;
1da177e4	371
ca366a32	372	BUG_ON(!timer->function);
1da177e4	373
1da177e4 LT	374	if (timer->expires == expires && vtimer_pending(timer))
1da177e4 LT	375	return 1;
27f6b416 MS	376	spin_lock_irqsave(&virt_timer_lock, flags);
	377	rc = vtimer_pending(timer);
	378	if (rc)
	379	list_del_init(&timer->entry);
	380	timer->interval = periodic ? expires : 0;
1da177e4	381	timer->expires = expires;
1da177e4	382	internal_add_vtimer(timer);
27f6b416 MS	383	spin_unlock_irqrestore(&virt_timer_lock, flags);
27f6b416 MS	384	return rc;
1da177e4	385	}
b6ecfa92 JG	386
b6ecfa92 JG	387	/*
b6ecfa92 JG	388	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	389	*/
27f6b416	390	int mod_virt_timer(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	391	{
	392	return __mod_vtimer(timer, expires, 0);
	393	}
1da177e4 LT	394	EXPORT_SYMBOL(mod_virt_timer);
1da177e4 LT	395
b6ecfa92	396	/*
b6ecfa92 JG	397	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	398	*/
27f6b416	399	int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	400	{
	401	return __mod_vtimer(timer, expires, 1);
	402	}
	403	EXPORT_SYMBOL(mod_virt_timer_periodic);
	404
1da177e4	405	/*
27f6b416	406	* Delete a virtual timer.
1da177e4 LT	407	*
	408	* returns whether the deleted timer was pending (1) or not (0)
	409	*/
	410	int del_virt_timer(struct vtimer_list *timer)
	411	{
	412	unsigned long flags;
1da177e4	413
1da177e4 LT	414	if (!vtimer_pending(timer))
1da177e4 LT	415	return 0;
27f6b416	416	spin_lock_irqsave(&virt_timer_lock, flags);
1da177e4	417	list_del_init(&timer->entry);
27f6b416	418	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	419	return 1;
	420	}
	421	EXPORT_SYMBOL(del_virt_timer);
	422
	423	/*
	424	* Start the virtual CPU timer on the current CPU.
	425	*/
b5f87f15	426	void vtime_init(void)
1da177e4	427	{
8b646bd7	428	/* set initial cpu timer */
27f6b416	429	set_vtimer(VTIMER_MAX_SLICE);
f341b8df MS	430	/* Setup initial MT scaling values */
	431	if (smp_cpu_mtid) {
	432	__this_cpu_write(mt_scaling_jiffies, jiffies);
	433	__this_cpu_write(mt_scaling_mult, 1);
	434	__this_cpu_write(mt_scaling_div, 1);
	435	stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
	436	}
1da177e4	437	}