[mirror_ubuntu-kernels.git] / arch / s390 / kernel / vtime.c

// SPDX-License-Identifier: GPL-2.0
/*
 *    Virtual cpu timer based timer functions.
 *
 *    Copyright IBM Corp. 2004, 2012
 *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
 */

#include <linux/kernel_stat.h>
#include <linux/sched/cputime.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/timex.h>
#include <linux/types.h>
#include <linux/time.h>

#include <asm/vtimer.h>
#include <asm/vtime.h>
#include <asm/cpu_mf.h>
#include <asm/smp.h>

#include "entry.h"

static void virt_timer_expire(void);

static LIST_HEAD(virt_timer_list);
static DEFINE_SPINLOCK(virt_timer_lock);
static atomic64_t virt_timer_current;
static atomic64_t virt_timer_elapsed;

DEFINE_PER_CPU(u64, mt_cycles[8]);
static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
static DEFINE_PER_CPU(u64, mt_scaling_jiffies);

static inline u64 get_vtimer(void)
{
	u64 timer;

	asm volatile("stpt %0" : "=m" (timer));
	return timer;
}

static inline void set_vtimer(u64 expires)
{
	u64 timer;

	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
		"	spt	%1"	/* Set new value imm. afterwards */
		: "=m" (timer) : "m" (expires));
	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	S390_lowcore.last_update_timer = expires;
}

static inline int virt_timer_forward(u64 elapsed)
{
	BUG_ON(!irqs_disabled());

	if (list_empty(&virt_timer_list))
		return 0;
	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	return elapsed >= atomic64_read(&virt_timer_current);
}

static void update_mt_scaling(void)
{
	u64 cycles_new[8], *cycles_old;
	u64 delta, fac, mult, div;
	int i;

	stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
	cycles_old = this_cpu_ptr(mt_cycles);
	fac = 1;
	mult = div = 0;
	for (i = 0; i <= smp_cpu_mtid; i++) {
		delta = cycles_new[i] - cycles_old[i];
		div += delta;
		mult *= i + 1;
		mult += delta * fac;
		fac *= i + 1;
	}
	div *= fac;
	if (div > 0) {
		/* Update scaling factor */
		__this_cpu_write(mt_scaling_mult, mult);
		__this_cpu_write(mt_scaling_div, div);
		memcpy(cycles_old, cycles_new,
		       sizeof(u64) * (smp_cpu_mtid + 1));
	}
	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
}

static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
{
	u64 delta;

	delta = new - *tsk_vtime;
	*tsk_vtime = new;
	return delta;
}


static inline u64 scale_vtime(u64 vtime)
{
	u64 mult = __this_cpu_read(mt_scaling_mult);
	u64 div = __this_cpu_read(mt_scaling_div);

	if (smp_cpu_mtid)
		return vtime * mult / div;
	return vtime;
}

static void account_system_index_scaled(struct task_struct *p, u64 cputime,
					enum cpu_usage_stat index)
{
	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
	account_system_index_time(p, cputime_to_nsecs(cputime), index);
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
static int do_account_vtime(struct task_struct *tsk)
{
	u64 timer, clock, user, guest, system, hardirq, softirq;

	timer = S390_lowcore.last_update_timer;
	clock = S390_lowcore.last_update_clock;
	asm volatile(
		"	stpt	%0\n"	/* Store current cpu timer value */
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
		"	stckf	%1"	/* Store current tod clock value */
#else
		"	stck	%1"	/* Store current tod clock value */
#endif
		: "=m" (S390_lowcore.last_update_timer),
		  "=m" (S390_lowcore.last_update_clock));
	clock = S390_lowcore.last_update_clock - clock;
	timer -= S390_lowcore.last_update_timer;

	if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	/* Update MT utilization calculation */
	if (smp_cpu_mtid &&
	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
		update_mt_scaling();

	/* Calculate cputime delta */
	user = update_tsk_timer(&tsk->thread.user_timer,
				READ_ONCE(S390_lowcore.user_timer));
	guest = update_tsk_timer(&tsk->thread.guest_timer,
				 READ_ONCE(S390_lowcore.guest_timer));
	system = update_tsk_timer(&tsk->thread.system_timer,
				  READ_ONCE(S390_lowcore.system_timer));
	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
				   READ_ONCE(S390_lowcore.hardirq_timer));
	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
				   READ_ONCE(S390_lowcore.softirq_timer));
	S390_lowcore.steal_timer +=
		clock - user - guest - system - hardirq - softirq;

	/* Push account value */
	if (user) {
		account_user_time(tsk, cputime_to_nsecs(user));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
	}

	if (guest) {
		account_guest_time(tsk, cputime_to_nsecs(guest));
		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
	}

	if (system)
		account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
	if (hardirq)
		account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
	if (softirq)
		account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);

	return virt_timer_forward(user + guest + system + hardirq + softirq);
}

void vtime_task_switch(struct task_struct *prev)
{
	do_account_vtime(prev);
	prev->thread.user_timer = S390_lowcore.user_timer;
	prev->thread.guest_timer = S390_lowcore.guest_timer;
	prev->thread.system_timer = S390_lowcore.system_timer;
	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
	S390_lowcore.user_timer = current->thread.user_timer;
	S390_lowcore.guest_timer = current->thread.guest_timer;
	S390_lowcore.system_timer = current->thread.system_timer;
	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
	S390_lowcore.softirq_timer = current->thread.softirq_timer;
}

/*
 * In s390, accounting pending user time also implies
 * accounting system time in order to correctly compute
 * the stolen time accounting.
 */
void vtime_flush(struct task_struct *tsk)
{
	u64 steal, avg_steal;

	if (do_account_vtime(tsk))
		virt_timer_expire();

	steal = S390_lowcore.steal_timer;
	avg_steal = S390_lowcore.avg_steal_timer / 2;
	if ((s64) steal > 0) {
		S390_lowcore.steal_timer = 0;
		account_steal_time(steal);
		avg_steal += steal;
	}
	S390_lowcore.avg_steal_timer = avg_steal;
}

/*
 * Update process times based on virtual cpu times stored by entry.S
 * to the lowcore fields user_timer, system_timer & steal_clock.
 */
void vtime_account_irq_enter(struct task_struct *tsk)
{
	u64 timer;

	timer = S390_lowcore.last_update_timer;
	S390_lowcore.last_update_timer = get_vtimer();
	timer -= S390_lowcore.last_update_timer;

	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
		S390_lowcore.guest_timer += timer;
	else if (hardirq_count())
		S390_lowcore.hardirq_timer += timer;
	else if (in_serving_softirq())
		S390_lowcore.softirq_timer += timer;
	else
		S390_lowcore.system_timer += timer;

	virt_timer_forward(timer);
}
EXPORT_SYMBOL_GPL(vtime_account_irq_enter);

void vtime_account_system(struct task_struct *tsk)
__attribute__((alias("vtime_account_irq_enter")));
EXPORT_SYMBOL_GPL(vtime_account_system);

/*
 * Sorted add to a list. List is linear searched until first bigger
 * element is found.
 */
static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
{
	struct vtimer_list *tmp;

	list_for_each_entry(tmp, head, entry) {
		if (tmp->expires > timer->expires) {
			list_add_tail(&timer->entry, &tmp->entry);
			return;
		}
	}
	list_add_tail(&timer->entry, head);
}

/*
 * Handler for expired virtual CPU timer.
 */
static void virt_timer_expire(void)
{
	struct vtimer_list *timer, *tmp;
	unsigned long elapsed;
	LIST_HEAD(cb_list);

	/* walk timer list, fire all expired timers */
	spin_lock(&virt_timer_lock);
	elapsed = atomic64_read(&virt_timer_elapsed);
	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
		if (timer->expires < elapsed)
			/* move expired timer to the callback queue */
			list_move_tail(&timer->entry, &cb_list);
		else
			timer->expires -= elapsed;
	}
	if (!list_empty(&virt_timer_list)) {
		timer = list_first_entry(&virt_timer_list,
					 struct vtimer_list, entry);
		atomic64_set(&virt_timer_current, timer->expires);
	}
	atomic64_sub(elapsed, &virt_timer_elapsed);
	spin_unlock(&virt_timer_lock);

	/* Do callbacks and recharge periodic timers */
	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
		list_del_init(&timer->entry);
		timer->function(timer->data);
		if (timer->interval) {
			/* Recharge interval timer */
			timer->expires = timer->interval +
				atomic64_read(&virt_timer_elapsed);
			spin_lock(&virt_timer_lock);
			list_add_sorted(timer, &virt_timer_list);
			spin_unlock(&virt_timer_lock);
		}
	}
}

void init_virt_timer(struct vtimer_list *timer)
{
	timer->function = NULL;
	INIT_LIST_HEAD(&timer->entry);
}
EXPORT_SYMBOL(init_virt_timer);

static inline int vtimer_pending(struct vtimer_list *timer)
{
	return !list_empty(&timer->entry);
}

static void internal_add_vtimer(struct vtimer_list *timer)
{
	if (list_empty(&virt_timer_list)) {
		/* First timer, just program it. */
		atomic64_set(&virt_timer_current, timer->expires);
		atomic64_set(&virt_timer_elapsed, 0);
		list_add(&timer->entry, &virt_timer_list);
	} else {
		/* Update timer against current base. */
		timer->expires += atomic64_read(&virt_timer_elapsed);
		if (likely((s64) timer->expires <
			   (s64) atomic64_read(&virt_timer_current)))
			/* The new timer expires before the current timer. */
			atomic64_set(&virt_timer_current, timer->expires);
		/* Insert new timer into the list. */
		list_add_sorted(timer, &virt_timer_list);
	}
}

static void __add_vtimer(struct vtimer_list *timer, int periodic)
{
	unsigned long flags;

	timer->interval = periodic ? timer->expires : 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
}

/*
 * add_virt_timer - add a oneshot virtual CPU timer
 */
void add_virt_timer(struct vtimer_list *timer)
{
	__add_vtimer(timer, 0);
}
EXPORT_SYMBOL(add_virt_timer);

/*
 * add_virt_timer_int - add an interval virtual CPU timer
 */
void add_virt_timer_periodic(struct vtimer_list *timer)
{
	__add_vtimer(timer, 1);
}
EXPORT_SYMBOL(add_virt_timer_periodic);

static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
{
	unsigned long flags;
	int rc;

	BUG_ON(!timer->function);

	if (timer->expires == expires && vtimer_pending(timer))
		return 1;
	spin_lock_irqsave(&virt_timer_lock, flags);
	rc = vtimer_pending(timer);
	if (rc)
		list_del_init(&timer->entry);
	timer->interval = periodic ? expires : 0;
	timer->expires = expires;
	internal_add_vtimer(timer);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return rc;
}

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 0);
}
EXPORT_SYMBOL(mod_virt_timer);

/*
 * returns whether it has modified a pending timer (1) or not (0)
 */
int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
{
	return __mod_vtimer(timer, expires, 1);
}
EXPORT_SYMBOL(mod_virt_timer_periodic);

/*
 * Delete a virtual timer.
 *
 * returns whether the deleted timer was pending (1) or not (0)
 */
int del_virt_timer(struct vtimer_list *timer)
{
	unsigned long flags;

	if (!vtimer_pending(timer))
		return 0;
	spin_lock_irqsave(&virt_timer_lock, flags);
	list_del_init(&timer->entry);
	spin_unlock_irqrestore(&virt_timer_lock, flags);
	return 1;
}
EXPORT_SYMBOL(del_virt_timer);

/*
 * Start the virtual CPU timer on the current CPU.
 */
void vtime_init(void)
{
	/* set initial cpu timer */
	set_vtimer(VTIMER_MAX_SLICE);
	/* Setup initial MT scaling values */
	if (smp_cpu_mtid) {
		__this_cpu_write(mt_scaling_jiffies, jiffies);
		__this_cpu_write(mt_scaling_mult, 1);
		__this_cpu_write(mt_scaling_div, 1);
		stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
	}
}
Commit	Line	Data
a17ae4c3	1	// SPDX-License-Identifier: GPL-2.0
1da177e4	2	/*
1da177e4 LT	3	* Virtual cpu timer based timer functions.
1da177e4 LT	4	*
27f6b416	5	* Copyright IBM Corp. 2004, 2012
1da177e4 LT	6	* Author(s): Jan Glauber <jan.glauber@de.ibm.com>
	7	*/
	8
27f6b416	9	#include <linux/kernel_stat.h>
32ef5517	10	#include <linux/sched/cputime.h>
27f6b416	11	#include <linux/export.h>
1da177e4	12	#include <linux/kernel.h>
1da177e4	13	#include <linux/timex.h>
27f6b416 MS	14	#include <linux/types.h>
27f6b416 MS	15	#include <linux/time.h>
1da177e4	16
27f6b416	17	#include <asm/vtimer.h>
a5725ac2	18	#include <asm/vtime.h>
10ad34bc MS	19	#include <asm/cpu_mf.h>
10ad34bc MS	20	#include <asm/smp.h>
1da177e4	21
521b00cd HC	22	#include "entry.h"
521b00cd HC	23
27f6b416	24	static void virt_timer_expire(void);
1da177e4	25
27f6b416 MS	26	static LIST_HEAD(virt_timer_list);
	27	static DEFINE_SPINLOCK(virt_timer_lock);
	28	static atomic64_t virt_timer_current;
	29	static atomic64_t virt_timer_elapsed;
	30
72d38b19	31	DEFINE_PER_CPU(u64, mt_cycles[8]);
10ad34bc MS	32	static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
10ad34bc MS	33	static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
f341b8df	34	static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
10ad34bc	35
27f6b416	36	static inline u64 get_vtimer(void)
9cfb9b3c	37	{
27f6b416	38	u64 timer;
9cfb9b3c	39
27f6b416	40	asm volatile("stpt %0" : "=m" (timer));
9cfb9b3c MS	41	return timer;
	42	}
	43
27f6b416	44	static inline void set_vtimer(u64 expires)
9cfb9b3c	45	{
27f6b416	46	u64 timer;
9cfb9b3c	47
27f6b416 MS	48	asm volatile(
	49	" stpt %0\n" /* Store current cpu timer value */
	50	" spt %1" /* Set new value imm. afterwards */
	51	: "=m" (timer) : "m" (expires));
9cfb9b3c MS	52	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
	53	S390_lowcore.last_update_timer = expires;
	54	}
	55
27f6b416 MS	56	static inline int virt_timer_forward(u64 elapsed)
	57	{
	58	BUG_ON(!irqs_disabled());
	59
	60	if (list_empty(&virt_timer_list))
	61	return 0;
	62	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
	63	return elapsed >= atomic64_read(&virt_timer_current);
	64	}
	65
72d38b19 MS	66	static void update_mt_scaling(void)
	67	{
	68	u64 cycles_new[8], *cycles_old;
	69	u64 delta, fac, mult, div;
	70	int i;
	71
346d034d	72	stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
72d38b19 MS	73	cycles_old = this_cpu_ptr(mt_cycles);
	74	fac = 1;
	75	mult = div = 0;
	76	for (i = 0; i <= smp_cpu_mtid; i++) {
	77	delta = cycles_new[i] - cycles_old[i];
	78	div += delta;
	79	mult *= i + 1;
	80	mult += delta * fac;
	81	fac *= i + 1;
	82	}
	83	div *= fac;
	84	if (div > 0) {
	85	/* Update scaling factor */
	86	__this_cpu_write(mt_scaling_mult, mult);
	87	__this_cpu_write(mt_scaling_div, div);
	88	memcpy(cycles_old, cycles_new,
	89	sizeof(u64) * (smp_cpu_mtid + 1));
	90	}
	91	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
	92	}
	93
b7394a5f MS	94	static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
	95	{
	96	u64 delta;
	97
	98	delta = new - *tsk_vtime;
	99	*tsk_vtime = new;
	100	return delta;
	101	}
	102
	103
	104	static inline u64 scale_vtime(u64 vtime)
	105	{
	106	u64 mult = __this_cpu_read(mt_scaling_mult);
	107	u64 div = __this_cpu_read(mt_scaling_div);
	108
	109	if (smp_cpu_mtid)
	110	return vtime * mult / div;
	111	return vtime;
	112	}
	113
b29e061b	114	static void account_system_index_scaled(struct task_struct *p, u64 cputime,
b7394a5f MS	115	enum cpu_usage_stat index)
b7394a5f MS	116	{
b29e061b	117	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
fb8b049c	118	account_system_index_time(p, cputime_to_nsecs(cputime), index);
b7394a5f MS	119	}
b7394a5f MS	120
1da177e4 LT	121	/*
	122	* Update process times based on virtual cpu times stored by entry.S
	123	* to the lowcore fields user_timer, system_timer & steal_clock.
	124	*/
8f2b468a	125	static int do_account_vtime(struct task_struct *tsk)
1da177e4	126	{
152e9b86	127	u64 timer, clock, user, guest, system, hardirq, softirq;
1da177e4 LT	128
	129	timer = S390_lowcore.last_update_timer;
	130	clock = S390_lowcore.last_update_clock;
27f6b416 MS	131	asm volatile(
27f6b416 MS	132	" stpt %0\n" /* Store current cpu timer value */
1f759bb3 MS	133	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	134	" stckf %1" /* Store current tod clock value */
	135	#else
27f6b416	136	" stck %1" /* Store current tod clock value */
1f759bb3	137	#endif
27f6b416 MS	138	: "=m" (S390_lowcore.last_update_timer),
27f6b416 MS	139	"=m" (S390_lowcore.last_update_clock));
b7394a5f MS	140	clock = S390_lowcore.last_update_clock - clock;
	141	timer -= S390_lowcore.last_update_timer;
	142
	143	if (hardirq_count())
	144	S390_lowcore.hardirq_timer += timer;
	145	else
	146	S390_lowcore.system_timer += timer;
1da177e4	147
72d38b19	148	/* Update MT utilization calculation */
f341b8df	149	if (smp_cpu_mtid &&
72d38b19 MS	150	time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
72d38b19 MS	151	update_mt_scaling();
10ad34bc	152
b7394a5f MS	153	/* Calculate cputime delta */
	154	user = update_tsk_timer(&tsk->thread.user_timer,
	155	READ_ONCE(S390_lowcore.user_timer));
	156	guest = update_tsk_timer(&tsk->thread.guest_timer,
	157	READ_ONCE(S390_lowcore.guest_timer));
	158	system = update_tsk_timer(&tsk->thread.system_timer,
	159	READ_ONCE(S390_lowcore.system_timer));
	160	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
	161	READ_ONCE(S390_lowcore.hardirq_timer));
	162	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
	163	READ_ONCE(S390_lowcore.softirq_timer));
	164	S390_lowcore.steal_timer +=
	165	clock - user - guest - system - hardirq - softirq;
	166
	167	/* Push account value */
	168	if (user) {
23244a5c	169	account_user_time(tsk, cputime_to_nsecs(user));
5613fda9	170	tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
b7394a5f	171	}
10ad34bc	172
b7394a5f	173	if (guest) {
fb8b049c	174	account_guest_time(tsk, cputime_to_nsecs(guest));
5613fda9	175	tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
10ad34bc	176	}
b7394a5f MS	177
b7394a5f MS	178	if (system)
b29e061b	179	account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
b7394a5f	180	if (hardirq)
b29e061b	181	account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
b7394a5f	182	if (softirq)
b29e061b	183	account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
1da177e4	184
b7394a5f	185	return virt_timer_forward(user + guest + system + hardirq + softirq);
1da177e4 LT	186	}
1da177e4 LT	187
bf9fae9f	188	void vtime_task_switch(struct task_struct *prev)
1f1c12af	189	{
8f2b468a	190	do_account_vtime(prev);
90c53e65	191	prev->thread.user_timer = S390_lowcore.user_timer;
b7394a5f	192	prev->thread.guest_timer = S390_lowcore.guest_timer;
90c53e65	193	prev->thread.system_timer = S390_lowcore.system_timer;
b7394a5f MS	194	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
b7394a5f MS	195	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
90c53e65	196	S390_lowcore.user_timer = current->thread.user_timer;
b7394a5f	197	S390_lowcore.guest_timer = current->thread.guest_timer;
90c53e65	198	S390_lowcore.system_timer = current->thread.system_timer;
b7394a5f MS	199	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
b7394a5f MS	200	S390_lowcore.softirq_timer = current->thread.softirq_timer;
aa5e97ce	201	}
1f1c12af	202
bcebdf84 FW	203	/*
	204	* In s390, accounting pending user time also implies
	205	* accounting system time in order to correctly compute
	206	* the stolen time accounting.
	207	*/
c8d7dabf	208	void vtime_flush(struct task_struct *tsk)
aa5e97ce	209	{
152e9b86 MS	210	u64 steal, avg_steal;
152e9b86 MS	211
8f2b468a	212	if (do_account_vtime(tsk))
27f6b416	213	virt_timer_expire();
152e9b86 MS	214
	215	steal = S390_lowcore.steal_timer;
	216	avg_steal = S390_lowcore.avg_steal_timer / 2;
	217	if ((s64) steal > 0) {
	218	S390_lowcore.steal_timer = 0;
	219	account_steal_time(steal);
	220	avg_steal += steal;
	221	}
	222	S390_lowcore.avg_steal_timer = avg_steal;
1f1c12af MS	223	}
1f1c12af MS	224
1da177e4 LT	225	/*
	226	* Update process times based on virtual cpu times stored by entry.S
	227	* to the lowcore fields user_timer, system_timer & steal_clock.
	228	*/
6a61671b	229	void vtime_account_irq_enter(struct task_struct *tsk)
1da177e4	230	{
b7394a5f	231	u64 timer;
1da177e4 LT	232
1da177e4 LT	233	timer = S390_lowcore.last_update_timer;
9cfb9b3c	234	S390_lowcore.last_update_timer = get_vtimer();
b7394a5f MS	235	timer -= S390_lowcore.last_update_timer;
	236
	237	if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
	238	S390_lowcore.guest_timer += timer;
	239	else if (hardirq_count())
	240	S390_lowcore.hardirq_timer += timer;
	241	else if (in_serving_softirq())
	242	S390_lowcore.softirq_timer += timer;
	243	else
	244	S390_lowcore.system_timer += timer;
	245
	246	virt_timer_forward(timer);
1da177e4	247	}
6a61671b	248	EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
1da177e4	249
fd25b4c2	250	void vtime_account_system(struct task_struct *tsk)
6a61671b	251	__attribute__((alias("vtime_account_irq_enter")));
fd25b4c2	252	EXPORT_SYMBOL_GPL(vtime_account_system);
11113334	253
1da177e4 LT	254	/*
	255	* Sorted add to a list. List is linear searched until first bigger
	256	* element is found.
	257	*/
	258	static void list_add_sorted(struct vtimer_list timer, struct list_head head)
	259	{
27f6b416	260	struct vtimer_list *tmp;
1da177e4	261
27f6b416 MS	262	list_for_each_entry(tmp, head, entry) {
	263	if (tmp->expires > timer->expires) {
	264	list_add_tail(&timer->entry, &tmp->entry);
1da177e4 LT	265	return;
	266	}
	267	}
	268	list_add_tail(&timer->entry, head);
	269	}
	270
	271	/*
27f6b416	272	* Handler for expired virtual CPU timer.
1da177e4	273	*/
27f6b416	274	static void virt_timer_expire(void)
1da177e4	275	{
27f6b416 MS	276	struct vtimer_list timer, tmp;
	277	unsigned long elapsed;
	278	LIST_HEAD(cb_list);
	279
	280	/* walk timer list, fire all expired timers */
	281	spin_lock(&virt_timer_lock);
	282	elapsed = atomic64_read(&virt_timer_elapsed);
	283	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
	284	if (timer->expires < elapsed)
9cfb9b3c	285	/* move expired timer to the callback queue */
27f6b416	286	list_move_tail(&timer->entry, &cb_list);
9cfb9b3c	287	else
27f6b416	288	timer->expires -= elapsed;
1da177e4	289	}
27f6b416 MS	290	if (!list_empty(&virt_timer_list)) {
	291	timer = list_first_entry(&virt_timer_list,
	292	struct vtimer_list, entry);
	293	atomic64_set(&virt_timer_current, timer->expires);
	294	}
	295	atomic64_sub(elapsed, &virt_timer_elapsed);
	296	spin_unlock(&virt_timer_lock);
	297
	298	/* Do callbacks and recharge periodic timers */
	299	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
	300	list_del_init(&timer->entry);
	301	timer->function(timer->data);
	302	if (timer->interval) {
	303	/* Recharge interval timer */
	304	timer->expires = timer->interval +
	305	atomic64_read(&virt_timer_elapsed);
	306	spin_lock(&virt_timer_lock);
	307	list_add_sorted(timer, &virt_timer_list);
	308	spin_unlock(&virt_timer_lock);
	309	}
4c1051e3	310	}
1da177e4 LT	311	}
	312
	313	void init_virt_timer(struct vtimer_list *timer)
	314	{
1da177e4 LT	315	timer->function = NULL;
1da177e4 LT	316	INIT_LIST_HEAD(&timer->entry);
1da177e4 LT	317	}
	318	EXPORT_SYMBOL(init_virt_timer);
	319
1da177e4 LT	320	static inline int vtimer_pending(struct vtimer_list *timer)
1da177e4 LT	321	{
27f6b416	322	return !list_empty(&timer->entry);
1da177e4 LT	323	}
1da177e4 LT	324
1da177e4 LT	325	static void internal_add_vtimer(struct vtimer_list *timer)
1da177e4 LT	326	{
27f6b416 MS	327	if (list_empty(&virt_timer_list)) {
	328	/* First timer, just program it. */
	329	atomic64_set(&virt_timer_current, timer->expires);
	330	atomic64_set(&virt_timer_elapsed, 0);
	331	list_add(&timer->entry, &virt_timer_list);
9cfb9b3c	332	} else {
27f6b416 MS	333	/* Update timer against current base. */
	334	timer->expires += atomic64_read(&virt_timer_elapsed);
	335	if (likely((s64) timer->expires <
	336	(s64) atomic64_read(&virt_timer_current)))
9cfb9b3c	337	/* The new timer expires before the current timer. */
27f6b416 MS	338	atomic64_set(&virt_timer_current, timer->expires);
	339	/* Insert new timer into the list. */
	340	list_add_sorted(timer, &virt_timer_list);
1da177e4	341	}
1da177e4 LT	342	}
1da177e4 LT	343
27f6b416	344	static void __add_vtimer(struct vtimer_list *timer, int periodic)
1da177e4	345	{
27f6b416 MS	346	unsigned long flags;
	347
	348	timer->interval = periodic ? timer->expires : 0;
	349	spin_lock_irqsave(&virt_timer_lock, flags);
	350	internal_add_vtimer(timer);
	351	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	352	}
	353
	354	/*
0f5e1558	355	* add_virt_timer - add a oneshot virtual CPU timer
1da177e4	356	*/
27f6b416	357	void add_virt_timer(struct vtimer_list *timer)
1da177e4	358	{
27f6b416	359	__add_vtimer(timer, 0);
1da177e4 LT	360	}
	361	EXPORT_SYMBOL(add_virt_timer);
	362
	363	/*
	364	* add_virt_timer_int - add an interval virtual CPU timer
	365	*/
27f6b416	366	void add_virt_timer_periodic(struct vtimer_list *timer)
1da177e4	367	{
27f6b416	368	__add_vtimer(timer, 1);
1da177e4 LT	369	}
	370	EXPORT_SYMBOL(add_virt_timer_periodic);
	371
27f6b416	372	static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
1da177e4	373	{
1da177e4	374	unsigned long flags;
27f6b416	375	int rc;
1da177e4	376
ca366a32	377	BUG_ON(!timer->function);
1da177e4	378
1da177e4 LT	379	if (timer->expires == expires && vtimer_pending(timer))
1da177e4 LT	380	return 1;
27f6b416 MS	381	spin_lock_irqsave(&virt_timer_lock, flags);
	382	rc = vtimer_pending(timer);
	383	if (rc)
	384	list_del_init(&timer->entry);
	385	timer->interval = periodic ? expires : 0;
1da177e4	386	timer->expires = expires;
1da177e4	387	internal_add_vtimer(timer);
27f6b416 MS	388	spin_unlock_irqrestore(&virt_timer_lock, flags);
27f6b416 MS	389	return rc;
1da177e4	390	}
b6ecfa92 JG	391
b6ecfa92 JG	392	/*
b6ecfa92 JG	393	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	394	*/
27f6b416	395	int mod_virt_timer(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	396	{
	397	return __mod_vtimer(timer, expires, 0);
	398	}
1da177e4 LT	399	EXPORT_SYMBOL(mod_virt_timer);
1da177e4 LT	400
b6ecfa92	401	/*
b6ecfa92 JG	402	* returns whether it has modified a pending timer (1) or not (0)
b6ecfa92 JG	403	*/
27f6b416	404	int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
b6ecfa92 JG	405	{
	406	return __mod_vtimer(timer, expires, 1);
	407	}
	408	EXPORT_SYMBOL(mod_virt_timer_periodic);
	409
1da177e4	410	/*
27f6b416	411	* Delete a virtual timer.
1da177e4 LT	412	*
	413	* returns whether the deleted timer was pending (1) or not (0)
	414	*/
	415	int del_virt_timer(struct vtimer_list *timer)
	416	{
	417	unsigned long flags;
1da177e4	418
1da177e4 LT	419	if (!vtimer_pending(timer))
1da177e4 LT	420	return 0;
27f6b416	421	spin_lock_irqsave(&virt_timer_lock, flags);
1da177e4	422	list_del_init(&timer->entry);
27f6b416	423	spin_unlock_irqrestore(&virt_timer_lock, flags);
1da177e4 LT	424	return 1;
	425	}
	426	EXPORT_SYMBOL(del_virt_timer);
	427
	428	/*
	429	* Start the virtual CPU timer on the current CPU.
	430	*/
b5f87f15	431	void vtime_init(void)
1da177e4	432	{
8b646bd7	433	/* set initial cpu timer */
27f6b416	434	set_vtimer(VTIMER_MAX_SLICE);
f341b8df MS	435	/* Setup initial MT scaling values */
	436	if (smp_cpu_mtid) {
	437	__this_cpu_write(mt_scaling_jiffies, jiffies);
	438	__this_cpu_write(mt_scaling_mult, 1);
	439	__this_cpu_write(mt_scaling_div, 1);
346d034d	440	stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
f341b8df	441	}
1da177e4	442	}