[mirror_ubuntu-jammy-kernel.git] / drivers / misc / sgi-gru / gruhandles.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *              GRU KERNEL MCS INSTRUCTIONS
 *
 *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
 */

#include <linux/kernel.h>
#include "gru.h"
#include "grulib.h"
#include "grutables.h"

/* 10 sec */
#ifdef CONFIG_IA64
#include <asm/processor.h>
#define GRU_OPERATION_TIMEOUT	(((cycles_t) local_cpu_data->itc_freq)*10)
#define CLKS2NSEC(c)		((c) *1000000000 / local_cpu_data->itc_freq)
#else
#include <asm/tsc.h>
#define GRU_OPERATION_TIMEOUT	((cycles_t) tsc_khz*10*1000)
#define CLKS2NSEC(c)		((c) * 1000000 / tsc_khz)
#endif

/* Extract the status field from a kernel handle */
#define GET_MSEG_HANDLE_STATUS(h)	(((*(unsigned long *)(h)) >> 16) & 3)

struct mcs_op_statistic mcs_op_statistics[mcsop_last];

static void update_mcs_stats(enum mcs_op op, unsigned long clks)
{
	unsigned long nsec;

	nsec = CLKS2NSEC(clks);
	atomic_long_inc(&mcs_op_statistics[op].count);
	atomic_long_add(nsec, &mcs_op_statistics[op].total);
	if (mcs_op_statistics[op].max < nsec)
		mcs_op_statistics[op].max = nsec;
}

static void start_instruction(void *h)
{
	unsigned long *w0 = h;

	wmb();		/* setting CMD/STATUS bits must be last */
	*w0 = *w0 | 0x20001;
	gru_flush_cache(h);
}

static void report_instruction_timeout(void *h)
{
	unsigned long goff = GSEGPOFF((unsigned long)h);
	char *id = "???";

	if (TYPE_IS(CCH, goff))
		id = "CCH";
	else if (TYPE_IS(TGH, goff))
		id = "TGH";
	else if (TYPE_IS(TFH, goff))
		id = "TFH";

	panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
}

static int wait_instruction_complete(void *h, enum mcs_op opc)
{
	int status;
	unsigned long start_time = get_cycles();

	while (1) {
		cpu_relax();
		status = GET_MSEG_HANDLE_STATUS(h);
		if (status != CCHSTATUS_ACTIVE)
			break;
		if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
			report_instruction_timeout(h);
			start_time = get_cycles();
		}
	}
	if (gru_options & OPT_STATS)
		update_mcs_stats(opc, get_cycles() - start_time);
	return status;
}

int cch_allocate(struct gru_context_configuration_handle *cch)
{
	int ret;

	cch->opc = CCHOP_ALLOCATE;
	start_instruction(cch);
	ret = wait_instruction_complete(cch, cchop_allocate);

	/*
	 * Stop speculation into the GSEG being mapped by the previous ALLOCATE.
	 * The GSEG memory does not exist until the ALLOCATE completes.
	 */
	sync_core();
	return ret;
}

int cch_start(struct gru_context_configuration_handle *cch)
{
	cch->opc = CCHOP_START;
	start_instruction(cch);
	return wait_instruction_complete(cch, cchop_start);
}

int cch_interrupt(struct gru_context_configuration_handle *cch)
{
	cch->opc = CCHOP_INTERRUPT;
	start_instruction(cch);
	return wait_instruction_complete(cch, cchop_interrupt);
}

int cch_deallocate(struct gru_context_configuration_handle *cch)
{
	int ret;

	cch->opc = CCHOP_DEALLOCATE;
	start_instruction(cch);
	ret = wait_instruction_complete(cch, cchop_deallocate);

	/*
	 * Stop speculation into the GSEG being unmapped by the previous
	 * DEALLOCATE.
	 */
	sync_core();
	return ret;
}

int cch_interrupt_sync(struct gru_context_configuration_handle
				     *cch)
{
	cch->opc = CCHOP_INTERRUPT_SYNC;
	start_instruction(cch);
	return wait_instruction_complete(cch, cchop_interrupt_sync);
}

int tgh_invalidate(struct gru_tlb_global_handle *tgh,
				 unsigned long vaddr, unsigned long vaddrmask,
				 int asid, int pagesize, int global, int n,
				 unsigned short ctxbitmap)
{
	tgh->vaddr = vaddr;
	tgh->asid = asid;
	tgh->pagesize = pagesize;
	tgh->n = n;
	tgh->global = global;
	tgh->vaddrmask = vaddrmask;
	tgh->ctxbitmap = ctxbitmap;
	tgh->opc = TGHOP_TLBINV;
	start_instruction(tgh);
	return wait_instruction_complete(tgh, tghop_invalidate);
}

int tfh_write_only(struct gru_tlb_fault_handle *tfh,
				  unsigned long paddr, int gaa,
				  unsigned long vaddr, int asid, int dirty,
				  int pagesize)
{
	tfh->fillasid = asid;
	tfh->fillvaddr = vaddr;
	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
	tfh->gaa = gaa;
	tfh->dirty = dirty;
	tfh->pagesize = pagesize;
	tfh->opc = TFHOP_WRITE_ONLY;
	start_instruction(tfh);
	return wait_instruction_complete(tfh, tfhop_write_only);
}

void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
				     unsigned long paddr, int gaa,
				     unsigned long vaddr, int asid, int dirty,
				     int pagesize)
{
	tfh->fillasid = asid;
	tfh->fillvaddr = vaddr;
	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
	tfh->gaa = gaa;
	tfh->dirty = dirty;
	tfh->pagesize = pagesize;
	tfh->opc = TFHOP_WRITE_RESTART;
	start_instruction(tfh);
}

void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
{
	tfh->opc = TFHOP_USER_POLLING_MODE;
	start_instruction(tfh);
}

void tfh_exception(struct gru_tlb_fault_handle *tfh)
{
	tfh->opc = TFHOP_EXCEPTION;
	start_instruction(tfh);
}
Commit	Line	Data
1a59d1b8	1	// SPDX-License-Identifier: GPL-2.0-or-later
a24e5e1c JS	2	/*
	3	* GRU KERNEL MCS INSTRUCTIONS
	4	*
	5	* Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
a24e5e1c JS	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include "gru.h"
	10	#include "grulib.h"
	11	#include "grutables.h"
	12
	13	/* 10 sec */
	14	#ifdef CONFIG_IA64
	15	#include <asm/processor.h>
	16	#define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10)
563447d7	17	#define CLKS2NSEC(c) ((c) *1000000000 / local_cpu_data->itc_freq)
a24e5e1c JS	18	#else
	19	#include <asm/tsc.h>
	20	#define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz101000)
563447d7	21	#define CLKS2NSEC(c) ((c) * 1000000 / tsc_khz)
a24e5e1c JS	22	#endif
	23
	24	/* Extract the status field from a kernel handle */
	25	#define GET_MSEG_HANDLE_STATUS(h) ((((unsigned long )(h)) >> 16) & 3)
	26
e56484da JS	27	struct mcs_op_statistic mcs_op_statistics[mcsop_last];
	28
	29	static void update_mcs_stats(enum mcs_op op, unsigned long clks)
	30	{
563447d7 JS	31	unsigned long nsec;
	32
	33	nsec = CLKS2NSEC(clks);
e56484da	34	atomic_long_inc(&mcs_op_statistics[op].count);
563447d7 JS	35	atomic_long_add(nsec, &mcs_op_statistics[op].total);
	36	if (mcs_op_statistics[op].max < nsec)
	37	mcs_op_statistics[op].max = nsec;
e56484da JS	38	}
e56484da JS	39
a24e5e1c JS	40	static void start_instruction(void *h)
	41	{
	42	unsigned long *w0 = h;
	43
57ebb034 JS	44	wmb(); /* setting CMD/STATUS bits must be last */
57ebb034 JS	45	w0 = w0 \| 0x20001;
a24e5e1c JS	46	gru_flush_cache(h);
	47	}
	48
648eb8e5 JS	49	static void report_instruction_timeout(void *h)
	50	{
	51	unsigned long goff = GSEGPOFF((unsigned long)h);
	52	char *id = "???";
	53
	54	if (TYPE_IS(CCH, goff))
	55	id = "CCH";
	56	else if (TYPE_IS(TGH, goff))
	57	id = "TGH";
	58	else if (TYPE_IS(TFH, goff))
	59	id = "TFH";
	60
	61	panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
	62	}
	63
a24e5e1c JS	64	static int wait_instruction_complete(void *h, enum mcs_op opc)
	65	{
	66	int status;
836ce679	67	unsigned long start_time = get_cycles();
a24e5e1c JS	68
	69	while (1) {
	70	cpu_relax();
	71	status = GET_MSEG_HANDLE_STATUS(h);
	72	if (status != CCHSTATUS_ACTIVE)
	73	break;
648eb8e5 JS	74	if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
	75	report_instruction_timeout(h);
	76	start_time = get_cycles();
	77	}
a24e5e1c	78	}
e56484da JS	79	if (gru_options & OPT_STATS)
e56484da JS	80	update_mcs_stats(opc, get_cycles() - start_time);
a24e5e1c JS	81	return status;
	82	}
	83
6e910074	84	int cch_allocate(struct gru_context_configuration_handle *cch)
a24e5e1c	85	{
67bf04a5 JS	86	int ret;
67bf04a5 JS	87
a24e5e1c JS	88	cch->opc = CCHOP_ALLOCATE;
a24e5e1c JS	89	start_instruction(cch);
67bf04a5 JS	90	ret = wait_instruction_complete(cch, cchop_allocate);
	91
	92	/*
	93	* Stop speculation into the GSEG being mapped by the previous ALLOCATE.
	94	* The GSEG memory does not exist until the ALLOCATE completes.
	95	*/
	96	sync_core();
	97	return ret;
a24e5e1c JS	98	}
	99
	100	int cch_start(struct gru_context_configuration_handle *cch)
	101	{
	102	cch->opc = CCHOP_START;
	103	start_instruction(cch);
	104	return wait_instruction_complete(cch, cchop_start);
	105	}
	106
	107	int cch_interrupt(struct gru_context_configuration_handle *cch)
	108	{
	109	cch->opc = CCHOP_INTERRUPT;
	110	start_instruction(cch);
	111	return wait_instruction_complete(cch, cchop_interrupt);
	112	}
	113
	114	int cch_deallocate(struct gru_context_configuration_handle *cch)
	115	{
67bf04a5 JS	116	int ret;
67bf04a5 JS	117
a24e5e1c JS	118	cch->opc = CCHOP_DEALLOCATE;
a24e5e1c JS	119	start_instruction(cch);
67bf04a5 JS	120	ret = wait_instruction_complete(cch, cchop_deallocate);
	121
	122	/*
	123	* Stop speculation into the GSEG being unmapped by the previous
	124	* DEALLOCATE.
	125	*/
	126	sync_core();
	127	return ret;
a24e5e1c JS	128	}
	129
	130	int cch_interrupt_sync(struct gru_context_configuration_handle
	131	*cch)
	132	{
	133	cch->opc = CCHOP_INTERRUPT_SYNC;
	134	start_instruction(cch);
	135	return wait_instruction_complete(cch, cchop_interrupt_sync);
	136	}
	137
	138	int tgh_invalidate(struct gru_tlb_global_handle *tgh,
	139	unsigned long vaddr, unsigned long vaddrmask,
	140	int asid, int pagesize, int global, int n,
	141	unsigned short ctxbitmap)
	142	{
	143	tgh->vaddr = vaddr;
	144	tgh->asid = asid;
	145	tgh->pagesize = pagesize;
	146	tgh->n = n;
	147	tgh->global = global;
	148	tgh->vaddrmask = vaddrmask;
	149	tgh->ctxbitmap = ctxbitmap;
	150	tgh->opc = TGHOP_TLBINV;
	151	start_instruction(tgh);
	152	return wait_instruction_complete(tgh, tghop_invalidate);
	153	}
	154
c550222f JS	155	int tfh_write_only(struct gru_tlb_fault_handle *tfh,
	156	unsigned long paddr, int gaa,
	157	unsigned long vaddr, int asid, int dirty,
	158	int pagesize)
a24e5e1c JS	159	{
	160	tfh->fillasid = asid;
	161	tfh->fillvaddr = vaddr;
c550222f JS	162	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
c550222f JS	163	tfh->gaa = gaa;
a24e5e1c JS	164	tfh->dirty = dirty;
	165	tfh->pagesize = pagesize;
	166	tfh->opc = TFHOP_WRITE_ONLY;
	167	start_instruction(tfh);
c550222f	168	return wait_instruction_complete(tfh, tfhop_write_only);
a24e5e1c JS	169	}
	170
	171	void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
	172	unsigned long paddr, int gaa,
	173	unsigned long vaddr, int asid, int dirty,
	174	int pagesize)
	175	{
	176	tfh->fillasid = asid;
	177	tfh->fillvaddr = vaddr;
	178	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
	179	tfh->gaa = gaa;
	180	tfh->dirty = dirty;
	181	tfh->pagesize = pagesize;
	182	tfh->opc = TFHOP_WRITE_RESTART;
	183	start_instruction(tfh);
	184	}
	185
a24e5e1c JS	186	void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
	187	{
	188	tfh->opc = TFHOP_USER_POLLING_MODE;
	189	start_instruction(tfh);
	190	}
	191
	192	void tfh_exception(struct gru_tlb_fault_handle *tfh)
	193	{
	194	tfh->opc = TFHOP_EXCEPTION;
	195	start_instruction(tfh);
	196	}
	197