[mirror_ubuntu-artful-kernel.git] / arch / x86 / include / asm / intel_rdt.h

#ifndef _ASM_X86_INTEL_RDT_H
#define _ASM_X86_INTEL_RDT_H

#ifdef CONFIG_INTEL_RDT

#include <linux/sched.h>
#include <linux/kernfs.h>
#include <linux/jump_label.h>

#include <asm/intel_rdt_common.h>

#define IA32_L3_QOS_CFG		0xc81
#define IA32_L3_CBM_BASE	0xc90
#define IA32_L2_CBM_BASE	0xd10
#define IA32_MBA_THRTL_BASE	0xd50

#define L3_QOS_CDP_ENABLE	0x01ULL

/**
 * struct rdtgroup - store rdtgroup's data in resctrl file system.
 * @kn:				kernfs node
 * @rdtgroup_list:		linked list for all rdtgroups
 * @closid:			closid for this rdtgroup
 * @cpu_mask:			CPUs assigned to this rdtgroup
 * @flags:			status bits
 * @waitcount:			how many cpus expect to find this
 *				group when they acquire rdtgroup_mutex
 */
struct rdtgroup {
	struct kernfs_node	*kn;
	struct list_head	rdtgroup_list;
	int			closid;
	struct cpumask		cpu_mask;
	int			flags;
	atomic_t		waitcount;
};

/* rdtgroup.flags */
#define	RDT_DELETED		1

/* rftype.flags */
#define RFTYPE_FLAGS_CPUS_LIST	1

/* List of all resource groups */
extern struct list_head rdt_all_groups;

extern int max_name_width, max_data_width;

int __init rdtgroup_init(void);

/**
 * struct rftype - describe each file in the resctrl file system
 * @name:	File name
 * @mode:	Access mode
 * @kf_ops:	File operations
 * @flags:	File specific RFTYPE_FLAGS_* flags
 * @seq_show:	Show content of the file
 * @write:	Write to the file
 */
struct rftype {
	char			*name;
	umode_t			mode;
	struct kernfs_ops	*kf_ops;
	unsigned long		flags;

	int (*seq_show)(struct kernfs_open_file *of,
			struct seq_file *sf, void *v);
	/*
	 * write() is the generic write callback which maps directly to
	 * kernfs write operation and overrides all other operations.
	 * Maximum write size is determined by ->max_write_len.
	 */
	ssize_t (*write)(struct kernfs_open_file *of,
			 char *buf, size_t nbytes, loff_t off);
};

/**
 * struct rdt_domain - group of cpus sharing an RDT resource
 * @list:	all instances of this resource
 * @id:		unique id for this instance
 * @cpu_mask:	which cpus share this resource
 * @ctrl_val:	array of cache or mem ctrl values (indexed by CLOSID)
 * @new_ctrl:	new ctrl value to be loaded
 * @have_new_ctrl: did user provide new_ctrl for this domain
 */
struct rdt_domain {
	struct list_head	list;
	int			id;
	struct cpumask		cpu_mask;
	u32			*ctrl_val;
	u32			new_ctrl;
	bool			have_new_ctrl;
};

/**
 * struct msr_param - set a range of MSRs from a domain
 * @res:       The resource to use
 * @low:       Beginning index from base MSR
 * @high:      End index
 */
struct msr_param {
	struct rdt_resource	*res;
	int			low;
	int			high;
};

/**
 * struct rdt_cache - Cache allocation related data
 * @cbm_len:		Length of the cache bit mask
 * @min_cbm_bits:	Minimum number of consecutive bits to be set
 * @cbm_idx_mult:	Multiplier of CBM index
 * @cbm_idx_offset:	Offset of CBM index. CBM index is computed by:
 *			closid * cbm_idx_multi + cbm_idx_offset
 *			in a cache bit mask
 */
struct rdt_cache {
	unsigned int	cbm_len;
	unsigned int	min_cbm_bits;
	unsigned int	cbm_idx_mult;
	unsigned int	cbm_idx_offset;
};

/**
 * struct rdt_membw - Memory bandwidth allocation related data
 * @max_delay:		Max throttle delay. Delay is the hardware
 *			representation for memory bandwidth.
 * @min_bw:		Minimum memory bandwidth percentage user can request
 * @bw_gran:		Granularity at which the memory bandwidth is allocated
 * @delay_linear:	True if memory B/W delay is in linear scale
 * @mb_map:		Mapping of memory B/W percentage to memory B/W delay
 */
struct rdt_membw {
	u32		max_delay;
	u32		min_bw;
	u32		bw_gran;
	u32		delay_linear;
	u32		*mb_map;
};

/**
 * struct rdt_resource - attributes of an RDT resource
 * @enabled:		Is this feature enabled on this machine
 * @capable:		Is this feature available on this machine
 * @name:		Name to use in "schemata" file
 * @num_closid:		Number of CLOSIDs available
 * @cache_level:	Which cache level defines scope of this resource
 * @default_ctrl:	Specifies default cache cbm or memory B/W percent.
 * @msr_base:		Base MSR address for CBMs
 * @msr_update:		Function pointer to update QOS MSRs
 * @data_width:		Character width of data when displaying
 * @domains:		All domains for this resource
 * @cache:		Cache allocation related data
 * @info_files:		resctrl info files for the resource
 * @nr_info_files:	Number of info files
 * @format_str:		Per resource format string to show domain value
 * @parse_ctrlval:	Per resource function pointer to parse control values
 */
struct rdt_resource {
	bool			enabled;
	bool			capable;
	char			*name;
	int			num_closid;
	int			cache_level;
	u32			default_ctrl;
	unsigned int		msr_base;
	void (*msr_update)	(struct rdt_domain *d, struct msr_param *m,
				 struct rdt_resource *r);
	int			data_width;
	struct list_head	domains;
	struct rdt_cache	cache;
	struct rdt_membw	membw;
	struct rftype		*info_files;
	int			nr_info_files;
	const char		*format_str;
	int (*parse_ctrlval)	(char *buf, struct rdt_resource *r,
				 struct rdt_domain *d);
};

void rdt_get_cache_infofile(struct rdt_resource *r);
void rdt_get_mba_infofile(struct rdt_resource *r);
int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d);
int parse_bw(char *buf, struct rdt_resource *r,  struct rdt_domain *d);

extern struct mutex rdtgroup_mutex;

extern struct rdt_resource rdt_resources_all[];
extern struct rdtgroup rdtgroup_default;
DECLARE_STATIC_KEY_FALSE(rdt_enable_key);

int __init rdtgroup_init(void);

enum {
	RDT_RESOURCE_L3,
	RDT_RESOURCE_L3DATA,
	RDT_RESOURCE_L3CODE,
	RDT_RESOURCE_L2,
	RDT_RESOURCE_MBA,

	/* Must be the last */
	RDT_NUM_RESOURCES,
};

#define for_each_capable_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->capable)

#define for_each_enabled_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->enabled)

/* CPUID.(EAX=10H, ECX=ResID=1).EAX */
union cpuid_0x10_1_eax {
	struct {
		unsigned int cbm_len:5;
	} split;
	unsigned int full;
};

/* CPUID.(EAX=10H, ECX=ResID=3).EAX */
union cpuid_0x10_3_eax {
	struct {
		unsigned int max_delay:12;
	} split;
	unsigned int full;
};

/* CPUID.(EAX=10H, ECX=ResID).EDX */
union cpuid_0x10_x_edx {
	struct {
		unsigned int cos_max:16;
	} split;
	unsigned int full;
};

DECLARE_PER_CPU_READ_MOSTLY(int, cpu_closid);

void rdt_ctrl_update(void *arg);
struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
void rdtgroup_kn_unlock(struct kernfs_node *kn);
ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off);
int rdtgroup_schemata_show(struct kernfs_open_file *of,
			   struct seq_file *s, void *v);

/*
 * intel_rdt_sched_in() - Writes the task's CLOSid to IA32_PQR_MSR
 *
 * Following considerations are made so that this has minimal impact
 * on scheduler hot path:
 * - This will stay as no-op unless we are running on an Intel SKU
 *   which supports resource control and we enable by mounting the
 *   resctrl file system.
 * - Caches the per cpu CLOSid values and does the MSR write only
 *   when a task with a different CLOSid is scheduled in.
 *
 * Must be called with preemption disabled.
 */
static inline void intel_rdt_sched_in(void)
{
	if (static_branch_likely(&rdt_enable_key)) {
		struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
		int closid;

		/*
		 * If this task has a closid assigned, use it.
		 * Else use the closid assigned to this cpu.
		 */
		closid = current->closid;
		if (closid == 0)
			closid = this_cpu_read(cpu_closid);

		if (closid != state->closid) {
			state->closid = closid;
			wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, closid);
		}
	}
}

#else

static inline void intel_rdt_sched_in(void) {}

#endif /* CONFIG_INTEL_RDT */
#endif /* _ASM_X86_INTEL_RDT_H */
Commit	Line	Data
113c6097 FY	1	#ifndef _ASM_X86_INTEL_RDT_H
	2	#define _ASM_X86_INTEL_RDT_H
	3
8c2bc51f	4	#ifdef CONFIG_INTEL_RDT
4f341a5e	5
5b825c3a	6	#include <linux/sched.h>
8ff42c02	7	#include <linux/kernfs.h>
5ff193fb FY	8	#include <linux/jump_label.h>
5ff193fb FY	9
4f341a5e FY	10	#include <asm/intel_rdt_common.h>
4f341a5e FY	11
5ff193fb	12	#define IA32_L3_QOS_CFG 0xc81
113c6097	13	#define IA32_L3_CBM_BASE 0xc90
c1c7c3f9	14	#define IA32_L2_CBM_BASE 0xd10
05b93417	15	#define IA32_MBA_THRTL_BASE 0xd50
113c6097	16
5ff193fb FY	17	#define L3_QOS_CDP_ENABLE 0x01ULL
	18
	19	/**
	20	* struct rdtgroup - store rdtgroup's data in resctrl file system.
	21	* @kn: kernfs node
	22	* @rdtgroup_list: linked list for all rdtgroups
	23	* @closid: closid for this rdtgroup
12e0110c	24	* @cpu_mask: CPUs assigned to this rdtgroup
60cf5e10 FY	25	* @flags: status bits
60cf5e10 FY	26	* @waitcount: how many cpus expect to find this
12e0110c	27	* group when they acquire rdtgroup_mutex
5ff193fb FY	28	*/
	29	struct rdtgroup {
	30	struct kernfs_node *kn;
	31	struct list_head rdtgroup_list;
	32	int closid;
12e0110c	33	struct cpumask cpu_mask;
60cf5e10 FY	34	int flags;
60cf5e10 FY	35	atomic_t waitcount;
5ff193fb FY	36	};
5ff193fb FY	37
60cf5e10 FY	38	/* rdtgroup.flags */
	39	#define RDT_DELETED 1
	40
4ffa3c97 JO	41	/* rftype.flags */
	42	#define RFTYPE_FLAGS_CPUS_LIST 1
	43
5ff193fb FY	44	/* List of all resource groups */
	45	extern struct list_head rdt_all_groups;
	46
de016df8 VS	47	extern int max_name_width, max_data_width;
de016df8 VS	48
5ff193fb FY	49	int __init rdtgroup_init(void);
5ff193fb FY	50
4e978d06 FY	51	/**
4e978d06 FY	52	* struct rftype - describe each file in the resctrl file system
17f8ba1d TG	53	* @name: File name
	54	* @mode: Access mode
	55	* @kf_ops: File operations
4ffa3c97	56	* @flags: File specific RFTYPE_FLAGS_* flags
17f8ba1d TG	57	* @seq_show: Show content of the file
17f8ba1d TG	58	* @write: Write to the file
4e978d06 FY	59	*/
	60	struct rftype {
	61	char *name;
	62	umode_t mode;
	63	struct kernfs_ops *kf_ops;
4ffa3c97	64	unsigned long flags;
4e978d06 FY	65
	66	int (seq_show)(struct kernfs_open_file of,
	67	struct seq_file sf, void v);
	68	/*
	69	* write() is the generic write callback which maps directly to
	70	* kernfs write operation and overrides all other operations.
	71	* Maximum write size is determined by ->max_write_len.
	72	*/
	73	ssize_t (write)(struct kernfs_open_file of,
	74	char *buf, size_t nbytes, loff_t off);
	75	};
	76
0921c547 TG	77	/**
	78	* struct rdt_domain - group of cpus sharing an RDT resource
	79	* @list: all instances of this resource
	80	* @id: unique id for this instance
	81	* @cpu_mask: which cpus share this resource
	82	* @ctrl_val: array of cache or mem ctrl values (indexed by CLOSID)
	83	* @new_ctrl: new ctrl value to be loaded
	84	* @have_new_ctrl: did user provide new_ctrl for this domain
	85	*/
	86	struct rdt_domain {
	87	struct list_head list;
	88	int id;
	89	struct cpumask cpu_mask;
	90	u32 *ctrl_val;
	91	u32 new_ctrl;
	92	bool have_new_ctrl;
	93	};
	94
	95	/**
	96	* struct msr_param - set a range of MSRs from a domain
	97	* @res: The resource to use
	98	* @low: Beginning index from base MSR
	99	* @high: End index
	100	*/
	101	struct msr_param {
	102	struct rdt_resource *res;
	103	int low;
	104	int high;
	105	};
	106
d3e11b4d TG	107	/**
	108	* struct rdt_cache - Cache allocation related data
	109	* @cbm_len: Length of the cache bit mask
	110	* @min_cbm_bits: Minimum number of consecutive bits to be set
	111	* @cbm_idx_mult: Multiplier of CBM index
	112	* @cbm_idx_offset: Offset of CBM index. CBM index is computed by:
	113	* closid * cbm_idx_multi + cbm_idx_offset
	114	* in a cache bit mask
	115	*/
	116	struct rdt_cache {
	117	unsigned int cbm_len;
	118	unsigned int min_cbm_bits;
	119	unsigned int cbm_idx_mult;
	120	unsigned int cbm_idx_offset;
	121	};
	122
05b93417 VS	123	/**
	124	* struct rdt_membw - Memory bandwidth allocation related data
	125	* @max_delay: Max throttle delay. Delay is the hardware
	126	* representation for memory bandwidth.
	127	* @min_bw: Minimum memory bandwidth percentage user can request
	128	* @bw_gran: Granularity at which the memory bandwidth is allocated
	129	* @delay_linear: True if memory B/W delay is in linear scale
	130	* @mb_map: Mapping of memory B/W percentage to memory B/W delay
	131	*/
	132	struct rdt_membw {
	133	u32 max_delay;
	134	u32 min_bw;
	135	u32 bw_gran;
	136	u32 delay_linear;
	137	u32 *mb_map;
	138	};
	139
c1c7c3f9 FY	140	/**
c1c7c3f9 FY	141	* struct rdt_resource - attributes of an RDT resource
d3e11b4d TG	142	* @enabled: Is this feature enabled on this machine
	143	* @capable: Is this feature available on this machine
	144	* @name: Name to use in "schemata" file
	145	* @num_closid: Number of CLOSIDs available
	146	* @cache_level: Which cache level defines scope of this resource
	147	* @default_ctrl: Specifies default cache cbm or memory B/W percent.
	148	* @msr_base: Base MSR address for CBMs
0921c547	149	* @msr_update: Function pointer to update QOS MSRs
d3e11b4d TG	150	* @data_width: Character width of data when displaying
	151	* @domains: All domains for this resource
	152	* @cache: Cache allocation related data
6a507a6a VS	153	* @info_files: resctrl info files for the resource
6a507a6a VS	154	* @nr_info_files: Number of info files
c6ea67de VS	155	* @format_str: Per resource format string to show domain value
c6ea67de VS	156	* @parse_ctrlval: Per resource function pointer to parse control values
c1c7c3f9 FY	157	*/
	158	struct rdt_resource {
	159	bool enabled;
	160	bool capable;
	161	char *name;
	162	int num_closid;
d3e11b4d	163	int cache_level;
2545e9f5	164	u32 default_ctrl;
d3e11b4d	165	unsigned int msr_base;
0921c547 TG	166	void (msr_update) (struct rdt_domain d, struct msr_param *m,
0921c547 TG	167	struct rdt_resource *r);
de016df8	168	int data_width;
c1c7c3f9	169	struct list_head domains;
a83827d0 TG	170	struct rdt_cache cache;
a83827d0 TG	171	struct rdt_membw membw;
6a507a6a VS	172	struct rftype *info_files;
6a507a6a VS	173	int nr_info_files;
c6ea67de VS	174	const char *format_str;
	175	int (parse_ctrlval) (char buf, struct rdt_resource *r,
	176	struct rdt_domain *d);
c1c7c3f9 FY	177	};
c1c7c3f9 FY	178
6a507a6a	179	void rdt_get_cache_infofile(struct rdt_resource *r);
db69ef65	180	void rdt_get_mba_infofile(struct rdt_resource *r);
c6ea67de	181	int parse_cbm(char buf, struct rdt_resource r, struct rdt_domain *d);
64e8ed3d	182	int parse_bw(char buf, struct rdt_resource r, struct rdt_domain *d);
6a507a6a	183
2264d9c7 TL	184	extern struct mutex rdtgroup_mutex;
2264d9c7 TL	185
c1c7c3f9	186	extern struct rdt_resource rdt_resources_all[];
5ff193fb FY	187	extern struct rdtgroup rdtgroup_default;
	188	DECLARE_STATIC_KEY_FALSE(rdt_enable_key);
	189
	190	int __init rdtgroup_init(void);
c1c7c3f9 FY	191
	192	enum {
	193	RDT_RESOURCE_L3,
	194	RDT_RESOURCE_L3DATA,
	195	RDT_RESOURCE_L3CODE,
	196	RDT_RESOURCE_L2,
05b93417	197	RDT_RESOURCE_MBA,
c1c7c3f9 FY	198
	199	/* Must be the last */
	200	RDT_NUM_RESOURCES,
	201	};
	202
	203	#define for_each_capable_rdt_resource(r) \
	204	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
17f8ba1d	205	r++) \
c1c7c3f9 FY	206	if (r->capable)
c1c7c3f9 FY	207
2264d9c7 TL	208	#define for_each_enabled_rdt_resource(r) \
	209	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	210	r++) \
	211	if (r->enabled)
	212
c1c7c3f9 FY	213	/* CPUID.(EAX=10H, ECX=ResID=1).EAX */
	214	union cpuid_0x10_1_eax {
	215	struct {
	216	unsigned int cbm_len:5;
	217	} split;
	218	unsigned int full;
	219	};
	220
ab66a33b VS	221	/* CPUID.(EAX=10H, ECX=ResID=3).EAX */
	222	union cpuid_0x10_3_eax {
	223	struct {
	224	unsigned int max_delay:12;
	225	} split;
	226	unsigned int full;
	227	};
	228
2545e9f5 VS	229	/* CPUID.(EAX=10H, ECX=ResID).EDX */
2545e9f5 VS	230	union cpuid_0x10_x_edx {
c1c7c3f9 FY	231	struct {
	232	unsigned int cos_max:16;
	233	} split;
	234	unsigned int full;
	235	};
2264d9c7	236
12e0110c TL	237	DECLARE_PER_CPU_READ_MOSTLY(int, cpu_closid);
12e0110c TL	238
2545e9f5	239	void rdt_ctrl_update(void *arg);
60cf5e10 FY	240	struct rdtgroup rdtgroup_kn_lock_live(struct kernfs_node kn);
60cf5e10 FY	241	void rdtgroup_kn_unlock(struct kernfs_node *kn);
60ec2440 TL	242	ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
	243	char *buf, size_t nbytes, loff_t off);
	244	int rdtgroup_schemata_show(struct kernfs_open_file *of,
	245	struct seq_file s, void v);
4f341a5e FY	246
	247	/*
	248	* intel_rdt_sched_in() - Writes the task's CLOSid to IA32_PQR_MSR
	249	*
	250	* Following considerations are made so that this has minimal impact
	251	* on scheduler hot path:
	252	* - This will stay as no-op unless we are running on an Intel SKU
	253	* which supports resource control and we enable by mounting the
	254	* resctrl file system.
	255	* - Caches the per cpu CLOSid values and does the MSR write only
	256	* when a task with a different CLOSid is scheduled in.
74fcdae1 FY	257	*
74fcdae1 FY	258	* Must be called with preemption disabled.
4f341a5e FY	259	*/
	260	static inline void intel_rdt_sched_in(void)
	261	{
	262	if (static_branch_likely(&rdt_enable_key)) {
	263	struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
	264	int closid;
	265
	266	/*
	267	* If this task has a closid assigned, use it.
	268	* Else use the closid assigned to this cpu.
	269	*/
	270	closid = current->closid;
	271	if (closid == 0)
	272	closid = this_cpu_read(cpu_closid);
	273
	274	if (closid != state->closid) {
	275	state->closid = closid;
	276	wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, closid);
	277	}
	278	}
	279	}
	280
	281	#else
	282
	283	static inline void intel_rdt_sched_in(void) {}
	284
8c2bc51f	285	#endif /* CONFIG_INTEL_RDT */
113c6097	286	#endif /* _ASM_X86_INTEL_RDT_H */