]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Resource Director Technology(RDT) | |
3 | * - Cache Allocation code. | |
4 | * | |
5 | * Copyright (C) 2016 Intel Corporation | |
6 | * | |
7 | * Authors: | |
8 | * Fenghua Yu <fenghua.yu@intel.com> | |
9 | * Tony Luck <tony.luck@intel.com> | |
10 | * Vikas Shivappa <vikas.shivappa@intel.com> | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify it | |
13 | * under the terms and conditions of the GNU General Public License, | |
14 | * version 2, as published by the Free Software Foundation. | |
15 | * | |
16 | * This program is distributed in the hope it will be useful, but WITHOUT | |
17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
19 | * more details. | |
20 | * | |
21 | * More information about RDT be found in the Intel (R) x86 Architecture | |
22 | * Software Developer Manual June 2016, volume 3, section 17.17. | |
23 | */ | |
24 | ||
25 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
26 | ||
27 | #include <linux/slab.h> | |
28 | #include <linux/err.h> | |
29 | #include <linux/cacheinfo.h> | |
30 | #include <linux/cpuhotplug.h> | |
31 | ||
32 | #include <asm/intel-family.h> | |
33 | #include <asm/intel_rdt.h> | |
34 | ||
35 | /* Mutex to protect rdtgroup access. */ | |
36 | DEFINE_MUTEX(rdtgroup_mutex); | |
37 | ||
38 | DEFINE_PER_CPU_READ_MOSTLY(int, cpu_closid); | |
39 | ||
40 | /* | |
41 | * Used to store the max resource name width and max resource data width | |
42 | * to display the schemata in a tabular format | |
43 | */ | |
44 | int max_name_width, max_data_width; | |
45 | ||
46 | static void | |
47 | cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r); | |
48 | ||
49 | #define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].domains) | |
50 | ||
51 | struct rdt_resource rdt_resources_all[] = { | |
52 | { | |
53 | .name = "L3", | |
54 | .domains = domain_init(RDT_RESOURCE_L3), | |
55 | .msr_base = IA32_L3_CBM_BASE, | |
56 | .msr_update = cat_wrmsr, | |
57 | .cache_level = 3, | |
58 | .cache = { | |
59 | .min_cbm_bits = 1, | |
60 | .cbm_idx_mult = 1, | |
61 | .cbm_idx_offset = 0, | |
62 | }, | |
63 | }, | |
64 | { | |
65 | .name = "L3DATA", | |
66 | .domains = domain_init(RDT_RESOURCE_L3DATA), | |
67 | .msr_base = IA32_L3_CBM_BASE, | |
68 | .msr_update = cat_wrmsr, | |
69 | .cache_level = 3, | |
70 | .cache = { | |
71 | .min_cbm_bits = 1, | |
72 | .cbm_idx_mult = 2, | |
73 | .cbm_idx_offset = 0, | |
74 | }, | |
75 | }, | |
76 | { | |
77 | .name = "L3CODE", | |
78 | .domains = domain_init(RDT_RESOURCE_L3CODE), | |
79 | .msr_base = IA32_L3_CBM_BASE, | |
80 | .msr_update = cat_wrmsr, | |
81 | .cache_level = 3, | |
82 | .cache = { | |
83 | .min_cbm_bits = 1, | |
84 | .cbm_idx_mult = 2, | |
85 | .cbm_idx_offset = 1, | |
86 | }, | |
87 | }, | |
88 | { | |
89 | .name = "L2", | |
90 | .domains = domain_init(RDT_RESOURCE_L2), | |
91 | .msr_base = IA32_L2_CBM_BASE, | |
92 | .msr_update = cat_wrmsr, | |
93 | .cache_level = 2, | |
94 | .cache = { | |
95 | .min_cbm_bits = 1, | |
96 | .cbm_idx_mult = 1, | |
97 | .cbm_idx_offset = 0, | |
98 | }, | |
99 | }, | |
100 | }; | |
101 | ||
102 | static unsigned int cbm_idx(struct rdt_resource *r, unsigned int closid) | |
103 | { | |
104 | return closid * r->cache.cbm_idx_mult + r->cache.cbm_idx_offset; | |
105 | } | |
106 | ||
107 | /* | |
108 | * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs | |
109 | * as they do not have CPUID enumeration support for Cache allocation. | |
110 | * The check for Vendor/Family/Model is not enough to guarantee that | |
111 | * the MSRs won't #GP fault because only the following SKUs support | |
112 | * CAT: | |
113 | * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz | |
114 | * Intel(R) Xeon(R) CPU E5-2648L v3 @ 1.80GHz | |
115 | * Intel(R) Xeon(R) CPU E5-2628L v3 @ 2.00GHz | |
116 | * Intel(R) Xeon(R) CPU E5-2618L v3 @ 2.30GHz | |
117 | * Intel(R) Xeon(R) CPU E5-2608L v3 @ 2.00GHz | |
118 | * Intel(R) Xeon(R) CPU E5-2658A v3 @ 2.20GHz | |
119 | * | |
120 | * Probe by trying to write the first of the L3 cach mask registers | |
121 | * and checking that the bits stick. Max CLOSids is always 4 and max cbm length | |
122 | * is always 20 on hsw server parts. The minimum cache bitmask length | |
123 | * allowed for HSW server is always 2 bits. Hardcode all of them. | |
124 | */ | |
125 | static inline bool cache_alloc_hsw_probe(void) | |
126 | { | |
127 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
128 | boot_cpu_data.x86 == 6 && | |
129 | boot_cpu_data.x86_model == INTEL_FAM6_HASWELL_X) { | |
130 | struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3]; | |
131 | u32 l, h, max_cbm = BIT_MASK(20) - 1; | |
132 | ||
133 | if (wrmsr_safe(IA32_L3_CBM_BASE, max_cbm, 0)) | |
134 | return false; | |
135 | rdmsr(IA32_L3_CBM_BASE, l, h); | |
136 | ||
137 | /* If all the bits were set in MSR, return success */ | |
138 | if (l != max_cbm) | |
139 | return false; | |
140 | ||
141 | r->num_closid = 4; | |
142 | r->default_ctrl = max_cbm; | |
143 | r->cache.cbm_len = 20; | |
144 | r->cache.min_cbm_bits = 2; | |
145 | r->capable = true; | |
146 | r->enabled = true; | |
147 | ||
148 | return true; | |
149 | } | |
150 | ||
151 | return false; | |
152 | } | |
153 | ||
154 | static void rdt_get_cache_config(int idx, struct rdt_resource *r) | |
155 | { | |
156 | union cpuid_0x10_1_eax eax; | |
157 | union cpuid_0x10_x_edx edx; | |
158 | u32 ebx, ecx; | |
159 | ||
160 | cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx, &edx.full); | |
161 | r->num_closid = edx.split.cos_max + 1; | |
162 | r->cache.cbm_len = eax.split.cbm_len + 1; | |
163 | r->default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; | |
164 | r->data_width = (r->cache.cbm_len + 3) / 4; | |
165 | r->capable = true; | |
166 | r->enabled = true; | |
167 | } | |
168 | ||
169 | static void rdt_get_cdp_l3_config(int type) | |
170 | { | |
171 | struct rdt_resource *r_l3 = &rdt_resources_all[RDT_RESOURCE_L3]; | |
172 | struct rdt_resource *r = &rdt_resources_all[type]; | |
173 | ||
174 | r->num_closid = r_l3->num_closid / 2; | |
175 | r->cache.cbm_len = r_l3->cache.cbm_len; | |
176 | r->default_ctrl = r_l3->default_ctrl; | |
177 | r->data_width = (r->cache.cbm_len + 3) / 4; | |
178 | r->capable = true; | |
179 | /* | |
180 | * By default, CDP is disabled. CDP can be enabled by mount parameter | |
181 | * "cdp" during resctrl file system mount time. | |
182 | */ | |
183 | r->enabled = false; | |
184 | } | |
185 | ||
186 | static int get_cache_id(int cpu, int level) | |
187 | { | |
188 | struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); | |
189 | int i; | |
190 | ||
191 | for (i = 0; i < ci->num_leaves; i++) { | |
192 | if (ci->info_list[i].level == level) | |
193 | return ci->info_list[i].id; | |
194 | } | |
195 | ||
196 | return -1; | |
197 | } | |
198 | ||
199 | static void | |
200 | cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) | |
201 | { | |
202 | unsigned int i; | |
203 | ||
204 | for (i = m->low; i < m->high; i++) | |
205 | wrmsrl(r->msr_base + cbm_idx(r, i), d->ctrl_val[i]); | |
206 | } | |
207 | ||
208 | void rdt_ctrl_update(void *arg) | |
209 | { | |
210 | struct msr_param *m = arg; | |
211 | struct rdt_resource *r = m->res; | |
212 | int cpu = smp_processor_id(); | |
213 | struct rdt_domain *d; | |
214 | ||
215 | list_for_each_entry(d, &r->domains, list) { | |
216 | /* Find the domain that contains this CPU */ | |
217 | if (cpumask_test_cpu(cpu, &d->cpu_mask)) { | |
218 | r->msr_update(d, m, r); | |
219 | return; | |
220 | } | |
221 | } | |
222 | pr_warn_once("cpu %d not found in any domain for resource %s\n", | |
223 | cpu, r->name); | |
224 | } | |
225 | ||
226 | /* | |
227 | * rdt_find_domain - Find a domain in a resource that matches input resource id | |
228 | * | |
229 | * Search resource r's domain list to find the resource id. If the resource | |
230 | * id is found in a domain, return the domain. Otherwise, if requested by | |
231 | * caller, return the first domain whose id is bigger than the input id. | |
232 | * The domain list is sorted by id in ascending order. | |
233 | */ | |
234 | static struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, | |
235 | struct list_head **pos) | |
236 | { | |
237 | struct rdt_domain *d; | |
238 | struct list_head *l; | |
239 | ||
240 | if (id < 0) | |
241 | return ERR_PTR(id); | |
242 | ||
243 | list_for_each(l, &r->domains) { | |
244 | d = list_entry(l, struct rdt_domain, list); | |
245 | /* When id is found, return its domain. */ | |
246 | if (id == d->id) | |
247 | return d; | |
248 | /* Stop searching when finding id's position in sorted list. */ | |
249 | if (id < d->id) | |
250 | break; | |
251 | } | |
252 | ||
253 | if (pos) | |
254 | *pos = l; | |
255 | ||
256 | return NULL; | |
257 | } | |
258 | ||
259 | static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) | |
260 | { | |
261 | struct msr_param m; | |
262 | u32 *dc; | |
263 | int i; | |
264 | ||
265 | dc = kmalloc_array(r->num_closid, sizeof(*d->ctrl_val), GFP_KERNEL); | |
266 | if (!dc) | |
267 | return -ENOMEM; | |
268 | ||
269 | d->ctrl_val = dc; | |
270 | ||
271 | /* | |
272 | * Initialize the Control MSRs to having no control. | |
273 | * For Cache Allocation: Set all bits in cbm | |
274 | * For Memory Allocation: Set b/w requested to 100 | |
275 | */ | |
276 | for (i = 0; i < r->num_closid; i++, dc++) | |
277 | *dc = r->default_ctrl; | |
278 | ||
279 | m.low = 0; | |
280 | m.high = r->num_closid; | |
281 | r->msr_update(d, &m, r); | |
282 | return 0; | |
283 | } | |
284 | ||
285 | /* | |
286 | * domain_add_cpu - Add a cpu to a resource's domain list. | |
287 | * | |
288 | * If an existing domain in the resource r's domain list matches the cpu's | |
289 | * resource id, add the cpu in the domain. | |
290 | * | |
291 | * Otherwise, a new domain is allocated and inserted into the right position | |
292 | * in the domain list sorted by id in ascending order. | |
293 | * | |
294 | * The order in the domain list is visible to users when we print entries | |
295 | * in the schemata file and schemata input is validated to have the same order | |
296 | * as this list. | |
297 | */ | |
298 | static void domain_add_cpu(int cpu, struct rdt_resource *r) | |
299 | { | |
300 | int id = get_cache_id(cpu, r->cache_level); | |
301 | struct list_head *add_pos = NULL; | |
302 | struct rdt_domain *d; | |
303 | ||
304 | d = rdt_find_domain(r, id, &add_pos); | |
305 | if (IS_ERR(d)) { | |
306 | pr_warn("Could't find cache id for cpu %d\n", cpu); | |
307 | return; | |
308 | } | |
309 | ||
310 | if (d) { | |
311 | cpumask_set_cpu(cpu, &d->cpu_mask); | |
312 | return; | |
313 | } | |
314 | ||
315 | d = kzalloc_node(sizeof(*d), GFP_KERNEL, cpu_to_node(cpu)); | |
316 | if (!d) | |
317 | return; | |
318 | ||
319 | d->id = id; | |
320 | ||
321 | if (domain_setup_ctrlval(r, d)) { | |
322 | kfree(d); | |
323 | return; | |
324 | } | |
325 | ||
326 | cpumask_set_cpu(cpu, &d->cpu_mask); | |
327 | list_add_tail(&d->list, add_pos); | |
328 | } | |
329 | ||
330 | static void domain_remove_cpu(int cpu, struct rdt_resource *r) | |
331 | { | |
332 | int id = get_cache_id(cpu, r->cache_level); | |
333 | struct rdt_domain *d; | |
334 | ||
335 | d = rdt_find_domain(r, id, NULL); | |
336 | if (IS_ERR_OR_NULL(d)) { | |
337 | pr_warn("Could't find cache id for cpu %d\n", cpu); | |
338 | return; | |
339 | } | |
340 | ||
341 | cpumask_clear_cpu(cpu, &d->cpu_mask); | |
342 | if (cpumask_empty(&d->cpu_mask)) { | |
343 | kfree(d->ctrl_val); | |
344 | list_del(&d->list); | |
345 | kfree(d); | |
346 | } | |
347 | } | |
348 | ||
349 | static void clear_closid(int cpu) | |
350 | { | |
351 | struct intel_pqr_state *state = this_cpu_ptr(&pqr_state); | |
352 | ||
353 | per_cpu(cpu_closid, cpu) = 0; | |
354 | state->closid = 0; | |
355 | wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, 0); | |
356 | } | |
357 | ||
358 | static int intel_rdt_online_cpu(unsigned int cpu) | |
359 | { | |
360 | struct rdt_resource *r; | |
361 | ||
362 | mutex_lock(&rdtgroup_mutex); | |
363 | for_each_capable_rdt_resource(r) | |
364 | domain_add_cpu(cpu, r); | |
365 | /* The cpu is set in default rdtgroup after online. */ | |
366 | cpumask_set_cpu(cpu, &rdtgroup_default.cpu_mask); | |
367 | clear_closid(cpu); | |
368 | mutex_unlock(&rdtgroup_mutex); | |
369 | ||
370 | return 0; | |
371 | } | |
372 | ||
373 | static int intel_rdt_offline_cpu(unsigned int cpu) | |
374 | { | |
375 | struct rdtgroup *rdtgrp; | |
376 | struct rdt_resource *r; | |
377 | ||
378 | mutex_lock(&rdtgroup_mutex); | |
379 | for_each_capable_rdt_resource(r) | |
380 | domain_remove_cpu(cpu, r); | |
381 | list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { | |
382 | if (cpumask_test_and_clear_cpu(cpu, &rdtgrp->cpu_mask)) | |
383 | break; | |
384 | } | |
385 | clear_closid(cpu); | |
386 | mutex_unlock(&rdtgroup_mutex); | |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
391 | /* | |
392 | * Choose a width for the resource name and resource data based on the | |
393 | * resource that has widest name and cbm. | |
394 | */ | |
395 | static __init void rdt_init_padding(void) | |
396 | { | |
397 | struct rdt_resource *r; | |
398 | int cl; | |
399 | ||
400 | for_each_enabled_rdt_resource(r) { | |
401 | cl = strlen(r->name); | |
402 | if (cl > max_name_width) | |
403 | max_name_width = cl; | |
404 | ||
405 | if (r->data_width > max_data_width) | |
406 | max_data_width = r->data_width; | |
407 | } | |
408 | } | |
409 | ||
410 | static __init bool get_rdt_resources(void) | |
411 | { | |
412 | bool ret = false; | |
413 | ||
414 | if (cache_alloc_hsw_probe()) | |
415 | return true; | |
416 | ||
417 | if (!boot_cpu_has(X86_FEATURE_RDT_A)) | |
418 | return false; | |
419 | ||
420 | if (boot_cpu_has(X86_FEATURE_CAT_L3)) { | |
421 | rdt_get_cache_config(1, &rdt_resources_all[RDT_RESOURCE_L3]); | |
422 | if (boot_cpu_has(X86_FEATURE_CDP_L3)) { | |
423 | rdt_get_cdp_l3_config(RDT_RESOURCE_L3DATA); | |
424 | rdt_get_cdp_l3_config(RDT_RESOURCE_L3CODE); | |
425 | } | |
426 | ret = true; | |
427 | } | |
428 | if (boot_cpu_has(X86_FEATURE_CAT_L2)) { | |
429 | /* CPUID 0x10.2 fields are same format at 0x10.1 */ | |
430 | rdt_get_cache_config(2, &rdt_resources_all[RDT_RESOURCE_L2]); | |
431 | ret = true; | |
432 | } | |
433 | ||
434 | if (boot_cpu_has(X86_FEATURE_MBA)) | |
435 | ret = true; | |
436 | ||
437 | return ret; | |
438 | } | |
439 | ||
440 | static int __init intel_rdt_late_init(void) | |
441 | { | |
442 | struct rdt_resource *r; | |
443 | int state, ret; | |
444 | ||
445 | if (!get_rdt_resources()) | |
446 | return -ENODEV; | |
447 | ||
448 | rdt_init_padding(); | |
449 | ||
450 | state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, | |
451 | "x86/rdt/cat:online:", | |
452 | intel_rdt_online_cpu, intel_rdt_offline_cpu); | |
453 | if (state < 0) | |
454 | return state; | |
455 | ||
456 | ret = rdtgroup_init(); | |
457 | if (ret) { | |
458 | cpuhp_remove_state(state); | |
459 | return ret; | |
460 | } | |
461 | ||
462 | for_each_capable_rdt_resource(r) | |
463 | pr_info("Intel RDT %s allocation detected\n", r->name); | |
464 | ||
465 | return 0; | |
466 | } | |
467 | ||
468 | late_initcall(intel_rdt_late_init); |