]>
Commit | Line | Data |
---|---|---|
78e99b4a FY |
1 | /* |
2 | * Resource Director Technology(RDT) | |
3 | * - Cache Allocation code. | |
4 | * | |
5 | * Copyright (C) 2016 Intel Corporation | |
6 | * | |
7 | * Authors: | |
8 | * Fenghua Yu <fenghua.yu@intel.com> | |
9 | * Tony Luck <tony.luck@intel.com> | |
10 | * Vikas Shivappa <vikas.shivappa@intel.com> | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify it | |
13 | * under the terms and conditions of the GNU General Public License, | |
14 | * version 2, as published by the Free Software Foundation. | |
15 | * | |
16 | * This program is distributed in the hope it will be useful, but WITHOUT | |
17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
19 | * more details. | |
20 | * | |
21 | * More information about RDT be found in the Intel (R) x86 Architecture | |
22 | * Software Developer Manual June 2016, volume 3, section 17.17. | |
23 | */ | |
24 | ||
25 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
26 | ||
27 | #include <linux/slab.h> | |
28 | #include <linux/err.h> | |
2264d9c7 TL |
29 | #include <linux/cacheinfo.h> |
30 | #include <linux/cpuhotplug.h> | |
78e99b4a | 31 | |
113c6097 | 32 | #include <asm/intel-family.h> |
05830204 VS |
33 | #include <asm/intel_rdt_sched.h> |
34 | #include "intel_rdt.h" | |
113c6097 | 35 | |
05b93417 VS |
36 | #define MAX_MBA_BW 100u |
37 | #define MBA_IS_LINEAR 0x4 | |
38 | ||
2264d9c7 TL |
39 | /* Mutex to protect rdtgroup access. */ |
40 | DEFINE_MUTEX(rdtgroup_mutex); | |
41 | ||
c39a0e2c VS |
42 | /* |
43 | * The cached intel_pqr_state is strictly per CPU and can never be | |
44 | * updated from a remote CPU. Functions which modify the state | |
45 | * are called with interrupts disabled and no preemption, which | |
46 | * is sufficient for the protection. | |
47 | */ | |
48 | DEFINE_PER_CPU(struct intel_pqr_state, pqr_state); | |
49 | ||
de016df8 VS |
50 | /* |
51 | * Used to store the max resource name width and max resource data width | |
52 | * to display the schemata in a tabular format | |
53 | */ | |
54 | int max_name_width, max_data_width; | |
55 | ||
6a445edc VS |
56 | /* |
57 | * Global boolean for rdt_alloc which is true if any | |
58 | * resource allocation is enabled. | |
59 | */ | |
60 | bool rdt_alloc_capable; | |
61 | ||
05b93417 VS |
62 | static void |
63 | mba_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r); | |
0921c547 TG |
64 | static void |
65 | cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r); | |
66 | ||
d3e11b4d TG |
67 | #define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].domains) |
68 | ||
c1c7c3f9 | 69 | struct rdt_resource rdt_resources_all[] = { |
dd131853 | 70 | [RDT_RESOURCE_L3] = |
c1c7c3f9 | 71 | { |
d89b7379 | 72 | .rid = RDT_RESOURCE_L3, |
d3e11b4d TG |
73 | .name = "L3", |
74 | .domains = domain_init(RDT_RESOURCE_L3), | |
75 | .msr_base = IA32_L3_CBM_BASE, | |
0921c547 | 76 | .msr_update = cat_wrmsr, |
d3e11b4d TG |
77 | .cache_level = 3, |
78 | .cache = { | |
79 | .min_cbm_bits = 1, | |
80 | .cbm_idx_mult = 1, | |
81 | .cbm_idx_offset = 0, | |
82 | }, | |
c6ea67de VS |
83 | .parse_ctrlval = parse_cbm, |
84 | .format_str = "%d=%0*x", | |
5dc1d5c6 | 85 | .fflags = RFTYPE_RES_CACHE, |
c1c7c3f9 | 86 | }, |
dd131853 | 87 | [RDT_RESOURCE_L3DATA] = |
c1c7c3f9 | 88 | { |
d89b7379 | 89 | .rid = RDT_RESOURCE_L3DATA, |
d3e11b4d TG |
90 | .name = "L3DATA", |
91 | .domains = domain_init(RDT_RESOURCE_L3DATA), | |
92 | .msr_base = IA32_L3_CBM_BASE, | |
0921c547 | 93 | .msr_update = cat_wrmsr, |
d3e11b4d TG |
94 | .cache_level = 3, |
95 | .cache = { | |
96 | .min_cbm_bits = 1, | |
97 | .cbm_idx_mult = 2, | |
98 | .cbm_idx_offset = 0, | |
99 | }, | |
c6ea67de VS |
100 | .parse_ctrlval = parse_cbm, |
101 | .format_str = "%d=%0*x", | |
5dc1d5c6 | 102 | .fflags = RFTYPE_RES_CACHE, |
c1c7c3f9 | 103 | }, |
dd131853 | 104 | [RDT_RESOURCE_L3CODE] = |
c1c7c3f9 | 105 | { |
d89b7379 | 106 | .rid = RDT_RESOURCE_L3CODE, |
d3e11b4d TG |
107 | .name = "L3CODE", |
108 | .domains = domain_init(RDT_RESOURCE_L3CODE), | |
109 | .msr_base = IA32_L3_CBM_BASE, | |
0921c547 | 110 | .msr_update = cat_wrmsr, |
d3e11b4d TG |
111 | .cache_level = 3, |
112 | .cache = { | |
113 | .min_cbm_bits = 1, | |
114 | .cbm_idx_mult = 2, | |
115 | .cbm_idx_offset = 1, | |
116 | }, | |
c6ea67de VS |
117 | .parse_ctrlval = parse_cbm, |
118 | .format_str = "%d=%0*x", | |
5dc1d5c6 | 119 | .fflags = RFTYPE_RES_CACHE, |
c1c7c3f9 | 120 | }, |
dd131853 | 121 | [RDT_RESOURCE_L2] = |
c1c7c3f9 | 122 | { |
d89b7379 | 123 | .rid = RDT_RESOURCE_L2, |
d3e11b4d TG |
124 | .name = "L2", |
125 | .domains = domain_init(RDT_RESOURCE_L2), | |
126 | .msr_base = IA32_L2_CBM_BASE, | |
0921c547 | 127 | .msr_update = cat_wrmsr, |
d3e11b4d TG |
128 | .cache_level = 2, |
129 | .cache = { | |
130 | .min_cbm_bits = 1, | |
131 | .cbm_idx_mult = 1, | |
132 | .cbm_idx_offset = 0, | |
133 | }, | |
c6ea67de VS |
134 | .parse_ctrlval = parse_cbm, |
135 | .format_str = "%d=%0*x", | |
5dc1d5c6 | 136 | .fflags = RFTYPE_RES_CACHE, |
c1c7c3f9 | 137 | }, |
dd131853 | 138 | [RDT_RESOURCE_MBA] = |
05b93417 | 139 | { |
d89b7379 | 140 | .rid = RDT_RESOURCE_MBA, |
05b93417 VS |
141 | .name = "MB", |
142 | .domains = domain_init(RDT_RESOURCE_MBA), | |
143 | .msr_base = IA32_MBA_THRTL_BASE, | |
144 | .msr_update = mba_wrmsr, | |
145 | .cache_level = 3, | |
64e8ed3d VS |
146 | .parse_ctrlval = parse_bw, |
147 | .format_str = "%d=%*d", | |
5dc1d5c6 | 148 | .fflags = RFTYPE_RES_MB, |
05b93417 | 149 | }, |
c1c7c3f9 FY |
150 | }; |
151 | ||
d3e11b4d | 152 | static unsigned int cbm_idx(struct rdt_resource *r, unsigned int closid) |
2264d9c7 | 153 | { |
d3e11b4d | 154 | return closid * r->cache.cbm_idx_mult + r->cache.cbm_idx_offset; |
2264d9c7 TL |
155 | } |
156 | ||
113c6097 FY |
157 | /* |
158 | * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs | |
159 | * as they do not have CPUID enumeration support for Cache allocation. | |
160 | * The check for Vendor/Family/Model is not enough to guarantee that | |
161 | * the MSRs won't #GP fault because only the following SKUs support | |
162 | * CAT: | |
163 | * Intel(R) Xeon(R) CPU E5-2658 v3 @ 2.20GHz | |
164 | * Intel(R) Xeon(R) CPU E5-2648L v3 @ 1.80GHz | |
165 | * Intel(R) Xeon(R) CPU E5-2628L v3 @ 2.00GHz | |
166 | * Intel(R) Xeon(R) CPU E5-2618L v3 @ 2.30GHz | |
167 | * Intel(R) Xeon(R) CPU E5-2608L v3 @ 2.00GHz | |
168 | * Intel(R) Xeon(R) CPU E5-2658A v3 @ 2.20GHz | |
169 | * | |
170 | * Probe by trying to write the first of the L3 cach mask registers | |
171 | * and checking that the bits stick. Max CLOSids is always 4 and max cbm length | |
172 | * is always 20 on hsw server parts. The minimum cache bitmask length | |
173 | * allowed for HSW server is always 2 bits. Hardcode all of them. | |
174 | */ | |
175 | static inline bool cache_alloc_hsw_probe(void) | |
176 | { | |
177 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
178 | boot_cpu_data.x86 == 6 && | |
179 | boot_cpu_data.x86_model == INTEL_FAM6_HASWELL_X) { | |
c1c7c3f9 | 180 | struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3]; |
113c6097 FY |
181 | u32 l, h, max_cbm = BIT_MASK(20) - 1; |
182 | ||
183 | if (wrmsr_safe(IA32_L3_CBM_BASE, max_cbm, 0)) | |
184 | return false; | |
185 | rdmsr(IA32_L3_CBM_BASE, l, h); | |
186 | ||
187 | /* If all the bits were set in MSR, return success */ | |
c1c7c3f9 FY |
188 | if (l != max_cbm) |
189 | return false; | |
190 | ||
191 | r->num_closid = 4; | |
2545e9f5 | 192 | r->default_ctrl = max_cbm; |
d3e11b4d | 193 | r->cache.cbm_len = 20; |
0dd2d749 | 194 | r->cache.shareable_bits = 0xc0000; |
d3e11b4d | 195 | r->cache.min_cbm_bits = 2; |
1b5c0b75 VS |
196 | r->alloc_capable = true; |
197 | r->alloc_enabled = true; | |
c1c7c3f9 FY |
198 | |
199 | return true; | |
113c6097 FY |
200 | } |
201 | ||
202 | return false; | |
203 | } | |
204 | ||
05b93417 VS |
205 | /* |
206 | * rdt_get_mb_table() - get a mapping of bandwidth(b/w) percentage values | |
207 | * exposed to user interface and the h/w understandable delay values. | |
208 | * | |
209 | * The non-linear delay values have the granularity of power of two | |
210 | * and also the h/w does not guarantee a curve for configured delay | |
211 | * values vs. actual b/w enforced. | |
212 | * Hence we need a mapping that is pre calibrated so the user can | |
213 | * express the memory b/w as a percentage value. | |
214 | */ | |
215 | static inline bool rdt_get_mb_table(struct rdt_resource *r) | |
216 | { | |
217 | /* | |
218 | * There are no Intel SKUs as of now to support non-linear delay. | |
219 | */ | |
220 | pr_info("MBA b/w map not implemented for cpu:%d, model:%d", | |
221 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
222 | ||
223 | return false; | |
224 | } | |
225 | ||
226 | static bool rdt_get_mem_config(struct rdt_resource *r) | |
227 | { | |
228 | union cpuid_0x10_3_eax eax; | |
229 | union cpuid_0x10_x_edx edx; | |
230 | u32 ebx, ecx; | |
231 | ||
232 | cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full); | |
233 | r->num_closid = edx.split.cos_max + 1; | |
234 | r->membw.max_delay = eax.split.max_delay + 1; | |
235 | r->default_ctrl = MAX_MBA_BW; | |
236 | if (ecx & MBA_IS_LINEAR) { | |
237 | r->membw.delay_linear = true; | |
238 | r->membw.min_bw = MAX_MBA_BW - r->membw.max_delay; | |
239 | r->membw.bw_gran = MAX_MBA_BW - r->membw.max_delay; | |
240 | } else { | |
241 | if (!rdt_get_mb_table(r)) | |
242 | return false; | |
243 | } | |
244 | r->data_width = 3; | |
245 | ||
1b5c0b75 VS |
246 | r->alloc_capable = true; |
247 | r->alloc_enabled = true; | |
05b93417 VS |
248 | |
249 | return true; | |
250 | } | |
251 | ||
6a445edc | 252 | static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r) |
c1c7c3f9 FY |
253 | { |
254 | union cpuid_0x10_1_eax eax; | |
2545e9f5 | 255 | union cpuid_0x10_x_edx edx; |
c1c7c3f9 FY |
256 | u32 ebx, ecx; |
257 | ||
258 | cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx, &edx.full); | |
259 | r->num_closid = edx.split.cos_max + 1; | |
d3e11b4d | 260 | r->cache.cbm_len = eax.split.cbm_len + 1; |
2545e9f5 | 261 | r->default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1; |
0dd2d749 | 262 | r->cache.shareable_bits = ebx & r->default_ctrl; |
d3e11b4d | 263 | r->data_width = (r->cache.cbm_len + 3) / 4; |
1b5c0b75 VS |
264 | r->alloc_capable = true; |
265 | r->alloc_enabled = true; | |
c1c7c3f9 FY |
266 | } |
267 | ||
268 | static void rdt_get_cdp_l3_config(int type) | |
269 | { | |
270 | struct rdt_resource *r_l3 = &rdt_resources_all[RDT_RESOURCE_L3]; | |
271 | struct rdt_resource *r = &rdt_resources_all[type]; | |
272 | ||
273 | r->num_closid = r_l3->num_closid / 2; | |
d3e11b4d | 274 | r->cache.cbm_len = r_l3->cache.cbm_len; |
2545e9f5 | 275 | r->default_ctrl = r_l3->default_ctrl; |
d3e11b4d | 276 | r->data_width = (r->cache.cbm_len + 3) / 4; |
1b5c0b75 | 277 | r->alloc_capable = true; |
c1c7c3f9 FY |
278 | /* |
279 | * By default, CDP is disabled. CDP can be enabled by mount parameter | |
280 | * "cdp" during resctrl file system mount time. | |
281 | */ | |
1b5c0b75 | 282 | r->alloc_enabled = false; |
c1c7c3f9 FY |
283 | } |
284 | ||
2264d9c7 TL |
285 | static int get_cache_id(int cpu, int level) |
286 | { | |
287 | struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu); | |
288 | int i; | |
289 | ||
290 | for (i = 0; i < ci->num_leaves; i++) { | |
291 | if (ci->info_list[i].level == level) | |
292 | return ci->info_list[i].id; | |
293 | } | |
294 | ||
295 | return -1; | |
296 | } | |
297 | ||
05b93417 VS |
298 | /* |
299 | * Map the memory b/w percentage value to delay values | |
300 | * that can be written to QOS_MSRs. | |
301 | * There are currently no SKUs which support non linear delay values. | |
302 | */ | |
303 | static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) | |
304 | { | |
305 | if (r->membw.delay_linear) | |
306 | return MAX_MBA_BW - bw; | |
307 | ||
308 | pr_warn_once("Non Linear delay-bw map not supported but queried\n"); | |
309 | return r->default_ctrl; | |
310 | } | |
311 | ||
312 | static void | |
313 | mba_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) | |
314 | { | |
315 | unsigned int i; | |
316 | ||
317 | /* Write the delay values for mba. */ | |
318 | for (i = m->low; i < m->high; i++) | |
319 | wrmsrl(r->msr_base + i, delay_bw_map(d->ctrl_val[i], r)); | |
320 | } | |
321 | ||
0921c547 TG |
322 | static void |
323 | cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) | |
324 | { | |
325 | unsigned int i; | |
326 | ||
327 | for (i = m->low; i < m->high; i++) | |
328 | wrmsrl(r->msr_base + cbm_idx(r, i), d->ctrl_val[i]); | |
329 | } | |
330 | ||
edf6fa1c VS |
331 | struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r) |
332 | { | |
333 | struct rdt_domain *d; | |
334 | ||
335 | list_for_each_entry(d, &r->domains, list) { | |
336 | /* Find the domain that contains this CPU */ | |
337 | if (cpumask_test_cpu(cpu, &d->cpu_mask)) | |
338 | return d; | |
339 | } | |
340 | ||
341 | return NULL; | |
342 | } | |
343 | ||
2545e9f5 | 344 | void rdt_ctrl_update(void *arg) |
2264d9c7 | 345 | { |
0921c547 | 346 | struct msr_param *m = arg; |
2264d9c7 | 347 | struct rdt_resource *r = m->res; |
0921c547 | 348 | int cpu = smp_processor_id(); |
2264d9c7 TL |
349 | struct rdt_domain *d; |
350 | ||
e3302683 VS |
351 | d = get_domain_from_cpu(cpu, r); |
352 | if (d) { | |
353 | r->msr_update(d, m, r); | |
354 | return; | |
2264d9c7 | 355 | } |
0921c547 | 356 | pr_warn_once("cpu %d not found in any domain for resource %s\n", |
2264d9c7 | 357 | cpu, r->name); |
2264d9c7 TL |
358 | } |
359 | ||
360 | /* | |
361 | * rdt_find_domain - Find a domain in a resource that matches input resource id | |
362 | * | |
363 | * Search resource r's domain list to find the resource id. If the resource | |
364 | * id is found in a domain, return the domain. Otherwise, if requested by | |
365 | * caller, return the first domain whose id is bigger than the input id. | |
366 | * The domain list is sorted by id in ascending order. | |
367 | */ | |
d89b7379 VS |
368 | struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, |
369 | struct list_head **pos) | |
2264d9c7 TL |
370 | { |
371 | struct rdt_domain *d; | |
372 | struct list_head *l; | |
373 | ||
374 | if (id < 0) | |
375 | return ERR_PTR(id); | |
376 | ||
377 | list_for_each(l, &r->domains) { | |
378 | d = list_entry(l, struct rdt_domain, list); | |
379 | /* When id is found, return its domain. */ | |
380 | if (id == d->id) | |
381 | return d; | |
382 | /* Stop searching when finding id's position in sorted list. */ | |
383 | if (id < d->id) | |
384 | break; | |
385 | } | |
386 | ||
387 | if (pos) | |
388 | *pos = l; | |
389 | ||
390 | return NULL; | |
391 | } | |
392 | ||
0921c547 TG |
393 | static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) |
394 | { | |
395 | struct msr_param m; | |
396 | u32 *dc; | |
397 | int i; | |
398 | ||
399 | dc = kmalloc_array(r->num_closid, sizeof(*d->ctrl_val), GFP_KERNEL); | |
400 | if (!dc) | |
401 | return -ENOMEM; | |
402 | ||
403 | d->ctrl_val = dc; | |
404 | ||
405 | /* | |
406 | * Initialize the Control MSRs to having no control. | |
407 | * For Cache Allocation: Set all bits in cbm | |
408 | * For Memory Allocation: Set b/w requested to 100 | |
409 | */ | |
410 | for (i = 0; i < r->num_closid; i++, dc++) | |
411 | *dc = r->default_ctrl; | |
412 | ||
413 | m.low = 0; | |
414 | m.high = r->num_closid; | |
415 | r->msr_update(d, &m, r); | |
416 | return 0; | |
417 | } | |
418 | ||
edf6fa1c VS |
419 | static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d) |
420 | { | |
9f52425b TL |
421 | size_t tsize; |
422 | ||
edf6fa1c VS |
423 | if (is_llc_occupancy_enabled()) { |
424 | d->rmid_busy_llc = kcalloc(BITS_TO_LONGS(r->num_rmid), | |
425 | sizeof(unsigned long), | |
426 | GFP_KERNEL); | |
427 | if (!d->rmid_busy_llc) | |
428 | return -ENOMEM; | |
24247aee | 429 | INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo); |
edf6fa1c | 430 | } |
9f52425b TL |
431 | if (is_mbm_total_enabled()) { |
432 | tsize = sizeof(*d->mbm_total); | |
433 | d->mbm_total = kcalloc(r->num_rmid, tsize, GFP_KERNEL); | |
434 | if (!d->mbm_total) { | |
435 | kfree(d->rmid_busy_llc); | |
436 | return -ENOMEM; | |
437 | } | |
438 | } | |
439 | if (is_mbm_local_enabled()) { | |
440 | tsize = sizeof(*d->mbm_local); | |
441 | d->mbm_local = kcalloc(r->num_rmid, tsize, GFP_KERNEL); | |
442 | if (!d->mbm_local) { | |
443 | kfree(d->rmid_busy_llc); | |
444 | kfree(d->mbm_total); | |
445 | return -ENOMEM; | |
446 | } | |
447 | } | |
edf6fa1c | 448 | |
e3302683 VS |
449 | if (is_mbm_enabled()) { |
450 | INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow); | |
bbc4615e | 451 | mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL); |
e3302683 VS |
452 | } |
453 | ||
edf6fa1c VS |
454 | return 0; |
455 | } | |
456 | ||
2264d9c7 TL |
457 | /* |
458 | * domain_add_cpu - Add a cpu to a resource's domain list. | |
459 | * | |
460 | * If an existing domain in the resource r's domain list matches the cpu's | |
461 | * resource id, add the cpu in the domain. | |
462 | * | |
463 | * Otherwise, a new domain is allocated and inserted into the right position | |
464 | * in the domain list sorted by id in ascending order. | |
465 | * | |
466 | * The order in the domain list is visible to users when we print entries | |
467 | * in the schemata file and schemata input is validated to have the same order | |
468 | * as this list. | |
469 | */ | |
470 | static void domain_add_cpu(int cpu, struct rdt_resource *r) | |
471 | { | |
0921c547 | 472 | int id = get_cache_id(cpu, r->cache_level); |
2264d9c7 TL |
473 | struct list_head *add_pos = NULL; |
474 | struct rdt_domain *d; | |
475 | ||
476 | d = rdt_find_domain(r, id, &add_pos); | |
477 | if (IS_ERR(d)) { | |
478 | pr_warn("Could't find cache id for cpu %d\n", cpu); | |
479 | return; | |
480 | } | |
481 | ||
482 | if (d) { | |
483 | cpumask_set_cpu(cpu, &d->cpu_mask); | |
484 | return; | |
485 | } | |
486 | ||
487 | d = kzalloc_node(sizeof(*d), GFP_KERNEL, cpu_to_node(cpu)); | |
488 | if (!d) | |
489 | return; | |
490 | ||
491 | d->id = id; | |
9f52425b | 492 | cpumask_set_cpu(cpu, &d->cpu_mask); |
2264d9c7 | 493 | |
1b5c0b75 | 494 | if (r->alloc_capable && domain_setup_ctrlval(r, d)) { |
2264d9c7 TL |
495 | kfree(d); |
496 | return; | |
497 | } | |
498 | ||
edf6fa1c VS |
499 | if (r->mon_capable && domain_setup_mon_state(r, d)) { |
500 | kfree(d); | |
501 | return; | |
502 | } | |
503 | ||
2264d9c7 | 504 | list_add_tail(&d->list, add_pos); |
895c663e VS |
505 | |
506 | /* | |
507 | * If resctrl is mounted, add | |
508 | * per domain monitor data directories. | |
509 | */ | |
510 | if (static_branch_unlikely(&rdt_mon_enable_key)) | |
511 | mkdir_mondata_subdir_allrdtgrp(r, d); | |
2264d9c7 TL |
512 | } |
513 | ||
514 | static void domain_remove_cpu(int cpu, struct rdt_resource *r) | |
515 | { | |
516 | int id = get_cache_id(cpu, r->cache_level); | |
517 | struct rdt_domain *d; | |
518 | ||
519 | d = rdt_find_domain(r, id, NULL); | |
520 | if (IS_ERR_OR_NULL(d)) { | |
521 | pr_warn("Could't find cache id for cpu %d\n", cpu); | |
522 | return; | |
523 | } | |
524 | ||
525 | cpumask_clear_cpu(cpu, &d->cpu_mask); | |
526 | if (cpumask_empty(&d->cpu_mask)) { | |
895c663e VS |
527 | /* |
528 | * If resctrl is mounted, remove all the | |
529 | * per domain monitor data directories. | |
530 | */ | |
531 | if (static_branch_unlikely(&rdt_mon_enable_key)) | |
532 | rmdir_mondata_subdir_allrdtgrp(r, d->id); | |
2545e9f5 | 533 | kfree(d->ctrl_val); |
edf6fa1c | 534 | kfree(d->rmid_busy_llc); |
9f52425b TL |
535 | kfree(d->mbm_total); |
536 | kfree(d->mbm_local); | |
2264d9c7 | 537 | list_del(&d->list); |
e3302683 VS |
538 | if (is_mbm_enabled()) |
539 | cancel_delayed_work(&d->mbm_over); | |
24247aee VS |
540 | if (is_llc_occupancy_enabled() && has_busy_rmid(r, d)) { |
541 | /* | |
542 | * When a package is going down, forcefully | |
543 | * decrement rmid->ebusy. There is no way to know | |
544 | * that the L3 was flushed and hence may lead to | |
545 | * incorrect counts in rare scenarios, but leaving | |
546 | * the RMID as busy creates RMID leaks if the | |
547 | * package never comes back. | |
548 | */ | |
549 | __check_limbo(d, true); | |
550 | cancel_delayed_work(&d->cqm_limbo); | |
551 | } | |
552 | ||
2264d9c7 | 553 | kfree(d); |
24247aee VS |
554 | return; |
555 | } | |
556 | ||
557 | if (r == &rdt_resources_all[RDT_RESOURCE_L3]) { | |
558 | if (is_mbm_enabled() && cpu == d->mbm_work_cpu) { | |
559 | cancel_delayed_work(&d->mbm_over); | |
560 | mbm_setup_overflow_handler(d, 0); | |
561 | } | |
562 | if (is_llc_occupancy_enabled() && cpu == d->cqm_work_cpu && | |
563 | has_busy_rmid(r, d)) { | |
564 | cancel_delayed_work(&d->cqm_limbo); | |
565 | cqm_setup_limbo_handler(d, 0); | |
566 | } | |
2264d9c7 TL |
567 | } |
568 | } | |
569 | ||
895c663e | 570 | static void clear_closid_rmid(int cpu) |
2264d9c7 TL |
571 | { |
572 | struct intel_pqr_state *state = this_cpu_ptr(&pqr_state); | |
12e0110c | 573 | |
a9110b55 VS |
574 | state->default_closid = 0; |
575 | state->default_rmid = 0; | |
576 | state->cur_closid = 0; | |
577 | state->cur_rmid = 0; | |
895c663e | 578 | wrmsr(IA32_PQR_ASSOC, 0, 0); |
12e0110c TL |
579 | } |
580 | ||
581 | static int intel_rdt_online_cpu(unsigned int cpu) | |
582 | { | |
2264d9c7 TL |
583 | struct rdt_resource *r; |
584 | ||
585 | mutex_lock(&rdtgroup_mutex); | |
895c663e | 586 | for_each_capable_rdt_resource(r) |
2264d9c7 | 587 | domain_add_cpu(cpu, r); |
12e0110c TL |
588 | /* The cpu is set in default rdtgroup after online. */ |
589 | cpumask_set_cpu(cpu, &rdtgroup_default.cpu_mask); | |
895c663e | 590 | clear_closid_rmid(cpu); |
2264d9c7 TL |
591 | mutex_unlock(&rdtgroup_mutex); |
592 | ||
593 | return 0; | |
594 | } | |
595 | ||
895c663e VS |
596 | static void clear_childcpus(struct rdtgroup *r, unsigned int cpu) |
597 | { | |
598 | struct rdtgroup *cr; | |
599 | ||
600 | list_for_each_entry(cr, &r->mon.crdtgrp_list, mon.crdtgrp_list) { | |
601 | if (cpumask_test_and_clear_cpu(cpu, &cr->cpu_mask)) { | |
602 | break; | |
603 | } | |
604 | } | |
605 | } | |
606 | ||
2264d9c7 TL |
607 | static int intel_rdt_offline_cpu(unsigned int cpu) |
608 | { | |
12e0110c | 609 | struct rdtgroup *rdtgrp; |
2264d9c7 TL |
610 | struct rdt_resource *r; |
611 | ||
612 | mutex_lock(&rdtgroup_mutex); | |
895c663e | 613 | for_each_capable_rdt_resource(r) |
2264d9c7 | 614 | domain_remove_cpu(cpu, r); |
12e0110c | 615 | list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { |
895c663e VS |
616 | if (cpumask_test_and_clear_cpu(cpu, &rdtgrp->cpu_mask)) { |
617 | clear_childcpus(rdtgrp, cpu); | |
12e0110c | 618 | break; |
895c663e | 619 | } |
12e0110c | 620 | } |
895c663e | 621 | clear_closid_rmid(cpu); |
2264d9c7 TL |
622 | mutex_unlock(&rdtgroup_mutex); |
623 | ||
624 | return 0; | |
625 | } | |
626 | ||
70a1ee92 TG |
627 | /* |
628 | * Choose a width for the resource name and resource data based on the | |
629 | * resource that has widest name and cbm. | |
630 | */ | |
631 | static __init void rdt_init_padding(void) | |
632 | { | |
633 | struct rdt_resource *r; | |
634 | int cl; | |
635 | ||
1b5c0b75 | 636 | for_each_alloc_capable_rdt_resource(r) { |
70a1ee92 TG |
637 | cl = strlen(r->name); |
638 | if (cl > max_name_width) | |
639 | max_name_width = cl; | |
640 | ||
641 | if (r->data_width > max_data_width) | |
642 | max_data_width = r->data_width; | |
643 | } | |
644 | } | |
645 | ||
6a445edc | 646 | static __init bool get_rdt_alloc_resources(void) |
70a1ee92 TG |
647 | { |
648 | bool ret = false; | |
649 | ||
650 | if (cache_alloc_hsw_probe()) | |
651 | return true; | |
652 | ||
653 | if (!boot_cpu_has(X86_FEATURE_RDT_A)) | |
654 | return false; | |
655 | ||
656 | if (boot_cpu_has(X86_FEATURE_CAT_L3)) { | |
6a445edc | 657 | rdt_get_cache_alloc_cfg(1, &rdt_resources_all[RDT_RESOURCE_L3]); |
70a1ee92 TG |
658 | if (boot_cpu_has(X86_FEATURE_CDP_L3)) { |
659 | rdt_get_cdp_l3_config(RDT_RESOURCE_L3DATA); | |
660 | rdt_get_cdp_l3_config(RDT_RESOURCE_L3CODE); | |
661 | } | |
662 | ret = true; | |
663 | } | |
664 | if (boot_cpu_has(X86_FEATURE_CAT_L2)) { | |
665 | /* CPUID 0x10.2 fields are same format at 0x10.1 */ | |
6a445edc | 666 | rdt_get_cache_alloc_cfg(2, &rdt_resources_all[RDT_RESOURCE_L2]); |
70a1ee92 TG |
667 | ret = true; |
668 | } | |
ab66a33b | 669 | |
05b93417 VS |
670 | if (boot_cpu_has(X86_FEATURE_MBA)) { |
671 | if (rdt_get_mem_config(&rdt_resources_all[RDT_RESOURCE_MBA])) | |
672 | ret = true; | |
673 | } | |
70a1ee92 TG |
674 | return ret; |
675 | } | |
676 | ||
6a445edc VS |
677 | static __init bool get_rdt_mon_resources(void) |
678 | { | |
679 | if (boot_cpu_has(X86_FEATURE_CQM_OCCUP_LLC)) | |
680 | rdt_mon_features |= (1 << QOS_L3_OCCUP_EVENT_ID); | |
681 | if (boot_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) | |
682 | rdt_mon_features |= (1 << QOS_L3_MBM_TOTAL_EVENT_ID); | |
683 | if (boot_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) | |
684 | rdt_mon_features |= (1 << QOS_L3_MBM_LOCAL_EVENT_ID); | |
685 | ||
686 | if (!rdt_mon_features) | |
687 | return false; | |
688 | ||
689 | return !rdt_get_mon_l3_config(&rdt_resources_all[RDT_RESOURCE_L3]); | |
690 | } | |
691 | ||
692 | static __init bool get_rdt_resources(void) | |
693 | { | |
694 | rdt_alloc_capable = get_rdt_alloc_resources(); | |
695 | rdt_mon_capable = get_rdt_mon_resources(); | |
696 | ||
697 | return (rdt_mon_capable || rdt_alloc_capable); | |
698 | } | |
699 | ||
78e99b4a FY |
700 | static int __init intel_rdt_late_init(void) |
701 | { | |
c1c7c3f9 | 702 | struct rdt_resource *r; |
5ff193fb | 703 | int state, ret; |
c1c7c3f9 | 704 | |
78e99b4a FY |
705 | if (!get_rdt_resources()) |
706 | return -ENODEV; | |
707 | ||
06b57e45 TG |
708 | rdt_init_padding(); |
709 | ||
2264d9c7 TL |
710 | state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, |
711 | "x86/rdt/cat:online:", | |
712 | intel_rdt_online_cpu, intel_rdt_offline_cpu); | |
713 | if (state < 0) | |
714 | return state; | |
715 | ||
5ff193fb FY |
716 | ret = rdtgroup_init(); |
717 | if (ret) { | |
718 | cpuhp_remove_state(state); | |
719 | return ret; | |
720 | } | |
721 | ||
1b5c0b75 | 722 | for_each_alloc_capable_rdt_resource(r) |
c1c7c3f9 | 723 | pr_info("Intel RDT %s allocation detected\n", r->name); |
78e99b4a | 724 | |
6a445edc VS |
725 | for_each_mon_capable_rdt_resource(r) |
726 | pr_info("Intel RDT %s monitoring detected\n", r->name); | |
727 | ||
78e99b4a FY |
728 | return 0; |
729 | } | |
730 | ||
731 | late_initcall(intel_rdt_late_init); |