2 * Intel Cache Quality-of-Service Monitoring (CQM) support.
4 * Based very, very heavily on work by Peter Zijlstra.
7 #include <linux/perf_event.h>
8 #include <linux/slab.h>
9 #include <asm/cpu_device_id.h>
10 #include "../perf_event.h"
12 #define MSR_IA32_PQR_ASSOC 0x0c8f
13 #define MSR_IA32_QM_CTR 0x0c8e
14 #define MSR_IA32_QM_EVTSEL 0x0c8d
16 static u32 cqm_max_rmid
= -1;
17 static unsigned int cqm_l3_scale
; /* supposedly cacheline size */
20 * struct intel_pqr_state - State cache for the PQR MSR
21 * @rmid: The cached Resource Monitoring ID
22 * @closid: The cached Class Of Service ID
23 * @rmid_usecnt: The usage counter for rmid
25 * The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
26 * lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
27 * contains both parts, so we need to cache them.
29 * The cache also helps to avoid pointless updates if the value does
32 struct intel_pqr_state
{
39 * The cached intel_pqr_state is strictly per CPU and can never be
40 * updated from a remote CPU. Both functions which modify the state
41 * (intel_cqm_event_start and intel_cqm_event_stop) are called with
42 * interrupts disabled, which is sufficient for the protection.
44 static DEFINE_PER_CPU(struct intel_pqr_state
, pqr_state
);
47 * Protects cache_cgroups and cqm_rmid_free_lru and cqm_rmid_limbo_lru.
48 * Also protects event->hw.cqm_rmid
50 * Hold either for stability, both for modification of ->hw.cqm_rmid.
52 static DEFINE_MUTEX(cache_mutex
);
53 static DEFINE_RAW_SPINLOCK(cache_lock
);
56 * Groups of events that have the same target(s), one RMID per group.
58 static LIST_HEAD(cache_groups
);
61 * Mask of CPUs for reading CQM values. We only need one per-socket.
63 static cpumask_t cqm_cpumask
;
65 #define RMID_VAL_ERROR (1ULL << 63)
66 #define RMID_VAL_UNAVAIL (1ULL << 62)
68 #define QOS_L3_OCCUP_EVENT_ID (1 << 0)
70 #define QOS_EVENT_MASK QOS_L3_OCCUP_EVENT_ID
73 * This is central to the rotation algorithm in __intel_cqm_rmid_rotate().
75 * This rmid is always free and is guaranteed to have an associated
76 * near-zero occupancy value, i.e. no cachelines are tagged with this
77 * RMID, once __intel_cqm_rmid_rotate() returns.
79 static u32 intel_cqm_rotation_rmid
;
81 #define INVALID_RMID (-1)
84 * Is @rmid valid for programming the hardware?
86 * rmid 0 is reserved by the hardware for all non-monitored tasks, which
87 * means that we should never come across an rmid with that value.
88 * Likewise, an rmid value of -1 is used to indicate "no rmid currently
89 * assigned" and is used as part of the rotation code.
91 static inline bool __rmid_valid(u32 rmid
)
93 if (!rmid
|| rmid
== INVALID_RMID
)
99 static u64
__rmid_read(u32 rmid
)
104 * Ignore the SDM, this thing is _NOTHING_ like a regular perfcnt,
105 * it just says that to increase confusion.
107 wrmsr(MSR_IA32_QM_EVTSEL
, QOS_L3_OCCUP_EVENT_ID
, rmid
);
108 rdmsrl(MSR_IA32_QM_CTR
, val
);
111 * Aside from the ERROR and UNAVAIL bits, assume this thing returns
112 * the number of cachelines tagged with @rmid.
117 enum rmid_recycle_state
{
123 struct cqm_rmid_entry
{
125 enum rmid_recycle_state state
;
126 struct list_head list
;
127 unsigned long queue_time
;
131 * cqm_rmid_free_lru - A least recently used list of RMIDs.
133 * Oldest entry at the head, newest (most recently used) entry at the
134 * tail. This list is never traversed, it's only used to keep track of
135 * the lru order. That is, we only pick entries of the head or insert
138 * All entries on the list are 'free', and their RMIDs are not currently
139 * in use. To mark an RMID as in use, remove its entry from the lru
143 * cqm_rmid_limbo_lru - list of currently unused but (potentially) dirty RMIDs.
145 * This list is contains RMIDs that no one is currently using but that
146 * may have a non-zero occupancy value associated with them. The
147 * rotation worker moves RMIDs from the limbo list to the free list once
148 * the occupancy value drops below __intel_cqm_threshold.
150 * Both lists are protected by cache_mutex.
152 static LIST_HEAD(cqm_rmid_free_lru
);
153 static LIST_HEAD(cqm_rmid_limbo_lru
);
156 * We use a simple array of pointers so that we can lookup a struct
157 * cqm_rmid_entry in O(1). This alleviates the callers of __get_rmid()
158 * and __put_rmid() from having to worry about dealing with struct
159 * cqm_rmid_entry - they just deal with rmids, i.e. integers.
161 * Once this array is initialized it is read-only. No locks are required
164 * All entries for all RMIDs can be looked up in the this array at all
167 static struct cqm_rmid_entry
**cqm_rmid_ptrs
;
169 static inline struct cqm_rmid_entry
*__rmid_entry(u32 rmid
)
171 struct cqm_rmid_entry
*entry
;
173 entry
= cqm_rmid_ptrs
[rmid
];
174 WARN_ON(entry
->rmid
!= rmid
);
180 * Returns < 0 on fail.
182 * We expect to be called with cache_mutex held.
184 static u32
__get_rmid(void)
186 struct cqm_rmid_entry
*entry
;
188 lockdep_assert_held(&cache_mutex
);
190 if (list_empty(&cqm_rmid_free_lru
))
193 entry
= list_first_entry(&cqm_rmid_free_lru
, struct cqm_rmid_entry
, list
);
194 list_del(&entry
->list
);
199 static void __put_rmid(u32 rmid
)
201 struct cqm_rmid_entry
*entry
;
203 lockdep_assert_held(&cache_mutex
);
205 WARN_ON(!__rmid_valid(rmid
));
206 entry
= __rmid_entry(rmid
);
208 entry
->queue_time
= jiffies
;
209 entry
->state
= RMID_YOUNG
;
211 list_add_tail(&entry
->list
, &cqm_rmid_limbo_lru
);
214 static int intel_cqm_setup_rmid_cache(void)
216 struct cqm_rmid_entry
*entry
;
217 unsigned int nr_rmids
;
220 nr_rmids
= cqm_max_rmid
+ 1;
221 cqm_rmid_ptrs
= kmalloc(sizeof(struct cqm_rmid_entry
*) *
222 nr_rmids
, GFP_KERNEL
);
226 for (; r
<= cqm_max_rmid
; r
++) {
227 struct cqm_rmid_entry
*entry
;
229 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
233 INIT_LIST_HEAD(&entry
->list
);
235 cqm_rmid_ptrs
[r
] = entry
;
237 list_add_tail(&entry
->list
, &cqm_rmid_free_lru
);
241 * RMID 0 is special and is always allocated. It's used for all
242 * tasks that are not monitored.
244 entry
= __rmid_entry(0);
245 list_del(&entry
->list
);
247 mutex_lock(&cache_mutex
);
248 intel_cqm_rotation_rmid
= __get_rmid();
249 mutex_unlock(&cache_mutex
);
254 kfree(cqm_rmid_ptrs
[r
]);
256 kfree(cqm_rmid_ptrs
);
261 * Determine if @a and @b measure the same set of tasks.
263 * If @a and @b measure the same set of tasks then we want to share a
266 static bool __match_event(struct perf_event
*a
, struct perf_event
*b
)
268 /* Per-cpu and task events don't mix */
269 if ((a
->attach_state
& PERF_ATTACH_TASK
) !=
270 (b
->attach_state
& PERF_ATTACH_TASK
))
273 #ifdef CONFIG_CGROUP_PERF
274 if (a
->cgrp
!= b
->cgrp
)
278 /* If not task event, we're machine wide */
279 if (!(b
->attach_state
& PERF_ATTACH_TASK
))
283 * Events that target same task are placed into the same cache group.
284 * Mark it as a multi event group, so that we update ->count
285 * for every event rather than just the group leader later.
287 if (a
->hw
.target
== b
->hw
.target
) {
288 b
->hw
.is_group_event
= true;
293 * Are we an inherited event?
301 #ifdef CONFIG_CGROUP_PERF
302 static inline struct perf_cgroup
*event_to_cgroup(struct perf_event
*event
)
304 if (event
->attach_state
& PERF_ATTACH_TASK
)
305 return perf_cgroup_from_task(event
->hw
.target
, event
->ctx
);
312 * Determine if @a's tasks intersect with @b's tasks
314 * There are combinations of events that we explicitly prohibit,
317 * system-wide -> cgroup and task
318 * cgroup -> system-wide
320 * task -> system-wide
323 * Call this function before allocating an RMID.
325 static bool __conflict_event(struct perf_event
*a
, struct perf_event
*b
)
327 #ifdef CONFIG_CGROUP_PERF
329 * We can have any number of cgroups but only one system-wide
332 if (a
->cgrp
&& b
->cgrp
) {
333 struct perf_cgroup
*ac
= a
->cgrp
;
334 struct perf_cgroup
*bc
= b
->cgrp
;
337 * This condition should have been caught in
338 * __match_event() and we should be sharing an RMID.
340 WARN_ON_ONCE(ac
== bc
);
342 if (cgroup_is_descendant(ac
->css
.cgroup
, bc
->css
.cgroup
) ||
343 cgroup_is_descendant(bc
->css
.cgroup
, ac
->css
.cgroup
))
349 if (a
->cgrp
|| b
->cgrp
) {
350 struct perf_cgroup
*ac
, *bc
;
353 * cgroup and system-wide events are mutually exclusive
355 if ((a
->cgrp
&& !(b
->attach_state
& PERF_ATTACH_TASK
)) ||
356 (b
->cgrp
&& !(a
->attach_state
& PERF_ATTACH_TASK
)))
360 * Ensure neither event is part of the other's cgroup
362 ac
= event_to_cgroup(a
);
363 bc
= event_to_cgroup(b
);
368 * Must have cgroup and non-intersecting task events.
374 * We have cgroup and task events, and the task belongs
375 * to a cgroup. Check for for overlap.
377 if (cgroup_is_descendant(ac
->css
.cgroup
, bc
->css
.cgroup
) ||
378 cgroup_is_descendant(bc
->css
.cgroup
, ac
->css
.cgroup
))
385 * If one of them is not a task, same story as above with cgroups.
387 if (!(a
->attach_state
& PERF_ATTACH_TASK
) ||
388 !(b
->attach_state
& PERF_ATTACH_TASK
))
392 * Must be non-overlapping.
402 static void __intel_cqm_event_count(void *info
);
405 * Exchange the RMID of a group of events.
407 static u32
intel_cqm_xchg_rmid(struct perf_event
*group
, u32 rmid
)
409 struct perf_event
*event
;
410 struct list_head
*head
= &group
->hw
.cqm_group_entry
;
411 u32 old_rmid
= group
->hw
.cqm_rmid
;
413 lockdep_assert_held(&cache_mutex
);
416 * If our RMID is being deallocated, perform a read now.
418 if (__rmid_valid(old_rmid
) && !__rmid_valid(rmid
)) {
419 struct rmid_read rr
= {
420 .value
= ATOMIC64_INIT(0),
424 on_each_cpu_mask(&cqm_cpumask
, __intel_cqm_event_count
,
426 local64_set(&group
->count
, atomic64_read(&rr
.value
));
429 raw_spin_lock_irq(&cache_lock
);
431 group
->hw
.cqm_rmid
= rmid
;
432 list_for_each_entry(event
, head
, hw
.cqm_group_entry
)
433 event
->hw
.cqm_rmid
= rmid
;
435 raw_spin_unlock_irq(&cache_lock
);
441 * If we fail to assign a new RMID for intel_cqm_rotation_rmid because
442 * cachelines are still tagged with RMIDs in limbo, we progressively
443 * increment the threshold until we find an RMID in limbo with <=
444 * __intel_cqm_threshold lines tagged. This is designed to mitigate the
445 * problem where cachelines tagged with an RMID are not steadily being
448 * On successful rotations we decrease the threshold back towards zero.
450 * __intel_cqm_max_threshold provides an upper bound on the threshold,
451 * and is measured in bytes because it's exposed to userland.
453 static unsigned int __intel_cqm_threshold
;
454 static unsigned int __intel_cqm_max_threshold
;
457 * Test whether an RMID has a zero occupancy value on this cpu.
459 static void intel_cqm_stable(void *arg
)
461 struct cqm_rmid_entry
*entry
;
463 list_for_each_entry(entry
, &cqm_rmid_limbo_lru
, list
) {
464 if (entry
->state
!= RMID_AVAILABLE
)
467 if (__rmid_read(entry
->rmid
) > __intel_cqm_threshold
)
468 entry
->state
= RMID_DIRTY
;
473 * If we have group events waiting for an RMID that don't conflict with
474 * events already running, assign @rmid.
476 static bool intel_cqm_sched_in_event(u32 rmid
)
478 struct perf_event
*leader
, *event
;
480 lockdep_assert_held(&cache_mutex
);
482 leader
= list_first_entry(&cache_groups
, struct perf_event
,
483 hw
.cqm_groups_entry
);
486 list_for_each_entry_continue(event
, &cache_groups
,
487 hw
.cqm_groups_entry
) {
488 if (__rmid_valid(event
->hw
.cqm_rmid
))
491 if (__conflict_event(event
, leader
))
494 intel_cqm_xchg_rmid(event
, rmid
);
502 * Initially use this constant for both the limbo queue time and the
503 * rotation timer interval, pmu::hrtimer_interval_ms.
505 * They don't need to be the same, but the two are related since if you
506 * rotate faster than you recycle RMIDs, you may run out of available
509 #define RMID_DEFAULT_QUEUE_TIME 250 /* ms */
511 static unsigned int __rmid_queue_time_ms
= RMID_DEFAULT_QUEUE_TIME
;
514 * intel_cqm_rmid_stabilize - move RMIDs from limbo to free list
515 * @nr_available: number of freeable RMIDs on the limbo list
517 * Quiescent state; wait for all 'freed' RMIDs to become unused, i.e. no
518 * cachelines are tagged with those RMIDs. After this we can reuse them
519 * and know that the current set of active RMIDs is stable.
521 * Return %true or %false depending on whether stabilization needs to be
524 * If we return %true then @nr_available is updated to indicate the
525 * number of RMIDs on the limbo list that have been queued for the
526 * minimum queue time (RMID_AVAILABLE), but whose data occupancy values
527 * are above __intel_cqm_threshold.
529 static bool intel_cqm_rmid_stabilize(unsigned int *available
)
531 struct cqm_rmid_entry
*entry
, *tmp
;
533 lockdep_assert_held(&cache_mutex
);
536 list_for_each_entry(entry
, &cqm_rmid_limbo_lru
, list
) {
537 unsigned long min_queue_time
;
538 unsigned long now
= jiffies
;
541 * We hold RMIDs placed into limbo for a minimum queue
542 * time. Before the minimum queue time has elapsed we do
545 * The reasoning is that until a sufficient time has
546 * passed since we stopped using an RMID, any RMID
547 * placed onto the limbo list will likely still have
548 * data tagged in the cache, which means we'll probably
549 * fail to recycle it anyway.
551 * We can save ourselves an expensive IPI by skipping
552 * any RMIDs that have not been queued for the minimum
555 min_queue_time
= entry
->queue_time
+
556 msecs_to_jiffies(__rmid_queue_time_ms
);
558 if (time_after(min_queue_time
, now
))
561 entry
->state
= RMID_AVAILABLE
;
566 * Fast return if none of the RMIDs on the limbo list have been
567 * sitting on the queue for the minimum queue time.
573 * Test whether an RMID is free for each package.
575 on_each_cpu_mask(&cqm_cpumask
, intel_cqm_stable
, NULL
, true);
577 list_for_each_entry_safe(entry
, tmp
, &cqm_rmid_limbo_lru
, list
) {
579 * Exhausted all RMIDs that have waited min queue time.
581 if (entry
->state
== RMID_YOUNG
)
584 if (entry
->state
== RMID_DIRTY
)
587 list_del(&entry
->list
); /* remove from limbo */
590 * The rotation RMID gets priority if it's
591 * currently invalid. In which case, skip adding
592 * the RMID to the the free lru.
594 if (!__rmid_valid(intel_cqm_rotation_rmid
)) {
595 intel_cqm_rotation_rmid
= entry
->rmid
;
600 * If we have groups waiting for RMIDs, hand
601 * them one now provided they don't conflict.
603 if (intel_cqm_sched_in_event(entry
->rmid
))
607 * Otherwise place it onto the free list.
609 list_add_tail(&entry
->list
, &cqm_rmid_free_lru
);
613 return __rmid_valid(intel_cqm_rotation_rmid
);
617 * Pick a victim group and move it to the tail of the group list.
618 * @next: The first group without an RMID
620 static void __intel_cqm_pick_and_rotate(struct perf_event
*next
)
622 struct perf_event
*rotor
;
625 lockdep_assert_held(&cache_mutex
);
627 rotor
= list_first_entry(&cache_groups
, struct perf_event
,
628 hw
.cqm_groups_entry
);
631 * The group at the front of the list should always have a valid
632 * RMID. If it doesn't then no groups have RMIDs assigned and we
633 * don't need to rotate the list.
638 rmid
= intel_cqm_xchg_rmid(rotor
, INVALID_RMID
);
641 list_rotate_left(&cache_groups
);
645 * Deallocate the RMIDs from any events that conflict with @event, and
646 * place them on the back of the group list.
648 static void intel_cqm_sched_out_conflicting_events(struct perf_event
*event
)
650 struct perf_event
*group
, *g
;
653 lockdep_assert_held(&cache_mutex
);
655 list_for_each_entry_safe(group
, g
, &cache_groups
, hw
.cqm_groups_entry
) {
659 rmid
= group
->hw
.cqm_rmid
;
662 * Skip events that don't have a valid RMID.
664 if (!__rmid_valid(rmid
))
668 * No conflict? No problem! Leave the event alone.
670 if (!__conflict_event(group
, event
))
673 intel_cqm_xchg_rmid(group
, INVALID_RMID
);
679 * Attempt to rotate the groups and assign new RMIDs.
681 * We rotate for two reasons,
682 * 1. To handle the scheduling of conflicting events
683 * 2. To recycle RMIDs
685 * Rotating RMIDs is complicated because the hardware doesn't give us
688 * There's problems with the hardware interface; when you change the
689 * task:RMID map cachelines retain their 'old' tags, giving a skewed
690 * picture. In order to work around this, we must always keep one free
691 * RMID - intel_cqm_rotation_rmid.
693 * Rotation works by taking away an RMID from a group (the old RMID),
694 * and assigning the free RMID to another group (the new RMID). We must
695 * then wait for the old RMID to not be used (no cachelines tagged).
696 * This ensure that all cachelines are tagged with 'active' RMIDs. At
697 * this point we can start reading values for the new RMID and treat the
698 * old RMID as the free RMID for the next rotation.
700 * Return %true or %false depending on whether we did any rotating.
702 static bool __intel_cqm_rmid_rotate(void)
704 struct perf_event
*group
, *start
= NULL
;
705 unsigned int threshold_limit
;
706 unsigned int nr_needed
= 0;
707 unsigned int nr_available
;
708 bool rotated
= false;
710 mutex_lock(&cache_mutex
);
714 * Fast path through this function if there are no groups and no
715 * RMIDs that need cleaning.
717 if (list_empty(&cache_groups
) && list_empty(&cqm_rmid_limbo_lru
))
720 list_for_each_entry(group
, &cache_groups
, hw
.cqm_groups_entry
) {
721 if (!__rmid_valid(group
->hw
.cqm_rmid
)) {
729 * We have some event groups, but they all have RMIDs assigned
730 * and no RMIDs need cleaning.
732 if (!nr_needed
&& list_empty(&cqm_rmid_limbo_lru
))
739 * We have more event groups without RMIDs than available RMIDs,
740 * or we have event groups that conflict with the ones currently
743 * We force deallocate the rmid of the group at the head of
744 * cache_groups. The first event group without an RMID then gets
745 * assigned intel_cqm_rotation_rmid. This ensures we always make
748 * Rotate the cache_groups list so the previous head is now the
751 __intel_cqm_pick_and_rotate(start
);
754 * If the rotation is going to succeed, reduce the threshold so
755 * that we don't needlessly reuse dirty RMIDs.
757 if (__rmid_valid(intel_cqm_rotation_rmid
)) {
758 intel_cqm_xchg_rmid(start
, intel_cqm_rotation_rmid
);
759 intel_cqm_rotation_rmid
= __get_rmid();
761 intel_cqm_sched_out_conflicting_events(start
);
763 if (__intel_cqm_threshold
)
764 __intel_cqm_threshold
--;
771 * We now need to stablize the RMID we freed above (if any) to
772 * ensure that the next time we rotate we have an RMID with zero
775 * Alternatively, if we didn't need to perform any rotation,
776 * we'll have a bunch of RMIDs in limbo that need stabilizing.
778 threshold_limit
= __intel_cqm_max_threshold
/ cqm_l3_scale
;
780 while (intel_cqm_rmid_stabilize(&nr_available
) &&
781 __intel_cqm_threshold
< threshold_limit
) {
782 unsigned int steal_limit
;
785 * Don't spin if nobody is actively waiting for an RMID,
786 * the rotation worker will be kicked as soon as an
787 * event needs an RMID anyway.
792 /* Allow max 25% of RMIDs to be in limbo. */
793 steal_limit
= (cqm_max_rmid
+ 1) / 4;
796 * We failed to stabilize any RMIDs so our rotation
797 * logic is now stuck. In order to make forward progress
798 * we have a few options:
800 * 1. rotate ("steal") another RMID
801 * 2. increase the threshold
804 * We do both of 1. and 2. until we hit the steal limit.
806 * The steal limit prevents all RMIDs ending up on the
807 * limbo list. This can happen if every RMID has a
808 * non-zero occupancy above threshold_limit, and the
809 * occupancy values aren't dropping fast enough.
811 * Note that there is prioritisation at work here - we'd
812 * rather increase the number of RMIDs on the limbo list
813 * than increase the threshold, because increasing the
814 * threshold skews the event data (because we reuse
815 * dirty RMIDs) - threshold bumps are a last resort.
817 if (nr_available
< steal_limit
)
820 __intel_cqm_threshold
++;
824 mutex_unlock(&cache_mutex
);
828 static void intel_cqm_rmid_rotate(struct work_struct
*work
);
830 static DECLARE_DELAYED_WORK(intel_cqm_rmid_work
, intel_cqm_rmid_rotate
);
832 static struct pmu intel_cqm_pmu
;
834 static void intel_cqm_rmid_rotate(struct work_struct
*work
)
838 __intel_cqm_rmid_rotate();
840 delay
= msecs_to_jiffies(intel_cqm_pmu
.hrtimer_interval_ms
);
841 schedule_delayed_work(&intel_cqm_rmid_work
, delay
);
845 * Find a group and setup RMID.
847 * If we're part of a group, we use the group's RMID.
849 static void intel_cqm_setup_event(struct perf_event
*event
,
850 struct perf_event
**group
)
852 struct perf_event
*iter
;
853 bool conflict
= false;
856 event
->hw
.is_group_event
= false;
857 list_for_each_entry(iter
, &cache_groups
, hw
.cqm_groups_entry
) {
858 rmid
= iter
->hw
.cqm_rmid
;
860 if (__match_event(iter
, event
)) {
861 /* All tasks in a group share an RMID */
862 event
->hw
.cqm_rmid
= rmid
;
868 * We only care about conflicts for events that are
869 * actually scheduled in (and hence have a valid RMID).
871 if (__conflict_event(iter
, event
) && __rmid_valid(rmid
))
880 event
->hw
.cqm_rmid
= rmid
;
883 static void intel_cqm_event_read(struct perf_event
*event
)
890 * Task events are handled by intel_cqm_event_count().
892 if (event
->cpu
== -1)
895 raw_spin_lock_irqsave(&cache_lock
, flags
);
896 rmid
= event
->hw
.cqm_rmid
;
898 if (!__rmid_valid(rmid
))
901 val
= __rmid_read(rmid
);
904 * Ignore this reading on error states and do not update the value.
906 if (val
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
))
909 local64_set(&event
->count
, val
);
911 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
914 static void __intel_cqm_event_count(void *info
)
916 struct rmid_read
*rr
= info
;
919 val
= __rmid_read(rr
->rmid
);
921 if (val
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
))
924 atomic64_add(val
, &rr
->value
);
927 static inline bool cqm_group_leader(struct perf_event
*event
)
929 return !list_empty(&event
->hw
.cqm_groups_entry
);
932 static u64
intel_cqm_event_count(struct perf_event
*event
)
935 struct rmid_read rr
= {
936 .value
= ATOMIC64_INIT(0),
940 * We only need to worry about task events. System-wide events
941 * are handled like usual, i.e. entirely with
942 * intel_cqm_event_read().
944 if (event
->cpu
!= -1)
945 return __perf_event_count(event
);
948 * Only the group leader gets to report values except in case of
949 * multiple events in the same group, we still need to read the
950 * other events.This stops us
951 * reporting duplicate values to userspace, and gives us a clear
952 * rule for which task gets to report the values.
954 * Note that it is impossible to attribute these values to
955 * specific packages - we forfeit that ability when we create
958 if (!cqm_group_leader(event
) && !event
->hw
.is_group_event
)
962 * Getting up-to-date values requires an SMP IPI which is not
963 * possible if we're being called in interrupt context. Return
964 * the cached values instead.
966 if (unlikely(in_interrupt()))
970 * Notice that we don't perform the reading of an RMID
971 * atomically, because we can't hold a spin lock across the
974 * Speculatively perform the read, since @event might be
975 * assigned a different (possibly invalid) RMID while we're
976 * busying performing the IPI calls. It's therefore necessary to
977 * check @event's RMID afterwards, and if it has changed,
978 * discard the result of the read.
980 rr
.rmid
= ACCESS_ONCE(event
->hw
.cqm_rmid
);
982 if (!__rmid_valid(rr
.rmid
))
985 on_each_cpu_mask(&cqm_cpumask
, __intel_cqm_event_count
, &rr
, 1);
987 raw_spin_lock_irqsave(&cache_lock
, flags
);
988 if (event
->hw
.cqm_rmid
== rr
.rmid
)
989 local64_set(&event
->count
, atomic64_read(&rr
.value
));
990 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
992 return __perf_event_count(event
);
995 static void intel_cqm_event_start(struct perf_event
*event
, int mode
)
997 struct intel_pqr_state
*state
= this_cpu_ptr(&pqr_state
);
998 u32 rmid
= event
->hw
.cqm_rmid
;
1000 if (!(event
->hw
.cqm_state
& PERF_HES_STOPPED
))
1003 event
->hw
.cqm_state
&= ~PERF_HES_STOPPED
;
1005 if (state
->rmid_usecnt
++) {
1006 if (!WARN_ON_ONCE(state
->rmid
!= rmid
))
1009 WARN_ON_ONCE(state
->rmid
);
1013 wrmsr(MSR_IA32_PQR_ASSOC
, rmid
, state
->closid
);
1016 static void intel_cqm_event_stop(struct perf_event
*event
, int mode
)
1018 struct intel_pqr_state
*state
= this_cpu_ptr(&pqr_state
);
1020 if (event
->hw
.cqm_state
& PERF_HES_STOPPED
)
1023 event
->hw
.cqm_state
|= PERF_HES_STOPPED
;
1025 intel_cqm_event_read(event
);
1027 if (!--state
->rmid_usecnt
) {
1029 wrmsr(MSR_IA32_PQR_ASSOC
, 0, state
->closid
);
1031 WARN_ON_ONCE(!state
->rmid
);
1035 static int intel_cqm_event_add(struct perf_event
*event
, int mode
)
1037 unsigned long flags
;
1040 raw_spin_lock_irqsave(&cache_lock
, flags
);
1042 event
->hw
.cqm_state
= PERF_HES_STOPPED
;
1043 rmid
= event
->hw
.cqm_rmid
;
1045 if (__rmid_valid(rmid
) && (mode
& PERF_EF_START
))
1046 intel_cqm_event_start(event
, mode
);
1048 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
1053 static void intel_cqm_event_destroy(struct perf_event
*event
)
1055 struct perf_event
*group_other
= NULL
;
1057 mutex_lock(&cache_mutex
);
1060 * If there's another event in this group...
1062 if (!list_empty(&event
->hw
.cqm_group_entry
)) {
1063 group_other
= list_first_entry(&event
->hw
.cqm_group_entry
,
1065 hw
.cqm_group_entry
);
1066 list_del(&event
->hw
.cqm_group_entry
);
1070 * And we're the group leader..
1072 if (cqm_group_leader(event
)) {
1074 * If there was a group_other, make that leader, otherwise
1075 * destroy the group and return the RMID.
1078 list_replace(&event
->hw
.cqm_groups_entry
,
1079 &group_other
->hw
.cqm_groups_entry
);
1081 u32 rmid
= event
->hw
.cqm_rmid
;
1083 if (__rmid_valid(rmid
))
1085 list_del(&event
->hw
.cqm_groups_entry
);
1089 mutex_unlock(&cache_mutex
);
1092 static int intel_cqm_event_init(struct perf_event
*event
)
1094 struct perf_event
*group
= NULL
;
1095 bool rotate
= false;
1097 if (event
->attr
.type
!= intel_cqm_pmu
.type
)
1100 if (event
->attr
.config
& ~QOS_EVENT_MASK
)
1103 /* unsupported modes and filters */
1104 if (event
->attr
.exclude_user
||
1105 event
->attr
.exclude_kernel
||
1106 event
->attr
.exclude_hv
||
1107 event
->attr
.exclude_idle
||
1108 event
->attr
.exclude_host
||
1109 event
->attr
.exclude_guest
||
1110 event
->attr
.sample_period
) /* no sampling */
1113 INIT_LIST_HEAD(&event
->hw
.cqm_group_entry
);
1114 INIT_LIST_HEAD(&event
->hw
.cqm_groups_entry
);
1116 event
->destroy
= intel_cqm_event_destroy
;
1118 mutex_lock(&cache_mutex
);
1120 /* Will also set rmid */
1121 intel_cqm_setup_event(event
, &group
);
1124 list_add_tail(&event
->hw
.cqm_group_entry
,
1125 &group
->hw
.cqm_group_entry
);
1127 list_add_tail(&event
->hw
.cqm_groups_entry
,
1131 * All RMIDs are either in use or have recently been
1132 * used. Kick the rotation worker to clean/free some.
1134 * We only do this for the group leader, rather than for
1135 * every event in a group to save on needless work.
1137 if (!__rmid_valid(event
->hw
.cqm_rmid
))
1141 mutex_unlock(&cache_mutex
);
1144 schedule_delayed_work(&intel_cqm_rmid_work
, 0);
1149 EVENT_ATTR_STR(llc_occupancy
, intel_cqm_llc
, "event=0x01");
1150 EVENT_ATTR_STR(llc_occupancy
.per
-pkg
, intel_cqm_llc_pkg
, "1");
1151 EVENT_ATTR_STR(llc_occupancy
.unit
, intel_cqm_llc_unit
, "Bytes");
1152 EVENT_ATTR_STR(llc_occupancy
.scale
, intel_cqm_llc_scale
, NULL
);
1153 EVENT_ATTR_STR(llc_occupancy
.snapshot
, intel_cqm_llc_snapshot
, "1");
1155 static struct attribute
*intel_cqm_events_attr
[] = {
1156 EVENT_PTR(intel_cqm_llc
),
1157 EVENT_PTR(intel_cqm_llc_pkg
),
1158 EVENT_PTR(intel_cqm_llc_unit
),
1159 EVENT_PTR(intel_cqm_llc_scale
),
1160 EVENT_PTR(intel_cqm_llc_snapshot
),
1164 static struct attribute_group intel_cqm_events_group
= {
1166 .attrs
= intel_cqm_events_attr
,
1169 PMU_FORMAT_ATTR(event
, "config:0-7");
1170 static struct attribute
*intel_cqm_formats_attr
[] = {
1171 &format_attr_event
.attr
,
1175 static struct attribute_group intel_cqm_format_group
= {
1177 .attrs
= intel_cqm_formats_attr
,
1181 max_recycle_threshold_show(struct device
*dev
, struct device_attribute
*attr
,
1186 mutex_lock(&cache_mutex
);
1187 rv
= snprintf(page
, PAGE_SIZE
-1, "%u\n", __intel_cqm_max_threshold
);
1188 mutex_unlock(&cache_mutex
);
1194 max_recycle_threshold_store(struct device
*dev
,
1195 struct device_attribute
*attr
,
1196 const char *buf
, size_t count
)
1198 unsigned int bytes
, cachelines
;
1201 ret
= kstrtouint(buf
, 0, &bytes
);
1205 mutex_lock(&cache_mutex
);
1207 __intel_cqm_max_threshold
= bytes
;
1208 cachelines
= bytes
/ cqm_l3_scale
;
1211 * The new maximum takes effect immediately.
1213 if (__intel_cqm_threshold
> cachelines
)
1214 __intel_cqm_threshold
= cachelines
;
1216 mutex_unlock(&cache_mutex
);
1221 static DEVICE_ATTR_RW(max_recycle_threshold
);
1223 static struct attribute
*intel_cqm_attrs
[] = {
1224 &dev_attr_max_recycle_threshold
.attr
,
1228 static const struct attribute_group intel_cqm_group
= {
1229 .attrs
= intel_cqm_attrs
,
1232 static const struct attribute_group
*intel_cqm_attr_groups
[] = {
1233 &intel_cqm_events_group
,
1234 &intel_cqm_format_group
,
1239 static struct pmu intel_cqm_pmu
= {
1240 .hrtimer_interval_ms
= RMID_DEFAULT_QUEUE_TIME
,
1241 .attr_groups
= intel_cqm_attr_groups
,
1242 .task_ctx_nr
= perf_sw_context
,
1243 .event_init
= intel_cqm_event_init
,
1244 .add
= intel_cqm_event_add
,
1245 .del
= intel_cqm_event_stop
,
1246 .start
= intel_cqm_event_start
,
1247 .stop
= intel_cqm_event_stop
,
1248 .read
= intel_cqm_event_read
,
1249 .count
= intel_cqm_event_count
,
1252 static inline void cqm_pick_event_reader(int cpu
)
1256 /* First online cpu in package becomes the reader */
1257 reader
= cpumask_any_and(&cqm_cpumask
, topology_core_cpumask(cpu
));
1258 if (reader
>= nr_cpu_ids
)
1259 cpumask_set_cpu(cpu
, &cqm_cpumask
);
1262 static void intel_cqm_cpu_starting(unsigned int cpu
)
1264 struct intel_pqr_state
*state
= &per_cpu(pqr_state
, cpu
);
1265 struct cpuinfo_x86
*c
= &cpu_data(cpu
);
1269 state
->rmid_usecnt
= 0;
1271 WARN_ON(c
->x86_cache_max_rmid
!= cqm_max_rmid
);
1272 WARN_ON(c
->x86_cache_occ_scale
!= cqm_l3_scale
);
1275 static void intel_cqm_cpu_exit(unsigned int cpu
)
1279 /* Is @cpu the current cqm reader for this package ? */
1280 if (!cpumask_test_and_clear_cpu(cpu
, &cqm_cpumask
))
1283 /* Find another online reader in this package */
1284 target
= cpumask_any_but(topology_core_cpumask(cpu
), cpu
);
1286 if (target
< nr_cpu_ids
)
1287 cpumask_set_cpu(target
, &cqm_cpumask
);
1290 static int intel_cqm_cpu_notifier(struct notifier_block
*nb
,
1291 unsigned long action
, void *hcpu
)
1293 unsigned int cpu
= (unsigned long)hcpu
;
1295 switch (action
& ~CPU_TASKS_FROZEN
) {
1296 case CPU_DOWN_PREPARE
:
1297 intel_cqm_cpu_exit(cpu
);
1300 intel_cqm_cpu_starting(cpu
);
1301 cqm_pick_event_reader(cpu
);
1308 static const struct x86_cpu_id intel_cqm_match
[] = {
1309 { .vendor
= X86_VENDOR_INTEL
, .feature
= X86_FEATURE_CQM_OCCUP_LLC
},
1313 static int __init
intel_cqm_init(void)
1315 char *str
, scale
[20];
1318 if (!x86_match_cpu(intel_cqm_match
))
1321 cqm_l3_scale
= boot_cpu_data
.x86_cache_occ_scale
;
1324 * It's possible that not all resources support the same number
1325 * of RMIDs. Instead of making scheduling much more complicated
1326 * (where we have to match a task's RMID to a cpu that supports
1327 * that many RMIDs) just find the minimum RMIDs supported across
1330 * Also, check that the scales match on all cpus.
1332 cpu_notifier_register_begin();
1334 for_each_online_cpu(cpu
) {
1335 struct cpuinfo_x86
*c
= &cpu_data(cpu
);
1337 if (c
->x86_cache_max_rmid
< cqm_max_rmid
)
1338 cqm_max_rmid
= c
->x86_cache_max_rmid
;
1340 if (c
->x86_cache_occ_scale
!= cqm_l3_scale
) {
1341 pr_err("Multiple LLC scale values, disabling\n");
1348 * A reasonable upper limit on the max threshold is the number
1349 * of lines tagged per RMID if all RMIDs have the same number of
1350 * lines tagged in the LLC.
1352 * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC.
1354 __intel_cqm_max_threshold
=
1355 boot_cpu_data
.x86_cache_size
* 1024 / (cqm_max_rmid
+ 1);
1357 snprintf(scale
, sizeof(scale
), "%u", cqm_l3_scale
);
1358 str
= kstrdup(scale
, GFP_KERNEL
);
1364 event_attr_intel_cqm_llc_scale
.event_str
= str
;
1366 ret
= intel_cqm_setup_rmid_cache();
1370 for_each_online_cpu(i
) {
1371 intel_cqm_cpu_starting(i
);
1372 cqm_pick_event_reader(i
);
1375 __perf_cpu_notifier(intel_cqm_cpu_notifier
);
1377 ret
= perf_pmu_register(&intel_cqm_pmu
, "intel_cqm", -1);
1379 pr_err("Intel CQM perf registration failed: %d\n", ret
);
1381 pr_info("Intel CQM monitoring enabled\n");
1384 cpu_notifier_register_done();
1388 device_initcall(intel_cqm_init
);