2 * Intel Cache Quality-of-Service Monitoring (CQM) support.
4 * Based very, very heavily on work by Peter Zijlstra.
7 #include <linux/perf_event.h>
8 #include <linux/slab.h>
9 #include <asm/cpu_device_id.h>
10 #include "../perf_event.h"
12 #define MSR_IA32_PQR_ASSOC 0x0c8f
13 #define MSR_IA32_QM_CTR 0x0c8e
14 #define MSR_IA32_QM_EVTSEL 0x0c8d
16 static u32 cqm_max_rmid
= -1;
17 static unsigned int cqm_l3_scale
; /* supposedly cacheline size */
20 * struct intel_pqr_state - State cache for the PQR MSR
21 * @rmid: The cached Resource Monitoring ID
22 * @closid: The cached Class Of Service ID
23 * @rmid_usecnt: The usage counter for rmid
25 * The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
26 * lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
27 * contains both parts, so we need to cache them.
29 * The cache also helps to avoid pointless updates if the value does
32 struct intel_pqr_state
{
39 * The cached intel_pqr_state is strictly per CPU and can never be
40 * updated from a remote CPU. Both functions which modify the state
41 * (intel_cqm_event_start and intel_cqm_event_stop) are called with
42 * interrupts disabled, which is sufficient for the protection.
44 static DEFINE_PER_CPU(struct intel_pqr_state
, pqr_state
);
47 * Protects cache_cgroups and cqm_rmid_free_lru and cqm_rmid_limbo_lru.
48 * Also protects event->hw.cqm_rmid
50 * Hold either for stability, both for modification of ->hw.cqm_rmid.
52 static DEFINE_MUTEX(cache_mutex
);
53 static DEFINE_RAW_SPINLOCK(cache_lock
);
56 * Groups of events that have the same target(s), one RMID per group.
58 static LIST_HEAD(cache_groups
);
61 * Mask of CPUs for reading CQM values. We only need one per-socket.
63 static cpumask_t cqm_cpumask
;
65 #define RMID_VAL_ERROR (1ULL << 63)
66 #define RMID_VAL_UNAVAIL (1ULL << 62)
68 #define QOS_L3_OCCUP_EVENT_ID (1 << 0)
70 #define QOS_EVENT_MASK QOS_L3_OCCUP_EVENT_ID
73 * This is central to the rotation algorithm in __intel_cqm_rmid_rotate().
75 * This rmid is always free and is guaranteed to have an associated
76 * near-zero occupancy value, i.e. no cachelines are tagged with this
77 * RMID, once __intel_cqm_rmid_rotate() returns.
79 static u32 intel_cqm_rotation_rmid
;
81 #define INVALID_RMID (-1)
84 * Is @rmid valid for programming the hardware?
86 * rmid 0 is reserved by the hardware for all non-monitored tasks, which
87 * means that we should never come across an rmid with that value.
88 * Likewise, an rmid value of -1 is used to indicate "no rmid currently
89 * assigned" and is used as part of the rotation code.
91 static inline bool __rmid_valid(u32 rmid
)
93 if (!rmid
|| rmid
== INVALID_RMID
)
99 static u64
__rmid_read(u32 rmid
)
104 * Ignore the SDM, this thing is _NOTHING_ like a regular perfcnt,
105 * it just says that to increase confusion.
107 wrmsr(MSR_IA32_QM_EVTSEL
, QOS_L3_OCCUP_EVENT_ID
, rmid
);
108 rdmsrl(MSR_IA32_QM_CTR
, val
);
111 * Aside from the ERROR and UNAVAIL bits, assume this thing returns
112 * the number of cachelines tagged with @rmid.
117 enum rmid_recycle_state
{
123 struct cqm_rmid_entry
{
125 enum rmid_recycle_state state
;
126 struct list_head list
;
127 unsigned long queue_time
;
131 * cqm_rmid_free_lru - A least recently used list of RMIDs.
133 * Oldest entry at the head, newest (most recently used) entry at the
134 * tail. This list is never traversed, it's only used to keep track of
135 * the lru order. That is, we only pick entries of the head or insert
138 * All entries on the list are 'free', and their RMIDs are not currently
139 * in use. To mark an RMID as in use, remove its entry from the lru
143 * cqm_rmid_limbo_lru - list of currently unused but (potentially) dirty RMIDs.
145 * This list is contains RMIDs that no one is currently using but that
146 * may have a non-zero occupancy value associated with them. The
147 * rotation worker moves RMIDs from the limbo list to the free list once
148 * the occupancy value drops below __intel_cqm_threshold.
150 * Both lists are protected by cache_mutex.
152 static LIST_HEAD(cqm_rmid_free_lru
);
153 static LIST_HEAD(cqm_rmid_limbo_lru
);
156 * We use a simple array of pointers so that we can lookup a struct
157 * cqm_rmid_entry in O(1). This alleviates the callers of __get_rmid()
158 * and __put_rmid() from having to worry about dealing with struct
159 * cqm_rmid_entry - they just deal with rmids, i.e. integers.
161 * Once this array is initialized it is read-only. No locks are required
164 * All entries for all RMIDs can be looked up in the this array at all
167 static struct cqm_rmid_entry
**cqm_rmid_ptrs
;
169 static inline struct cqm_rmid_entry
*__rmid_entry(u32 rmid
)
171 struct cqm_rmid_entry
*entry
;
173 entry
= cqm_rmid_ptrs
[rmid
];
174 WARN_ON(entry
->rmid
!= rmid
);
180 * Returns < 0 on fail.
182 * We expect to be called with cache_mutex held.
184 static u32
__get_rmid(void)
186 struct cqm_rmid_entry
*entry
;
188 lockdep_assert_held(&cache_mutex
);
190 if (list_empty(&cqm_rmid_free_lru
))
193 entry
= list_first_entry(&cqm_rmid_free_lru
, struct cqm_rmid_entry
, list
);
194 list_del(&entry
->list
);
199 static void __put_rmid(u32 rmid
)
201 struct cqm_rmid_entry
*entry
;
203 lockdep_assert_held(&cache_mutex
);
205 WARN_ON(!__rmid_valid(rmid
));
206 entry
= __rmid_entry(rmid
);
208 entry
->queue_time
= jiffies
;
209 entry
->state
= RMID_YOUNG
;
211 list_add_tail(&entry
->list
, &cqm_rmid_limbo_lru
);
214 static void cqm_cleanup(void)
221 for (i
= 0; i
< cqm_max_rmid
; i
++)
222 kfree(cqm_rmid_ptrs
[i
]);
224 kfree(cqm_rmid_ptrs
);
225 cqm_rmid_ptrs
= NULL
;
228 static int intel_cqm_setup_rmid_cache(void)
230 struct cqm_rmid_entry
*entry
;
231 unsigned int nr_rmids
;
234 nr_rmids
= cqm_max_rmid
+ 1;
235 cqm_rmid_ptrs
= kzalloc(sizeof(struct cqm_rmid_entry
*) *
236 nr_rmids
, GFP_KERNEL
);
240 for (; r
<= cqm_max_rmid
; r
++) {
241 struct cqm_rmid_entry
*entry
;
243 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
247 INIT_LIST_HEAD(&entry
->list
);
249 cqm_rmid_ptrs
[r
] = entry
;
251 list_add_tail(&entry
->list
, &cqm_rmid_free_lru
);
255 * RMID 0 is special and is always allocated. It's used for all
256 * tasks that are not monitored.
258 entry
= __rmid_entry(0);
259 list_del(&entry
->list
);
261 mutex_lock(&cache_mutex
);
262 intel_cqm_rotation_rmid
= __get_rmid();
263 mutex_unlock(&cache_mutex
);
273 * Determine if @a and @b measure the same set of tasks.
275 * If @a and @b measure the same set of tasks then we want to share a
278 static bool __match_event(struct perf_event
*a
, struct perf_event
*b
)
280 /* Per-cpu and task events don't mix */
281 if ((a
->attach_state
& PERF_ATTACH_TASK
) !=
282 (b
->attach_state
& PERF_ATTACH_TASK
))
285 #ifdef CONFIG_CGROUP_PERF
286 if (a
->cgrp
!= b
->cgrp
)
290 /* If not task event, we're machine wide */
291 if (!(b
->attach_state
& PERF_ATTACH_TASK
))
295 * Events that target same task are placed into the same cache group.
296 * Mark it as a multi event group, so that we update ->count
297 * for every event rather than just the group leader later.
299 if (a
->hw
.target
== b
->hw
.target
) {
300 b
->hw
.is_group_event
= true;
305 * Are we an inherited event?
313 #ifdef CONFIG_CGROUP_PERF
314 static inline struct perf_cgroup
*event_to_cgroup(struct perf_event
*event
)
316 if (event
->attach_state
& PERF_ATTACH_TASK
)
317 return perf_cgroup_from_task(event
->hw
.target
, event
->ctx
);
324 * Determine if @a's tasks intersect with @b's tasks
326 * There are combinations of events that we explicitly prohibit,
329 * system-wide -> cgroup and task
330 * cgroup -> system-wide
332 * task -> system-wide
335 * Call this function before allocating an RMID.
337 static bool __conflict_event(struct perf_event
*a
, struct perf_event
*b
)
339 #ifdef CONFIG_CGROUP_PERF
341 * We can have any number of cgroups but only one system-wide
344 if (a
->cgrp
&& b
->cgrp
) {
345 struct perf_cgroup
*ac
= a
->cgrp
;
346 struct perf_cgroup
*bc
= b
->cgrp
;
349 * This condition should have been caught in
350 * __match_event() and we should be sharing an RMID.
352 WARN_ON_ONCE(ac
== bc
);
354 if (cgroup_is_descendant(ac
->css
.cgroup
, bc
->css
.cgroup
) ||
355 cgroup_is_descendant(bc
->css
.cgroup
, ac
->css
.cgroup
))
361 if (a
->cgrp
|| b
->cgrp
) {
362 struct perf_cgroup
*ac
, *bc
;
365 * cgroup and system-wide events are mutually exclusive
367 if ((a
->cgrp
&& !(b
->attach_state
& PERF_ATTACH_TASK
)) ||
368 (b
->cgrp
&& !(a
->attach_state
& PERF_ATTACH_TASK
)))
372 * Ensure neither event is part of the other's cgroup
374 ac
= event_to_cgroup(a
);
375 bc
= event_to_cgroup(b
);
380 * Must have cgroup and non-intersecting task events.
386 * We have cgroup and task events, and the task belongs
387 * to a cgroup. Check for for overlap.
389 if (cgroup_is_descendant(ac
->css
.cgroup
, bc
->css
.cgroup
) ||
390 cgroup_is_descendant(bc
->css
.cgroup
, ac
->css
.cgroup
))
397 * If one of them is not a task, same story as above with cgroups.
399 if (!(a
->attach_state
& PERF_ATTACH_TASK
) ||
400 !(b
->attach_state
& PERF_ATTACH_TASK
))
404 * Must be non-overlapping.
414 static void __intel_cqm_event_count(void *info
);
417 * Exchange the RMID of a group of events.
419 static u32
intel_cqm_xchg_rmid(struct perf_event
*group
, u32 rmid
)
421 struct perf_event
*event
;
422 struct list_head
*head
= &group
->hw
.cqm_group_entry
;
423 u32 old_rmid
= group
->hw
.cqm_rmid
;
425 lockdep_assert_held(&cache_mutex
);
428 * If our RMID is being deallocated, perform a read now.
430 if (__rmid_valid(old_rmid
) && !__rmid_valid(rmid
)) {
431 struct rmid_read rr
= {
432 .value
= ATOMIC64_INIT(0),
436 on_each_cpu_mask(&cqm_cpumask
, __intel_cqm_event_count
,
438 local64_set(&group
->count
, atomic64_read(&rr
.value
));
441 raw_spin_lock_irq(&cache_lock
);
443 group
->hw
.cqm_rmid
= rmid
;
444 list_for_each_entry(event
, head
, hw
.cqm_group_entry
)
445 event
->hw
.cqm_rmid
= rmid
;
447 raw_spin_unlock_irq(&cache_lock
);
453 * If we fail to assign a new RMID for intel_cqm_rotation_rmid because
454 * cachelines are still tagged with RMIDs in limbo, we progressively
455 * increment the threshold until we find an RMID in limbo with <=
456 * __intel_cqm_threshold lines tagged. This is designed to mitigate the
457 * problem where cachelines tagged with an RMID are not steadily being
460 * On successful rotations we decrease the threshold back towards zero.
462 * __intel_cqm_max_threshold provides an upper bound on the threshold,
463 * and is measured in bytes because it's exposed to userland.
465 static unsigned int __intel_cqm_threshold
;
466 static unsigned int __intel_cqm_max_threshold
;
469 * Test whether an RMID has a zero occupancy value on this cpu.
471 static void intel_cqm_stable(void *arg
)
473 struct cqm_rmid_entry
*entry
;
475 list_for_each_entry(entry
, &cqm_rmid_limbo_lru
, list
) {
476 if (entry
->state
!= RMID_AVAILABLE
)
479 if (__rmid_read(entry
->rmid
) > __intel_cqm_threshold
)
480 entry
->state
= RMID_DIRTY
;
485 * If we have group events waiting for an RMID that don't conflict with
486 * events already running, assign @rmid.
488 static bool intel_cqm_sched_in_event(u32 rmid
)
490 struct perf_event
*leader
, *event
;
492 lockdep_assert_held(&cache_mutex
);
494 leader
= list_first_entry(&cache_groups
, struct perf_event
,
495 hw
.cqm_groups_entry
);
498 list_for_each_entry_continue(event
, &cache_groups
,
499 hw
.cqm_groups_entry
) {
500 if (__rmid_valid(event
->hw
.cqm_rmid
))
503 if (__conflict_event(event
, leader
))
506 intel_cqm_xchg_rmid(event
, rmid
);
514 * Initially use this constant for both the limbo queue time and the
515 * rotation timer interval, pmu::hrtimer_interval_ms.
517 * They don't need to be the same, but the two are related since if you
518 * rotate faster than you recycle RMIDs, you may run out of available
521 #define RMID_DEFAULT_QUEUE_TIME 250 /* ms */
523 static unsigned int __rmid_queue_time_ms
= RMID_DEFAULT_QUEUE_TIME
;
526 * intel_cqm_rmid_stabilize - move RMIDs from limbo to free list
527 * @nr_available: number of freeable RMIDs on the limbo list
529 * Quiescent state; wait for all 'freed' RMIDs to become unused, i.e. no
530 * cachelines are tagged with those RMIDs. After this we can reuse them
531 * and know that the current set of active RMIDs is stable.
533 * Return %true or %false depending on whether stabilization needs to be
536 * If we return %true then @nr_available is updated to indicate the
537 * number of RMIDs on the limbo list that have been queued for the
538 * minimum queue time (RMID_AVAILABLE), but whose data occupancy values
539 * are above __intel_cqm_threshold.
541 static bool intel_cqm_rmid_stabilize(unsigned int *available
)
543 struct cqm_rmid_entry
*entry
, *tmp
;
545 lockdep_assert_held(&cache_mutex
);
548 list_for_each_entry(entry
, &cqm_rmid_limbo_lru
, list
) {
549 unsigned long min_queue_time
;
550 unsigned long now
= jiffies
;
553 * We hold RMIDs placed into limbo for a minimum queue
554 * time. Before the minimum queue time has elapsed we do
557 * The reasoning is that until a sufficient time has
558 * passed since we stopped using an RMID, any RMID
559 * placed onto the limbo list will likely still have
560 * data tagged in the cache, which means we'll probably
561 * fail to recycle it anyway.
563 * We can save ourselves an expensive IPI by skipping
564 * any RMIDs that have not been queued for the minimum
567 min_queue_time
= entry
->queue_time
+
568 msecs_to_jiffies(__rmid_queue_time_ms
);
570 if (time_after(min_queue_time
, now
))
573 entry
->state
= RMID_AVAILABLE
;
578 * Fast return if none of the RMIDs on the limbo list have been
579 * sitting on the queue for the minimum queue time.
585 * Test whether an RMID is free for each package.
587 on_each_cpu_mask(&cqm_cpumask
, intel_cqm_stable
, NULL
, true);
589 list_for_each_entry_safe(entry
, tmp
, &cqm_rmid_limbo_lru
, list
) {
591 * Exhausted all RMIDs that have waited min queue time.
593 if (entry
->state
== RMID_YOUNG
)
596 if (entry
->state
== RMID_DIRTY
)
599 list_del(&entry
->list
); /* remove from limbo */
602 * The rotation RMID gets priority if it's
603 * currently invalid. In which case, skip adding
604 * the RMID to the the free lru.
606 if (!__rmid_valid(intel_cqm_rotation_rmid
)) {
607 intel_cqm_rotation_rmid
= entry
->rmid
;
612 * If we have groups waiting for RMIDs, hand
613 * them one now provided they don't conflict.
615 if (intel_cqm_sched_in_event(entry
->rmid
))
619 * Otherwise place it onto the free list.
621 list_add_tail(&entry
->list
, &cqm_rmid_free_lru
);
625 return __rmid_valid(intel_cqm_rotation_rmid
);
629 * Pick a victim group and move it to the tail of the group list.
630 * @next: The first group without an RMID
632 static void __intel_cqm_pick_and_rotate(struct perf_event
*next
)
634 struct perf_event
*rotor
;
637 lockdep_assert_held(&cache_mutex
);
639 rotor
= list_first_entry(&cache_groups
, struct perf_event
,
640 hw
.cqm_groups_entry
);
643 * The group at the front of the list should always have a valid
644 * RMID. If it doesn't then no groups have RMIDs assigned and we
645 * don't need to rotate the list.
650 rmid
= intel_cqm_xchg_rmid(rotor
, INVALID_RMID
);
653 list_rotate_left(&cache_groups
);
657 * Deallocate the RMIDs from any events that conflict with @event, and
658 * place them on the back of the group list.
660 static void intel_cqm_sched_out_conflicting_events(struct perf_event
*event
)
662 struct perf_event
*group
, *g
;
665 lockdep_assert_held(&cache_mutex
);
667 list_for_each_entry_safe(group
, g
, &cache_groups
, hw
.cqm_groups_entry
) {
671 rmid
= group
->hw
.cqm_rmid
;
674 * Skip events that don't have a valid RMID.
676 if (!__rmid_valid(rmid
))
680 * No conflict? No problem! Leave the event alone.
682 if (!__conflict_event(group
, event
))
685 intel_cqm_xchg_rmid(group
, INVALID_RMID
);
691 * Attempt to rotate the groups and assign new RMIDs.
693 * We rotate for two reasons,
694 * 1. To handle the scheduling of conflicting events
695 * 2. To recycle RMIDs
697 * Rotating RMIDs is complicated because the hardware doesn't give us
700 * There's problems with the hardware interface; when you change the
701 * task:RMID map cachelines retain their 'old' tags, giving a skewed
702 * picture. In order to work around this, we must always keep one free
703 * RMID - intel_cqm_rotation_rmid.
705 * Rotation works by taking away an RMID from a group (the old RMID),
706 * and assigning the free RMID to another group (the new RMID). We must
707 * then wait for the old RMID to not be used (no cachelines tagged).
708 * This ensure that all cachelines are tagged with 'active' RMIDs. At
709 * this point we can start reading values for the new RMID and treat the
710 * old RMID as the free RMID for the next rotation.
712 * Return %true or %false depending on whether we did any rotating.
714 static bool __intel_cqm_rmid_rotate(void)
716 struct perf_event
*group
, *start
= NULL
;
717 unsigned int threshold_limit
;
718 unsigned int nr_needed
= 0;
719 unsigned int nr_available
;
720 bool rotated
= false;
722 mutex_lock(&cache_mutex
);
726 * Fast path through this function if there are no groups and no
727 * RMIDs that need cleaning.
729 if (list_empty(&cache_groups
) && list_empty(&cqm_rmid_limbo_lru
))
732 list_for_each_entry(group
, &cache_groups
, hw
.cqm_groups_entry
) {
733 if (!__rmid_valid(group
->hw
.cqm_rmid
)) {
741 * We have some event groups, but they all have RMIDs assigned
742 * and no RMIDs need cleaning.
744 if (!nr_needed
&& list_empty(&cqm_rmid_limbo_lru
))
751 * We have more event groups without RMIDs than available RMIDs,
752 * or we have event groups that conflict with the ones currently
755 * We force deallocate the rmid of the group at the head of
756 * cache_groups. The first event group without an RMID then gets
757 * assigned intel_cqm_rotation_rmid. This ensures we always make
760 * Rotate the cache_groups list so the previous head is now the
763 __intel_cqm_pick_and_rotate(start
);
766 * If the rotation is going to succeed, reduce the threshold so
767 * that we don't needlessly reuse dirty RMIDs.
769 if (__rmid_valid(intel_cqm_rotation_rmid
)) {
770 intel_cqm_xchg_rmid(start
, intel_cqm_rotation_rmid
);
771 intel_cqm_rotation_rmid
= __get_rmid();
773 intel_cqm_sched_out_conflicting_events(start
);
775 if (__intel_cqm_threshold
)
776 __intel_cqm_threshold
--;
783 * We now need to stablize the RMID we freed above (if any) to
784 * ensure that the next time we rotate we have an RMID with zero
787 * Alternatively, if we didn't need to perform any rotation,
788 * we'll have a bunch of RMIDs in limbo that need stabilizing.
790 threshold_limit
= __intel_cqm_max_threshold
/ cqm_l3_scale
;
792 while (intel_cqm_rmid_stabilize(&nr_available
) &&
793 __intel_cqm_threshold
< threshold_limit
) {
794 unsigned int steal_limit
;
797 * Don't spin if nobody is actively waiting for an RMID,
798 * the rotation worker will be kicked as soon as an
799 * event needs an RMID anyway.
804 /* Allow max 25% of RMIDs to be in limbo. */
805 steal_limit
= (cqm_max_rmid
+ 1) / 4;
808 * We failed to stabilize any RMIDs so our rotation
809 * logic is now stuck. In order to make forward progress
810 * we have a few options:
812 * 1. rotate ("steal") another RMID
813 * 2. increase the threshold
816 * We do both of 1. and 2. until we hit the steal limit.
818 * The steal limit prevents all RMIDs ending up on the
819 * limbo list. This can happen if every RMID has a
820 * non-zero occupancy above threshold_limit, and the
821 * occupancy values aren't dropping fast enough.
823 * Note that there is prioritisation at work here - we'd
824 * rather increase the number of RMIDs on the limbo list
825 * than increase the threshold, because increasing the
826 * threshold skews the event data (because we reuse
827 * dirty RMIDs) - threshold bumps are a last resort.
829 if (nr_available
< steal_limit
)
832 __intel_cqm_threshold
++;
836 mutex_unlock(&cache_mutex
);
840 static void intel_cqm_rmid_rotate(struct work_struct
*work
);
842 static DECLARE_DELAYED_WORK(intel_cqm_rmid_work
, intel_cqm_rmid_rotate
);
844 static struct pmu intel_cqm_pmu
;
846 static void intel_cqm_rmid_rotate(struct work_struct
*work
)
850 __intel_cqm_rmid_rotate();
852 delay
= msecs_to_jiffies(intel_cqm_pmu
.hrtimer_interval_ms
);
853 schedule_delayed_work(&intel_cqm_rmid_work
, delay
);
857 * Find a group and setup RMID.
859 * If we're part of a group, we use the group's RMID.
861 static void intel_cqm_setup_event(struct perf_event
*event
,
862 struct perf_event
**group
)
864 struct perf_event
*iter
;
865 bool conflict
= false;
868 event
->hw
.is_group_event
= false;
869 list_for_each_entry(iter
, &cache_groups
, hw
.cqm_groups_entry
) {
870 rmid
= iter
->hw
.cqm_rmid
;
872 if (__match_event(iter
, event
)) {
873 /* All tasks in a group share an RMID */
874 event
->hw
.cqm_rmid
= rmid
;
880 * We only care about conflicts for events that are
881 * actually scheduled in (and hence have a valid RMID).
883 if (__conflict_event(iter
, event
) && __rmid_valid(rmid
))
892 event
->hw
.cqm_rmid
= rmid
;
895 static void intel_cqm_event_read(struct perf_event
*event
)
902 * Task events are handled by intel_cqm_event_count().
904 if (event
->cpu
== -1)
907 raw_spin_lock_irqsave(&cache_lock
, flags
);
908 rmid
= event
->hw
.cqm_rmid
;
910 if (!__rmid_valid(rmid
))
913 val
= __rmid_read(rmid
);
916 * Ignore this reading on error states and do not update the value.
918 if (val
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
))
921 local64_set(&event
->count
, val
);
923 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
926 static void __intel_cqm_event_count(void *info
)
928 struct rmid_read
*rr
= info
;
931 val
= __rmid_read(rr
->rmid
);
933 if (val
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
))
936 atomic64_add(val
, &rr
->value
);
939 static inline bool cqm_group_leader(struct perf_event
*event
)
941 return !list_empty(&event
->hw
.cqm_groups_entry
);
944 static u64
intel_cqm_event_count(struct perf_event
*event
)
947 struct rmid_read rr
= {
948 .value
= ATOMIC64_INIT(0),
952 * We only need to worry about task events. System-wide events
953 * are handled like usual, i.e. entirely with
954 * intel_cqm_event_read().
956 if (event
->cpu
!= -1)
957 return __perf_event_count(event
);
960 * Only the group leader gets to report values except in case of
961 * multiple events in the same group, we still need to read the
962 * other events.This stops us
963 * reporting duplicate values to userspace, and gives us a clear
964 * rule for which task gets to report the values.
966 * Note that it is impossible to attribute these values to
967 * specific packages - we forfeit that ability when we create
970 if (!cqm_group_leader(event
) && !event
->hw
.is_group_event
)
974 * Getting up-to-date values requires an SMP IPI which is not
975 * possible if we're being called in interrupt context. Return
976 * the cached values instead.
978 if (unlikely(in_interrupt()))
982 * Notice that we don't perform the reading of an RMID
983 * atomically, because we can't hold a spin lock across the
986 * Speculatively perform the read, since @event might be
987 * assigned a different (possibly invalid) RMID while we're
988 * busying performing the IPI calls. It's therefore necessary to
989 * check @event's RMID afterwards, and if it has changed,
990 * discard the result of the read.
992 rr
.rmid
= ACCESS_ONCE(event
->hw
.cqm_rmid
);
994 if (!__rmid_valid(rr
.rmid
))
997 on_each_cpu_mask(&cqm_cpumask
, __intel_cqm_event_count
, &rr
, 1);
999 raw_spin_lock_irqsave(&cache_lock
, flags
);
1000 if (event
->hw
.cqm_rmid
== rr
.rmid
)
1001 local64_set(&event
->count
, atomic64_read(&rr
.value
));
1002 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
1004 return __perf_event_count(event
);
1007 static void intel_cqm_event_start(struct perf_event
*event
, int mode
)
1009 struct intel_pqr_state
*state
= this_cpu_ptr(&pqr_state
);
1010 u32 rmid
= event
->hw
.cqm_rmid
;
1012 if (!(event
->hw
.cqm_state
& PERF_HES_STOPPED
))
1015 event
->hw
.cqm_state
&= ~PERF_HES_STOPPED
;
1017 if (state
->rmid_usecnt
++) {
1018 if (!WARN_ON_ONCE(state
->rmid
!= rmid
))
1021 WARN_ON_ONCE(state
->rmid
);
1025 wrmsr(MSR_IA32_PQR_ASSOC
, rmid
, state
->closid
);
1028 static void intel_cqm_event_stop(struct perf_event
*event
, int mode
)
1030 struct intel_pqr_state
*state
= this_cpu_ptr(&pqr_state
);
1032 if (event
->hw
.cqm_state
& PERF_HES_STOPPED
)
1035 event
->hw
.cqm_state
|= PERF_HES_STOPPED
;
1037 intel_cqm_event_read(event
);
1039 if (!--state
->rmid_usecnt
) {
1041 wrmsr(MSR_IA32_PQR_ASSOC
, 0, state
->closid
);
1043 WARN_ON_ONCE(!state
->rmid
);
1047 static int intel_cqm_event_add(struct perf_event
*event
, int mode
)
1049 unsigned long flags
;
1052 raw_spin_lock_irqsave(&cache_lock
, flags
);
1054 event
->hw
.cqm_state
= PERF_HES_STOPPED
;
1055 rmid
= event
->hw
.cqm_rmid
;
1057 if (__rmid_valid(rmid
) && (mode
& PERF_EF_START
))
1058 intel_cqm_event_start(event
, mode
);
1060 raw_spin_unlock_irqrestore(&cache_lock
, flags
);
1065 static void intel_cqm_event_destroy(struct perf_event
*event
)
1067 struct perf_event
*group_other
= NULL
;
1069 mutex_lock(&cache_mutex
);
1072 * If there's another event in this group...
1074 if (!list_empty(&event
->hw
.cqm_group_entry
)) {
1075 group_other
= list_first_entry(&event
->hw
.cqm_group_entry
,
1077 hw
.cqm_group_entry
);
1078 list_del(&event
->hw
.cqm_group_entry
);
1082 * And we're the group leader..
1084 if (cqm_group_leader(event
)) {
1086 * If there was a group_other, make that leader, otherwise
1087 * destroy the group and return the RMID.
1090 list_replace(&event
->hw
.cqm_groups_entry
,
1091 &group_other
->hw
.cqm_groups_entry
);
1093 u32 rmid
= event
->hw
.cqm_rmid
;
1095 if (__rmid_valid(rmid
))
1097 list_del(&event
->hw
.cqm_groups_entry
);
1101 mutex_unlock(&cache_mutex
);
1104 static int intel_cqm_event_init(struct perf_event
*event
)
1106 struct perf_event
*group
= NULL
;
1107 bool rotate
= false;
1109 if (event
->attr
.type
!= intel_cqm_pmu
.type
)
1112 if (event
->attr
.config
& ~QOS_EVENT_MASK
)
1115 /* unsupported modes and filters */
1116 if (event
->attr
.exclude_user
||
1117 event
->attr
.exclude_kernel
||
1118 event
->attr
.exclude_hv
||
1119 event
->attr
.exclude_idle
||
1120 event
->attr
.exclude_host
||
1121 event
->attr
.exclude_guest
||
1122 event
->attr
.sample_period
) /* no sampling */
1125 INIT_LIST_HEAD(&event
->hw
.cqm_group_entry
);
1126 INIT_LIST_HEAD(&event
->hw
.cqm_groups_entry
);
1128 event
->destroy
= intel_cqm_event_destroy
;
1130 mutex_lock(&cache_mutex
);
1132 /* Will also set rmid */
1133 intel_cqm_setup_event(event
, &group
);
1136 list_add_tail(&event
->hw
.cqm_group_entry
,
1137 &group
->hw
.cqm_group_entry
);
1139 list_add_tail(&event
->hw
.cqm_groups_entry
,
1143 * All RMIDs are either in use or have recently been
1144 * used. Kick the rotation worker to clean/free some.
1146 * We only do this for the group leader, rather than for
1147 * every event in a group to save on needless work.
1149 if (!__rmid_valid(event
->hw
.cqm_rmid
))
1153 mutex_unlock(&cache_mutex
);
1156 schedule_delayed_work(&intel_cqm_rmid_work
, 0);
1161 EVENT_ATTR_STR(llc_occupancy
, intel_cqm_llc
, "event=0x01");
1162 EVENT_ATTR_STR(llc_occupancy
.per
-pkg
, intel_cqm_llc_pkg
, "1");
1163 EVENT_ATTR_STR(llc_occupancy
.unit
, intel_cqm_llc_unit
, "Bytes");
1164 EVENT_ATTR_STR(llc_occupancy
.scale
, intel_cqm_llc_scale
, NULL
);
1165 EVENT_ATTR_STR(llc_occupancy
.snapshot
, intel_cqm_llc_snapshot
, "1");
1167 static struct attribute
*intel_cqm_events_attr
[] = {
1168 EVENT_PTR(intel_cqm_llc
),
1169 EVENT_PTR(intel_cqm_llc_pkg
),
1170 EVENT_PTR(intel_cqm_llc_unit
),
1171 EVENT_PTR(intel_cqm_llc_scale
),
1172 EVENT_PTR(intel_cqm_llc_snapshot
),
1176 static struct attribute_group intel_cqm_events_group
= {
1178 .attrs
= intel_cqm_events_attr
,
1181 PMU_FORMAT_ATTR(event
, "config:0-7");
1182 static struct attribute
*intel_cqm_formats_attr
[] = {
1183 &format_attr_event
.attr
,
1187 static struct attribute_group intel_cqm_format_group
= {
1189 .attrs
= intel_cqm_formats_attr
,
1193 max_recycle_threshold_show(struct device
*dev
, struct device_attribute
*attr
,
1198 mutex_lock(&cache_mutex
);
1199 rv
= snprintf(page
, PAGE_SIZE
-1, "%u\n", __intel_cqm_max_threshold
);
1200 mutex_unlock(&cache_mutex
);
1206 max_recycle_threshold_store(struct device
*dev
,
1207 struct device_attribute
*attr
,
1208 const char *buf
, size_t count
)
1210 unsigned int bytes
, cachelines
;
1213 ret
= kstrtouint(buf
, 0, &bytes
);
1217 mutex_lock(&cache_mutex
);
1219 __intel_cqm_max_threshold
= bytes
;
1220 cachelines
= bytes
/ cqm_l3_scale
;
1223 * The new maximum takes effect immediately.
1225 if (__intel_cqm_threshold
> cachelines
)
1226 __intel_cqm_threshold
= cachelines
;
1228 mutex_unlock(&cache_mutex
);
1233 static DEVICE_ATTR_RW(max_recycle_threshold
);
1235 static struct attribute
*intel_cqm_attrs
[] = {
1236 &dev_attr_max_recycle_threshold
.attr
,
1240 static const struct attribute_group intel_cqm_group
= {
1241 .attrs
= intel_cqm_attrs
,
1244 static const struct attribute_group
*intel_cqm_attr_groups
[] = {
1245 &intel_cqm_events_group
,
1246 &intel_cqm_format_group
,
1251 static struct pmu intel_cqm_pmu
= {
1252 .hrtimer_interval_ms
= RMID_DEFAULT_QUEUE_TIME
,
1253 .attr_groups
= intel_cqm_attr_groups
,
1254 .task_ctx_nr
= perf_sw_context
,
1255 .event_init
= intel_cqm_event_init
,
1256 .add
= intel_cqm_event_add
,
1257 .del
= intel_cqm_event_stop
,
1258 .start
= intel_cqm_event_start
,
1259 .stop
= intel_cqm_event_stop
,
1260 .read
= intel_cqm_event_read
,
1261 .count
= intel_cqm_event_count
,
1264 static inline void cqm_pick_event_reader(int cpu
)
1268 /* First online cpu in package becomes the reader */
1269 reader
= cpumask_any_and(&cqm_cpumask
, topology_core_cpumask(cpu
));
1270 if (reader
>= nr_cpu_ids
)
1271 cpumask_set_cpu(cpu
, &cqm_cpumask
);
1274 static void intel_cqm_cpu_starting(unsigned int cpu
)
1276 struct intel_pqr_state
*state
= &per_cpu(pqr_state
, cpu
);
1277 struct cpuinfo_x86
*c
= &cpu_data(cpu
);
1281 state
->rmid_usecnt
= 0;
1283 WARN_ON(c
->x86_cache_max_rmid
!= cqm_max_rmid
);
1284 WARN_ON(c
->x86_cache_occ_scale
!= cqm_l3_scale
);
1287 static void intel_cqm_cpu_exit(unsigned int cpu
)
1291 /* Is @cpu the current cqm reader for this package ? */
1292 if (!cpumask_test_and_clear_cpu(cpu
, &cqm_cpumask
))
1295 /* Find another online reader in this package */
1296 target
= cpumask_any_but(topology_core_cpumask(cpu
), cpu
);
1298 if (target
< nr_cpu_ids
)
1299 cpumask_set_cpu(target
, &cqm_cpumask
);
1302 static int intel_cqm_cpu_notifier(struct notifier_block
*nb
,
1303 unsigned long action
, void *hcpu
)
1305 unsigned int cpu
= (unsigned long)hcpu
;
1307 switch (action
& ~CPU_TASKS_FROZEN
) {
1308 case CPU_DOWN_PREPARE
:
1309 intel_cqm_cpu_exit(cpu
);
1312 intel_cqm_cpu_starting(cpu
);
1313 cqm_pick_event_reader(cpu
);
1320 static const struct x86_cpu_id intel_cqm_match
[] = {
1321 { .vendor
= X86_VENDOR_INTEL
, .feature
= X86_FEATURE_CQM_OCCUP_LLC
},
1325 static int __init
intel_cqm_init(void)
1327 char *str
= NULL
, scale
[20];
1330 if (!x86_match_cpu(intel_cqm_match
))
1333 cqm_l3_scale
= boot_cpu_data
.x86_cache_occ_scale
;
1336 * It's possible that not all resources support the same number
1337 * of RMIDs. Instead of making scheduling much more complicated
1338 * (where we have to match a task's RMID to a cpu that supports
1339 * that many RMIDs) just find the minimum RMIDs supported across
1342 * Also, check that the scales match on all cpus.
1344 cpu_notifier_register_begin();
1346 for_each_online_cpu(cpu
) {
1347 struct cpuinfo_x86
*c
= &cpu_data(cpu
);
1349 if (c
->x86_cache_max_rmid
< cqm_max_rmid
)
1350 cqm_max_rmid
= c
->x86_cache_max_rmid
;
1352 if (c
->x86_cache_occ_scale
!= cqm_l3_scale
) {
1353 pr_err("Multiple LLC scale values, disabling\n");
1360 * A reasonable upper limit on the max threshold is the number
1361 * of lines tagged per RMID if all RMIDs have the same number of
1362 * lines tagged in the LLC.
1364 * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC.
1366 __intel_cqm_max_threshold
=
1367 boot_cpu_data
.x86_cache_size
* 1024 / (cqm_max_rmid
+ 1);
1369 snprintf(scale
, sizeof(scale
), "%u", cqm_l3_scale
);
1370 str
= kstrdup(scale
, GFP_KERNEL
);
1376 event_attr_intel_cqm_llc_scale
.event_str
= str
;
1378 ret
= intel_cqm_setup_rmid_cache();
1382 for_each_online_cpu(i
) {
1383 intel_cqm_cpu_starting(i
);
1384 cqm_pick_event_reader(i
);
1387 ret
= perf_pmu_register(&intel_cqm_pmu
, "intel_cqm", -1);
1389 pr_err("Intel CQM perf registration failed: %d\n", ret
);
1393 pr_info("Intel CQM monitoring enabled\n");
1396 * Register the hot cpu notifier once we are sure cqm
1397 * is enabled to avoid notifier leak.
1399 __perf_cpu_notifier(intel_cqm_cpu_notifier
);
1401 cpu_notifier_register_done();
1409 device_initcall(intel_cqm_init
);