1 // SPDX-License-Identifier: GPL-2.0
3 * Performance events ring-buffer code:
5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
11 #include <linux/perf_event.h>
12 #include <linux/vmalloc.h>
13 #include <linux/slab.h>
14 #include <linux/circ_buf.h>
15 #include <linux/poll.h>
16 #include <linux/nospec.h>
20 static void perf_output_wakeup(struct perf_output_handle
*handle
)
22 atomic_set(&handle
->rb
->poll
, EPOLLIN
);
24 handle
->event
->pending_wakeup
= 1;
25 irq_work_queue(&handle
->event
->pending
);
29 * We need to ensure a later event_id doesn't publish a head when a former
30 * event isn't done writing. However since we need to deal with NMIs we
31 * cannot fully serialize things.
33 * We only publish the head (and generate a wakeup) when the outer-most
36 static void perf_output_get_handle(struct perf_output_handle
*handle
)
38 struct ring_buffer
*rb
= handle
->rb
;
42 handle
->wakeup
= local_read(&rb
->wakeup
);
45 static void perf_output_put_handle(struct perf_output_handle
*handle
)
47 struct ring_buffer
*rb
= handle
->rb
;
51 head
= local_read(&rb
->head
);
54 * IRQ/NMI can happen here, which means we can miss a head update.
57 if (!local_dec_and_test(&rb
->nest
))
61 * Since the mmap() consumer (userspace) can run on a different CPU:
65 * if (LOAD ->data_tail) { LOAD ->data_head
67 * STORE $data LOAD $data
68 * smp_wmb() (B) smp_mb() (D)
69 * STORE ->data_head STORE ->data_tail
72 * Where A pairs with D, and B pairs with C.
74 * In our case (A) is a control dependency that separates the load of
75 * the ->data_tail and the stores of $data. In case ->data_tail
76 * indicates there is no room in the buffer to store $data we do not.
78 * D needs to be a full barrier since it separates the data READ
79 * from the tail WRITE.
81 * For B a WMB is sufficient since it separates two WRITEs, and for C
82 * an RMB is sufficient since it separates two READs.
84 * See perf_output_begin().
86 smp_wmb(); /* B, matches C */
87 rb
->user_page
->data_head
= head
;
90 * Now check if we missed an update -- rely on previous implied
91 * compiler barriers to force a re-read.
93 if (unlikely(head
!= local_read(&rb
->head
))) {
98 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
99 perf_output_wakeup(handle
);
105 static __always_inline
bool
106 ring_buffer_has_space(unsigned long head
, unsigned long tail
,
107 unsigned long data_size
, unsigned int size
,
111 return CIRC_SPACE(head
, tail
, data_size
) >= size
;
113 return CIRC_SPACE(tail
, head
, data_size
) >= size
;
116 static __always_inline
int
117 __perf_output_begin(struct perf_output_handle
*handle
,
118 struct perf_event
*event
, unsigned int size
,
121 struct ring_buffer
*rb
;
122 unsigned long tail
, offset
, head
;
123 int have_lost
, page_shift
;
125 struct perf_event_header header
;
132 * For inherited events we send all the output towards the parent.
135 event
= event
->parent
;
137 rb
= rcu_dereference(event
->rb
);
141 if (unlikely(rb
->paused
)) {
143 local_inc(&rb
->lost
);
148 handle
->event
= event
;
150 have_lost
= local_read(&rb
->lost
);
151 if (unlikely(have_lost
)) {
152 size
+= sizeof(lost_event
);
153 if (event
->attr
.sample_id_all
)
154 size
+= event
->id_header_size
;
157 perf_output_get_handle(handle
);
160 tail
= READ_ONCE(rb
->user_page
->data_tail
);
161 offset
= head
= local_read(&rb
->head
);
162 if (!rb
->overwrite
) {
163 if (unlikely(!ring_buffer_has_space(head
, tail
,
170 * The above forms a control dependency barrier separating the
171 * @tail load above from the data stores below. Since the @tail
172 * load is required to compute the branch to fail below.
174 * A, matches D; the full memory barrier userspace SHOULD issue
175 * after reading the data and before storing the new tail
178 * See perf_output_put_handle().
185 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
193 * We rely on the implied barrier() by local_cmpxchg() to ensure
194 * none of the data stores below can be lifted up by the compiler.
197 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
198 local_add(rb
->watermark
, &rb
->wakeup
);
200 page_shift
= PAGE_SHIFT
+ page_order(rb
);
202 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
203 offset
&= (1UL << page_shift
) - 1;
204 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
205 handle
->size
= (1UL << page_shift
) - offset
;
207 if (unlikely(have_lost
)) {
208 struct perf_sample_data sample_data
;
210 lost_event
.header
.size
= sizeof(lost_event
);
211 lost_event
.header
.type
= PERF_RECORD_LOST
;
212 lost_event
.header
.misc
= 0;
213 lost_event
.id
= event
->id
;
214 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
216 perf_event_header__init_id(&lost_event
.header
,
217 &sample_data
, event
);
218 perf_output_put(handle
, lost_event
);
219 perf_event__output_id_sample(event
, handle
, &sample_data
);
225 local_inc(&rb
->lost
);
226 perf_output_put_handle(handle
);
233 int perf_output_begin_forward(struct perf_output_handle
*handle
,
234 struct perf_event
*event
, unsigned int size
)
236 return __perf_output_begin(handle
, event
, size
, false);
239 int perf_output_begin_backward(struct perf_output_handle
*handle
,
240 struct perf_event
*event
, unsigned int size
)
242 return __perf_output_begin(handle
, event
, size
, true);
245 int perf_output_begin(struct perf_output_handle
*handle
,
246 struct perf_event
*event
, unsigned int size
)
249 return __perf_output_begin(handle
, event
, size
,
250 unlikely(is_write_backward(event
)));
253 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
254 const void *buf
, unsigned int len
)
256 return __output_copy(handle
, buf
, len
);
259 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
262 return __output_skip(handle
, NULL
, len
);
265 void perf_output_end(struct perf_output_handle
*handle
)
267 perf_output_put_handle(handle
);
272 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
274 long max_size
= perf_data_size(rb
);
277 rb
->watermark
= min(max_size
, watermark
);
280 rb
->watermark
= max_size
/ 2;
282 if (flags
& RING_BUFFER_WRITABLE
)
287 refcount_set(&rb
->refcount
, 1);
289 INIT_LIST_HEAD(&rb
->event_list
);
290 spin_lock_init(&rb
->event_lock
);
293 * perf_output_begin() only checks rb->paused, therefore
294 * rb->paused must be true if we have no pages for output.
300 void perf_aux_output_flag(struct perf_output_handle
*handle
, u64 flags
)
303 * OVERWRITE is determined by perf_aux_output_end() and can't
304 * be passed in directly.
306 if (WARN_ON_ONCE(flags
& PERF_AUX_FLAG_OVERWRITE
))
309 handle
->aux_flags
|= flags
;
311 EXPORT_SYMBOL_GPL(perf_aux_output_flag
);
314 * This is called before hardware starts writing to the AUX area to
315 * obtain an output handle and make sure there's room in the buffer.
316 * When the capture completes, call perf_aux_output_end() to commit
317 * the recorded data to the buffer.
319 * The ordering is similar to that of perf_output_{begin,end}, with
320 * the exception of (B), which should be taken care of by the pmu
321 * driver, since ordering rules will differ depending on hardware.
323 * Call this from pmu::start(); see the comment in perf_aux_output_end()
324 * about its use in pmu callbacks. Both can also be called from the PMI
327 void *perf_aux_output_begin(struct perf_output_handle
*handle
,
328 struct perf_event
*event
)
330 struct perf_event
*output_event
= event
;
331 unsigned long aux_head
, aux_tail
;
332 struct ring_buffer
*rb
;
334 if (output_event
->parent
)
335 output_event
= output_event
->parent
;
338 * Since this will typically be open across pmu::add/pmu::del, we
339 * grab ring_buffer's refcount instead of holding rcu read lock
340 * to make sure it doesn't disappear under us.
342 rb
= ring_buffer_get(output_event
);
350 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
351 * about to get freed, so we leave immediately.
353 * Checking rb::aux_mmap_count and rb::refcount has to be done in
354 * the same order, see perf_mmap_close. Otherwise we end up freeing
355 * aux pages in this path, which is a bug, because in_atomic().
357 if (!atomic_read(&rb
->aux_mmap_count
))
360 if (!refcount_inc_not_zero(&rb
->aux_refcount
))
364 * Nesting is not supported for AUX area, make sure nested
365 * writers are caught early
367 if (WARN_ON_ONCE(local_xchg(&rb
->aux_nest
, 1)))
370 aux_head
= rb
->aux_head
;
373 handle
->event
= event
;
374 handle
->head
= aux_head
;
376 handle
->aux_flags
= 0;
379 * In overwrite mode, AUX data stores do not depend on aux_tail,
380 * therefore (A) control dependency barrier does not exist. The
381 * (B) <-> (C) ordering is still observed by the pmu driver.
383 if (!rb
->aux_overwrite
) {
384 aux_tail
= READ_ONCE(rb
->user_page
->aux_tail
);
385 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
386 if (aux_head
- aux_tail
< perf_aux_size(rb
))
387 handle
->size
= CIRC_SPACE(aux_head
, aux_tail
, perf_aux_size(rb
));
390 * handle->size computation depends on aux_tail load; this forms a
391 * control dependency barrier separating aux_tail load from aux data
392 * store that will be enabled on successful return
394 if (!handle
->size
) { /* A, matches D */
395 event
->pending_disable
= 1;
396 perf_output_wakeup(handle
);
397 local_set(&rb
->aux_nest
, 0);
402 return handle
->rb
->aux_priv
;
410 handle
->event
= NULL
;
414 EXPORT_SYMBOL_GPL(perf_aux_output_begin
);
416 static __always_inline
bool rb_need_aux_wakeup(struct ring_buffer
*rb
)
418 if (rb
->aux_overwrite
)
421 if (rb
->aux_head
- rb
->aux_wakeup
>= rb
->aux_watermark
) {
422 rb
->aux_wakeup
= rounddown(rb
->aux_head
, rb
->aux_watermark
);
430 * Commit the data written by hardware into the ring buffer by adjusting
431 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
432 * pmu driver's responsibility to observe ordering rules of the hardware,
433 * so that all the data is externally visible before this is called.
435 * Note: this has to be called from pmu::stop() callback, as the assumption
436 * of the AUX buffer management code is that after pmu::stop(), the AUX
437 * transaction must be stopped and therefore drop the AUX reference count.
439 void perf_aux_output_end(struct perf_output_handle
*handle
, unsigned long size
)
441 bool wakeup
= !!(handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
);
442 struct ring_buffer
*rb
= handle
->rb
;
443 unsigned long aux_head
;
445 /* in overwrite mode, driver provides aux_head via handle */
446 if (rb
->aux_overwrite
) {
447 handle
->aux_flags
|= PERF_AUX_FLAG_OVERWRITE
;
449 aux_head
= handle
->head
;
450 rb
->aux_head
= aux_head
;
452 handle
->aux_flags
&= ~PERF_AUX_FLAG_OVERWRITE
;
454 aux_head
= rb
->aux_head
;
455 rb
->aux_head
+= size
;
458 if (size
|| handle
->aux_flags
) {
460 * Only send RECORD_AUX if we have something useful to communicate
462 * Note: the OVERWRITE records by themselves are not considered
463 * useful, as they don't communicate any *new* information,
464 * aside from the short-lived offset, that becomes history at
465 * the next event sched-in and therefore isn't useful.
466 * The userspace that needs to copy out AUX data in overwrite
467 * mode should know to use user_page::aux_head for the actual
468 * offset. So, from now on we don't output AUX records that
469 * have *only* OVERWRITE flag set.
472 if (handle
->aux_flags
& ~(u64
)PERF_AUX_FLAG_OVERWRITE
)
473 perf_event_aux_event(handle
->event
, aux_head
, size
,
477 rb
->user_page
->aux_head
= rb
->aux_head
;
478 if (rb_need_aux_wakeup(rb
))
482 if (handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
)
483 handle
->event
->pending_disable
= 1;
484 perf_output_wakeup(handle
);
487 handle
->event
= NULL
;
489 local_set(&rb
->aux_nest
, 0);
494 EXPORT_SYMBOL_GPL(perf_aux_output_end
);
497 * Skip over a given number of bytes in the AUX buffer, due to, for example,
498 * hardware's alignment constraints.
500 int perf_aux_output_skip(struct perf_output_handle
*handle
, unsigned long size
)
502 struct ring_buffer
*rb
= handle
->rb
;
504 if (size
> handle
->size
)
507 rb
->aux_head
+= size
;
509 rb
->user_page
->aux_head
= rb
->aux_head
;
510 if (rb_need_aux_wakeup(rb
)) {
511 perf_output_wakeup(handle
);
512 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
515 handle
->head
= rb
->aux_head
;
516 handle
->size
-= size
;
520 EXPORT_SYMBOL_GPL(perf_aux_output_skip
);
522 void *perf_get_aux(struct perf_output_handle
*handle
)
524 /* this is only valid between perf_aux_output_begin and *_end */
528 return handle
->rb
->aux_priv
;
530 EXPORT_SYMBOL_GPL(perf_get_aux
);
532 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
534 static struct page
*rb_alloc_aux_page(int node
, int order
)
538 if (order
> MAX_ORDER
)
542 page
= alloc_pages_node(node
, PERF_AUX_GFP
, order
);
543 } while (!page
&& order
--);
547 * Communicate the allocation size to the driver:
548 * if we managed to secure a high-order allocation,
549 * set its first page's private to this order;
550 * !PagePrivate(page) means it's just a normal page.
552 split_page(page
, order
);
553 SetPagePrivate(page
);
554 set_page_private(page
, order
);
560 static void rb_free_aux_page(struct ring_buffer
*rb
, int idx
)
562 struct page
*page
= virt_to_page(rb
->aux_pages
[idx
]);
564 ClearPagePrivate(page
);
565 page
->mapping
= NULL
;
569 static void __rb_free_aux(struct ring_buffer
*rb
)
574 * Should never happen, the last reference should be dropped from
575 * perf_mmap_close() path, which first stops aux transactions (which
576 * in turn are the atomic holders of aux_refcount) and then does the
577 * last rb_free_aux().
579 WARN_ON_ONCE(in_atomic());
582 rb
->free_aux(rb
->aux_priv
);
587 if (rb
->aux_nr_pages
) {
588 for (pg
= 0; pg
< rb
->aux_nr_pages
; pg
++)
589 rb_free_aux_page(rb
, pg
);
591 kfree(rb
->aux_pages
);
592 rb
->aux_nr_pages
= 0;
596 int rb_alloc_aux(struct ring_buffer
*rb
, struct perf_event
*event
,
597 pgoff_t pgoff
, int nr_pages
, long watermark
, int flags
)
599 bool overwrite
= !(flags
& RING_BUFFER_WRITABLE
);
600 int node
= (event
->cpu
== -1) ? -1 : cpu_to_node(event
->cpu
);
601 int ret
= -ENOMEM
, max_order
;
607 * We need to start with the max_order that fits in nr_pages,
608 * not the other way around, hence ilog2() and not get_order.
610 max_order
= ilog2(nr_pages
);
613 * PMU requests more than one contiguous chunks of memory
614 * for SW double buffering
616 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_SW_DOUBLEBUF
) &&
624 rb
->aux_pages
= kcalloc_node(nr_pages
, sizeof(void *), GFP_KERNEL
,
629 rb
->free_aux
= event
->pmu
->free_aux
;
630 for (rb
->aux_nr_pages
= 0; rb
->aux_nr_pages
< nr_pages
;) {
634 order
= min(max_order
, ilog2(nr_pages
- rb
->aux_nr_pages
));
635 page
= rb_alloc_aux_page(node
, order
);
639 for (last
= rb
->aux_nr_pages
+ (1 << page_private(page
));
640 last
> rb
->aux_nr_pages
; rb
->aux_nr_pages
++)
641 rb
->aux_pages
[rb
->aux_nr_pages
] = page_address(page
++);
645 * In overwrite mode, PMUs that don't support SG may not handle more
646 * than one contiguous allocation, since they rely on PMI to do double
647 * buffering. In this case, the entire buffer has to be one contiguous
650 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) &&
652 struct page
*page
= virt_to_page(rb
->aux_pages
[0]);
654 if (page_private(page
) != max_order
)
658 rb
->aux_priv
= event
->pmu
->setup_aux(event
, rb
->aux_pages
, nr_pages
,
666 * aux_pages (and pmu driver's private data, aux_priv) will be
667 * referenced in both producer's and consumer's contexts, thus
668 * we keep a refcount here to make sure either of the two can
669 * reference them safely.
671 refcount_set(&rb
->aux_refcount
, 1);
673 rb
->aux_overwrite
= overwrite
;
674 rb
->aux_watermark
= watermark
;
676 if (!rb
->aux_watermark
&& !rb
->aux_overwrite
)
677 rb
->aux_watermark
= nr_pages
<< (PAGE_SHIFT
- 1);
681 rb
->aux_pgoff
= pgoff
;
688 void rb_free_aux(struct ring_buffer
*rb
)
690 if (refcount_dec_and_test(&rb
->aux_refcount
))
694 #ifndef CONFIG_PERF_USE_VMALLOC
697 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
701 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
703 if (pgoff
> rb
->nr_pages
)
707 return virt_to_page(rb
->user_page
);
709 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
712 static void *perf_mmap_alloc_page(int cpu
)
717 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
718 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
722 return page_address(page
);
725 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
727 struct ring_buffer
*rb
;
731 size
= sizeof(struct ring_buffer
);
732 size
+= nr_pages
* sizeof(void *);
734 if (order_base_2(size
) >= PAGE_SHIFT
+MAX_ORDER
)
737 rb
= kzalloc(size
, GFP_KERNEL
);
741 rb
->user_page
= perf_mmap_alloc_page(cpu
);
745 for (i
= 0; i
< nr_pages
; i
++) {
746 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
747 if (!rb
->data_pages
[i
])
748 goto fail_data_pages
;
751 rb
->nr_pages
= nr_pages
;
753 ring_buffer_init(rb
, watermark
, flags
);
758 for (i
--; i
>= 0; i
--)
759 free_page((unsigned long)rb
->data_pages
[i
]);
761 free_page((unsigned long)rb
->user_page
);
770 static void perf_mmap_free_page(unsigned long addr
)
772 struct page
*page
= virt_to_page((void *)addr
);
774 page
->mapping
= NULL
;
778 void rb_free(struct ring_buffer
*rb
)
782 perf_mmap_free_page((unsigned long)rb
->user_page
);
783 for (i
= 0; i
< rb
->nr_pages
; i
++)
784 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
789 static int data_page_nr(struct ring_buffer
*rb
)
791 return rb
->nr_pages
<< page_order(rb
);
795 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
797 /* The '>' counts in the user page. */
798 if (pgoff
> data_page_nr(rb
))
801 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
804 static void perf_mmap_unmark_page(void *addr
)
806 struct page
*page
= vmalloc_to_page(addr
);
808 page
->mapping
= NULL
;
811 static void rb_free_work(struct work_struct
*work
)
813 struct ring_buffer
*rb
;
817 rb
= container_of(work
, struct ring_buffer
, work
);
818 nr
= data_page_nr(rb
);
820 base
= rb
->user_page
;
821 /* The '<=' counts in the user page. */
822 for (i
= 0; i
<= nr
; i
++)
823 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
829 void rb_free(struct ring_buffer
*rb
)
831 schedule_work(&rb
->work
);
834 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
836 struct ring_buffer
*rb
;
840 size
= sizeof(struct ring_buffer
);
841 size
+= sizeof(void *);
843 rb
= kzalloc(size
, GFP_KERNEL
);
847 INIT_WORK(&rb
->work
, rb_free_work
);
849 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
853 rb
->user_page
= all_buf
;
854 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
857 rb
->page_order
= ilog2(nr_pages
);
860 ring_buffer_init(rb
, watermark
, flags
);
874 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
876 if (rb
->aux_nr_pages
) {
877 /* above AUX space */
878 if (pgoff
> rb
->aux_pgoff
+ rb
->aux_nr_pages
)
882 if (pgoff
>= rb
->aux_pgoff
) {
883 int aux_pgoff
= array_index_nospec(pgoff
- rb
->aux_pgoff
, rb
->aux_nr_pages
);
884 return virt_to_page(rb
->aux_pages
[aux_pgoff
]);
888 return __perf_mmap_to_page(rb
, pgoff
);