2 * Performance events ring-buffer code:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/vmalloc.h>
14 #include <linux/slab.h>
15 #include <linux/circ_buf.h>
16 #include <linux/poll.h>
20 static void perf_output_wakeup(struct perf_output_handle
*handle
)
22 atomic_set(&handle
->rb
->poll
, POLLIN
);
24 handle
->event
->pending_wakeup
= 1;
25 irq_work_queue(&handle
->event
->pending
);
29 * We need to ensure a later event_id doesn't publish a head when a former
30 * event isn't done writing. However since we need to deal with NMIs we
31 * cannot fully serialize things.
33 * We only publish the head (and generate a wakeup) when the outer-most
36 static void perf_output_get_handle(struct perf_output_handle
*handle
)
38 struct ring_buffer
*rb
= handle
->rb
;
42 handle
->wakeup
= local_read(&rb
->wakeup
);
45 static void perf_output_put_handle(struct perf_output_handle
*handle
)
47 struct ring_buffer
*rb
= handle
->rb
;
51 head
= local_read(&rb
->head
);
54 * IRQ/NMI can happen here, which means we can miss a head update.
57 if (!local_dec_and_test(&rb
->nest
))
61 * Since the mmap() consumer (userspace) can run on a different CPU:
65 * if (LOAD ->data_tail) { LOAD ->data_head
67 * STORE $data LOAD $data
68 * smp_wmb() (B) smp_mb() (D)
69 * STORE ->data_head STORE ->data_tail
72 * Where A pairs with D, and B pairs with C.
74 * In our case (A) is a control dependency that separates the load of
75 * the ->data_tail and the stores of $data. In case ->data_tail
76 * indicates there is no room in the buffer to store $data we do not.
78 * D needs to be a full barrier since it separates the data READ
79 * from the tail WRITE.
81 * For B a WMB is sufficient since it separates two WRITEs, and for C
82 * an RMB is sufficient since it separates two READs.
84 * See perf_output_begin().
86 smp_wmb(); /* B, matches C */
87 rb
->user_page
->data_head
= head
;
90 * Now check if we missed an update -- rely on previous implied
91 * compiler barriers to force a re-read.
93 if (unlikely(head
!= local_read(&rb
->head
))) {
98 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
99 perf_output_wakeup(handle
);
105 static bool __always_inline
106 ring_buffer_has_space(unsigned long head
, unsigned long tail
,
107 unsigned long data_size
, unsigned int size
,
111 return CIRC_SPACE(head
, tail
, data_size
) >= size
;
113 return CIRC_SPACE(tail
, head
, data_size
) >= size
;
116 static int __always_inline
117 __perf_output_begin(struct perf_output_handle
*handle
,
118 struct perf_event
*event
, unsigned int size
,
121 struct ring_buffer
*rb
;
122 unsigned long tail
, offset
, head
;
123 int have_lost
, page_shift
;
125 struct perf_event_header header
;
132 * For inherited events we send all the output towards the parent.
135 event
= event
->parent
;
137 rb
= rcu_dereference(event
->rb
);
141 if (unlikely(rb
->paused
)) {
143 local_inc(&rb
->lost
);
148 handle
->event
= event
;
150 have_lost
= local_read(&rb
->lost
);
151 if (unlikely(have_lost
)) {
152 size
+= sizeof(lost_event
);
153 if (event
->attr
.sample_id_all
)
154 size
+= event
->id_header_size
;
157 perf_output_get_handle(handle
);
160 tail
= READ_ONCE(rb
->user_page
->data_tail
);
161 offset
= head
= local_read(&rb
->head
);
162 if (!rb
->overwrite
) {
163 if (unlikely(!ring_buffer_has_space(head
, tail
,
170 * The above forms a control dependency barrier separating the
171 * @tail load above from the data stores below. Since the @tail
172 * load is required to compute the branch to fail below.
174 * A, matches D; the full memory barrier userspace SHOULD issue
175 * after reading the data and before storing the new tail
178 * See perf_output_put_handle().
185 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
193 * We rely on the implied barrier() by local_cmpxchg() to ensure
194 * none of the data stores below can be lifted up by the compiler.
197 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
198 local_add(rb
->watermark
, &rb
->wakeup
);
200 page_shift
= PAGE_SHIFT
+ page_order(rb
);
202 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
203 offset
&= (1UL << page_shift
) - 1;
204 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
205 handle
->size
= (1UL << page_shift
) - offset
;
207 if (unlikely(have_lost
)) {
208 struct perf_sample_data sample_data
;
210 lost_event
.header
.size
= sizeof(lost_event
);
211 lost_event
.header
.type
= PERF_RECORD_LOST
;
212 lost_event
.header
.misc
= 0;
213 lost_event
.id
= event
->id
;
214 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
216 perf_event_header__init_id(&lost_event
.header
,
217 &sample_data
, event
);
218 perf_output_put(handle
, lost_event
);
219 perf_event__output_id_sample(event
, handle
, &sample_data
);
225 local_inc(&rb
->lost
);
226 perf_output_put_handle(handle
);
233 int perf_output_begin_forward(struct perf_output_handle
*handle
,
234 struct perf_event
*event
, unsigned int size
)
236 return __perf_output_begin(handle
, event
, size
, false);
239 int perf_output_begin_backward(struct perf_output_handle
*handle
,
240 struct perf_event
*event
, unsigned int size
)
242 return __perf_output_begin(handle
, event
, size
, true);
245 int perf_output_begin(struct perf_output_handle
*handle
,
246 struct perf_event
*event
, unsigned int size
)
249 return __perf_output_begin(handle
, event
, size
,
250 unlikely(is_write_backward(event
)));
253 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
254 const void *buf
, unsigned int len
)
256 return __output_copy(handle
, buf
, len
);
259 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
262 return __output_skip(handle
, NULL
, len
);
265 void perf_output_end(struct perf_output_handle
*handle
)
267 perf_output_put_handle(handle
);
272 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
274 long max_size
= perf_data_size(rb
);
277 rb
->watermark
= min(max_size
, watermark
);
280 rb
->watermark
= max_size
/ 2;
282 if (flags
& RING_BUFFER_WRITABLE
)
287 atomic_set(&rb
->refcount
, 1);
289 INIT_LIST_HEAD(&rb
->event_list
);
290 spin_lock_init(&rb
->event_lock
);
293 * perf_output_begin() only checks rb->paused, therefore
294 * rb->paused must be true if we have no pages for output.
300 void perf_aux_output_flag(struct perf_output_handle
*handle
, u64 flags
)
303 * OVERWRITE is determined by perf_aux_output_end() and can't
304 * be passed in directly.
306 if (WARN_ON_ONCE(flags
& PERF_AUX_FLAG_OVERWRITE
))
309 handle
->aux_flags
|= flags
;
311 EXPORT_SYMBOL_GPL(perf_aux_output_flag
);
314 * This is called before hardware starts writing to the AUX area to
315 * obtain an output handle and make sure there's room in the buffer.
316 * When the capture completes, call perf_aux_output_end() to commit
317 * the recorded data to the buffer.
319 * The ordering is similar to that of perf_output_{begin,end}, with
320 * the exception of (B), which should be taken care of by the pmu
321 * driver, since ordering rules will differ depending on hardware.
323 * Call this from pmu::start(); see the comment in perf_aux_output_end()
324 * about its use in pmu callbacks. Both can also be called from the PMI
327 void *perf_aux_output_begin(struct perf_output_handle
*handle
,
328 struct perf_event
*event
)
330 struct perf_event
*output_event
= event
;
331 unsigned long aux_head
, aux_tail
;
332 struct ring_buffer
*rb
;
334 if (output_event
->parent
)
335 output_event
= output_event
->parent
;
338 * Since this will typically be open across pmu::add/pmu::del, we
339 * grab ring_buffer's refcount instead of holding rcu read lock
340 * to make sure it doesn't disappear under us.
342 rb
= ring_buffer_get(output_event
);
350 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
351 * about to get freed, so we leave immediately.
353 * Checking rb::aux_mmap_count and rb::refcount has to be done in
354 * the same order, see perf_mmap_close. Otherwise we end up freeing
355 * aux pages in this path, which is a bug, because in_atomic().
357 if (!atomic_read(&rb
->aux_mmap_count
))
360 if (!atomic_inc_not_zero(&rb
->aux_refcount
))
364 * Nesting is not supported for AUX area, make sure nested
365 * writers are caught early
367 if (WARN_ON_ONCE(local_xchg(&rb
->aux_nest
, 1)))
370 aux_head
= rb
->aux_head
;
373 handle
->event
= event
;
374 handle
->head
= aux_head
;
376 handle
->aux_flags
= 0;
379 * In overwrite mode, AUX data stores do not depend on aux_tail,
380 * therefore (A) control dependency barrier does not exist. The
381 * (B) <-> (C) ordering is still observed by the pmu driver.
383 if (!rb
->aux_overwrite
) {
384 aux_tail
= ACCESS_ONCE(rb
->user_page
->aux_tail
);
385 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
386 if (aux_head
- aux_tail
< perf_aux_size(rb
))
387 handle
->size
= CIRC_SPACE(aux_head
, aux_tail
, perf_aux_size(rb
));
390 * handle->size computation depends on aux_tail load; this forms a
391 * control dependency barrier separating aux_tail load from aux data
392 * store that will be enabled on successful return
394 if (!handle
->size
) { /* A, matches D */
395 event
->pending_disable
= 1;
396 perf_output_wakeup(handle
);
397 local_set(&rb
->aux_nest
, 0);
402 return handle
->rb
->aux_priv
;
410 handle
->event
= NULL
;
415 static bool __always_inline
rb_need_aux_wakeup(struct ring_buffer
*rb
)
417 if (rb
->aux_overwrite
)
420 if (rb
->aux_head
- rb
->aux_wakeup
>= rb
->aux_watermark
) {
421 rb
->aux_wakeup
= rounddown(rb
->aux_head
, rb
->aux_watermark
);
429 * Commit the data written by hardware into the ring buffer by adjusting
430 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
431 * pmu driver's responsibility to observe ordering rules of the hardware,
432 * so that all the data is externally visible before this is called.
434 * Note: this has to be called from pmu::stop() callback, as the assumption
435 * of the AUX buffer management code is that after pmu::stop(), the AUX
436 * transaction must be stopped and therefore drop the AUX reference count.
438 void perf_aux_output_end(struct perf_output_handle
*handle
, unsigned long size
)
440 bool wakeup
= !!(handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
);
441 struct ring_buffer
*rb
= handle
->rb
;
442 unsigned long aux_head
;
444 /* in overwrite mode, driver provides aux_head via handle */
445 if (rb
->aux_overwrite
) {
446 handle
->aux_flags
|= PERF_AUX_FLAG_OVERWRITE
;
448 aux_head
= handle
->head
;
449 rb
->aux_head
= aux_head
;
451 handle
->aux_flags
&= ~PERF_AUX_FLAG_OVERWRITE
;
453 aux_head
= rb
->aux_head
;
454 rb
->aux_head
+= size
;
457 if (size
|| handle
->aux_flags
) {
459 * Only send RECORD_AUX if we have something useful to communicate
462 perf_event_aux_event(handle
->event
, aux_head
, size
,
466 rb
->user_page
->aux_head
= rb
->aux_head
;
467 if (rb_need_aux_wakeup(rb
))
471 if (handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
)
472 handle
->event
->pending_disable
= 1;
473 perf_output_wakeup(handle
);
476 handle
->event
= NULL
;
478 local_set(&rb
->aux_nest
, 0);
485 * Skip over a given number of bytes in the AUX buffer, due to, for example,
486 * hardware's alignment constraints.
488 int perf_aux_output_skip(struct perf_output_handle
*handle
, unsigned long size
)
490 struct ring_buffer
*rb
= handle
->rb
;
492 if (size
> handle
->size
)
495 rb
->aux_head
+= size
;
497 rb
->user_page
->aux_head
= rb
->aux_head
;
498 if (rb_need_aux_wakeup(rb
)) {
499 perf_output_wakeup(handle
);
500 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
503 handle
->head
= rb
->aux_head
;
504 handle
->size
-= size
;
509 void *perf_get_aux(struct perf_output_handle
*handle
)
511 /* this is only valid between perf_aux_output_begin and *_end */
515 return handle
->rb
->aux_priv
;
518 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
520 static struct page
*rb_alloc_aux_page(int node
, int order
)
524 if (order
> MAX_ORDER
)
528 page
= alloc_pages_node(node
, PERF_AUX_GFP
, order
);
529 } while (!page
&& order
--);
533 * Communicate the allocation size to the driver:
534 * if we managed to secure a high-order allocation,
535 * set its first page's private to this order;
536 * !PagePrivate(page) means it's just a normal page.
538 split_page(page
, order
);
539 SetPagePrivate(page
);
540 set_page_private(page
, order
);
546 static void rb_free_aux_page(struct ring_buffer
*rb
, int idx
)
548 struct page
*page
= virt_to_page(rb
->aux_pages
[idx
]);
550 ClearPagePrivate(page
);
551 page
->mapping
= NULL
;
555 static void __rb_free_aux(struct ring_buffer
*rb
)
560 * Should never happen, the last reference should be dropped from
561 * perf_mmap_close() path, which first stops aux transactions (which
562 * in turn are the atomic holders of aux_refcount) and then does the
563 * last rb_free_aux().
565 WARN_ON_ONCE(in_atomic());
568 rb
->free_aux(rb
->aux_priv
);
573 if (rb
->aux_nr_pages
) {
574 for (pg
= 0; pg
< rb
->aux_nr_pages
; pg
++)
575 rb_free_aux_page(rb
, pg
);
577 kfree(rb
->aux_pages
);
578 rb
->aux_nr_pages
= 0;
582 int rb_alloc_aux(struct ring_buffer
*rb
, struct perf_event
*event
,
583 pgoff_t pgoff
, int nr_pages
, long watermark
, int flags
)
585 bool overwrite
= !(flags
& RING_BUFFER_WRITABLE
);
586 int node
= (event
->cpu
== -1) ? -1 : cpu_to_node(event
->cpu
);
587 int ret
= -ENOMEM
, max_order
= 0;
592 if (event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) {
594 * We need to start with the max_order that fits in nr_pages,
595 * not the other way around, hence ilog2() and not get_order.
597 max_order
= ilog2(nr_pages
);
600 * PMU requests more than one contiguous chunks of memory
601 * for SW double buffering
603 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_SW_DOUBLEBUF
) &&
612 rb
->aux_pages
= kzalloc_node(nr_pages
* sizeof(void *), GFP_KERNEL
, node
);
616 rb
->free_aux
= event
->pmu
->free_aux
;
617 for (rb
->aux_nr_pages
= 0; rb
->aux_nr_pages
< nr_pages
;) {
621 order
= min(max_order
, ilog2(nr_pages
- rb
->aux_nr_pages
));
622 page
= rb_alloc_aux_page(node
, order
);
626 for (last
= rb
->aux_nr_pages
+ (1 << page_private(page
));
627 last
> rb
->aux_nr_pages
; rb
->aux_nr_pages
++)
628 rb
->aux_pages
[rb
->aux_nr_pages
] = page_address(page
++);
632 * In overwrite mode, PMUs that don't support SG may not handle more
633 * than one contiguous allocation, since they rely on PMI to do double
634 * buffering. In this case, the entire buffer has to be one contiguous
637 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) &&
639 struct page
*page
= virt_to_page(rb
->aux_pages
[0]);
641 if (page_private(page
) != max_order
)
645 rb
->aux_priv
= event
->pmu
->setup_aux(event
->cpu
, rb
->aux_pages
, nr_pages
,
653 * aux_pages (and pmu driver's private data, aux_priv) will be
654 * referenced in both producer's and consumer's contexts, thus
655 * we keep a refcount here to make sure either of the two can
656 * reference them safely.
658 atomic_set(&rb
->aux_refcount
, 1);
660 rb
->aux_overwrite
= overwrite
;
661 rb
->aux_watermark
= watermark
;
663 if (!rb
->aux_watermark
&& !rb
->aux_overwrite
)
664 rb
->aux_watermark
= nr_pages
<< (PAGE_SHIFT
- 1);
668 rb
->aux_pgoff
= pgoff
;
675 void rb_free_aux(struct ring_buffer
*rb
)
677 if (atomic_dec_and_test(&rb
->aux_refcount
))
681 #ifndef CONFIG_PERF_USE_VMALLOC
684 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
688 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
690 if (pgoff
> rb
->nr_pages
)
694 return virt_to_page(rb
->user_page
);
696 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
699 static void *perf_mmap_alloc_page(int cpu
)
704 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
705 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
709 return page_address(page
);
712 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
714 struct ring_buffer
*rb
;
718 size
= sizeof(struct ring_buffer
);
719 size
+= nr_pages
* sizeof(void *);
721 rb
= kzalloc(size
, GFP_KERNEL
);
725 rb
->user_page
= perf_mmap_alloc_page(cpu
);
729 for (i
= 0; i
< nr_pages
; i
++) {
730 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
731 if (!rb
->data_pages
[i
])
732 goto fail_data_pages
;
735 rb
->nr_pages
= nr_pages
;
737 ring_buffer_init(rb
, watermark
, flags
);
742 for (i
--; i
>= 0; i
--)
743 free_page((unsigned long)rb
->data_pages
[i
]);
745 free_page((unsigned long)rb
->user_page
);
754 static void perf_mmap_free_page(unsigned long addr
)
756 struct page
*page
= virt_to_page((void *)addr
);
758 page
->mapping
= NULL
;
762 void rb_free(struct ring_buffer
*rb
)
766 perf_mmap_free_page((unsigned long)rb
->user_page
);
767 for (i
= 0; i
< rb
->nr_pages
; i
++)
768 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
773 static int data_page_nr(struct ring_buffer
*rb
)
775 return rb
->nr_pages
<< page_order(rb
);
779 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
781 /* The '>' counts in the user page. */
782 if (pgoff
> data_page_nr(rb
))
785 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
788 static void perf_mmap_unmark_page(void *addr
)
790 struct page
*page
= vmalloc_to_page(addr
);
792 page
->mapping
= NULL
;
795 static void rb_free_work(struct work_struct
*work
)
797 struct ring_buffer
*rb
;
801 rb
= container_of(work
, struct ring_buffer
, work
);
802 nr
= data_page_nr(rb
);
804 base
= rb
->user_page
;
805 /* The '<=' counts in the user page. */
806 for (i
= 0; i
<= nr
; i
++)
807 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
813 void rb_free(struct ring_buffer
*rb
)
815 schedule_work(&rb
->work
);
818 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
820 struct ring_buffer
*rb
;
824 size
= sizeof(struct ring_buffer
);
825 size
+= sizeof(void *);
827 rb
= kzalloc(size
, GFP_KERNEL
);
831 INIT_WORK(&rb
->work
, rb_free_work
);
833 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
837 rb
->user_page
= all_buf
;
838 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
841 rb
->page_order
= ilog2(nr_pages
);
844 ring_buffer_init(rb
, watermark
, flags
);
858 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
860 if (rb
->aux_nr_pages
) {
861 /* above AUX space */
862 if (pgoff
> rb
->aux_pgoff
+ rb
->aux_nr_pages
)
866 if (pgoff
>= rb
->aux_pgoff
)
867 return virt_to_page(rb
->aux_pages
[pgoff
- rb
->aux_pgoff
]);
870 return __perf_mmap_to_page(rb
, pgoff
);