2 * Public API and common code for kernel->userspace relay file support.
4 * See Documentation/filesystems/relay.txt for an overview.
6 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
7 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
9 * Moved to kernel/relay.c by Paul Mundt, 2006.
10 * November 2006 - CPU hotplug support by Mathieu Desnoyers
11 * (mathieu.desnoyers@polymtl.ca)
13 * This file is released under the GPL.
15 #include <linux/errno.h>
16 #include <linux/stddef.h>
17 #include <linux/slab.h>
18 #include <linux/export.h>
19 #include <linux/string.h>
20 #include <linux/relay.h>
21 #include <linux/vmalloc.h>
23 #include <linux/cpu.h>
24 #include <linux/splice.h>
26 /* list of open channels, for cpu hotplug */
27 static DEFINE_MUTEX(relay_channels_mutex
);
28 static LIST_HEAD(relay_channels
);
31 * close() vm_op implementation for relay file mapping.
33 static void relay_file_mmap_close(struct vm_area_struct
*vma
)
35 struct rchan_buf
*buf
= vma
->vm_private_data
;
36 buf
->chan
->cb
->buf_unmapped(buf
, vma
->vm_file
);
40 * fault() vm_op implementation for relay file mapping.
42 static int relay_buf_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
45 struct rchan_buf
*buf
= vma
->vm_private_data
;
46 pgoff_t pgoff
= vmf
->pgoff
;
51 page
= vmalloc_to_page(buf
->start
+ (pgoff
<< PAGE_SHIFT
));
53 return VM_FAULT_SIGBUS
;
61 * vm_ops for relay file mappings.
63 static const struct vm_operations_struct relay_file_mmap_ops
= {
64 .fault
= relay_buf_fault
,
65 .close
= relay_file_mmap_close
,
69 * allocate an array of pointers of struct page
71 static struct page
**relay_alloc_page_array(unsigned int n_pages
)
73 const size_t pa_size
= n_pages
* sizeof(struct page
*);
74 if (pa_size
> PAGE_SIZE
)
75 return vzalloc(pa_size
);
76 return kzalloc(pa_size
, GFP_KERNEL
);
80 * free an array of pointers of struct page
82 static void relay_free_page_array(struct page
**array
)
88 * relay_mmap_buf: - mmap channel buffer to process address space
89 * @buf: relay channel buffer
90 * @vma: vm_area_struct describing memory to be mapped
92 * Returns 0 if ok, negative on error
94 * Caller should already have grabbed mmap_sem.
96 static int relay_mmap_buf(struct rchan_buf
*buf
, struct vm_area_struct
*vma
)
98 unsigned long length
= vma
->vm_end
- vma
->vm_start
;
99 struct file
*filp
= vma
->vm_file
;
104 if (length
!= (unsigned long)buf
->chan
->alloc_size
)
107 vma
->vm_ops
= &relay_file_mmap_ops
;
108 vma
->vm_flags
|= VM_DONTEXPAND
;
109 vma
->vm_private_data
= buf
;
110 buf
->chan
->cb
->buf_mapped(buf
, filp
);
116 * relay_alloc_buf - allocate a channel buffer
117 * @buf: the buffer struct
118 * @size: total size of the buffer
120 * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
121 * passed in size will get page aligned, if it isn't already.
123 static void *relay_alloc_buf(struct rchan_buf
*buf
, size_t *size
)
126 unsigned int i
, j
, n_pages
;
128 *size
= PAGE_ALIGN(*size
);
129 n_pages
= *size
>> PAGE_SHIFT
;
131 buf
->page_array
= relay_alloc_page_array(n_pages
);
132 if (!buf
->page_array
)
135 for (i
= 0; i
< n_pages
; i
++) {
136 buf
->page_array
[i
] = alloc_page(GFP_KERNEL
);
137 if (unlikely(!buf
->page_array
[i
]))
139 set_page_private(buf
->page_array
[i
], (unsigned long)buf
);
141 mem
= vmap(buf
->page_array
, n_pages
, VM_MAP
, PAGE_KERNEL
);
145 memset(mem
, 0, *size
);
146 buf
->page_count
= n_pages
;
150 for (j
= 0; j
< i
; j
++)
151 __free_page(buf
->page_array
[j
]);
152 relay_free_page_array(buf
->page_array
);
157 * relay_create_buf - allocate and initialize a channel buffer
158 * @chan: the relay channel
160 * Returns channel buffer if successful, %NULL otherwise.
162 static struct rchan_buf
*relay_create_buf(struct rchan
*chan
)
164 struct rchan_buf
*buf
;
166 if (chan
->n_subbufs
> UINT_MAX
/ sizeof(size_t *))
169 buf
= kzalloc(sizeof(struct rchan_buf
), GFP_KERNEL
);
172 buf
->padding
= kmalloc(chan
->n_subbufs
* sizeof(size_t *), GFP_KERNEL
);
176 buf
->start
= relay_alloc_buf(buf
, &chan
->alloc_size
);
181 kref_get(&buf
->chan
->kref
);
191 * relay_destroy_channel - free the channel struct
192 * @kref: target kernel reference that contains the relay channel
194 * Should only be called from kref_put().
196 static void relay_destroy_channel(struct kref
*kref
)
198 struct rchan
*chan
= container_of(kref
, struct rchan
, kref
);
203 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
204 * @buf: the buffer struct
206 static void relay_destroy_buf(struct rchan_buf
*buf
)
208 struct rchan
*chan
= buf
->chan
;
211 if (likely(buf
->start
)) {
213 for (i
= 0; i
< buf
->page_count
; i
++)
214 __free_page(buf
->page_array
[i
]);
215 relay_free_page_array(buf
->page_array
);
217 *per_cpu_ptr(chan
->buf
, buf
->cpu
) = NULL
;
220 kref_put(&chan
->kref
, relay_destroy_channel
);
224 * relay_remove_buf - remove a channel buffer
225 * @kref: target kernel reference that contains the relay buffer
227 * Removes the file from the filesystem, which also frees the
228 * rchan_buf_struct and the channel buffer. Should only be called from
231 static void relay_remove_buf(struct kref
*kref
)
233 struct rchan_buf
*buf
= container_of(kref
, struct rchan_buf
, kref
);
234 relay_destroy_buf(buf
);
238 * relay_buf_empty - boolean, is the channel buffer empty?
239 * @buf: channel buffer
241 * Returns 1 if the buffer is empty, 0 otherwise.
243 static int relay_buf_empty(struct rchan_buf
*buf
)
245 return (buf
->subbufs_produced
- buf
->subbufs_consumed
) ? 0 : 1;
249 * relay_buf_full - boolean, is the channel buffer full?
250 * @buf: channel buffer
252 * Returns 1 if the buffer is full, 0 otherwise.
254 int relay_buf_full(struct rchan_buf
*buf
)
256 size_t ready
= buf
->subbufs_produced
- buf
->subbufs_consumed
;
257 return (ready
>= buf
->chan
->n_subbufs
) ? 1 : 0;
259 EXPORT_SYMBOL_GPL(relay_buf_full
);
262 * High-level relay kernel API and associated functions.
266 * rchan_callback implementations defining default channel behavior. Used
267 * in place of corresponding NULL values in client callback struct.
271 * subbuf_start() default callback. Does nothing.
273 static int subbuf_start_default_callback (struct rchan_buf
*buf
,
278 if (relay_buf_full(buf
))
285 * buf_mapped() default callback. Does nothing.
287 static void buf_mapped_default_callback(struct rchan_buf
*buf
,
293 * buf_unmapped() default callback. Does nothing.
295 static void buf_unmapped_default_callback(struct rchan_buf
*buf
,
301 * create_buf_file_create() default callback. Does nothing.
303 static struct dentry
*create_buf_file_default_callback(const char *filename
,
304 struct dentry
*parent
,
306 struct rchan_buf
*buf
,
313 * remove_buf_file() default callback. Does nothing.
315 static int remove_buf_file_default_callback(struct dentry
*dentry
)
320 /* relay channel default callbacks */
321 static struct rchan_callbacks default_channel_callbacks
= {
322 .subbuf_start
= subbuf_start_default_callback
,
323 .buf_mapped
= buf_mapped_default_callback
,
324 .buf_unmapped
= buf_unmapped_default_callback
,
325 .create_buf_file
= create_buf_file_default_callback
,
326 .remove_buf_file
= remove_buf_file_default_callback
,
330 * wakeup_readers - wake up readers waiting on a channel
331 * @data: contains the channel buffer
333 * This is the timer function used to defer reader waking.
335 static void wakeup_readers(unsigned long data
)
337 struct rchan_buf
*buf
= (struct rchan_buf
*)data
;
338 wake_up_interruptible(&buf
->read_wait
);
342 * __relay_reset - reset a channel buffer
343 * @buf: the channel buffer
344 * @init: 1 if this is a first-time initialization
346 * See relay_reset() for description of effect.
348 static void __relay_reset(struct rchan_buf
*buf
, unsigned int init
)
353 init_waitqueue_head(&buf
->read_wait
);
354 kref_init(&buf
->kref
);
355 setup_timer(&buf
->timer
, wakeup_readers
, (unsigned long)buf
);
357 del_timer_sync(&buf
->timer
);
359 buf
->subbufs_produced
= 0;
360 buf
->subbufs_consumed
= 0;
361 buf
->bytes_consumed
= 0;
363 buf
->data
= buf
->start
;
366 for (i
= 0; i
< buf
->chan
->n_subbufs
; i
++)
369 buf
->chan
->cb
->subbuf_start(buf
, buf
->data
, NULL
, 0);
373 * relay_reset - reset the channel
376 * This has the effect of erasing all data from all channel buffers
377 * and restarting the channel in its initial state. The buffers
378 * are not freed, so any mappings are still in effect.
380 * NOTE. Care should be taken that the channel isn't actually
381 * being used by anything when this call is made.
383 void relay_reset(struct rchan
*chan
)
385 struct rchan_buf
*buf
;
391 if (chan
->is_global
&& (buf
= *per_cpu_ptr(chan
->buf
, 0))) {
392 __relay_reset(buf
, 0);
396 mutex_lock(&relay_channels_mutex
);
397 for_each_possible_cpu(i
)
398 if ((buf
= *per_cpu_ptr(chan
->buf
, i
)))
399 __relay_reset(buf
, 0);
400 mutex_unlock(&relay_channels_mutex
);
402 EXPORT_SYMBOL_GPL(relay_reset
);
404 static inline void relay_set_buf_dentry(struct rchan_buf
*buf
,
405 struct dentry
*dentry
)
407 buf
->dentry
= dentry
;
408 d_inode(buf
->dentry
)->i_size
= buf
->early_bytes
;
411 static struct dentry
*relay_create_buf_file(struct rchan
*chan
,
412 struct rchan_buf
*buf
,
415 struct dentry
*dentry
;
418 tmpname
= kzalloc(NAME_MAX
+ 1, GFP_KERNEL
);
421 snprintf(tmpname
, NAME_MAX
, "%s%d", chan
->base_filename
, cpu
);
423 /* Create file in fs */
424 dentry
= chan
->cb
->create_buf_file(tmpname
, chan
->parent
,
434 * relay_open_buf - create a new relay channel buffer
436 * used by relay_open() and CPU hotplug.
438 static struct rchan_buf
*relay_open_buf(struct rchan
*chan
, unsigned int cpu
)
440 struct rchan_buf
*buf
= NULL
;
441 struct dentry
*dentry
;
444 return *per_cpu_ptr(chan
->buf
, 0);
446 buf
= relay_create_buf(chan
);
450 if (chan
->has_base_filename
) {
451 dentry
= relay_create_buf_file(chan
, buf
, cpu
);
454 relay_set_buf_dentry(buf
, dentry
);
456 /* Only retrieve global info, nothing more, nothing less */
457 dentry
= chan
->cb
->create_buf_file(NULL
, NULL
,
465 __relay_reset(buf
, 1);
467 if(chan
->is_global
) {
468 *per_cpu_ptr(chan
->buf
, 0) = buf
;
475 relay_destroy_buf(buf
);
480 * relay_close_buf - close a channel buffer
481 * @buf: channel buffer
483 * Marks the buffer finalized and restores the default callbacks.
484 * The channel buffer and channel buffer data structure are then freed
485 * automatically when the last reference is given up.
487 static void relay_close_buf(struct rchan_buf
*buf
)
490 del_timer_sync(&buf
->timer
);
491 buf
->chan
->cb
->remove_buf_file(buf
->dentry
);
492 kref_put(&buf
->kref
, relay_remove_buf
);
495 static void setup_callbacks(struct rchan
*chan
,
496 struct rchan_callbacks
*cb
)
499 chan
->cb
= &default_channel_callbacks
;
503 if (!cb
->subbuf_start
)
504 cb
->subbuf_start
= subbuf_start_default_callback
;
506 cb
->buf_mapped
= buf_mapped_default_callback
;
507 if (!cb
->buf_unmapped
)
508 cb
->buf_unmapped
= buf_unmapped_default_callback
;
509 if (!cb
->create_buf_file
)
510 cb
->create_buf_file
= create_buf_file_default_callback
;
511 if (!cb
->remove_buf_file
)
512 cb
->remove_buf_file
= remove_buf_file_default_callback
;
516 int relay_prepare_cpu(unsigned int cpu
)
519 struct rchan_buf
*buf
;
521 mutex_lock(&relay_channels_mutex
);
522 list_for_each_entry(chan
, &relay_channels
, list
) {
523 if ((buf
= *per_cpu_ptr(chan
->buf
, cpu
)))
525 buf
= relay_open_buf(chan
, cpu
);
527 pr_err("relay: cpu %d buffer creation failed\n", cpu
);
528 mutex_unlock(&relay_channels_mutex
);
531 *per_cpu_ptr(chan
->buf
, cpu
) = buf
;
533 mutex_unlock(&relay_channels_mutex
);
538 * relay_open - create a new relay channel
539 * @base_filename: base name of files to create, %NULL for buffering only
540 * @parent: dentry of parent directory, %NULL for root directory or buffer
541 * @subbuf_size: size of sub-buffers
542 * @n_subbufs: number of sub-buffers
543 * @cb: client callback functions
544 * @private_data: user-defined data
546 * Returns channel pointer if successful, %NULL otherwise.
548 * Creates a channel buffer for each cpu using the sizes and
549 * attributes specified. The created channel buffer files
550 * will be named base_filename0...base_filenameN-1. File
551 * permissions will be %S_IRUSR.
553 * If opening a buffer (@parent = NULL) that you later wish to register
554 * in a filesystem, call relay_late_setup_files() once the @parent dentry
557 struct rchan
*relay_open(const char *base_filename
,
558 struct dentry
*parent
,
561 struct rchan_callbacks
*cb
,
566 struct rchan_buf
*buf
;
568 if (!(subbuf_size
&& n_subbufs
))
570 if (subbuf_size
> UINT_MAX
/ n_subbufs
)
573 chan
= kzalloc(sizeof(struct rchan
), GFP_KERNEL
);
577 chan
->buf
= alloc_percpu(struct rchan_buf
*);
578 chan
->version
= RELAYFS_CHANNEL_VERSION
;
579 chan
->n_subbufs
= n_subbufs
;
580 chan
->subbuf_size
= subbuf_size
;
581 chan
->alloc_size
= PAGE_ALIGN(subbuf_size
* n_subbufs
);
582 chan
->parent
= parent
;
583 chan
->private_data
= private_data
;
585 chan
->has_base_filename
= 1;
586 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
588 setup_callbacks(chan
, cb
);
589 kref_init(&chan
->kref
);
591 mutex_lock(&relay_channels_mutex
);
592 for_each_online_cpu(i
) {
593 buf
= relay_open_buf(chan
, i
);
596 *per_cpu_ptr(chan
->buf
, i
) = buf
;
598 list_add(&chan
->list
, &relay_channels
);
599 mutex_unlock(&relay_channels_mutex
);
604 for_each_possible_cpu(i
) {
605 if ((buf
= *per_cpu_ptr(chan
->buf
, i
)))
606 relay_close_buf(buf
);
609 kref_put(&chan
->kref
, relay_destroy_channel
);
610 mutex_unlock(&relay_channels_mutex
);
614 EXPORT_SYMBOL_GPL(relay_open
);
616 struct rchan_percpu_buf_dispatcher
{
617 struct rchan_buf
*buf
;
618 struct dentry
*dentry
;
621 /* Called in atomic context. */
622 static void __relay_set_buf_dentry(void *info
)
624 struct rchan_percpu_buf_dispatcher
*p
= info
;
626 relay_set_buf_dentry(p
->buf
, p
->dentry
);
630 * relay_late_setup_files - triggers file creation
631 * @chan: channel to operate on
632 * @base_filename: base name of files to create
633 * @parent: dentry of parent directory, %NULL for root directory
635 * Returns 0 if successful, non-zero otherwise.
637 * Use to setup files for a previously buffer-only channel created
638 * by relay_open() with a NULL parent dentry.
640 * For example, this is useful for perfomring early tracing in kernel,
641 * before VFS is up and then exposing the early results once the dentry
644 int relay_late_setup_files(struct rchan
*chan
,
645 const char *base_filename
,
646 struct dentry
*parent
)
649 unsigned int i
, curr_cpu
;
651 struct dentry
*dentry
;
652 struct rchan_buf
*buf
;
653 struct rchan_percpu_buf_dispatcher disp
;
655 if (!chan
|| !base_filename
)
658 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
660 mutex_lock(&relay_channels_mutex
);
661 /* Is chan already set up? */
662 if (unlikely(chan
->has_base_filename
)) {
663 mutex_unlock(&relay_channels_mutex
);
666 chan
->has_base_filename
= 1;
667 chan
->parent
= parent
;
669 if (chan
->is_global
) {
671 buf
= *per_cpu_ptr(chan
->buf
, 0);
672 if (!WARN_ON_ONCE(!buf
)) {
673 dentry
= relay_create_buf_file(chan
, buf
, 0);
674 if (dentry
&& !WARN_ON_ONCE(!chan
->is_global
)) {
675 relay_set_buf_dentry(buf
, dentry
);
679 mutex_unlock(&relay_channels_mutex
);
683 curr_cpu
= get_cpu();
685 * The CPU hotplug notifier ran before us and created buffers with
686 * no files associated. So it's safe to call relay_setup_buf_file()
687 * on all currently online CPUs.
689 for_each_online_cpu(i
) {
690 buf
= *per_cpu_ptr(chan
->buf
, i
);
691 if (unlikely(!buf
)) {
692 WARN_ONCE(1, KERN_ERR
"CPU has no buffer!\n");
697 dentry
= relay_create_buf_file(chan
, buf
, i
);
698 if (unlikely(!dentry
)) {
704 local_irq_save(flags
);
705 relay_set_buf_dentry(buf
, dentry
);
706 local_irq_restore(flags
);
709 disp
.dentry
= dentry
;
711 /* relay_channels_mutex must be held, so wait. */
712 err
= smp_call_function_single(i
,
713 __relay_set_buf_dentry
,
720 mutex_unlock(&relay_channels_mutex
);
724 EXPORT_SYMBOL_GPL(relay_late_setup_files
);
727 * relay_switch_subbuf - switch to a new sub-buffer
728 * @buf: channel buffer
729 * @length: size of current event
731 * Returns either the length passed in or 0 if full.
733 * Performs sub-buffer-switch tasks such as invoking callbacks,
734 * updating padding counts, waking up readers, etc.
736 size_t relay_switch_subbuf(struct rchan_buf
*buf
, size_t length
)
739 size_t old_subbuf
, new_subbuf
;
741 if (unlikely(length
> buf
->chan
->subbuf_size
))
744 if (buf
->offset
!= buf
->chan
->subbuf_size
+ 1) {
745 buf
->prev_padding
= buf
->chan
->subbuf_size
- buf
->offset
;
746 old_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
747 buf
->padding
[old_subbuf
] = buf
->prev_padding
;
748 buf
->subbufs_produced
++;
750 d_inode(buf
->dentry
)->i_size
+=
751 buf
->chan
->subbuf_size
-
752 buf
->padding
[old_subbuf
];
754 buf
->early_bytes
+= buf
->chan
->subbuf_size
-
755 buf
->padding
[old_subbuf
];
757 if (waitqueue_active(&buf
->read_wait
))
759 * Calling wake_up_interruptible() from here
760 * will deadlock if we happen to be logging
761 * from the scheduler (trying to re-grab
762 * rq->lock), so defer it.
764 mod_timer(&buf
->timer
, jiffies
+ 1);
768 new_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
769 new = buf
->start
+ new_subbuf
* buf
->chan
->subbuf_size
;
771 if (!buf
->chan
->cb
->subbuf_start(buf
, new, old
, buf
->prev_padding
)) {
772 buf
->offset
= buf
->chan
->subbuf_size
+ 1;
776 buf
->padding
[new_subbuf
] = 0;
778 if (unlikely(length
+ buf
->offset
> buf
->chan
->subbuf_size
))
784 buf
->chan
->last_toobig
= length
;
787 EXPORT_SYMBOL_GPL(relay_switch_subbuf
);
790 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
792 * @cpu: the cpu associated with the channel buffer to update
793 * @subbufs_consumed: number of sub-buffers to add to current buf's count
795 * Adds to the channel buffer's consumed sub-buffer count.
796 * subbufs_consumed should be the number of sub-buffers newly consumed,
797 * not the total consumed.
799 * NOTE. Kernel clients don't need to call this function if the channel
800 * mode is 'overwrite'.
802 void relay_subbufs_consumed(struct rchan
*chan
,
804 size_t subbufs_consumed
)
806 struct rchan_buf
*buf
;
811 buf
= *per_cpu_ptr(chan
->buf
, cpu
);
812 if (cpu
>= NR_CPUS
|| !buf
|| subbufs_consumed
> chan
->n_subbufs
)
815 if (subbufs_consumed
> buf
->subbufs_produced
- buf
->subbufs_consumed
)
816 buf
->subbufs_consumed
= buf
->subbufs_produced
;
818 buf
->subbufs_consumed
+= subbufs_consumed
;
820 EXPORT_SYMBOL_GPL(relay_subbufs_consumed
);
823 * relay_close - close the channel
826 * Closes all channel buffers and frees the channel.
828 void relay_close(struct rchan
*chan
)
830 struct rchan_buf
*buf
;
836 mutex_lock(&relay_channels_mutex
);
837 if (chan
->is_global
&& (buf
= *per_cpu_ptr(chan
->buf
, 0)))
838 relay_close_buf(buf
);
840 for_each_possible_cpu(i
)
841 if ((buf
= *per_cpu_ptr(chan
->buf
, i
)))
842 relay_close_buf(buf
);
844 if (chan
->last_toobig
)
845 printk(KERN_WARNING
"relay: one or more items not logged "
846 "[item size (%Zd) > sub-buffer size (%Zd)]\n",
847 chan
->last_toobig
, chan
->subbuf_size
);
849 list_del(&chan
->list
);
850 kref_put(&chan
->kref
, relay_destroy_channel
);
851 mutex_unlock(&relay_channels_mutex
);
853 EXPORT_SYMBOL_GPL(relay_close
);
856 * relay_flush - close the channel
859 * Flushes all channel buffers, i.e. forces buffer switch.
861 void relay_flush(struct rchan
*chan
)
863 struct rchan_buf
*buf
;
869 if (chan
->is_global
&& (buf
= *per_cpu_ptr(chan
->buf
, 0))) {
870 relay_switch_subbuf(buf
, 0);
874 mutex_lock(&relay_channels_mutex
);
875 for_each_possible_cpu(i
)
876 if ((buf
= *per_cpu_ptr(chan
->buf
, i
)))
877 relay_switch_subbuf(buf
, 0);
878 mutex_unlock(&relay_channels_mutex
);
880 EXPORT_SYMBOL_GPL(relay_flush
);
883 * relay_file_open - open file op for relay files
887 * Increments the channel buffer refcount.
889 static int relay_file_open(struct inode
*inode
, struct file
*filp
)
891 struct rchan_buf
*buf
= inode
->i_private
;
892 kref_get(&buf
->kref
);
893 filp
->private_data
= buf
;
895 return nonseekable_open(inode
, filp
);
899 * relay_file_mmap - mmap file op for relay files
901 * @vma: the vma describing what to map
903 * Calls upon relay_mmap_buf() to map the file into user space.
905 static int relay_file_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
907 struct rchan_buf
*buf
= filp
->private_data
;
908 return relay_mmap_buf(buf
, vma
);
912 * relay_file_poll - poll file op for relay files
918 static unsigned int relay_file_poll(struct file
*filp
, poll_table
*wait
)
920 unsigned int mask
= 0;
921 struct rchan_buf
*buf
= filp
->private_data
;
926 if (filp
->f_mode
& FMODE_READ
) {
927 poll_wait(filp
, &buf
->read_wait
, wait
);
928 if (!relay_buf_empty(buf
))
929 mask
|= POLLIN
| POLLRDNORM
;
936 * relay_file_release - release file op for relay files
940 * Decrements the channel refcount, as the filesystem is
941 * no longer using it.
943 static int relay_file_release(struct inode
*inode
, struct file
*filp
)
945 struct rchan_buf
*buf
= filp
->private_data
;
946 kref_put(&buf
->kref
, relay_remove_buf
);
952 * relay_file_read_consume - update the consumed count for the buffer
954 static void relay_file_read_consume(struct rchan_buf
*buf
,
956 size_t bytes_consumed
)
958 size_t subbuf_size
= buf
->chan
->subbuf_size
;
959 size_t n_subbufs
= buf
->chan
->n_subbufs
;
962 if (buf
->subbufs_produced
== buf
->subbufs_consumed
&&
963 buf
->offset
== buf
->bytes_consumed
)
966 if (buf
->bytes_consumed
+ bytes_consumed
> subbuf_size
) {
967 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
968 buf
->bytes_consumed
= 0;
971 buf
->bytes_consumed
+= bytes_consumed
;
973 read_subbuf
= buf
->subbufs_consumed
% n_subbufs
;
975 read_subbuf
= read_pos
/ buf
->chan
->subbuf_size
;
976 if (buf
->bytes_consumed
+ buf
->padding
[read_subbuf
] == subbuf_size
) {
977 if ((read_subbuf
== buf
->subbufs_produced
% n_subbufs
) &&
978 (buf
->offset
== subbuf_size
))
980 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
981 buf
->bytes_consumed
= 0;
986 * relay_file_read_avail - boolean, are there unconsumed bytes available?
988 static int relay_file_read_avail(struct rchan_buf
*buf
, size_t read_pos
)
990 size_t subbuf_size
= buf
->chan
->subbuf_size
;
991 size_t n_subbufs
= buf
->chan
->n_subbufs
;
992 size_t produced
= buf
->subbufs_produced
;
993 size_t consumed
= buf
->subbufs_consumed
;
995 relay_file_read_consume(buf
, read_pos
, 0);
997 consumed
= buf
->subbufs_consumed
;
999 if (unlikely(buf
->offset
> subbuf_size
)) {
1000 if (produced
== consumed
)
1005 if (unlikely(produced
- consumed
>= n_subbufs
)) {
1006 consumed
= produced
- n_subbufs
+ 1;
1007 buf
->subbufs_consumed
= consumed
;
1008 buf
->bytes_consumed
= 0;
1011 produced
= (produced
% n_subbufs
) * subbuf_size
+ buf
->offset
;
1012 consumed
= (consumed
% n_subbufs
) * subbuf_size
+ buf
->bytes_consumed
;
1014 if (consumed
> produced
)
1015 produced
+= n_subbufs
* subbuf_size
;
1017 if (consumed
== produced
) {
1018 if (buf
->offset
== subbuf_size
&&
1019 buf
->subbufs_produced
> buf
->subbufs_consumed
)
1028 * relay_file_read_subbuf_avail - return bytes available in sub-buffer
1029 * @read_pos: file read position
1030 * @buf: relay channel buffer
1032 static size_t relay_file_read_subbuf_avail(size_t read_pos
,
1033 struct rchan_buf
*buf
)
1035 size_t padding
, avail
= 0;
1036 size_t read_subbuf
, read_offset
, write_subbuf
, write_offset
;
1037 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1039 write_subbuf
= (buf
->data
- buf
->start
) / subbuf_size
;
1040 write_offset
= buf
->offset
> subbuf_size
? subbuf_size
: buf
->offset
;
1041 read_subbuf
= read_pos
/ subbuf_size
;
1042 read_offset
= read_pos
% subbuf_size
;
1043 padding
= buf
->padding
[read_subbuf
];
1045 if (read_subbuf
== write_subbuf
) {
1046 if (read_offset
+ padding
< write_offset
)
1047 avail
= write_offset
- (read_offset
+ padding
);
1049 avail
= (subbuf_size
- padding
) - read_offset
;
1055 * relay_file_read_start_pos - find the first available byte to read
1056 * @read_pos: file read position
1057 * @buf: relay channel buffer
1059 * If the @read_pos is in the middle of padding, return the
1060 * position of the first actually available byte, otherwise
1061 * return the original value.
1063 static size_t relay_file_read_start_pos(size_t read_pos
,
1064 struct rchan_buf
*buf
)
1066 size_t read_subbuf
, padding
, padding_start
, padding_end
;
1067 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1068 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1069 size_t consumed
= buf
->subbufs_consumed
% n_subbufs
;
1072 read_pos
= consumed
* subbuf_size
+ buf
->bytes_consumed
;
1073 read_subbuf
= read_pos
/ subbuf_size
;
1074 padding
= buf
->padding
[read_subbuf
];
1075 padding_start
= (read_subbuf
+ 1) * subbuf_size
- padding
;
1076 padding_end
= (read_subbuf
+ 1) * subbuf_size
;
1077 if (read_pos
>= padding_start
&& read_pos
< padding_end
) {
1078 read_subbuf
= (read_subbuf
+ 1) % n_subbufs
;
1079 read_pos
= read_subbuf
* subbuf_size
;
1086 * relay_file_read_end_pos - return the new read position
1087 * @read_pos: file read position
1088 * @buf: relay channel buffer
1089 * @count: number of bytes to be read
1091 static size_t relay_file_read_end_pos(struct rchan_buf
*buf
,
1095 size_t read_subbuf
, padding
, end_pos
;
1096 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1097 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1099 read_subbuf
= read_pos
/ subbuf_size
;
1100 padding
= buf
->padding
[read_subbuf
];
1101 if (read_pos
% subbuf_size
+ count
+ padding
== subbuf_size
)
1102 end_pos
= (read_subbuf
+ 1) * subbuf_size
;
1104 end_pos
= read_pos
+ count
;
1105 if (end_pos
>= subbuf_size
* n_subbufs
)
1111 static ssize_t
relay_file_read(struct file
*filp
,
1112 char __user
*buffer
,
1116 struct rchan_buf
*buf
= filp
->private_data
;
1117 size_t read_start
, avail
;
1124 inode_lock(file_inode(filp
));
1128 if (!relay_file_read_avail(buf
, *ppos
))
1131 read_start
= relay_file_read_start_pos(*ppos
, buf
);
1132 avail
= relay_file_read_subbuf_avail(read_start
, buf
);
1136 avail
= min(count
, avail
);
1137 from
= buf
->start
+ read_start
;
1139 if (copy_to_user(buffer
, from
, avail
))
1146 relay_file_read_consume(buf
, read_start
, ret
);
1147 *ppos
= relay_file_read_end_pos(buf
, read_start
, ret
);
1149 inode_unlock(file_inode(filp
));
1154 static void relay_consume_bytes(struct rchan_buf
*rbuf
, int bytes_consumed
)
1156 rbuf
->bytes_consumed
+= bytes_consumed
;
1158 if (rbuf
->bytes_consumed
>= rbuf
->chan
->subbuf_size
) {
1159 relay_subbufs_consumed(rbuf
->chan
, rbuf
->cpu
, 1);
1160 rbuf
->bytes_consumed
%= rbuf
->chan
->subbuf_size
;
1164 static void relay_pipe_buf_release(struct pipe_inode_info
*pipe
,
1165 struct pipe_buffer
*buf
)
1167 struct rchan_buf
*rbuf
;
1169 rbuf
= (struct rchan_buf
*)page_private(buf
->page
);
1170 relay_consume_bytes(rbuf
, buf
->private);
1173 static const struct pipe_buf_operations relay_pipe_buf_ops
= {
1175 .confirm
= generic_pipe_buf_confirm
,
1176 .release
= relay_pipe_buf_release
,
1177 .steal
= generic_pipe_buf_steal
,
1178 .get
= generic_pipe_buf_get
,
1181 static void relay_page_release(struct splice_pipe_desc
*spd
, unsigned int i
)
1186 * subbuf_splice_actor - splice up to one subbuf's worth of data
1188 static ssize_t
subbuf_splice_actor(struct file
*in
,
1190 struct pipe_inode_info
*pipe
,
1195 unsigned int pidx
, poff
, total_len
, subbuf_pages
, nr_pages
;
1196 struct rchan_buf
*rbuf
= in
->private_data
;
1197 unsigned int subbuf_size
= rbuf
->chan
->subbuf_size
;
1198 uint64_t pos
= (uint64_t) *ppos
;
1199 uint32_t alloc_size
= (uint32_t) rbuf
->chan
->alloc_size
;
1200 size_t read_start
= (size_t) do_div(pos
, alloc_size
);
1201 size_t read_subbuf
= read_start
/ subbuf_size
;
1202 size_t padding
= rbuf
->padding
[read_subbuf
];
1203 size_t nonpad_end
= read_subbuf
* subbuf_size
+ subbuf_size
- padding
;
1204 struct page
*pages
[PIPE_DEF_BUFFERS
];
1205 struct partial_page partial
[PIPE_DEF_BUFFERS
];
1206 struct splice_pipe_desc spd
= {
1209 .nr_pages_max
= PIPE_DEF_BUFFERS
,
1212 .ops
= &relay_pipe_buf_ops
,
1213 .spd_release
= relay_page_release
,
1217 if (rbuf
->subbufs_produced
== rbuf
->subbufs_consumed
)
1219 if (splice_grow_spd(pipe
, &spd
))
1223 * Adjust read len, if longer than what is available
1225 if (len
> (subbuf_size
- read_start
% subbuf_size
))
1226 len
= subbuf_size
- read_start
% subbuf_size
;
1228 subbuf_pages
= rbuf
->chan
->alloc_size
>> PAGE_SHIFT
;
1229 pidx
= (read_start
/ PAGE_SIZE
) % subbuf_pages
;
1230 poff
= read_start
& ~PAGE_MASK
;
1231 nr_pages
= min_t(unsigned int, subbuf_pages
, spd
.nr_pages_max
);
1233 for (total_len
= 0; spd
.nr_pages
< nr_pages
; spd
.nr_pages
++) {
1234 unsigned int this_len
, this_end
, private;
1235 unsigned int cur_pos
= read_start
+ total_len
;
1240 this_len
= min_t(unsigned long, len
, PAGE_SIZE
- poff
);
1243 spd
.pages
[spd
.nr_pages
] = rbuf
->page_array
[pidx
];
1244 spd
.partial
[spd
.nr_pages
].offset
= poff
;
1246 this_end
= cur_pos
+ this_len
;
1247 if (this_end
>= nonpad_end
) {
1248 this_len
= nonpad_end
- cur_pos
;
1249 private = this_len
+ padding
;
1251 spd
.partial
[spd
.nr_pages
].len
= this_len
;
1252 spd
.partial
[spd
.nr_pages
].private = private;
1255 total_len
+= this_len
;
1257 pidx
= (pidx
+ 1) % subbuf_pages
;
1259 if (this_end
>= nonpad_end
) {
1269 ret
= *nonpad_ret
= splice_to_pipe(pipe
, &spd
);
1270 if (ret
< 0 || ret
< total_len
)
1273 if (read_start
+ ret
== nonpad_end
)
1277 splice_shrink_spd(&spd
);
1281 static ssize_t
relay_file_splice_read(struct file
*in
,
1283 struct pipe_inode_info
*pipe
,
1294 while (len
&& !spliced
) {
1295 ret
= subbuf_splice_actor(in
, ppos
, pipe
, len
, flags
, &nonpad_ret
);
1299 if (flags
& SPLICE_F_NONBLOCK
)
1309 spliced
+= nonpad_ret
;
1319 const struct file_operations relay_file_operations
= {
1320 .open
= relay_file_open
,
1321 .poll
= relay_file_poll
,
1322 .mmap
= relay_file_mmap
,
1323 .read
= relay_file_read
,
1324 .llseek
= no_llseek
,
1325 .release
= relay_file_release
,
1326 .splice_read
= relay_file_splice_read
,
1328 EXPORT_SYMBOL_GPL(relay_file_operations
);