4 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org>
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/init.h>
12 #include <linux/sched/signal.h>
13 #include <linux/kernel.h>
14 #include <linux/slab.h>
15 #include <linux/list.h>
16 #include <linux/spinlock.h>
17 #include <linux/anon_inodes.h>
18 #include <linux/syscalls.h>
19 #include <linux/export.h>
20 #include <linux/kref.h>
21 #include <linux/eventfd.h>
22 #include <linux/proc_fs.h>
23 #include <linux/seq_file.h>
25 DEFINE_PER_CPU(int, eventfd_wake_count
);
29 wait_queue_head_t wqh
;
31 * Every time that a write(2) is performed on an eventfd, the
32 * value of the __u64 being written is added to "count" and a
33 * wakeup is performed on "wqh". A read(2) will return the "count"
34 * value to userspace, and will reset "count" to zero. The kernel
35 * side eventfd_signal() also, adds to the "count" counter and
43 * eventfd_signal - Adds @n to the eventfd counter.
44 * @ctx: [in] Pointer to the eventfd context.
45 * @n: [in] Value of the counter to be added to the eventfd internal counter.
46 * The value cannot be negative.
48 * This function is supposed to be called by the kernel in paths that do not
49 * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX
50 * value, and we signal this as overflow condition by returning a POLLERR
53 * Returns the amount by which the counter was incremented. This will be less
54 * than @n if the counter has overflowed.
56 __u64
eventfd_signal(struct eventfd_ctx
*ctx
, __u64 n
)
61 * Deadlock or stack overflow issues can happen if we recurse here
62 * through waitqueue wakeup handlers. If the caller users potentially
63 * nested waitqueues with custom wakeup handlers, then it should
64 * check eventfd_signal_count() before calling this function. If
65 * it returns true, the eventfd_signal() call should be deferred to a
68 if (WARN_ON_ONCE(this_cpu_read(eventfd_wake_count
)))
71 spin_lock_irqsave(&ctx
->wqh
.lock
, flags
);
72 this_cpu_inc(eventfd_wake_count
);
73 if (ULLONG_MAX
- ctx
->count
< n
)
74 n
= ULLONG_MAX
- ctx
->count
;
76 if (waitqueue_active(&ctx
->wqh
))
77 wake_up_locked_poll(&ctx
->wqh
, POLLIN
);
78 this_cpu_dec(eventfd_wake_count
);
79 spin_unlock_irqrestore(&ctx
->wqh
.lock
, flags
);
83 EXPORT_SYMBOL_GPL(eventfd_signal
);
85 static void eventfd_free_ctx(struct eventfd_ctx
*ctx
)
90 static void eventfd_free(struct kref
*kref
)
92 struct eventfd_ctx
*ctx
= container_of(kref
, struct eventfd_ctx
, kref
);
94 eventfd_free_ctx(ctx
);
98 * eventfd_ctx_get - Acquires a reference to the internal eventfd context.
99 * @ctx: [in] Pointer to the eventfd context.
101 * Returns: In case of success, returns a pointer to the eventfd context.
103 struct eventfd_ctx
*eventfd_ctx_get(struct eventfd_ctx
*ctx
)
105 kref_get(&ctx
->kref
);
108 EXPORT_SYMBOL_GPL(eventfd_ctx_get
);
111 * eventfd_ctx_put - Releases a reference to the internal eventfd context.
112 * @ctx: [in] Pointer to eventfd context.
114 * The eventfd context reference must have been previously acquired either
115 * with eventfd_ctx_get() or eventfd_ctx_fdget().
117 void eventfd_ctx_put(struct eventfd_ctx
*ctx
)
119 kref_put(&ctx
->kref
, eventfd_free
);
121 EXPORT_SYMBOL_GPL(eventfd_ctx_put
);
123 static int eventfd_release(struct inode
*inode
, struct file
*file
)
125 struct eventfd_ctx
*ctx
= file
->private_data
;
127 wake_up_poll(&ctx
->wqh
, POLLHUP
);
128 eventfd_ctx_put(ctx
);
132 static unsigned int eventfd_poll(struct file
*file
, poll_table
*wait
)
134 struct eventfd_ctx
*ctx
= file
->private_data
;
135 unsigned int events
= 0;
138 poll_wait(file
, &ctx
->wqh
, wait
);
141 * All writes to ctx->count occur within ctx->wqh.lock. This read
142 * can be done outside ctx->wqh.lock because we know that poll_wait
143 * takes that lock (through add_wait_queue) if our caller will sleep.
145 * The read _can_ therefore seep into add_wait_queue's critical
146 * section, but cannot move above it! add_wait_queue's spin_lock acts
147 * as an acquire barrier and ensures that the read be ordered properly
148 * against the writes. The following CAN happen and is safe:
151 * ----------------- ------------
152 * lock ctx->wqh.lock (in poll_wait)
155 * unlock ctx->wqh.lock
158 * if (waitqueue_active)
159 * wake_up_locked_poll
160 * unlock ctx->qwh.lock
161 * eventfd_poll returns 0
163 * but the following, which would miss a wakeup, cannot happen:
166 * ----------------- ------------
167 * count = ctx->count (INVALID!)
170 * **waitqueue_active is false**
171 * **no wake_up_locked_poll!**
172 * unlock ctx->qwh.lock
173 * lock ctx->wqh.lock (in poll_wait)
175 * unlock ctx->wqh.lock
176 * eventfd_poll returns 0
178 count
= READ_ONCE(ctx
->count
);
182 if (count
== ULLONG_MAX
)
184 if (ULLONG_MAX
- 1 > count
)
190 static void eventfd_ctx_do_read(struct eventfd_ctx
*ctx
, __u64
*cnt
)
192 *cnt
= (ctx
->flags
& EFD_SEMAPHORE
) ? 1 : ctx
->count
;
197 * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue.
198 * @ctx: [in] Pointer to eventfd context.
199 * @wait: [in] Wait queue to be removed.
200 * @cnt: [out] Pointer to the 64-bit counter value.
202 * Returns %0 if successful, or the following error codes:
204 * -EAGAIN : The operation would have blocked.
206 * This is used to atomically remove a wait queue entry from the eventfd wait
207 * queue head, and read/reset the counter value.
209 int eventfd_ctx_remove_wait_queue(struct eventfd_ctx
*ctx
, wait_queue_entry_t
*wait
,
214 spin_lock_irqsave(&ctx
->wqh
.lock
, flags
);
215 eventfd_ctx_do_read(ctx
, cnt
);
216 __remove_wait_queue(&ctx
->wqh
, wait
);
217 if (*cnt
!= 0 && waitqueue_active(&ctx
->wqh
))
218 wake_up_locked_poll(&ctx
->wqh
, POLLOUT
);
219 spin_unlock_irqrestore(&ctx
->wqh
.lock
, flags
);
221 return *cnt
!= 0 ? 0 : -EAGAIN
;
223 EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue
);
226 * eventfd_ctx_read - Reads the eventfd counter or wait if it is zero.
227 * @ctx: [in] Pointer to eventfd context.
228 * @no_wait: [in] Different from zero if the operation should not block.
229 * @cnt: [out] Pointer to the 64-bit counter value.
231 * Returns %0 if successful, or the following error codes:
233 * - -EAGAIN : The operation would have blocked but @no_wait was non-zero.
234 * - -ERESTARTSYS : A signal interrupted the wait operation.
236 * If @no_wait is zero, the function might sleep until the eventfd internal
237 * counter becomes greater than zero.
239 ssize_t
eventfd_ctx_read(struct eventfd_ctx
*ctx
, int no_wait
, __u64
*cnt
)
242 DECLARE_WAITQUEUE(wait
, current
);
244 spin_lock_irq(&ctx
->wqh
.lock
);
250 __add_wait_queue(&ctx
->wqh
, &wait
);
252 set_current_state(TASK_INTERRUPTIBLE
);
253 if (ctx
->count
> 0) {
257 if (signal_pending(current
)) {
261 spin_unlock_irq(&ctx
->wqh
.lock
);
263 spin_lock_irq(&ctx
->wqh
.lock
);
265 __remove_wait_queue(&ctx
->wqh
, &wait
);
266 __set_current_state(TASK_RUNNING
);
268 if (likely(res
== 0)) {
269 eventfd_ctx_do_read(ctx
, cnt
);
270 if (waitqueue_active(&ctx
->wqh
))
271 wake_up_locked_poll(&ctx
->wqh
, POLLOUT
);
273 spin_unlock_irq(&ctx
->wqh
.lock
);
277 EXPORT_SYMBOL_GPL(eventfd_ctx_read
);
279 static ssize_t
eventfd_read(struct file
*file
, char __user
*buf
, size_t count
,
282 struct eventfd_ctx
*ctx
= file
->private_data
;
286 if (count
< sizeof(cnt
))
288 res
= eventfd_ctx_read(ctx
, file
->f_flags
& O_NONBLOCK
, &cnt
);
292 return put_user(cnt
, (__u64 __user
*) buf
) ? -EFAULT
: sizeof(cnt
);
295 static ssize_t
eventfd_write(struct file
*file
, const char __user
*buf
, size_t count
,
298 struct eventfd_ctx
*ctx
= file
->private_data
;
301 DECLARE_WAITQUEUE(wait
, current
);
303 if (count
< sizeof(ucnt
))
305 if (copy_from_user(&ucnt
, buf
, sizeof(ucnt
)))
307 if (ucnt
== ULLONG_MAX
)
309 spin_lock_irq(&ctx
->wqh
.lock
);
311 if (ULLONG_MAX
- ctx
->count
> ucnt
)
313 else if (!(file
->f_flags
& O_NONBLOCK
)) {
314 __add_wait_queue(&ctx
->wqh
, &wait
);
316 set_current_state(TASK_INTERRUPTIBLE
);
317 if (ULLONG_MAX
- ctx
->count
> ucnt
) {
321 if (signal_pending(current
)) {
325 spin_unlock_irq(&ctx
->wqh
.lock
);
327 spin_lock_irq(&ctx
->wqh
.lock
);
329 __remove_wait_queue(&ctx
->wqh
, &wait
);
330 __set_current_state(TASK_RUNNING
);
332 if (likely(res
> 0)) {
334 if (waitqueue_active(&ctx
->wqh
))
335 wake_up_locked_poll(&ctx
->wqh
, POLLIN
);
337 spin_unlock_irq(&ctx
->wqh
.lock
);
342 #ifdef CONFIG_PROC_FS
343 static void eventfd_show_fdinfo(struct seq_file
*m
, struct file
*f
)
345 struct eventfd_ctx
*ctx
= f
->private_data
;
347 spin_lock_irq(&ctx
->wqh
.lock
);
348 seq_printf(m
, "eventfd-count: %16llx\n",
349 (unsigned long long)ctx
->count
);
350 spin_unlock_irq(&ctx
->wqh
.lock
);
354 static const struct file_operations eventfd_fops
= {
355 #ifdef CONFIG_PROC_FS
356 .show_fdinfo
= eventfd_show_fdinfo
,
358 .release
= eventfd_release
,
359 .poll
= eventfd_poll
,
360 .read
= eventfd_read
,
361 .write
= eventfd_write
,
362 .llseek
= noop_llseek
,
366 * eventfd_fget - Acquire a reference of an eventfd file descriptor.
367 * @fd: [in] Eventfd file descriptor.
369 * Returns a pointer to the eventfd file structure in case of success, or the
370 * following error pointer:
372 * -EBADF : Invalid @fd file descriptor.
373 * -EINVAL : The @fd file descriptor is not an eventfd file.
375 struct file
*eventfd_fget(int fd
)
381 return ERR_PTR(-EBADF
);
382 if (file
->f_op
!= &eventfd_fops
) {
384 return ERR_PTR(-EINVAL
);
389 EXPORT_SYMBOL_GPL(eventfd_fget
);
392 * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context.
393 * @fd: [in] Eventfd file descriptor.
395 * Returns a pointer to the internal eventfd context, otherwise the error
396 * pointers returned by the following functions:
400 struct eventfd_ctx
*eventfd_ctx_fdget(int fd
)
402 struct eventfd_ctx
*ctx
;
403 struct fd f
= fdget(fd
);
405 return ERR_PTR(-EBADF
);
406 ctx
= eventfd_ctx_fileget(f
.file
);
410 EXPORT_SYMBOL_GPL(eventfd_ctx_fdget
);
413 * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context.
414 * @file: [in] Eventfd file pointer.
416 * Returns a pointer to the internal eventfd context, otherwise the error
419 * -EINVAL : The @fd file descriptor is not an eventfd file.
421 struct eventfd_ctx
*eventfd_ctx_fileget(struct file
*file
)
423 if (file
->f_op
!= &eventfd_fops
)
424 return ERR_PTR(-EINVAL
);
426 return eventfd_ctx_get(file
->private_data
);
428 EXPORT_SYMBOL_GPL(eventfd_ctx_fileget
);
431 * eventfd_file_create - Creates an eventfd file pointer.
432 * @count: Initial eventfd counter value.
433 * @flags: Flags for the eventfd file.
435 * This function creates an eventfd file pointer, w/out installing it into
436 * the fd table. This is useful when the eventfd file is used during the
437 * initialization of data structures that require extra setup after the eventfd
438 * creation. So the eventfd creation is split into the file pointer creation
439 * phase, and the file descriptor installation phase.
440 * In this way races with userspace closing the newly installed file descriptor
442 * Returns an eventfd file pointer, or a proper error pointer.
444 struct file
*eventfd_file_create(unsigned int count
, int flags
)
447 struct eventfd_ctx
*ctx
;
449 /* Check the EFD_* constants for consistency. */
450 BUILD_BUG_ON(EFD_CLOEXEC
!= O_CLOEXEC
);
451 BUILD_BUG_ON(EFD_NONBLOCK
!= O_NONBLOCK
);
453 if (flags
& ~EFD_FLAGS_SET
)
454 return ERR_PTR(-EINVAL
);
456 ctx
= kmalloc(sizeof(*ctx
), GFP_KERNEL
);
458 return ERR_PTR(-ENOMEM
);
460 kref_init(&ctx
->kref
);
461 init_waitqueue_head(&ctx
->wqh
);
465 file
= anon_inode_getfile("[eventfd]", &eventfd_fops
, ctx
,
466 O_RDWR
| (flags
& EFD_SHARED_FCNTL_FLAGS
));
468 eventfd_free_ctx(ctx
);
473 SYSCALL_DEFINE2(eventfd2
, unsigned int, count
, int, flags
)
478 error
= get_unused_fd_flags(flags
& EFD_SHARED_FCNTL_FLAGS
);
483 file
= eventfd_file_create(count
, flags
);
485 error
= PTR_ERR(file
);
486 goto err_put_unused_fd
;
488 fd_install(fd
, file
);
498 SYSCALL_DEFINE1(eventfd
, unsigned int, count
)
500 return sys_eventfd2(count
, 0);