2 * An async IO implementation for Linux
3 * Written by Benjamin LaHaise <bcrl@kvack.org>
5 * Implements an efficient asynchronous io interface.
7 * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved.
8 * Copyright 2018 Christoph Hellwig.
10 * See ../COPYING for licensing terms.
12 #define pr_fmt(fmt) "%s: " fmt, __func__
14 #include <linux/kernel.h>
15 #include <linux/init.h>
16 #include <linux/errno.h>
17 #include <linux/time.h>
18 #include <linux/aio_abi.h>
19 #include <linux/export.h>
20 #include <linux/syscalls.h>
21 #include <linux/backing-dev.h>
22 #include <linux/refcount.h>
23 #include <linux/uio.h>
25 #include <linux/sched/signal.h>
27 #include <linux/file.h>
29 #include <linux/mman.h>
30 #include <linux/mmu_context.h>
31 #include <linux/percpu.h>
32 #include <linux/slab.h>
33 #include <linux/timer.h>
34 #include <linux/aio.h>
35 #include <linux/highmem.h>
36 #include <linux/workqueue.h>
37 #include <linux/security.h>
38 #include <linux/eventfd.h>
39 #include <linux/blkdev.h>
40 #include <linux/compat.h>
41 #include <linux/migrate.h>
42 #include <linux/ramfs.h>
43 #include <linux/percpu-refcount.h>
44 #include <linux/mount.h>
46 #include <asm/kmap_types.h>
47 #include <linux/uaccess.h>
53 #define AIO_RING_MAGIC 0xa10a10a1
54 #define AIO_RING_COMPAT_FEATURES 1
55 #define AIO_RING_INCOMPAT_FEATURES 0
57 unsigned id
; /* kernel internal index number */
58 unsigned nr
; /* number of io_events */
59 unsigned head
; /* Written to by userland or under ring_lock
60 * mutex by aio_read_events_ring(). */
64 unsigned compat_features
;
65 unsigned incompat_features
;
66 unsigned header_length
; /* size of aio_ring */
69 struct io_event io_events
[0];
70 }; /* 128 bytes + ring size */
72 #define AIO_RING_PAGES 8
77 struct kioctx __rcu
*table
[];
81 unsigned reqs_available
;
85 struct completion comp
;
90 struct percpu_ref users
;
93 struct percpu_ref reqs
;
95 unsigned long user_id
;
97 struct __percpu kioctx_cpu
*cpu
;
100 * For percpu reqs_available, number of slots we move to/from global
105 * This is what userspace passed to io_setup(), it's not used for
106 * anything but counting against the global max_reqs quota.
108 * The real limit is nr_events - 1, which will be larger (see
113 /* Size of ringbuffer, in units of struct io_event */
116 unsigned long mmap_base
;
117 unsigned long mmap_size
;
119 struct page
**ring_pages
;
122 struct rcu_work free_rwork
; /* see free_ioctx() */
125 * signals when all in-flight requests are done
127 struct ctx_rq_wait
*rq_wait
;
131 * This counts the number of available slots in the ringbuffer,
132 * so we avoid overflowing it: it's decremented (if positive)
133 * when allocating a kiocb and incremented when the resulting
134 * io_event is pulled off the ringbuffer.
136 * We batch accesses to it with a percpu version.
138 atomic_t reqs_available
;
139 } ____cacheline_aligned_in_smp
;
143 struct list_head active_reqs
; /* used for cancellation */
144 } ____cacheline_aligned_in_smp
;
147 struct mutex ring_lock
;
148 wait_queue_head_t wait
;
149 } ____cacheline_aligned_in_smp
;
153 unsigned completed_events
;
154 spinlock_t completion_lock
;
155 } ____cacheline_aligned_in_smp
;
157 struct page
*internal_pages
[AIO_RING_PAGES
];
158 struct file
*aio_ring_file
;
164 struct work_struct work
;
171 struct wait_queue_head
*head
;
175 struct wait_queue_entry wait
;
176 struct work_struct work
;
182 struct fsync_iocb fsync
;
183 struct poll_iocb poll
;
186 struct kioctx
*ki_ctx
;
187 kiocb_cancel_fn
*ki_cancel
;
189 struct iocb __user
*ki_user_iocb
; /* user's aiocb */
190 __u64 ki_user_data
; /* user's data for completion */
192 struct list_head ki_list
; /* the aio core uses this
193 * for cancellation */
194 refcount_t ki_refcnt
;
197 * If the aio_resfd field of the userspace iocb is not zero,
198 * this is the underlying eventfd context to deliver events to.
200 struct eventfd_ctx
*ki_eventfd
;
203 /*------ sysctl variables----*/
204 static DEFINE_SPINLOCK(aio_nr_lock
);
205 unsigned long aio_nr
; /* current system wide number of aio requests */
206 unsigned long aio_max_nr
= 0x10000; /* system wide maximum number of aio requests */
207 /*----end sysctl variables---*/
209 static struct kmem_cache
*kiocb_cachep
;
210 static struct kmem_cache
*kioctx_cachep
;
212 static struct vfsmount
*aio_mnt
;
214 static const struct file_operations aio_ring_fops
;
215 static const struct address_space_operations aio_ctx_aops
;
217 static struct file
*aio_private_file(struct kioctx
*ctx
, loff_t nr_pages
)
220 struct inode
*inode
= alloc_anon_inode(aio_mnt
->mnt_sb
);
222 return ERR_CAST(inode
);
224 inode
->i_mapping
->a_ops
= &aio_ctx_aops
;
225 inode
->i_mapping
->private_data
= ctx
;
226 inode
->i_size
= PAGE_SIZE
* nr_pages
;
228 file
= alloc_file_pseudo(inode
, aio_mnt
, "[aio]",
229 O_RDWR
, &aio_ring_fops
);
235 static struct dentry
*aio_mount(struct file_system_type
*fs_type
,
236 int flags
, const char *dev_name
, void *data
)
238 struct dentry
*root
= mount_pseudo(fs_type
, "aio:", NULL
, NULL
,
242 root
->d_sb
->s_iflags
|= SB_I_NOEXEC
;
247 * Creates the slab caches used by the aio routines, panic on
248 * failure as this is done early during the boot sequence.
250 static int __init
aio_setup(void)
252 static struct file_system_type aio_fs
= {
255 .kill_sb
= kill_anon_super
,
257 aio_mnt
= kern_mount(&aio_fs
);
259 panic("Failed to create aio fs mount.");
261 kiocb_cachep
= KMEM_CACHE(aio_kiocb
, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
);
262 kioctx_cachep
= KMEM_CACHE(kioctx
,SLAB_HWCACHE_ALIGN
|SLAB_PANIC
);
265 __initcall(aio_setup
);
267 static void put_aio_ring_file(struct kioctx
*ctx
)
269 struct file
*aio_ring_file
= ctx
->aio_ring_file
;
270 struct address_space
*i_mapping
;
273 truncate_setsize(file_inode(aio_ring_file
), 0);
275 /* Prevent further access to the kioctx from migratepages */
276 i_mapping
= aio_ring_file
->f_mapping
;
277 spin_lock(&i_mapping
->private_lock
);
278 i_mapping
->private_data
= NULL
;
279 ctx
->aio_ring_file
= NULL
;
280 spin_unlock(&i_mapping
->private_lock
);
286 static void aio_free_ring(struct kioctx
*ctx
)
290 /* Disconnect the kiotx from the ring file. This prevents future
291 * accesses to the kioctx from page migration.
293 put_aio_ring_file(ctx
);
295 for (i
= 0; i
< ctx
->nr_pages
; i
++) {
297 pr_debug("pid(%d) [%d] page->count=%d\n", current
->pid
, i
,
298 page_count(ctx
->ring_pages
[i
]));
299 page
= ctx
->ring_pages
[i
];
302 ctx
->ring_pages
[i
] = NULL
;
306 if (ctx
->ring_pages
&& ctx
->ring_pages
!= ctx
->internal_pages
) {
307 kfree(ctx
->ring_pages
);
308 ctx
->ring_pages
= NULL
;
312 static int aio_ring_mremap(struct vm_area_struct
*vma
)
314 struct file
*file
= vma
->vm_file
;
315 struct mm_struct
*mm
= vma
->vm_mm
;
316 struct kioctx_table
*table
;
317 int i
, res
= -EINVAL
;
319 spin_lock(&mm
->ioctx_lock
);
321 table
= rcu_dereference(mm
->ioctx_table
);
322 for (i
= 0; i
< table
->nr
; i
++) {
325 ctx
= rcu_dereference(table
->table
[i
]);
326 if (ctx
&& ctx
->aio_ring_file
== file
) {
327 if (!atomic_read(&ctx
->dead
)) {
328 ctx
->user_id
= ctx
->mmap_base
= vma
->vm_start
;
336 spin_unlock(&mm
->ioctx_lock
);
340 static const struct vm_operations_struct aio_ring_vm_ops
= {
341 .mremap
= aio_ring_mremap
,
342 #if IS_ENABLED(CONFIG_MMU)
343 .fault
= filemap_fault
,
344 .map_pages
= filemap_map_pages
,
345 .page_mkwrite
= filemap_page_mkwrite
,
349 static int aio_ring_mmap(struct file
*file
, struct vm_area_struct
*vma
)
351 vma
->vm_flags
|= VM_DONTEXPAND
;
352 vma
->vm_ops
= &aio_ring_vm_ops
;
356 static const struct file_operations aio_ring_fops
= {
357 .mmap
= aio_ring_mmap
,
360 #if IS_ENABLED(CONFIG_MIGRATION)
361 static int aio_migratepage(struct address_space
*mapping
, struct page
*new,
362 struct page
*old
, enum migrate_mode mode
)
370 * We cannot support the _NO_COPY case here, because copy needs to
371 * happen under the ctx->completion_lock. That does not work with the
372 * migration workflow of MIGRATE_SYNC_NO_COPY.
374 if (mode
== MIGRATE_SYNC_NO_COPY
)
379 /* mapping->private_lock here protects against the kioctx teardown. */
380 spin_lock(&mapping
->private_lock
);
381 ctx
= mapping
->private_data
;
387 /* The ring_lock mutex. The prevents aio_read_events() from writing
388 * to the ring's head, and prevents page migration from mucking in
389 * a partially initialized kiotx.
391 if (!mutex_trylock(&ctx
->ring_lock
)) {
397 if (idx
< (pgoff_t
)ctx
->nr_pages
) {
398 /* Make sure the old page hasn't already been changed */
399 if (ctx
->ring_pages
[idx
] != old
)
407 /* Writeback must be complete */
408 BUG_ON(PageWriteback(old
));
411 rc
= migrate_page_move_mapping(mapping
, new, old
, NULL
, mode
, 1);
412 if (rc
!= MIGRATEPAGE_SUCCESS
) {
417 /* Take completion_lock to prevent other writes to the ring buffer
418 * while the old page is copied to the new. This prevents new
419 * events from being lost.
421 spin_lock_irqsave(&ctx
->completion_lock
, flags
);
422 migrate_page_copy(new, old
);
423 BUG_ON(ctx
->ring_pages
[idx
] != old
);
424 ctx
->ring_pages
[idx
] = new;
425 spin_unlock_irqrestore(&ctx
->completion_lock
, flags
);
427 /* The old page is no longer accessible. */
431 mutex_unlock(&ctx
->ring_lock
);
433 spin_unlock(&mapping
->private_lock
);
438 static const struct address_space_operations aio_ctx_aops
= {
439 .set_page_dirty
= __set_page_dirty_no_writeback
,
440 #if IS_ENABLED(CONFIG_MIGRATION)
441 .migratepage
= aio_migratepage
,
445 static int aio_setup_ring(struct kioctx
*ctx
, unsigned int nr_events
)
447 struct aio_ring
*ring
;
448 struct mm_struct
*mm
= current
->mm
;
449 unsigned long size
, unused
;
454 /* Compensate for the ring buffer's head/tail overlap entry */
455 nr_events
+= 2; /* 1 is required, 2 for good luck */
457 size
= sizeof(struct aio_ring
);
458 size
+= sizeof(struct io_event
) * nr_events
;
460 nr_pages
= PFN_UP(size
);
464 file
= aio_private_file(ctx
, nr_pages
);
466 ctx
->aio_ring_file
= NULL
;
470 ctx
->aio_ring_file
= file
;
471 nr_events
= (PAGE_SIZE
* nr_pages
- sizeof(struct aio_ring
))
472 / sizeof(struct io_event
);
474 ctx
->ring_pages
= ctx
->internal_pages
;
475 if (nr_pages
> AIO_RING_PAGES
) {
476 ctx
->ring_pages
= kcalloc(nr_pages
, sizeof(struct page
*),
478 if (!ctx
->ring_pages
) {
479 put_aio_ring_file(ctx
);
484 for (i
= 0; i
< nr_pages
; i
++) {
486 page
= find_or_create_page(file
->f_mapping
,
487 i
, GFP_HIGHUSER
| __GFP_ZERO
);
490 pr_debug("pid(%d) page[%d]->count=%d\n",
491 current
->pid
, i
, page_count(page
));
492 SetPageUptodate(page
);
495 ctx
->ring_pages
[i
] = page
;
499 if (unlikely(i
!= nr_pages
)) {
504 ctx
->mmap_size
= nr_pages
* PAGE_SIZE
;
505 pr_debug("attempting mmap of %lu bytes\n", ctx
->mmap_size
);
507 if (down_write_killable(&mm
->mmap_sem
)) {
513 ctx
->mmap_base
= do_mmap_pgoff(ctx
->aio_ring_file
, 0, ctx
->mmap_size
,
514 PROT_READ
| PROT_WRITE
,
515 MAP_SHARED
, 0, &unused
, NULL
);
516 up_write(&mm
->mmap_sem
);
517 if (IS_ERR((void *)ctx
->mmap_base
)) {
523 pr_debug("mmap address: 0x%08lx\n", ctx
->mmap_base
);
525 ctx
->user_id
= ctx
->mmap_base
;
526 ctx
->nr_events
= nr_events
; /* trusted copy */
528 ring
= kmap_atomic(ctx
->ring_pages
[0]);
529 ring
->nr
= nr_events
; /* user copy */
531 ring
->head
= ring
->tail
= 0;
532 ring
->magic
= AIO_RING_MAGIC
;
533 ring
->compat_features
= AIO_RING_COMPAT_FEATURES
;
534 ring
->incompat_features
= AIO_RING_INCOMPAT_FEATURES
;
535 ring
->header_length
= sizeof(struct aio_ring
);
537 flush_dcache_page(ctx
->ring_pages
[0]);
542 #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
543 #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
544 #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
546 void kiocb_set_cancel_fn(struct kiocb
*iocb
, kiocb_cancel_fn
*cancel
)
548 struct aio_kiocb
*req
= container_of(iocb
, struct aio_kiocb
, rw
);
549 struct kioctx
*ctx
= req
->ki_ctx
;
552 if (WARN_ON_ONCE(!list_empty(&req
->ki_list
)))
555 spin_lock_irqsave(&ctx
->ctx_lock
, flags
);
556 list_add_tail(&req
->ki_list
, &ctx
->active_reqs
);
557 req
->ki_cancel
= cancel
;
558 spin_unlock_irqrestore(&ctx
->ctx_lock
, flags
);
560 EXPORT_SYMBOL(kiocb_set_cancel_fn
);
563 * free_ioctx() should be RCU delayed to synchronize against the RCU
564 * protected lookup_ioctx() and also needs process context to call
565 * aio_free_ring(). Use rcu_work.
567 static void free_ioctx(struct work_struct
*work
)
569 struct kioctx
*ctx
= container_of(to_rcu_work(work
), struct kioctx
,
571 pr_debug("freeing %p\n", ctx
);
574 free_percpu(ctx
->cpu
);
575 percpu_ref_exit(&ctx
->reqs
);
576 percpu_ref_exit(&ctx
->users
);
577 kmem_cache_free(kioctx_cachep
, ctx
);
580 static void free_ioctx_reqs(struct percpu_ref
*ref
)
582 struct kioctx
*ctx
= container_of(ref
, struct kioctx
, reqs
);
584 /* At this point we know that there are no any in-flight requests */
585 if (ctx
->rq_wait
&& atomic_dec_and_test(&ctx
->rq_wait
->count
))
586 complete(&ctx
->rq_wait
->comp
);
588 /* Synchronize against RCU protected table->table[] dereferences */
589 INIT_RCU_WORK(&ctx
->free_rwork
, free_ioctx
);
590 queue_rcu_work(system_wq
, &ctx
->free_rwork
);
594 * When this function runs, the kioctx has been removed from the "hash table"
595 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
596 * now it's safe to cancel any that need to be.
598 static void free_ioctx_users(struct percpu_ref
*ref
)
600 struct kioctx
*ctx
= container_of(ref
, struct kioctx
, users
);
601 struct aio_kiocb
*req
;
603 spin_lock_irq(&ctx
->ctx_lock
);
605 while (!list_empty(&ctx
->active_reqs
)) {
606 req
= list_first_entry(&ctx
->active_reqs
,
607 struct aio_kiocb
, ki_list
);
608 req
->ki_cancel(&req
->rw
);
609 list_del_init(&req
->ki_list
);
612 spin_unlock_irq(&ctx
->ctx_lock
);
614 percpu_ref_kill(&ctx
->reqs
);
615 percpu_ref_put(&ctx
->reqs
);
618 static int ioctx_add_table(struct kioctx
*ctx
, struct mm_struct
*mm
)
621 struct kioctx_table
*table
, *old
;
622 struct aio_ring
*ring
;
624 spin_lock(&mm
->ioctx_lock
);
625 table
= rcu_dereference_raw(mm
->ioctx_table
);
629 for (i
= 0; i
< table
->nr
; i
++)
630 if (!rcu_access_pointer(table
->table
[i
])) {
632 rcu_assign_pointer(table
->table
[i
], ctx
);
633 spin_unlock(&mm
->ioctx_lock
);
635 /* While kioctx setup is in progress,
636 * we are protected from page migration
637 * changes ring_pages by ->ring_lock.
639 ring
= kmap_atomic(ctx
->ring_pages
[0]);
645 new_nr
= (table
? table
->nr
: 1) * 4;
646 spin_unlock(&mm
->ioctx_lock
);
648 table
= kzalloc(sizeof(*table
) + sizeof(struct kioctx
*) *
655 spin_lock(&mm
->ioctx_lock
);
656 old
= rcu_dereference_raw(mm
->ioctx_table
);
659 rcu_assign_pointer(mm
->ioctx_table
, table
);
660 } else if (table
->nr
> old
->nr
) {
661 memcpy(table
->table
, old
->table
,
662 old
->nr
* sizeof(struct kioctx
*));
664 rcu_assign_pointer(mm
->ioctx_table
, table
);
673 static void aio_nr_sub(unsigned nr
)
675 spin_lock(&aio_nr_lock
);
676 if (WARN_ON(aio_nr
- nr
> aio_nr
))
680 spin_unlock(&aio_nr_lock
);
684 * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
686 static struct kioctx
*ioctx_alloc(unsigned nr_events
)
688 struct mm_struct
*mm
= current
->mm
;
693 * Store the original nr_events -- what userspace passed to io_setup(),
694 * for counting against the global limit -- before it changes.
696 unsigned int max_reqs
= nr_events
;
699 * We keep track of the number of available ringbuffer slots, to prevent
700 * overflow (reqs_available), and we also use percpu counters for this.
702 * So since up to half the slots might be on other cpu's percpu counters
703 * and unavailable, double nr_events so userspace sees what they
704 * expected: additionally, we move req_batch slots to/from percpu
705 * counters at a time, so make sure that isn't 0:
707 nr_events
= max(nr_events
, num_possible_cpus() * 4);
710 /* Prevent overflows */
711 if (nr_events
> (0x10000000U
/ sizeof(struct io_event
))) {
712 pr_debug("ENOMEM: nr_events too high\n");
713 return ERR_PTR(-EINVAL
);
716 if (!nr_events
|| (unsigned long)max_reqs
> aio_max_nr
)
717 return ERR_PTR(-EAGAIN
);
719 ctx
= kmem_cache_zalloc(kioctx_cachep
, GFP_KERNEL
);
721 return ERR_PTR(-ENOMEM
);
723 ctx
->max_reqs
= max_reqs
;
725 spin_lock_init(&ctx
->ctx_lock
);
726 spin_lock_init(&ctx
->completion_lock
);
727 mutex_init(&ctx
->ring_lock
);
728 /* Protect against page migration throughout kiotx setup by keeping
729 * the ring_lock mutex held until setup is complete. */
730 mutex_lock(&ctx
->ring_lock
);
731 init_waitqueue_head(&ctx
->wait
);
733 INIT_LIST_HEAD(&ctx
->active_reqs
);
735 if (percpu_ref_init(&ctx
->users
, free_ioctx_users
, 0, GFP_KERNEL
))
738 if (percpu_ref_init(&ctx
->reqs
, free_ioctx_reqs
, 0, GFP_KERNEL
))
741 ctx
->cpu
= alloc_percpu(struct kioctx_cpu
);
745 err
= aio_setup_ring(ctx
, nr_events
);
749 atomic_set(&ctx
->reqs_available
, ctx
->nr_events
- 1);
750 ctx
->req_batch
= (ctx
->nr_events
- 1) / (num_possible_cpus() * 4);
751 if (ctx
->req_batch
< 1)
754 /* limit the number of system wide aios */
755 spin_lock(&aio_nr_lock
);
756 if (aio_nr
+ ctx
->max_reqs
> aio_max_nr
||
757 aio_nr
+ ctx
->max_reqs
< aio_nr
) {
758 spin_unlock(&aio_nr_lock
);
762 aio_nr
+= ctx
->max_reqs
;
763 spin_unlock(&aio_nr_lock
);
765 percpu_ref_get(&ctx
->users
); /* io_setup() will drop this ref */
766 percpu_ref_get(&ctx
->reqs
); /* free_ioctx_users() will drop this */
768 err
= ioctx_add_table(ctx
, mm
);
772 /* Release the ring_lock mutex now that all setup is complete. */
773 mutex_unlock(&ctx
->ring_lock
);
775 pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
776 ctx
, ctx
->user_id
, mm
, ctx
->nr_events
);
780 aio_nr_sub(ctx
->max_reqs
);
782 atomic_set(&ctx
->dead
, 1);
784 vm_munmap(ctx
->mmap_base
, ctx
->mmap_size
);
787 mutex_unlock(&ctx
->ring_lock
);
788 free_percpu(ctx
->cpu
);
789 percpu_ref_exit(&ctx
->reqs
);
790 percpu_ref_exit(&ctx
->users
);
791 kmem_cache_free(kioctx_cachep
, ctx
);
792 pr_debug("error allocating ioctx %d\n", err
);
797 * Cancels all outstanding aio requests on an aio context. Used
798 * when the processes owning a context have all exited to encourage
799 * the rapid destruction of the kioctx.
801 static int kill_ioctx(struct mm_struct
*mm
, struct kioctx
*ctx
,
802 struct ctx_rq_wait
*wait
)
804 struct kioctx_table
*table
;
806 spin_lock(&mm
->ioctx_lock
);
807 if (atomic_xchg(&ctx
->dead
, 1)) {
808 spin_unlock(&mm
->ioctx_lock
);
812 table
= rcu_dereference_raw(mm
->ioctx_table
);
813 WARN_ON(ctx
!= rcu_access_pointer(table
->table
[ctx
->id
]));
814 RCU_INIT_POINTER(table
->table
[ctx
->id
], NULL
);
815 spin_unlock(&mm
->ioctx_lock
);
817 /* free_ioctx_reqs() will do the necessary RCU synchronization */
818 wake_up_all(&ctx
->wait
);
821 * It'd be more correct to do this in free_ioctx(), after all
822 * the outstanding kiocbs have finished - but by then io_destroy
823 * has already returned, so io_setup() could potentially return
824 * -EAGAIN with no ioctxs actually in use (as far as userspace
827 aio_nr_sub(ctx
->max_reqs
);
830 vm_munmap(ctx
->mmap_base
, ctx
->mmap_size
);
833 percpu_ref_kill(&ctx
->users
);
838 * exit_aio: called when the last user of mm goes away. At this point, there is
839 * no way for any new requests to be submited or any of the io_* syscalls to be
840 * called on the context.
842 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
845 void exit_aio(struct mm_struct
*mm
)
847 struct kioctx_table
*table
= rcu_dereference_raw(mm
->ioctx_table
);
848 struct ctx_rq_wait wait
;
854 atomic_set(&wait
.count
, table
->nr
);
855 init_completion(&wait
.comp
);
858 for (i
= 0; i
< table
->nr
; ++i
) {
860 rcu_dereference_protected(table
->table
[i
], true);
868 * We don't need to bother with munmap() here - exit_mmap(mm)
869 * is coming and it'll unmap everything. And we simply can't,
870 * this is not necessarily our ->mm.
871 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
872 * that it needs to unmap the area, just set it to 0.
875 kill_ioctx(mm
, ctx
, &wait
);
878 if (!atomic_sub_and_test(skipped
, &wait
.count
)) {
879 /* Wait until all IO for the context are done. */
880 wait_for_completion(&wait
.comp
);
883 RCU_INIT_POINTER(mm
->ioctx_table
, NULL
);
887 static void put_reqs_available(struct kioctx
*ctx
, unsigned nr
)
889 struct kioctx_cpu
*kcpu
;
892 local_irq_save(flags
);
893 kcpu
= this_cpu_ptr(ctx
->cpu
);
894 kcpu
->reqs_available
+= nr
;
896 while (kcpu
->reqs_available
>= ctx
->req_batch
* 2) {
897 kcpu
->reqs_available
-= ctx
->req_batch
;
898 atomic_add(ctx
->req_batch
, &ctx
->reqs_available
);
901 local_irq_restore(flags
);
904 static bool get_reqs_available(struct kioctx
*ctx
)
906 struct kioctx_cpu
*kcpu
;
910 local_irq_save(flags
);
911 kcpu
= this_cpu_ptr(ctx
->cpu
);
912 if (!kcpu
->reqs_available
) {
913 int old
, avail
= atomic_read(&ctx
->reqs_available
);
916 if (avail
< ctx
->req_batch
)
920 avail
= atomic_cmpxchg(&ctx
->reqs_available
,
921 avail
, avail
- ctx
->req_batch
);
922 } while (avail
!= old
);
924 kcpu
->reqs_available
+= ctx
->req_batch
;
928 kcpu
->reqs_available
--;
930 local_irq_restore(flags
);
934 /* refill_reqs_available
935 * Updates the reqs_available reference counts used for tracking the
936 * number of free slots in the completion ring. This can be called
937 * from aio_complete() (to optimistically update reqs_available) or
938 * from aio_get_req() (the we're out of events case). It must be
939 * called holding ctx->completion_lock.
941 static void refill_reqs_available(struct kioctx
*ctx
, unsigned head
,
944 unsigned events_in_ring
, completed
;
946 /* Clamp head since userland can write to it. */
947 head
%= ctx
->nr_events
;
949 events_in_ring
= tail
- head
;
951 events_in_ring
= ctx
->nr_events
- (head
- tail
);
953 completed
= ctx
->completed_events
;
954 if (events_in_ring
< completed
)
955 completed
-= events_in_ring
;
962 ctx
->completed_events
-= completed
;
963 put_reqs_available(ctx
, completed
);
966 /* user_refill_reqs_available
967 * Called to refill reqs_available when aio_get_req() encounters an
968 * out of space in the completion ring.
970 static void user_refill_reqs_available(struct kioctx
*ctx
)
972 spin_lock_irq(&ctx
->completion_lock
);
973 if (ctx
->completed_events
) {
974 struct aio_ring
*ring
;
977 /* Access of ring->head may race with aio_read_events_ring()
978 * here, but that's okay since whether we read the old version
979 * or the new version, and either will be valid. The important
980 * part is that head cannot pass tail since we prevent
981 * aio_complete() from updating tail by holding
982 * ctx->completion_lock. Even if head is invalid, the check
983 * against ctx->completed_events below will make sure we do the
986 ring
= kmap_atomic(ctx
->ring_pages
[0]);
990 refill_reqs_available(ctx
, head
, ctx
->tail
);
993 spin_unlock_irq(&ctx
->completion_lock
);
997 * Allocate a slot for an aio request.
998 * Returns NULL if no requests are free.
1000 static inline struct aio_kiocb
*aio_get_req(struct kioctx
*ctx
)
1002 struct aio_kiocb
*req
;
1004 if (!get_reqs_available(ctx
)) {
1005 user_refill_reqs_available(ctx
);
1006 if (!get_reqs_available(ctx
))
1010 req
= kmem_cache_alloc(kiocb_cachep
, GFP_KERNEL
|__GFP_ZERO
);
1014 percpu_ref_get(&ctx
->reqs
);
1015 INIT_LIST_HEAD(&req
->ki_list
);
1016 refcount_set(&req
->ki_refcnt
, 0);
1020 put_reqs_available(ctx
, 1);
1024 static struct kioctx
*lookup_ioctx(unsigned long ctx_id
)
1026 struct aio_ring __user
*ring
= (void __user
*)ctx_id
;
1027 struct mm_struct
*mm
= current
->mm
;
1028 struct kioctx
*ctx
, *ret
= NULL
;
1029 struct kioctx_table
*table
;
1032 if (get_user(id
, &ring
->id
))
1036 table
= rcu_dereference(mm
->ioctx_table
);
1038 if (!table
|| id
>= table
->nr
)
1041 ctx
= rcu_dereference(table
->table
[id
]);
1042 if (ctx
&& ctx
->user_id
== ctx_id
) {
1043 if (percpu_ref_tryget_live(&ctx
->users
))
1051 static inline void iocb_put(struct aio_kiocb
*iocb
)
1053 if (refcount_read(&iocb
->ki_refcnt
) == 0 ||
1054 refcount_dec_and_test(&iocb
->ki_refcnt
)) {
1055 percpu_ref_put(&iocb
->ki_ctx
->reqs
);
1056 kmem_cache_free(kiocb_cachep
, iocb
);
1061 * Called when the io request on the given iocb is complete.
1063 static void aio_complete(struct aio_kiocb
*iocb
, long res
, long res2
)
1065 struct kioctx
*ctx
= iocb
->ki_ctx
;
1066 struct aio_ring
*ring
;
1067 struct io_event
*ev_page
, *event
;
1068 unsigned tail
, pos
, head
;
1069 unsigned long flags
;
1072 * Add a completion event to the ring buffer. Must be done holding
1073 * ctx->completion_lock to prevent other code from messing with the tail
1074 * pointer since we might be called from irq context.
1076 spin_lock_irqsave(&ctx
->completion_lock
, flags
);
1079 pos
= tail
+ AIO_EVENTS_OFFSET
;
1081 if (++tail
>= ctx
->nr_events
)
1084 ev_page
= kmap_atomic(ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
]);
1085 event
= ev_page
+ pos
% AIO_EVENTS_PER_PAGE
;
1087 event
->obj
= (u64
)(unsigned long)iocb
->ki_user_iocb
;
1088 event
->data
= iocb
->ki_user_data
;
1092 kunmap_atomic(ev_page
);
1093 flush_dcache_page(ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
]);
1095 pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
1096 ctx
, tail
, iocb
, iocb
->ki_user_iocb
, iocb
->ki_user_data
,
1099 /* after flagging the request as done, we
1100 * must never even look at it again
1102 smp_wmb(); /* make event visible before updating tail */
1106 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1109 kunmap_atomic(ring
);
1110 flush_dcache_page(ctx
->ring_pages
[0]);
1112 ctx
->completed_events
++;
1113 if (ctx
->completed_events
> 1)
1114 refill_reqs_available(ctx
, head
, tail
);
1115 spin_unlock_irqrestore(&ctx
->completion_lock
, flags
);
1117 pr_debug("added to ring %p at [%u]\n", iocb
, tail
);
1120 * Check if the user asked us to deliver the result through an
1121 * eventfd. The eventfd_signal() function is safe to be called
1124 if (iocb
->ki_eventfd
) {
1125 eventfd_signal(iocb
->ki_eventfd
, 1);
1126 eventfd_ctx_put(iocb
->ki_eventfd
);
1130 * We have to order our ring_info tail store above and test
1131 * of the wait list below outside the wait lock. This is
1132 * like in wake_up_bit() where clearing a bit has to be
1133 * ordered with the unlocked test.
1137 if (waitqueue_active(&ctx
->wait
))
1138 wake_up(&ctx
->wait
);
1142 /* aio_read_events_ring
1143 * Pull an event off of the ioctx's event ring. Returns the number of
1146 static long aio_read_events_ring(struct kioctx
*ctx
,
1147 struct io_event __user
*event
, long nr
)
1149 struct aio_ring
*ring
;
1150 unsigned head
, tail
, pos
;
1155 * The mutex can block and wake us up and that will cause
1156 * wait_event_interruptible_hrtimeout() to schedule without sleeping
1157 * and repeat. This should be rare enough that it doesn't cause
1158 * peformance issues. See the comment in read_events() for more detail.
1160 sched_annotate_sleep();
1161 mutex_lock(&ctx
->ring_lock
);
1163 /* Access to ->ring_pages here is protected by ctx->ring_lock. */
1164 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1167 kunmap_atomic(ring
);
1170 * Ensure that once we've read the current tail pointer, that
1171 * we also see the events that were stored up to the tail.
1175 pr_debug("h%u t%u m%u\n", head
, tail
, ctx
->nr_events
);
1180 head
%= ctx
->nr_events
;
1181 tail
%= ctx
->nr_events
;
1185 struct io_event
*ev
;
1188 avail
= (head
<= tail
? tail
: ctx
->nr_events
) - head
;
1192 pos
= head
+ AIO_EVENTS_OFFSET
;
1193 page
= ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
];
1194 pos
%= AIO_EVENTS_PER_PAGE
;
1196 avail
= min(avail
, nr
- ret
);
1197 avail
= min_t(long, avail
, AIO_EVENTS_PER_PAGE
- pos
);
1200 copy_ret
= copy_to_user(event
+ ret
, ev
+ pos
,
1201 sizeof(*ev
) * avail
);
1204 if (unlikely(copy_ret
)) {
1211 head
%= ctx
->nr_events
;
1214 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1216 kunmap_atomic(ring
);
1217 flush_dcache_page(ctx
->ring_pages
[0]);
1219 pr_debug("%li h%u t%u\n", ret
, head
, tail
);
1221 mutex_unlock(&ctx
->ring_lock
);
1226 static bool aio_read_events(struct kioctx
*ctx
, long min_nr
, long nr
,
1227 struct io_event __user
*event
, long *i
)
1229 long ret
= aio_read_events_ring(ctx
, event
+ *i
, nr
- *i
);
1234 if (unlikely(atomic_read(&ctx
->dead
)))
1240 return ret
< 0 || *i
>= min_nr
;
1243 static long read_events(struct kioctx
*ctx
, long min_nr
, long nr
,
1244 struct io_event __user
*event
,
1250 * Note that aio_read_events() is being called as the conditional - i.e.
1251 * we're calling it after prepare_to_wait() has set task state to
1252 * TASK_INTERRUPTIBLE.
1254 * But aio_read_events() can block, and if it blocks it's going to flip
1255 * the task state back to TASK_RUNNING.
1257 * This should be ok, provided it doesn't flip the state back to
1258 * TASK_RUNNING and return 0 too much - that causes us to spin. That
1259 * will only happen if the mutex_lock() call blocks, and we then find
1260 * the ringbuffer empty. So in practice we should be ok, but it's
1261 * something to be aware of when touching this code.
1264 aio_read_events(ctx
, min_nr
, nr
, event
, &ret
);
1266 wait_event_interruptible_hrtimeout(ctx
->wait
,
1267 aio_read_events(ctx
, min_nr
, nr
, event
, &ret
),
1273 * Create an aio_context capable of receiving at least nr_events.
1274 * ctxp must not point to an aio_context that already exists, and
1275 * must be initialized to 0 prior to the call. On successful
1276 * creation of the aio_context, *ctxp is filled in with the resulting
1277 * handle. May fail with -EINVAL if *ctxp is not initialized,
1278 * if the specified nr_events exceeds internal limits. May fail
1279 * with -EAGAIN if the specified nr_events exceeds the user's limit
1280 * of available events. May fail with -ENOMEM if insufficient kernel
1281 * resources are available. May fail with -EFAULT if an invalid
1282 * pointer is passed for ctxp. Will fail with -ENOSYS if not
1285 SYSCALL_DEFINE2(io_setup
, unsigned, nr_events
, aio_context_t __user
*, ctxp
)
1287 struct kioctx
*ioctx
= NULL
;
1291 ret
= get_user(ctx
, ctxp
);
1296 if (unlikely(ctx
|| nr_events
== 0)) {
1297 pr_debug("EINVAL: ctx %lu nr_events %u\n",
1302 ioctx
= ioctx_alloc(nr_events
);
1303 ret
= PTR_ERR(ioctx
);
1304 if (!IS_ERR(ioctx
)) {
1305 ret
= put_user(ioctx
->user_id
, ctxp
);
1307 kill_ioctx(current
->mm
, ioctx
, NULL
);
1308 percpu_ref_put(&ioctx
->users
);
1315 #ifdef CONFIG_COMPAT
1316 COMPAT_SYSCALL_DEFINE2(io_setup
, unsigned, nr_events
, u32 __user
*, ctx32p
)
1318 struct kioctx
*ioctx
= NULL
;
1322 ret
= get_user(ctx
, ctx32p
);
1327 if (unlikely(ctx
|| nr_events
== 0)) {
1328 pr_debug("EINVAL: ctx %lu nr_events %u\n",
1333 ioctx
= ioctx_alloc(nr_events
);
1334 ret
= PTR_ERR(ioctx
);
1335 if (!IS_ERR(ioctx
)) {
1336 /* truncating is ok because it's a user address */
1337 ret
= put_user((u32
)ioctx
->user_id
, ctx32p
);
1339 kill_ioctx(current
->mm
, ioctx
, NULL
);
1340 percpu_ref_put(&ioctx
->users
);
1349 * Destroy the aio_context specified. May cancel any outstanding
1350 * AIOs and block on completion. Will fail with -ENOSYS if not
1351 * implemented. May fail with -EINVAL if the context pointed to
1354 SYSCALL_DEFINE1(io_destroy
, aio_context_t
, ctx
)
1356 struct kioctx
*ioctx
= lookup_ioctx(ctx
);
1357 if (likely(NULL
!= ioctx
)) {
1358 struct ctx_rq_wait wait
;
1361 init_completion(&wait
.comp
);
1362 atomic_set(&wait
.count
, 1);
1364 /* Pass requests_done to kill_ioctx() where it can be set
1365 * in a thread-safe way. If we try to set it here then we have
1366 * a race condition if two io_destroy() called simultaneously.
1368 ret
= kill_ioctx(current
->mm
, ioctx
, &wait
);
1369 percpu_ref_put(&ioctx
->users
);
1371 /* Wait until all IO for the context are done. Otherwise kernel
1372 * keep using user-space buffers even if user thinks the context
1376 wait_for_completion(&wait
.comp
);
1380 pr_debug("EINVAL: invalid context id\n");
1384 static void aio_remove_iocb(struct aio_kiocb
*iocb
)
1386 struct kioctx
*ctx
= iocb
->ki_ctx
;
1387 unsigned long flags
;
1389 spin_lock_irqsave(&ctx
->ctx_lock
, flags
);
1390 list_del(&iocb
->ki_list
);
1391 spin_unlock_irqrestore(&ctx
->ctx_lock
, flags
);
1394 static void aio_complete_rw(struct kiocb
*kiocb
, long res
, long res2
)
1396 struct aio_kiocb
*iocb
= container_of(kiocb
, struct aio_kiocb
, rw
);
1398 if (!list_empty_careful(&iocb
->ki_list
))
1399 aio_remove_iocb(iocb
);
1401 if (kiocb
->ki_flags
& IOCB_WRITE
) {
1402 struct inode
*inode
= file_inode(kiocb
->ki_filp
);
1405 * Tell lockdep we inherited freeze protection from submission
1408 if (S_ISREG(inode
->i_mode
))
1409 __sb_writers_acquired(inode
->i_sb
, SB_FREEZE_WRITE
);
1410 file_end_write(kiocb
->ki_filp
);
1413 fput(kiocb
->ki_filp
);
1414 aio_complete(iocb
, res
, res2
);
1417 static int aio_prep_rw(struct kiocb
*req
, struct iocb
*iocb
)
1421 req
->ki_filp
= fget(iocb
->aio_fildes
);
1422 if (unlikely(!req
->ki_filp
))
1424 req
->ki_complete
= aio_complete_rw
;
1425 req
->ki_pos
= iocb
->aio_offset
;
1426 req
->ki_flags
= iocb_flags(req
->ki_filp
);
1427 if (iocb
->aio_flags
& IOCB_FLAG_RESFD
)
1428 req
->ki_flags
|= IOCB_EVENTFD
;
1429 req
->ki_hint
= ki_hint_validate(file_write_hint(req
->ki_filp
));
1430 if (iocb
->aio_flags
& IOCB_FLAG_IOPRIO
) {
1432 * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then
1433 * aio_reqprio is interpreted as an I/O scheduling
1434 * class and priority.
1436 ret
= ioprio_check_cap(iocb
->aio_reqprio
);
1438 pr_debug("aio ioprio check cap error: %d\n", ret
);
1443 req
->ki_ioprio
= iocb
->aio_reqprio
;
1445 req
->ki_ioprio
= IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE
, 0);
1447 ret
= kiocb_set_rw_flags(req
, iocb
->aio_rw_flags
);
1453 static int aio_setup_rw(int rw
, struct iocb
*iocb
, struct iovec
**iovec
,
1454 bool vectored
, bool compat
, struct iov_iter
*iter
)
1456 void __user
*buf
= (void __user
*)(uintptr_t)iocb
->aio_buf
;
1457 size_t len
= iocb
->aio_nbytes
;
1460 ssize_t ret
= import_single_range(rw
, buf
, len
, *iovec
, iter
);
1464 #ifdef CONFIG_COMPAT
1466 return compat_import_iovec(rw
, buf
, len
, UIO_FASTIOV
, iovec
,
1469 return import_iovec(rw
, buf
, len
, UIO_FASTIOV
, iovec
, iter
);
1472 static inline void aio_rw_done(struct kiocb
*req
, ssize_t ret
)
1478 case -ERESTARTNOINTR
:
1479 case -ERESTARTNOHAND
:
1480 case -ERESTART_RESTARTBLOCK
:
1482 * There's no easy way to restart the syscall since other AIO's
1483 * may be already running. Just fail this IO with EINTR.
1488 aio_complete_rw(req
, ret
, 0);
1492 static ssize_t
aio_read(struct kiocb
*req
, struct iocb
*iocb
, bool vectored
,
1495 struct iovec inline_vecs
[UIO_FASTIOV
], *iovec
= inline_vecs
;
1496 struct iov_iter iter
;
1500 ret
= aio_prep_rw(req
, iocb
);
1503 file
= req
->ki_filp
;
1506 if (unlikely(!(file
->f_mode
& FMODE_READ
)))
1509 if (unlikely(!file
->f_op
->read_iter
))
1512 ret
= aio_setup_rw(READ
, iocb
, &iovec
, vectored
, compat
, &iter
);
1515 ret
= rw_verify_area(READ
, file
, &req
->ki_pos
, iov_iter_count(&iter
));
1517 aio_rw_done(req
, call_read_iter(file
, req
, &iter
));
1525 static ssize_t
aio_write(struct kiocb
*req
, struct iocb
*iocb
, bool vectored
,
1528 struct iovec inline_vecs
[UIO_FASTIOV
], *iovec
= inline_vecs
;
1529 struct iov_iter iter
;
1533 ret
= aio_prep_rw(req
, iocb
);
1536 file
= req
->ki_filp
;
1539 if (unlikely(!(file
->f_mode
& FMODE_WRITE
)))
1542 if (unlikely(!file
->f_op
->write_iter
))
1545 ret
= aio_setup_rw(WRITE
, iocb
, &iovec
, vectored
, compat
, &iter
);
1548 ret
= rw_verify_area(WRITE
, file
, &req
->ki_pos
, iov_iter_count(&iter
));
1551 * Open-code file_start_write here to grab freeze protection,
1552 * which will be released by another thread in
1553 * aio_complete_rw(). Fool lockdep by telling it the lock got
1554 * released so that it doesn't complain about the held lock when
1555 * we return to userspace.
1557 if (S_ISREG(file_inode(file
)->i_mode
)) {
1558 __sb_start_write(file_inode(file
)->i_sb
, SB_FREEZE_WRITE
, true);
1559 __sb_writers_release(file_inode(file
)->i_sb
, SB_FREEZE_WRITE
);
1561 req
->ki_flags
|= IOCB_WRITE
;
1562 aio_rw_done(req
, call_write_iter(file
, req
, &iter
));
1571 static void aio_fsync_work(struct work_struct
*work
)
1573 struct fsync_iocb
*req
= container_of(work
, struct fsync_iocb
, work
);
1576 ret
= vfs_fsync(req
->file
, req
->datasync
);
1578 aio_complete(container_of(req
, struct aio_kiocb
, fsync
), ret
, 0);
1581 static int aio_fsync(struct fsync_iocb
*req
, struct iocb
*iocb
, bool datasync
)
1583 if (unlikely(iocb
->aio_buf
|| iocb
->aio_offset
|| iocb
->aio_nbytes
||
1584 iocb
->aio_rw_flags
))
1587 req
->file
= fget(iocb
->aio_fildes
);
1588 if (unlikely(!req
->file
))
1590 if (unlikely(!req
->file
->f_op
->fsync
)) {
1595 req
->datasync
= datasync
;
1596 INIT_WORK(&req
->work
, aio_fsync_work
);
1597 schedule_work(&req
->work
);
1601 static inline void aio_poll_complete(struct aio_kiocb
*iocb
, __poll_t mask
)
1603 struct file
*file
= iocb
->poll
.file
;
1605 aio_complete(iocb
, mangle_poll(mask
), 0);
1609 static void aio_poll_complete_work(struct work_struct
*work
)
1611 struct poll_iocb
*req
= container_of(work
, struct poll_iocb
, work
);
1612 struct aio_kiocb
*iocb
= container_of(req
, struct aio_kiocb
, poll
);
1613 struct poll_table_struct pt
= { ._key
= req
->events
};
1614 struct kioctx
*ctx
= iocb
->ki_ctx
;
1617 if (!READ_ONCE(req
->cancelled
))
1618 mask
= vfs_poll(req
->file
, &pt
) & req
->events
;
1621 * Note that ->ki_cancel callers also delete iocb from active_reqs after
1622 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
1623 * synchronize with them. In the cancellation case the list_del_init
1624 * itself is not actually needed, but harmless so we keep it in to
1625 * avoid further branches in the fast path.
1627 spin_lock_irq(&ctx
->ctx_lock
);
1628 if (!mask
&& !READ_ONCE(req
->cancelled
)) {
1629 add_wait_queue(req
->head
, &req
->wait
);
1630 spin_unlock_irq(&ctx
->ctx_lock
);
1633 list_del_init(&iocb
->ki_list
);
1634 spin_unlock_irq(&ctx
->ctx_lock
);
1636 aio_poll_complete(iocb
, mask
);
1639 /* assumes we are called with irqs disabled */
1640 static int aio_poll_cancel(struct kiocb
*iocb
)
1642 struct aio_kiocb
*aiocb
= container_of(iocb
, struct aio_kiocb
, rw
);
1643 struct poll_iocb
*req
= &aiocb
->poll
;
1645 spin_lock(&req
->head
->lock
);
1646 WRITE_ONCE(req
->cancelled
, true);
1647 if (!list_empty(&req
->wait
.entry
)) {
1648 list_del_init(&req
->wait
.entry
);
1649 schedule_work(&aiocb
->poll
.work
);
1651 spin_unlock(&req
->head
->lock
);
1656 static int aio_poll_wake(struct wait_queue_entry
*wait
, unsigned mode
, int sync
,
1659 struct poll_iocb
*req
= container_of(wait
, struct poll_iocb
, wait
);
1660 struct aio_kiocb
*iocb
= container_of(req
, struct aio_kiocb
, poll
);
1661 __poll_t mask
= key_to_poll(key
);
1665 /* for instances that support it check for an event match first: */
1667 if (!(mask
& req
->events
))
1670 /* try to complete the iocb inline if we can: */
1671 if (spin_trylock(&iocb
->ki_ctx
->ctx_lock
)) {
1672 list_del(&iocb
->ki_list
);
1673 spin_unlock(&iocb
->ki_ctx
->ctx_lock
);
1675 list_del_init(&req
->wait
.entry
);
1676 aio_poll_complete(iocb
, mask
);
1681 list_del_init(&req
->wait
.entry
);
1682 schedule_work(&req
->work
);
1686 struct aio_poll_table
{
1687 struct poll_table_struct pt
;
1688 struct aio_kiocb
*iocb
;
1693 aio_poll_queue_proc(struct file
*file
, struct wait_queue_head
*head
,
1694 struct poll_table_struct
*p
)
1696 struct aio_poll_table
*pt
= container_of(p
, struct aio_poll_table
, pt
);
1698 /* multiple wait queues per file are not supported */
1699 if (unlikely(pt
->iocb
->poll
.head
)) {
1700 pt
->error
= -EINVAL
;
1705 pt
->iocb
->poll
.head
= head
;
1706 add_wait_queue(head
, &pt
->iocb
->poll
.wait
);
1709 static ssize_t
aio_poll(struct aio_kiocb
*aiocb
, struct iocb
*iocb
)
1711 struct kioctx
*ctx
= aiocb
->ki_ctx
;
1712 struct poll_iocb
*req
= &aiocb
->poll
;
1713 struct aio_poll_table apt
;
1716 /* reject any unknown events outside the normal event mask. */
1717 if ((u16
)iocb
->aio_buf
!= iocb
->aio_buf
)
1719 /* reject fields that are not defined for poll */
1720 if (iocb
->aio_offset
|| iocb
->aio_nbytes
|| iocb
->aio_rw_flags
)
1723 INIT_WORK(&req
->work
, aio_poll_complete_work
);
1724 req
->events
= demangle_poll(iocb
->aio_buf
) | EPOLLERR
| EPOLLHUP
;
1725 req
->file
= fget(iocb
->aio_fildes
);
1726 if (unlikely(!req
->file
))
1729 apt
.pt
._qproc
= aio_poll_queue_proc
;
1730 apt
.pt
._key
= req
->events
;
1732 apt
.error
= -EINVAL
; /* same as no support for IOCB_CMD_POLL */
1734 /* initialized the list so that we can do list_empty checks */
1735 INIT_LIST_HEAD(&req
->wait
.entry
);
1736 init_waitqueue_func_entry(&req
->wait
, aio_poll_wake
);
1738 /* one for removal from waitqueue, one for this function */
1739 refcount_set(&aiocb
->ki_refcnt
, 2);
1741 mask
= vfs_poll(req
->file
, &apt
.pt
) & req
->events
;
1742 if (unlikely(!req
->head
)) {
1743 /* we did not manage to set up a waitqueue, done */
1747 spin_lock_irq(&ctx
->ctx_lock
);
1748 spin_lock(&req
->head
->lock
);
1750 /* wake_up context handles the rest */
1753 } else if (mask
|| apt
.error
) {
1754 /* if we get an error or a mask we are done */
1755 WARN_ON_ONCE(list_empty(&req
->wait
.entry
));
1756 list_del_init(&req
->wait
.entry
);
1758 /* actually waiting for an event */
1759 list_add_tail(&aiocb
->ki_list
, &ctx
->active_reqs
);
1760 aiocb
->ki_cancel
= aio_poll_cancel
;
1762 spin_unlock(&req
->head
->lock
);
1763 spin_unlock_irq(&ctx
->ctx_lock
);
1766 if (unlikely(apt
.error
)) {
1772 aio_poll_complete(aiocb
, mask
);
1777 static int io_submit_one(struct kioctx
*ctx
, struct iocb __user
*user_iocb
,
1780 struct aio_kiocb
*req
;
1784 if (unlikely(copy_from_user(&iocb
, user_iocb
, sizeof(iocb
))))
1787 /* enforce forwards compatibility on users */
1788 if (unlikely(iocb
.aio_reserved2
)) {
1789 pr_debug("EINVAL: reserve field set\n");
1793 /* prevent overflows */
1795 (iocb
.aio_buf
!= (unsigned long)iocb
.aio_buf
) ||
1796 (iocb
.aio_nbytes
!= (size_t)iocb
.aio_nbytes
) ||
1797 ((ssize_t
)iocb
.aio_nbytes
< 0)
1799 pr_debug("EINVAL: overflow check\n");
1803 req
= aio_get_req(ctx
);
1807 if (iocb
.aio_flags
& IOCB_FLAG_RESFD
) {
1809 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
1810 * instance of the file* now. The file descriptor must be
1811 * an eventfd() fd, and will be signaled for each completed
1812 * event using the eventfd_signal() function.
1814 req
->ki_eventfd
= eventfd_ctx_fdget((int) iocb
.aio_resfd
);
1815 if (IS_ERR(req
->ki_eventfd
)) {
1816 ret
= PTR_ERR(req
->ki_eventfd
);
1817 req
->ki_eventfd
= NULL
;
1822 ret
= put_user(KIOCB_KEY
, &user_iocb
->aio_key
);
1823 if (unlikely(ret
)) {
1824 pr_debug("EFAULT: aio_key\n");
1828 req
->ki_user_iocb
= user_iocb
;
1829 req
->ki_user_data
= iocb
.aio_data
;
1831 switch (iocb
.aio_lio_opcode
) {
1832 case IOCB_CMD_PREAD
:
1833 ret
= aio_read(&req
->rw
, &iocb
, false, compat
);
1835 case IOCB_CMD_PWRITE
:
1836 ret
= aio_write(&req
->rw
, &iocb
, false, compat
);
1838 case IOCB_CMD_PREADV
:
1839 ret
= aio_read(&req
->rw
, &iocb
, true, compat
);
1841 case IOCB_CMD_PWRITEV
:
1842 ret
= aio_write(&req
->rw
, &iocb
, true, compat
);
1844 case IOCB_CMD_FSYNC
:
1845 ret
= aio_fsync(&req
->fsync
, &iocb
, false);
1847 case IOCB_CMD_FDSYNC
:
1848 ret
= aio_fsync(&req
->fsync
, &iocb
, true);
1851 ret
= aio_poll(req
, &iocb
);
1854 pr_debug("invalid aio operation %d\n", iocb
.aio_lio_opcode
);
1860 * If ret is 0, we'd either done aio_complete() ourselves or have
1861 * arranged for that to be done asynchronously. Anything non-zero
1862 * means that we need to destroy req ourselves.
1868 put_reqs_available(ctx
, 1);
1869 percpu_ref_put(&ctx
->reqs
);
1870 if (req
->ki_eventfd
)
1871 eventfd_ctx_put(req
->ki_eventfd
);
1872 kmem_cache_free(kiocb_cachep
, req
);
1877 * Queue the nr iocbs pointed to by iocbpp for processing. Returns
1878 * the number of iocbs queued. May return -EINVAL if the aio_context
1879 * specified by ctx_id is invalid, if nr is < 0, if the iocb at
1880 * *iocbpp[0] is not properly initialized, if the operation specified
1881 * is invalid for the file descriptor in the iocb. May fail with
1882 * -EFAULT if any of the data structures point to invalid data. May
1883 * fail with -EBADF if the file descriptor specified in the first
1884 * iocb is invalid. May fail with -EAGAIN if insufficient resources
1885 * are available to queue any iocbs. Will return 0 if nr is 0. Will
1886 * fail with -ENOSYS if not implemented.
1888 SYSCALL_DEFINE3(io_submit
, aio_context_t
, ctx_id
, long, nr
,
1889 struct iocb __user
* __user
*, iocbpp
)
1894 struct blk_plug plug
;
1896 if (unlikely(nr
< 0))
1899 ctx
= lookup_ioctx(ctx_id
);
1900 if (unlikely(!ctx
)) {
1901 pr_debug("EINVAL: invalid context id\n");
1905 if (nr
> ctx
->nr_events
)
1906 nr
= ctx
->nr_events
;
1908 blk_start_plug(&plug
);
1909 for (i
= 0; i
< nr
; i
++) {
1910 struct iocb __user
*user_iocb
;
1912 if (unlikely(get_user(user_iocb
, iocbpp
+ i
))) {
1917 ret
= io_submit_one(ctx
, user_iocb
, false);
1921 blk_finish_plug(&plug
);
1923 percpu_ref_put(&ctx
->users
);
1927 #ifdef CONFIG_COMPAT
1928 COMPAT_SYSCALL_DEFINE3(io_submit
, compat_aio_context_t
, ctx_id
,
1929 int, nr
, compat_uptr_t __user
*, iocbpp
)
1934 struct blk_plug plug
;
1936 if (unlikely(nr
< 0))
1939 ctx
= lookup_ioctx(ctx_id
);
1940 if (unlikely(!ctx
)) {
1941 pr_debug("EINVAL: invalid context id\n");
1945 if (nr
> ctx
->nr_events
)
1946 nr
= ctx
->nr_events
;
1948 blk_start_plug(&plug
);
1949 for (i
= 0; i
< nr
; i
++) {
1950 compat_uptr_t user_iocb
;
1952 if (unlikely(get_user(user_iocb
, iocbpp
+ i
))) {
1957 ret
= io_submit_one(ctx
, compat_ptr(user_iocb
), true);
1961 blk_finish_plug(&plug
);
1963 percpu_ref_put(&ctx
->users
);
1969 * Finds a given iocb for cancellation.
1971 static struct aio_kiocb
*
1972 lookup_kiocb(struct kioctx
*ctx
, struct iocb __user
*iocb
)
1974 struct aio_kiocb
*kiocb
;
1976 assert_spin_locked(&ctx
->ctx_lock
);
1978 /* TODO: use a hash or array, this sucks. */
1979 list_for_each_entry(kiocb
, &ctx
->active_reqs
, ki_list
) {
1980 if (kiocb
->ki_user_iocb
== iocb
)
1987 * Attempts to cancel an iocb previously passed to io_submit. If
1988 * the operation is successfully cancelled, the resulting event is
1989 * copied into the memory pointed to by result without being placed
1990 * into the completion queue and 0 is returned. May fail with
1991 * -EFAULT if any of the data structures pointed to are invalid.
1992 * May fail with -EINVAL if aio_context specified by ctx_id is
1993 * invalid. May fail with -EAGAIN if the iocb specified was not
1994 * cancelled. Will fail with -ENOSYS if not implemented.
1996 SYSCALL_DEFINE3(io_cancel
, aio_context_t
, ctx_id
, struct iocb __user
*, iocb
,
1997 struct io_event __user
*, result
)
2000 struct aio_kiocb
*kiocb
;
2004 if (unlikely(get_user(key
, &iocb
->aio_key
)))
2006 if (unlikely(key
!= KIOCB_KEY
))
2009 ctx
= lookup_ioctx(ctx_id
);
2013 spin_lock_irq(&ctx
->ctx_lock
);
2014 kiocb
= lookup_kiocb(ctx
, iocb
);
2016 ret
= kiocb
->ki_cancel(&kiocb
->rw
);
2017 list_del_init(&kiocb
->ki_list
);
2019 spin_unlock_irq(&ctx
->ctx_lock
);
2023 * The result argument is no longer used - the io_event is
2024 * always delivered via the ring buffer. -EINPROGRESS indicates
2025 * cancellation is progress:
2030 percpu_ref_put(&ctx
->users
);
2035 static long do_io_getevents(aio_context_t ctx_id
,
2038 struct io_event __user
*events
,
2039 struct timespec64
*ts
)
2041 ktime_t until
= ts
? timespec64_to_ktime(*ts
) : KTIME_MAX
;
2042 struct kioctx
*ioctx
= lookup_ioctx(ctx_id
);
2045 if (likely(ioctx
)) {
2046 if (likely(min_nr
<= nr
&& min_nr
>= 0))
2047 ret
= read_events(ioctx
, min_nr
, nr
, events
, until
);
2048 percpu_ref_put(&ioctx
->users
);
2055 * Attempts to read at least min_nr events and up to nr events from
2056 * the completion queue for the aio_context specified by ctx_id. If
2057 * it succeeds, the number of read events is returned. May fail with
2058 * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
2059 * out of range, if timeout is out of range. May fail with -EFAULT
2060 * if any of the memory specified is invalid. May return 0 or
2061 * < min_nr if the timeout specified by timeout has elapsed
2062 * before sufficient events are available, where timeout == NULL
2063 * specifies an infinite timeout. Note that the timeout pointed to by
2064 * timeout is relative. Will fail with -ENOSYS if not implemented.
2066 SYSCALL_DEFINE5(io_getevents
, aio_context_t
, ctx_id
,
2069 struct io_event __user
*, events
,
2070 struct timespec __user
*, timeout
)
2072 struct timespec64 ts
;
2075 if (timeout
&& unlikely(get_timespec64(&ts
, timeout
)))
2078 ret
= do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &ts
: NULL
);
2079 if (!ret
&& signal_pending(current
))
2084 struct __aio_sigset
{
2085 const sigset_t __user
*sigmask
;
2089 SYSCALL_DEFINE6(io_pgetevents
,
2090 aio_context_t
, ctx_id
,
2093 struct io_event __user
*, events
,
2094 struct timespec __user
*, timeout
,
2095 const struct __aio_sigset __user
*, usig
)
2097 struct __aio_sigset ksig
= { NULL
, };
2098 sigset_t ksigmask
, sigsaved
;
2099 struct timespec64 ts
;
2102 if (timeout
&& unlikely(get_timespec64(&ts
, timeout
)))
2105 if (usig
&& copy_from_user(&ksig
, usig
, sizeof(ksig
)))
2109 if (ksig
.sigsetsize
!= sizeof(sigset_t
))
2111 if (copy_from_user(&ksigmask
, ksig
.sigmask
, sizeof(ksigmask
)))
2113 sigdelsetmask(&ksigmask
, sigmask(SIGKILL
) | sigmask(SIGSTOP
));
2114 sigprocmask(SIG_SETMASK
, &ksigmask
, &sigsaved
);
2117 ret
= do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &ts
: NULL
);
2118 if (signal_pending(current
)) {
2120 current
->saved_sigmask
= sigsaved
;
2121 set_restore_sigmask();
2125 ret
= -ERESTARTNOHAND
;
2128 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
2134 #ifdef CONFIG_COMPAT
2135 COMPAT_SYSCALL_DEFINE5(io_getevents
, compat_aio_context_t
, ctx_id
,
2136 compat_long_t
, min_nr
,
2138 struct io_event __user
*, events
,
2139 struct old_timespec32 __user
*, timeout
)
2141 struct timespec64 t
;
2144 if (timeout
&& get_old_timespec32(&t
, timeout
))
2147 ret
= do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &t
: NULL
);
2148 if (!ret
&& signal_pending(current
))
2154 struct __compat_aio_sigset
{
2155 compat_sigset_t __user
*sigmask
;
2156 compat_size_t sigsetsize
;
2159 COMPAT_SYSCALL_DEFINE6(io_pgetevents
,
2160 compat_aio_context_t
, ctx_id
,
2161 compat_long_t
, min_nr
,
2163 struct io_event __user
*, events
,
2164 struct old_timespec32 __user
*, timeout
,
2165 const struct __compat_aio_sigset __user
*, usig
)
2167 struct __compat_aio_sigset ksig
= { NULL
, };
2168 sigset_t ksigmask
, sigsaved
;
2169 struct timespec64 t
;
2172 if (timeout
&& get_old_timespec32(&t
, timeout
))
2175 if (usig
&& copy_from_user(&ksig
, usig
, sizeof(ksig
)))
2179 if (ksig
.sigsetsize
!= sizeof(compat_sigset_t
))
2181 if (get_compat_sigset(&ksigmask
, ksig
.sigmask
))
2183 sigdelsetmask(&ksigmask
, sigmask(SIGKILL
) | sigmask(SIGSTOP
));
2184 sigprocmask(SIG_SETMASK
, &ksigmask
, &sigsaved
);
2187 ret
= do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &t
: NULL
);
2188 if (signal_pending(current
)) {
2190 current
->saved_sigmask
= sigsaved
;
2191 set_restore_sigmask();
2194 ret
= -ERESTARTNOHAND
;
2197 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);