2 * An async IO implementation for Linux
3 * Written by Benjamin LaHaise <bcrl@kvack.org>
5 * Implements an efficient asynchronous io interface.
7 * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved.
9 * See ../COPYING for licensing terms.
11 #define pr_fmt(fmt) "%s: " fmt, __func__
13 #include <linux/kernel.h>
14 #include <linux/init.h>
15 #include <linux/errno.h>
16 #include <linux/time.h>
17 #include <linux/aio_abi.h>
18 #include <linux/export.h>
19 #include <linux/syscalls.h>
20 #include <linux/backing-dev.h>
21 #include <linux/uio.h>
23 #include <linux/sched/signal.h>
25 #include <linux/file.h>
27 #include <linux/mman.h>
28 #include <linux/mmu_context.h>
29 #include <linux/percpu.h>
30 #include <linux/slab.h>
31 #include <linux/timer.h>
32 #include <linux/aio.h>
33 #include <linux/highmem.h>
34 #include <linux/workqueue.h>
35 #include <linux/security.h>
36 #include <linux/eventfd.h>
37 #include <linux/blkdev.h>
38 #include <linux/compat.h>
39 #include <linux/migrate.h>
40 #include <linux/ramfs.h>
41 #include <linux/percpu-refcount.h>
42 #include <linux/mount.h>
44 #include <asm/kmap_types.h>
45 #include <linux/uaccess.h>
49 #define AIO_RING_MAGIC 0xa10a10a1
50 #define AIO_RING_COMPAT_FEATURES 1
51 #define AIO_RING_INCOMPAT_FEATURES 0
53 unsigned id
; /* kernel internal index number */
54 unsigned nr
; /* number of io_events */
55 unsigned head
; /* Written to by userland or under ring_lock
56 * mutex by aio_read_events_ring(). */
60 unsigned compat_features
;
61 unsigned incompat_features
;
62 unsigned header_length
; /* size of aio_ring */
65 struct io_event io_events
[0];
66 }; /* 128 bytes + ring size */
68 #define AIO_RING_PAGES 8
73 struct kioctx __rcu
*table
[];
77 unsigned reqs_available
;
81 struct completion comp
;
86 struct percpu_ref users
;
89 struct percpu_ref reqs
;
91 unsigned long user_id
;
93 struct __percpu kioctx_cpu
*cpu
;
96 * For percpu reqs_available, number of slots we move to/from global
101 * This is what userspace passed to io_setup(), it's not used for
102 * anything but counting against the global max_reqs quota.
104 * The real limit is nr_events - 1, which will be larger (see
109 /* Size of ringbuffer, in units of struct io_event */
112 unsigned long mmap_base
;
113 unsigned long mmap_size
;
115 struct page
**ring_pages
;
118 struct rcu_head free_rcu
;
119 struct work_struct free_work
; /* see free_ioctx() */
122 * signals when all in-flight requests are done
124 struct ctx_rq_wait
*rq_wait
;
128 * This counts the number of available slots in the ringbuffer,
129 * so we avoid overflowing it: it's decremented (if positive)
130 * when allocating a kiocb and incremented when the resulting
131 * io_event is pulled off the ringbuffer.
133 * We batch accesses to it with a percpu version.
135 atomic_t reqs_available
;
136 } ____cacheline_aligned_in_smp
;
140 struct list_head active_reqs
; /* used for cancellation */
141 } ____cacheline_aligned_in_smp
;
144 struct mutex ring_lock
;
145 wait_queue_head_t wait
;
146 } ____cacheline_aligned_in_smp
;
150 unsigned completed_events
;
151 spinlock_t completion_lock
;
152 } ____cacheline_aligned_in_smp
;
154 struct page
*internal_pages
[AIO_RING_PAGES
];
155 struct file
*aio_ring_file
;
161 * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
162 * cancelled or completed (this makes a certain amount of sense because
163 * successful cancellation - io_cancel() - does deliver the completion to
166 * And since most things don't implement kiocb cancellation and we'd really like
167 * kiocb completion to be lockless when possible, we use ki_cancel to
168 * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
169 * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
171 #define KIOCB_CANCELLED ((void *) (~0ULL))
176 struct kioctx
*ki_ctx
;
177 kiocb_cancel_fn
*ki_cancel
;
179 struct iocb __user
*ki_user_iocb
; /* user's aiocb */
180 __u64 ki_user_data
; /* user's data for completion */
182 struct list_head ki_list
; /* the aio core uses this
183 * for cancellation */
186 * If the aio_resfd field of the userspace iocb is not zero,
187 * this is the underlying eventfd context to deliver events to.
189 struct eventfd_ctx
*ki_eventfd
;
192 /*------ sysctl variables----*/
193 static DEFINE_SPINLOCK(aio_nr_lock
);
194 unsigned long aio_nr
; /* current system wide number of aio requests */
195 unsigned long aio_max_nr
= 0x10000; /* system wide maximum number of aio requests */
196 /*----end sysctl variables---*/
198 static struct kmem_cache
*kiocb_cachep
;
199 static struct kmem_cache
*kioctx_cachep
;
201 static struct vfsmount
*aio_mnt
;
203 static const struct file_operations aio_ring_fops
;
204 static const struct address_space_operations aio_ctx_aops
;
206 static struct file
*aio_private_file(struct kioctx
*ctx
, loff_t nr_pages
)
208 struct qstr
this = QSTR_INIT("[aio]", 5);
211 struct inode
*inode
= alloc_anon_inode(aio_mnt
->mnt_sb
);
213 return ERR_CAST(inode
);
215 inode
->i_mapping
->a_ops
= &aio_ctx_aops
;
216 inode
->i_mapping
->private_data
= ctx
;
217 inode
->i_size
= PAGE_SIZE
* nr_pages
;
219 path
.dentry
= d_alloc_pseudo(aio_mnt
->mnt_sb
, &this);
222 return ERR_PTR(-ENOMEM
);
224 path
.mnt
= mntget(aio_mnt
);
226 d_instantiate(path
.dentry
, inode
);
227 file
= alloc_file(&path
, FMODE_READ
| FMODE_WRITE
, &aio_ring_fops
);
233 file
->f_flags
= O_RDWR
;
237 static struct dentry
*aio_mount(struct file_system_type
*fs_type
,
238 int flags
, const char *dev_name
, void *data
)
240 static const struct dentry_operations ops
= {
241 .d_dname
= simple_dname
,
243 struct dentry
*root
= mount_pseudo(fs_type
, "aio:", NULL
, &ops
,
247 root
->d_sb
->s_iflags
|= SB_I_NOEXEC
;
252 * Creates the slab caches used by the aio routines, panic on
253 * failure as this is done early during the boot sequence.
255 static int __init
aio_setup(void)
257 static struct file_system_type aio_fs
= {
260 .kill_sb
= kill_anon_super
,
262 aio_mnt
= kern_mount(&aio_fs
);
264 panic("Failed to create aio fs mount.");
266 kiocb_cachep
= KMEM_CACHE(aio_kiocb
, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
);
267 kioctx_cachep
= KMEM_CACHE(kioctx
,SLAB_HWCACHE_ALIGN
|SLAB_PANIC
);
269 pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page
));
273 __initcall(aio_setup
);
275 static void put_aio_ring_file(struct kioctx
*ctx
)
277 struct file
*aio_ring_file
= ctx
->aio_ring_file
;
278 struct address_space
*i_mapping
;
281 truncate_setsize(file_inode(aio_ring_file
), 0);
283 /* Prevent further access to the kioctx from migratepages */
284 i_mapping
= aio_ring_file
->f_mapping
;
285 spin_lock(&i_mapping
->private_lock
);
286 i_mapping
->private_data
= NULL
;
287 ctx
->aio_ring_file
= NULL
;
288 spin_unlock(&i_mapping
->private_lock
);
294 static void aio_free_ring(struct kioctx
*ctx
)
298 /* Disconnect the kiotx from the ring file. This prevents future
299 * accesses to the kioctx from page migration.
301 put_aio_ring_file(ctx
);
303 for (i
= 0; i
< ctx
->nr_pages
; i
++) {
305 pr_debug("pid(%d) [%d] page->count=%d\n", current
->pid
, i
,
306 page_count(ctx
->ring_pages
[i
]));
307 page
= ctx
->ring_pages
[i
];
310 ctx
->ring_pages
[i
] = NULL
;
314 if (ctx
->ring_pages
&& ctx
->ring_pages
!= ctx
->internal_pages
) {
315 kfree(ctx
->ring_pages
);
316 ctx
->ring_pages
= NULL
;
320 static int aio_ring_mremap(struct vm_area_struct
*vma
)
322 struct file
*file
= vma
->vm_file
;
323 struct mm_struct
*mm
= vma
->vm_mm
;
324 struct kioctx_table
*table
;
325 int i
, res
= -EINVAL
;
327 spin_lock(&mm
->ioctx_lock
);
329 table
= rcu_dereference(mm
->ioctx_table
);
330 for (i
= 0; i
< table
->nr
; i
++) {
333 ctx
= rcu_dereference(table
->table
[i
]);
334 if (ctx
&& ctx
->aio_ring_file
== file
) {
335 if (!atomic_read(&ctx
->dead
)) {
336 ctx
->user_id
= ctx
->mmap_base
= vma
->vm_start
;
344 spin_unlock(&mm
->ioctx_lock
);
348 static const struct vm_operations_struct aio_ring_vm_ops
= {
349 .mremap
= aio_ring_mremap
,
350 #if IS_ENABLED(CONFIG_MMU)
351 .fault
= filemap_fault
,
352 .map_pages
= filemap_map_pages
,
353 .page_mkwrite
= filemap_page_mkwrite
,
357 static int aio_ring_mmap(struct file
*file
, struct vm_area_struct
*vma
)
359 vma
->vm_flags
|= VM_DONTEXPAND
;
360 vma
->vm_ops
= &aio_ring_vm_ops
;
364 static const struct file_operations aio_ring_fops
= {
365 .mmap
= aio_ring_mmap
,
368 #if IS_ENABLED(CONFIG_MIGRATION)
369 static int aio_migratepage(struct address_space
*mapping
, struct page
*new,
370 struct page
*old
, enum migrate_mode mode
)
378 * We cannot support the _NO_COPY case here, because copy needs to
379 * happen under the ctx->completion_lock. That does not work with the
380 * migration workflow of MIGRATE_SYNC_NO_COPY.
382 if (mode
== MIGRATE_SYNC_NO_COPY
)
387 /* mapping->private_lock here protects against the kioctx teardown. */
388 spin_lock(&mapping
->private_lock
);
389 ctx
= mapping
->private_data
;
395 /* The ring_lock mutex. The prevents aio_read_events() from writing
396 * to the ring's head, and prevents page migration from mucking in
397 * a partially initialized kiotx.
399 if (!mutex_trylock(&ctx
->ring_lock
)) {
405 if (idx
< (pgoff_t
)ctx
->nr_pages
) {
406 /* Make sure the old page hasn't already been changed */
407 if (ctx
->ring_pages
[idx
] != old
)
415 /* Writeback must be complete */
416 BUG_ON(PageWriteback(old
));
419 rc
= migrate_page_move_mapping(mapping
, new, old
, NULL
, mode
, 1);
420 if (rc
!= MIGRATEPAGE_SUCCESS
) {
425 /* Take completion_lock to prevent other writes to the ring buffer
426 * while the old page is copied to the new. This prevents new
427 * events from being lost.
429 spin_lock_irqsave(&ctx
->completion_lock
, flags
);
430 migrate_page_copy(new, old
);
431 BUG_ON(ctx
->ring_pages
[idx
] != old
);
432 ctx
->ring_pages
[idx
] = new;
433 spin_unlock_irqrestore(&ctx
->completion_lock
, flags
);
435 /* The old page is no longer accessible. */
439 mutex_unlock(&ctx
->ring_lock
);
441 spin_unlock(&mapping
->private_lock
);
446 static const struct address_space_operations aio_ctx_aops
= {
447 .set_page_dirty
= __set_page_dirty_no_writeback
,
448 #if IS_ENABLED(CONFIG_MIGRATION)
449 .migratepage
= aio_migratepage
,
453 static int aio_setup_ring(struct kioctx
*ctx
, unsigned int nr_events
)
455 struct aio_ring
*ring
;
456 struct mm_struct
*mm
= current
->mm
;
457 unsigned long size
, unused
;
462 /* Compensate for the ring buffer's head/tail overlap entry */
463 nr_events
+= 2; /* 1 is required, 2 for good luck */
465 size
= sizeof(struct aio_ring
);
466 size
+= sizeof(struct io_event
) * nr_events
;
468 nr_pages
= PFN_UP(size
);
472 file
= aio_private_file(ctx
, nr_pages
);
474 ctx
->aio_ring_file
= NULL
;
478 ctx
->aio_ring_file
= file
;
479 nr_events
= (PAGE_SIZE
* nr_pages
- sizeof(struct aio_ring
))
480 / sizeof(struct io_event
);
482 ctx
->ring_pages
= ctx
->internal_pages
;
483 if (nr_pages
> AIO_RING_PAGES
) {
484 ctx
->ring_pages
= kcalloc(nr_pages
, sizeof(struct page
*),
486 if (!ctx
->ring_pages
) {
487 put_aio_ring_file(ctx
);
492 for (i
= 0; i
< nr_pages
; i
++) {
494 page
= find_or_create_page(file
->f_mapping
,
495 i
, GFP_HIGHUSER
| __GFP_ZERO
);
498 pr_debug("pid(%d) page[%d]->count=%d\n",
499 current
->pid
, i
, page_count(page
));
500 SetPageUptodate(page
);
503 ctx
->ring_pages
[i
] = page
;
507 if (unlikely(i
!= nr_pages
)) {
512 ctx
->mmap_size
= nr_pages
* PAGE_SIZE
;
513 pr_debug("attempting mmap of %lu bytes\n", ctx
->mmap_size
);
515 if (down_write_killable(&mm
->mmap_sem
)) {
521 ctx
->mmap_base
= do_mmap_pgoff(ctx
->aio_ring_file
, 0, ctx
->mmap_size
,
522 PROT_READ
| PROT_WRITE
,
523 MAP_SHARED
, 0, &unused
, NULL
);
524 up_write(&mm
->mmap_sem
);
525 if (IS_ERR((void *)ctx
->mmap_base
)) {
531 pr_debug("mmap address: 0x%08lx\n", ctx
->mmap_base
);
533 ctx
->user_id
= ctx
->mmap_base
;
534 ctx
->nr_events
= nr_events
; /* trusted copy */
536 ring
= kmap_atomic(ctx
->ring_pages
[0]);
537 ring
->nr
= nr_events
; /* user copy */
539 ring
->head
= ring
->tail
= 0;
540 ring
->magic
= AIO_RING_MAGIC
;
541 ring
->compat_features
= AIO_RING_COMPAT_FEATURES
;
542 ring
->incompat_features
= AIO_RING_INCOMPAT_FEATURES
;
543 ring
->header_length
= sizeof(struct aio_ring
);
545 flush_dcache_page(ctx
->ring_pages
[0]);
550 #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
551 #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
552 #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
554 void kiocb_set_cancel_fn(struct kiocb
*iocb
, kiocb_cancel_fn
*cancel
)
556 struct aio_kiocb
*req
= container_of(iocb
, struct aio_kiocb
, common
);
557 struct kioctx
*ctx
= req
->ki_ctx
;
560 spin_lock_irqsave(&ctx
->ctx_lock
, flags
);
562 if (!req
->ki_list
.next
)
563 list_add(&req
->ki_list
, &ctx
->active_reqs
);
565 req
->ki_cancel
= cancel
;
567 spin_unlock_irqrestore(&ctx
->ctx_lock
, flags
);
569 EXPORT_SYMBOL(kiocb_set_cancel_fn
);
571 static int kiocb_cancel(struct aio_kiocb
*kiocb
)
573 kiocb_cancel_fn
*old
, *cancel
;
576 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
577 * actually has a cancel function, hence the cmpxchg()
580 cancel
= READ_ONCE(kiocb
->ki_cancel
);
582 if (!cancel
|| cancel
== KIOCB_CANCELLED
)
586 cancel
= cmpxchg(&kiocb
->ki_cancel
, old
, KIOCB_CANCELLED
);
587 } while (cancel
!= old
);
589 return cancel(&kiocb
->common
);
593 * free_ioctx() should be RCU delayed to synchronize against the RCU
594 * protected lookup_ioctx() and also needs process context to call
595 * aio_free_ring(), so the double bouncing through kioctx->free_rcu and
598 static void free_ioctx(struct work_struct
*work
)
600 struct kioctx
*ctx
= container_of(work
, struct kioctx
, free_work
);
602 pr_debug("freeing %p\n", ctx
);
605 free_percpu(ctx
->cpu
);
606 percpu_ref_exit(&ctx
->reqs
);
607 percpu_ref_exit(&ctx
->users
);
608 kmem_cache_free(kioctx_cachep
, ctx
);
611 static void free_ioctx_rcufn(struct rcu_head
*head
)
613 struct kioctx
*ctx
= container_of(head
, struct kioctx
, free_rcu
);
615 INIT_WORK(&ctx
->free_work
, free_ioctx
);
616 schedule_work(&ctx
->free_work
);
619 static void free_ioctx_reqs(struct percpu_ref
*ref
)
621 struct kioctx
*ctx
= container_of(ref
, struct kioctx
, reqs
);
623 /* At this point we know that there are no any in-flight requests */
624 if (ctx
->rq_wait
&& atomic_dec_and_test(&ctx
->rq_wait
->count
))
625 complete(&ctx
->rq_wait
->comp
);
627 /* Synchronize against RCU protected table->table[] dereferences */
628 call_rcu(&ctx
->free_rcu
, free_ioctx_rcufn
);
632 * When this function runs, the kioctx has been removed from the "hash table"
633 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
634 * now it's safe to cancel any that need to be.
636 static void free_ioctx_users(struct percpu_ref
*ref
)
638 struct kioctx
*ctx
= container_of(ref
, struct kioctx
, users
);
639 struct aio_kiocb
*req
;
641 spin_lock_irq(&ctx
->ctx_lock
);
643 while (!list_empty(&ctx
->active_reqs
)) {
644 req
= list_first_entry(&ctx
->active_reqs
,
645 struct aio_kiocb
, ki_list
);
647 list_del_init(&req
->ki_list
);
650 spin_unlock_irq(&ctx
->ctx_lock
);
652 percpu_ref_kill(&ctx
->reqs
);
653 percpu_ref_put(&ctx
->reqs
);
656 static int ioctx_add_table(struct kioctx
*ctx
, struct mm_struct
*mm
)
659 struct kioctx_table
*table
, *old
;
660 struct aio_ring
*ring
;
662 spin_lock(&mm
->ioctx_lock
);
663 table
= rcu_dereference_raw(mm
->ioctx_table
);
667 for (i
= 0; i
< table
->nr
; i
++)
668 if (!rcu_access_pointer(table
->table
[i
])) {
670 rcu_assign_pointer(table
->table
[i
], ctx
);
671 spin_unlock(&mm
->ioctx_lock
);
673 /* While kioctx setup is in progress,
674 * we are protected from page migration
675 * changes ring_pages by ->ring_lock.
677 ring
= kmap_atomic(ctx
->ring_pages
[0]);
683 new_nr
= (table
? table
->nr
: 1) * 4;
684 spin_unlock(&mm
->ioctx_lock
);
686 table
= kzalloc(sizeof(*table
) + sizeof(struct kioctx
*) *
693 spin_lock(&mm
->ioctx_lock
);
694 old
= rcu_dereference_raw(mm
->ioctx_table
);
697 rcu_assign_pointer(mm
->ioctx_table
, table
);
698 } else if (table
->nr
> old
->nr
) {
699 memcpy(table
->table
, old
->table
,
700 old
->nr
* sizeof(struct kioctx
*));
702 rcu_assign_pointer(mm
->ioctx_table
, table
);
711 static void aio_nr_sub(unsigned nr
)
713 spin_lock(&aio_nr_lock
);
714 if (WARN_ON(aio_nr
- nr
> aio_nr
))
718 spin_unlock(&aio_nr_lock
);
722 * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed.
724 static struct kioctx
*ioctx_alloc(unsigned nr_events
)
726 struct mm_struct
*mm
= current
->mm
;
731 * Store the original nr_events -- what userspace passed to io_setup(),
732 * for counting against the global limit -- before it changes.
734 unsigned int max_reqs
= nr_events
;
737 * We keep track of the number of available ringbuffer slots, to prevent
738 * overflow (reqs_available), and we also use percpu counters for this.
740 * So since up to half the slots might be on other cpu's percpu counters
741 * and unavailable, double nr_events so userspace sees what they
742 * expected: additionally, we move req_batch slots to/from percpu
743 * counters at a time, so make sure that isn't 0:
745 nr_events
= max(nr_events
, num_possible_cpus() * 4);
748 /* Prevent overflows */
749 if (nr_events
> (0x10000000U
/ sizeof(struct io_event
))) {
750 pr_debug("ENOMEM: nr_events too high\n");
751 return ERR_PTR(-EINVAL
);
754 if (!nr_events
|| (unsigned long)max_reqs
> aio_max_nr
)
755 return ERR_PTR(-EAGAIN
);
757 ctx
= kmem_cache_zalloc(kioctx_cachep
, GFP_KERNEL
);
759 return ERR_PTR(-ENOMEM
);
761 ctx
->max_reqs
= max_reqs
;
763 spin_lock_init(&ctx
->ctx_lock
);
764 spin_lock_init(&ctx
->completion_lock
);
765 mutex_init(&ctx
->ring_lock
);
766 /* Protect against page migration throughout kiotx setup by keeping
767 * the ring_lock mutex held until setup is complete. */
768 mutex_lock(&ctx
->ring_lock
);
769 init_waitqueue_head(&ctx
->wait
);
771 INIT_LIST_HEAD(&ctx
->active_reqs
);
773 if (percpu_ref_init(&ctx
->users
, free_ioctx_users
, 0, GFP_KERNEL
))
776 if (percpu_ref_init(&ctx
->reqs
, free_ioctx_reqs
, 0, GFP_KERNEL
))
779 ctx
->cpu
= alloc_percpu(struct kioctx_cpu
);
783 err
= aio_setup_ring(ctx
, nr_events
);
787 atomic_set(&ctx
->reqs_available
, ctx
->nr_events
- 1);
788 ctx
->req_batch
= (ctx
->nr_events
- 1) / (num_possible_cpus() * 4);
789 if (ctx
->req_batch
< 1)
792 /* limit the number of system wide aios */
793 spin_lock(&aio_nr_lock
);
794 if (aio_nr
+ ctx
->max_reqs
> aio_max_nr
||
795 aio_nr
+ ctx
->max_reqs
< aio_nr
) {
796 spin_unlock(&aio_nr_lock
);
800 aio_nr
+= ctx
->max_reqs
;
801 spin_unlock(&aio_nr_lock
);
803 percpu_ref_get(&ctx
->users
); /* io_setup() will drop this ref */
804 percpu_ref_get(&ctx
->reqs
); /* free_ioctx_users() will drop this */
806 err
= ioctx_add_table(ctx
, mm
);
810 /* Release the ring_lock mutex now that all setup is complete. */
811 mutex_unlock(&ctx
->ring_lock
);
813 pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
814 ctx
, ctx
->user_id
, mm
, ctx
->nr_events
);
818 aio_nr_sub(ctx
->max_reqs
);
820 atomic_set(&ctx
->dead
, 1);
822 vm_munmap(ctx
->mmap_base
, ctx
->mmap_size
);
825 mutex_unlock(&ctx
->ring_lock
);
826 free_percpu(ctx
->cpu
);
827 percpu_ref_exit(&ctx
->reqs
);
828 percpu_ref_exit(&ctx
->users
);
829 kmem_cache_free(kioctx_cachep
, ctx
);
830 pr_debug("error allocating ioctx %d\n", err
);
835 * Cancels all outstanding aio requests on an aio context. Used
836 * when the processes owning a context have all exited to encourage
837 * the rapid destruction of the kioctx.
839 static int kill_ioctx(struct mm_struct
*mm
, struct kioctx
*ctx
,
840 struct ctx_rq_wait
*wait
)
842 struct kioctx_table
*table
;
844 spin_lock(&mm
->ioctx_lock
);
845 if (atomic_xchg(&ctx
->dead
, 1)) {
846 spin_unlock(&mm
->ioctx_lock
);
850 table
= rcu_dereference_raw(mm
->ioctx_table
);
851 WARN_ON(ctx
!= rcu_access_pointer(table
->table
[ctx
->id
]));
852 RCU_INIT_POINTER(table
->table
[ctx
->id
], NULL
);
853 spin_unlock(&mm
->ioctx_lock
);
855 /* free_ioctx_reqs() will do the necessary RCU synchronization */
856 wake_up_all(&ctx
->wait
);
859 * It'd be more correct to do this in free_ioctx(), after all
860 * the outstanding kiocbs have finished - but by then io_destroy
861 * has already returned, so io_setup() could potentially return
862 * -EAGAIN with no ioctxs actually in use (as far as userspace
865 aio_nr_sub(ctx
->max_reqs
);
868 vm_munmap(ctx
->mmap_base
, ctx
->mmap_size
);
871 percpu_ref_kill(&ctx
->users
);
876 * exit_aio: called when the last user of mm goes away. At this point, there is
877 * no way for any new requests to be submited or any of the io_* syscalls to be
878 * called on the context.
880 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
883 void exit_aio(struct mm_struct
*mm
)
885 struct kioctx_table
*table
= rcu_dereference_raw(mm
->ioctx_table
);
886 struct ctx_rq_wait wait
;
892 atomic_set(&wait
.count
, table
->nr
);
893 init_completion(&wait
.comp
);
896 for (i
= 0; i
< table
->nr
; ++i
) {
898 rcu_dereference_protected(table
->table
[i
], true);
906 * We don't need to bother with munmap() here - exit_mmap(mm)
907 * is coming and it'll unmap everything. And we simply can't,
908 * this is not necessarily our ->mm.
909 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
910 * that it needs to unmap the area, just set it to 0.
913 kill_ioctx(mm
, ctx
, &wait
);
916 if (!atomic_sub_and_test(skipped
, &wait
.count
)) {
917 /* Wait until all IO for the context are done. */
918 wait_for_completion(&wait
.comp
);
921 RCU_INIT_POINTER(mm
->ioctx_table
, NULL
);
925 static void put_reqs_available(struct kioctx
*ctx
, unsigned nr
)
927 struct kioctx_cpu
*kcpu
;
930 local_irq_save(flags
);
931 kcpu
= this_cpu_ptr(ctx
->cpu
);
932 kcpu
->reqs_available
+= nr
;
934 while (kcpu
->reqs_available
>= ctx
->req_batch
* 2) {
935 kcpu
->reqs_available
-= ctx
->req_batch
;
936 atomic_add(ctx
->req_batch
, &ctx
->reqs_available
);
939 local_irq_restore(flags
);
942 static bool get_reqs_available(struct kioctx
*ctx
)
944 struct kioctx_cpu
*kcpu
;
948 local_irq_save(flags
);
949 kcpu
= this_cpu_ptr(ctx
->cpu
);
950 if (!kcpu
->reqs_available
) {
951 int old
, avail
= atomic_read(&ctx
->reqs_available
);
954 if (avail
< ctx
->req_batch
)
958 avail
= atomic_cmpxchg(&ctx
->reqs_available
,
959 avail
, avail
- ctx
->req_batch
);
960 } while (avail
!= old
);
962 kcpu
->reqs_available
+= ctx
->req_batch
;
966 kcpu
->reqs_available
--;
968 local_irq_restore(flags
);
972 /* refill_reqs_available
973 * Updates the reqs_available reference counts used for tracking the
974 * number of free slots in the completion ring. This can be called
975 * from aio_complete() (to optimistically update reqs_available) or
976 * from aio_get_req() (the we're out of events case). It must be
977 * called holding ctx->completion_lock.
979 static void refill_reqs_available(struct kioctx
*ctx
, unsigned head
,
982 unsigned events_in_ring
, completed
;
984 /* Clamp head since userland can write to it. */
985 head
%= ctx
->nr_events
;
987 events_in_ring
= tail
- head
;
989 events_in_ring
= ctx
->nr_events
- (head
- tail
);
991 completed
= ctx
->completed_events
;
992 if (events_in_ring
< completed
)
993 completed
-= events_in_ring
;
1000 ctx
->completed_events
-= completed
;
1001 put_reqs_available(ctx
, completed
);
1004 /* user_refill_reqs_available
1005 * Called to refill reqs_available when aio_get_req() encounters an
1006 * out of space in the completion ring.
1008 static void user_refill_reqs_available(struct kioctx
*ctx
)
1010 spin_lock_irq(&ctx
->completion_lock
);
1011 if (ctx
->completed_events
) {
1012 struct aio_ring
*ring
;
1015 /* Access of ring->head may race with aio_read_events_ring()
1016 * here, but that's okay since whether we read the old version
1017 * or the new version, and either will be valid. The important
1018 * part is that head cannot pass tail since we prevent
1019 * aio_complete() from updating tail by holding
1020 * ctx->completion_lock. Even if head is invalid, the check
1021 * against ctx->completed_events below will make sure we do the
1024 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1026 kunmap_atomic(ring
);
1028 refill_reqs_available(ctx
, head
, ctx
->tail
);
1031 spin_unlock_irq(&ctx
->completion_lock
);
1035 * Allocate a slot for an aio request.
1036 * Returns NULL if no requests are free.
1038 static inline struct aio_kiocb
*aio_get_req(struct kioctx
*ctx
)
1040 struct aio_kiocb
*req
;
1042 if (!get_reqs_available(ctx
)) {
1043 user_refill_reqs_available(ctx
);
1044 if (!get_reqs_available(ctx
))
1048 req
= kmem_cache_alloc(kiocb_cachep
, GFP_KERNEL
|__GFP_ZERO
);
1052 percpu_ref_get(&ctx
->reqs
);
1057 put_reqs_available(ctx
, 1);
1061 static void kiocb_free(struct aio_kiocb
*req
)
1063 if (req
->common
.ki_filp
)
1064 fput(req
->common
.ki_filp
);
1065 if (req
->ki_eventfd
!= NULL
)
1066 eventfd_ctx_put(req
->ki_eventfd
);
1067 kmem_cache_free(kiocb_cachep
, req
);
1070 static struct kioctx
*lookup_ioctx(unsigned long ctx_id
)
1072 struct aio_ring __user
*ring
= (void __user
*)ctx_id
;
1073 struct mm_struct
*mm
= current
->mm
;
1074 struct kioctx
*ctx
, *ret
= NULL
;
1075 struct kioctx_table
*table
;
1078 if (get_user(id
, &ring
->id
))
1082 table
= rcu_dereference(mm
->ioctx_table
);
1084 if (!table
|| id
>= table
->nr
)
1087 ctx
= rcu_dereference(table
->table
[id
]);
1088 if (ctx
&& ctx
->user_id
== ctx_id
) {
1089 if (percpu_ref_tryget_live(&ctx
->users
))
1098 * Called when the io request on the given iocb is complete.
1100 static void aio_complete(struct kiocb
*kiocb
, long res
, long res2
)
1102 struct aio_kiocb
*iocb
= container_of(kiocb
, struct aio_kiocb
, common
);
1103 struct kioctx
*ctx
= iocb
->ki_ctx
;
1104 struct aio_ring
*ring
;
1105 struct io_event
*ev_page
, *event
;
1106 unsigned tail
, pos
, head
;
1107 unsigned long flags
;
1109 if (kiocb
->ki_flags
& IOCB_WRITE
) {
1110 struct file
*file
= kiocb
->ki_filp
;
1113 * Tell lockdep we inherited freeze protection from submission
1116 if (S_ISREG(file_inode(file
)->i_mode
))
1117 __sb_writers_acquired(file_inode(file
)->i_sb
, SB_FREEZE_WRITE
);
1118 file_end_write(file
);
1122 * Special case handling for sync iocbs:
1123 * - events go directly into the iocb for fast handling
1124 * - the sync task with the iocb in its stack holds the single iocb
1125 * ref, no other paths have a way to get another ref
1126 * - the sync task helpfully left a reference to itself in the iocb
1128 BUG_ON(is_sync_kiocb(kiocb
));
1130 if (iocb
->ki_list
.next
) {
1131 unsigned long flags
;
1133 spin_lock_irqsave(&ctx
->ctx_lock
, flags
);
1134 list_del(&iocb
->ki_list
);
1135 spin_unlock_irqrestore(&ctx
->ctx_lock
, flags
);
1139 * Add a completion event to the ring buffer. Must be done holding
1140 * ctx->completion_lock to prevent other code from messing with the tail
1141 * pointer since we might be called from irq context.
1143 spin_lock_irqsave(&ctx
->completion_lock
, flags
);
1146 pos
= tail
+ AIO_EVENTS_OFFSET
;
1148 if (++tail
>= ctx
->nr_events
)
1151 ev_page
= kmap_atomic(ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
]);
1152 event
= ev_page
+ pos
% AIO_EVENTS_PER_PAGE
;
1154 event
->obj
= (u64
)(unsigned long)iocb
->ki_user_iocb
;
1155 event
->data
= iocb
->ki_user_data
;
1159 kunmap_atomic(ev_page
);
1160 flush_dcache_page(ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
]);
1162 pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n",
1163 ctx
, tail
, iocb
, iocb
->ki_user_iocb
, iocb
->ki_user_data
,
1166 /* after flagging the request as done, we
1167 * must never even look at it again
1169 smp_wmb(); /* make event visible before updating tail */
1173 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1176 kunmap_atomic(ring
);
1177 flush_dcache_page(ctx
->ring_pages
[0]);
1179 ctx
->completed_events
++;
1180 if (ctx
->completed_events
> 1)
1181 refill_reqs_available(ctx
, head
, tail
);
1182 spin_unlock_irqrestore(&ctx
->completion_lock
, flags
);
1184 pr_debug("added to ring %p at [%u]\n", iocb
, tail
);
1187 * Check if the user asked us to deliver the result through an
1188 * eventfd. The eventfd_signal() function is safe to be called
1191 if (iocb
->ki_eventfd
!= NULL
)
1192 eventfd_signal(iocb
->ki_eventfd
, 1);
1194 /* everything turned out well, dispose of the aiocb. */
1198 * We have to order our ring_info tail store above and test
1199 * of the wait list below outside the wait lock. This is
1200 * like in wake_up_bit() where clearing a bit has to be
1201 * ordered with the unlocked test.
1205 if (waitqueue_active(&ctx
->wait
))
1206 wake_up(&ctx
->wait
);
1208 percpu_ref_put(&ctx
->reqs
);
1211 /* aio_read_events_ring
1212 * Pull an event off of the ioctx's event ring. Returns the number of
1215 static long aio_read_events_ring(struct kioctx
*ctx
,
1216 struct io_event __user
*event
, long nr
)
1218 struct aio_ring
*ring
;
1219 unsigned head
, tail
, pos
;
1224 * The mutex can block and wake us up and that will cause
1225 * wait_event_interruptible_hrtimeout() to schedule without sleeping
1226 * and repeat. This should be rare enough that it doesn't cause
1227 * peformance issues. See the comment in read_events() for more detail.
1229 sched_annotate_sleep();
1230 mutex_lock(&ctx
->ring_lock
);
1232 /* Access to ->ring_pages here is protected by ctx->ring_lock. */
1233 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1236 kunmap_atomic(ring
);
1239 * Ensure that once we've read the current tail pointer, that
1240 * we also see the events that were stored up to the tail.
1244 pr_debug("h%u t%u m%u\n", head
, tail
, ctx
->nr_events
);
1249 head
%= ctx
->nr_events
;
1250 tail
%= ctx
->nr_events
;
1254 struct io_event
*ev
;
1257 avail
= (head
<= tail
? tail
: ctx
->nr_events
) - head
;
1261 avail
= min(avail
, nr
- ret
);
1262 avail
= min_t(long, avail
, AIO_EVENTS_PER_PAGE
-
1263 ((head
+ AIO_EVENTS_OFFSET
) % AIO_EVENTS_PER_PAGE
));
1265 pos
= head
+ AIO_EVENTS_OFFSET
;
1266 page
= ctx
->ring_pages
[pos
/ AIO_EVENTS_PER_PAGE
];
1267 pos
%= AIO_EVENTS_PER_PAGE
;
1270 copy_ret
= copy_to_user(event
+ ret
, ev
+ pos
,
1271 sizeof(*ev
) * avail
);
1274 if (unlikely(copy_ret
)) {
1281 head
%= ctx
->nr_events
;
1284 ring
= kmap_atomic(ctx
->ring_pages
[0]);
1286 kunmap_atomic(ring
);
1287 flush_dcache_page(ctx
->ring_pages
[0]);
1289 pr_debug("%li h%u t%u\n", ret
, head
, tail
);
1291 mutex_unlock(&ctx
->ring_lock
);
1296 static bool aio_read_events(struct kioctx
*ctx
, long min_nr
, long nr
,
1297 struct io_event __user
*event
, long *i
)
1299 long ret
= aio_read_events_ring(ctx
, event
+ *i
, nr
- *i
);
1304 if (unlikely(atomic_read(&ctx
->dead
)))
1310 return ret
< 0 || *i
>= min_nr
;
1313 static long read_events(struct kioctx
*ctx
, long min_nr
, long nr
,
1314 struct io_event __user
*event
,
1320 * Note that aio_read_events() is being called as the conditional - i.e.
1321 * we're calling it after prepare_to_wait() has set task state to
1322 * TASK_INTERRUPTIBLE.
1324 * But aio_read_events() can block, and if it blocks it's going to flip
1325 * the task state back to TASK_RUNNING.
1327 * This should be ok, provided it doesn't flip the state back to
1328 * TASK_RUNNING and return 0 too much - that causes us to spin. That
1329 * will only happen if the mutex_lock() call blocks, and we then find
1330 * the ringbuffer empty. So in practice we should be ok, but it's
1331 * something to be aware of when touching this code.
1334 aio_read_events(ctx
, min_nr
, nr
, event
, &ret
);
1336 wait_event_interruptible_hrtimeout(ctx
->wait
,
1337 aio_read_events(ctx
, min_nr
, nr
, event
, &ret
),
1340 if (!ret
&& signal_pending(current
))
1347 * Create an aio_context capable of receiving at least nr_events.
1348 * ctxp must not point to an aio_context that already exists, and
1349 * must be initialized to 0 prior to the call. On successful
1350 * creation of the aio_context, *ctxp is filled in with the resulting
1351 * handle. May fail with -EINVAL if *ctxp is not initialized,
1352 * if the specified nr_events exceeds internal limits. May fail
1353 * with -EAGAIN if the specified nr_events exceeds the user's limit
1354 * of available events. May fail with -ENOMEM if insufficient kernel
1355 * resources are available. May fail with -EFAULT if an invalid
1356 * pointer is passed for ctxp. Will fail with -ENOSYS if not
1359 SYSCALL_DEFINE2(io_setup
, unsigned, nr_events
, aio_context_t __user
*, ctxp
)
1361 struct kioctx
*ioctx
= NULL
;
1365 ret
= get_user(ctx
, ctxp
);
1370 if (unlikely(ctx
|| nr_events
== 0)) {
1371 pr_debug("EINVAL: ctx %lu nr_events %u\n",
1376 ioctx
= ioctx_alloc(nr_events
);
1377 ret
= PTR_ERR(ioctx
);
1378 if (!IS_ERR(ioctx
)) {
1379 ret
= put_user(ioctx
->user_id
, ctxp
);
1381 kill_ioctx(current
->mm
, ioctx
, NULL
);
1382 percpu_ref_put(&ioctx
->users
);
1389 #ifdef CONFIG_COMPAT
1390 COMPAT_SYSCALL_DEFINE2(io_setup
, unsigned, nr_events
, u32 __user
*, ctx32p
)
1392 struct kioctx
*ioctx
= NULL
;
1396 ret
= get_user(ctx
, ctx32p
);
1401 if (unlikely(ctx
|| nr_events
== 0)) {
1402 pr_debug("EINVAL: ctx %lu nr_events %u\n",
1407 ioctx
= ioctx_alloc(nr_events
);
1408 ret
= PTR_ERR(ioctx
);
1409 if (!IS_ERR(ioctx
)) {
1410 /* truncating is ok because it's a user address */
1411 ret
= put_user((u32
)ioctx
->user_id
, ctx32p
);
1413 kill_ioctx(current
->mm
, ioctx
, NULL
);
1414 percpu_ref_put(&ioctx
->users
);
1423 * Destroy the aio_context specified. May cancel any outstanding
1424 * AIOs and block on completion. Will fail with -ENOSYS if not
1425 * implemented. May fail with -EINVAL if the context pointed to
1428 SYSCALL_DEFINE1(io_destroy
, aio_context_t
, ctx
)
1430 struct kioctx
*ioctx
= lookup_ioctx(ctx
);
1431 if (likely(NULL
!= ioctx
)) {
1432 struct ctx_rq_wait wait
;
1435 init_completion(&wait
.comp
);
1436 atomic_set(&wait
.count
, 1);
1438 /* Pass requests_done to kill_ioctx() where it can be set
1439 * in a thread-safe way. If we try to set it here then we have
1440 * a race condition if two io_destroy() called simultaneously.
1442 ret
= kill_ioctx(current
->mm
, ioctx
, &wait
);
1443 percpu_ref_put(&ioctx
->users
);
1445 /* Wait until all IO for the context are done. Otherwise kernel
1446 * keep using user-space buffers even if user thinks the context
1450 wait_for_completion(&wait
.comp
);
1454 pr_debug("EINVAL: invalid context id\n");
1458 static int aio_setup_rw(int rw
, struct iocb
*iocb
, struct iovec
**iovec
,
1459 bool vectored
, bool compat
, struct iov_iter
*iter
)
1461 void __user
*buf
= (void __user
*)(uintptr_t)iocb
->aio_buf
;
1462 size_t len
= iocb
->aio_nbytes
;
1465 ssize_t ret
= import_single_range(rw
, buf
, len
, *iovec
, iter
);
1469 #ifdef CONFIG_COMPAT
1471 return compat_import_iovec(rw
, buf
, len
, UIO_FASTIOV
, iovec
,
1474 return import_iovec(rw
, buf
, len
, UIO_FASTIOV
, iovec
, iter
);
1477 static inline ssize_t
aio_ret(struct kiocb
*req
, ssize_t ret
)
1483 case -ERESTARTNOINTR
:
1484 case -ERESTARTNOHAND
:
1485 case -ERESTART_RESTARTBLOCK
:
1487 * There's no easy way to restart the syscall since other AIO's
1488 * may be already running. Just fail this IO with EINTR.
1493 aio_complete(req
, ret
, 0);
1498 static ssize_t
aio_read(struct kiocb
*req
, struct iocb
*iocb
, bool vectored
,
1501 struct file
*file
= req
->ki_filp
;
1502 struct iovec inline_vecs
[UIO_FASTIOV
], *iovec
= inline_vecs
;
1503 struct iov_iter iter
;
1506 if (unlikely(!(file
->f_mode
& FMODE_READ
)))
1508 if (unlikely(!file
->f_op
->read_iter
))
1511 ret
= aio_setup_rw(READ
, iocb
, &iovec
, vectored
, compat
, &iter
);
1514 ret
= rw_verify_area(READ
, file
, &req
->ki_pos
, iov_iter_count(&iter
));
1516 ret
= aio_ret(req
, call_read_iter(file
, req
, &iter
));
1521 static ssize_t
aio_write(struct kiocb
*req
, struct iocb
*iocb
, bool vectored
,
1524 struct file
*file
= req
->ki_filp
;
1525 struct iovec inline_vecs
[UIO_FASTIOV
], *iovec
= inline_vecs
;
1526 struct iov_iter iter
;
1529 if (unlikely(!(file
->f_mode
& FMODE_WRITE
)))
1531 if (unlikely(!file
->f_op
->write_iter
))
1534 ret
= aio_setup_rw(WRITE
, iocb
, &iovec
, vectored
, compat
, &iter
);
1537 ret
= rw_verify_area(WRITE
, file
, &req
->ki_pos
, iov_iter_count(&iter
));
1539 req
->ki_flags
|= IOCB_WRITE
;
1540 file_start_write(file
);
1541 ret
= aio_ret(req
, call_write_iter(file
, req
, &iter
));
1543 * We release freeze protection in aio_complete(). Fool lockdep
1544 * by telling it the lock got released so that it doesn't
1545 * complain about held lock when we return to userspace.
1547 if (S_ISREG(file_inode(file
)->i_mode
))
1548 __sb_writers_release(file_inode(file
)->i_sb
, SB_FREEZE_WRITE
);
1554 static int io_submit_one(struct kioctx
*ctx
, struct iocb __user
*user_iocb
,
1555 struct iocb
*iocb
, bool compat
)
1557 struct aio_kiocb
*req
;
1561 /* enforce forwards compatibility on users */
1562 if (unlikely(iocb
->aio_reserved2
)) {
1563 pr_debug("EINVAL: reserve field set\n");
1567 /* prevent overflows */
1569 (iocb
->aio_buf
!= (unsigned long)iocb
->aio_buf
) ||
1570 (iocb
->aio_nbytes
!= (size_t)iocb
->aio_nbytes
) ||
1571 ((ssize_t
)iocb
->aio_nbytes
< 0)
1573 pr_debug("EINVAL: overflow check\n");
1577 req
= aio_get_req(ctx
);
1581 req
->common
.ki_filp
= file
= fget(iocb
->aio_fildes
);
1582 if (unlikely(!req
->common
.ki_filp
)) {
1586 req
->common
.ki_pos
= iocb
->aio_offset
;
1587 req
->common
.ki_complete
= aio_complete
;
1588 req
->common
.ki_flags
= iocb_flags(req
->common
.ki_filp
);
1589 req
->common
.ki_hint
= file_write_hint(file
);
1591 if (iocb
->aio_flags
& IOCB_FLAG_RESFD
) {
1593 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
1594 * instance of the file* now. The file descriptor must be
1595 * an eventfd() fd, and will be signaled for each completed
1596 * event using the eventfd_signal() function.
1598 req
->ki_eventfd
= eventfd_ctx_fdget((int) iocb
->aio_resfd
);
1599 if (IS_ERR(req
->ki_eventfd
)) {
1600 ret
= PTR_ERR(req
->ki_eventfd
);
1601 req
->ki_eventfd
= NULL
;
1605 req
->common
.ki_flags
|= IOCB_EVENTFD
;
1608 ret
= kiocb_set_rw_flags(&req
->common
, iocb
->aio_rw_flags
);
1609 if (unlikely(ret
)) {
1610 pr_debug("EINVAL: aio_rw_flags\n");
1614 ret
= put_user(KIOCB_KEY
, &user_iocb
->aio_key
);
1615 if (unlikely(ret
)) {
1616 pr_debug("EFAULT: aio_key\n");
1620 req
->ki_user_iocb
= user_iocb
;
1621 req
->ki_user_data
= iocb
->aio_data
;
1624 switch (iocb
->aio_lio_opcode
) {
1625 case IOCB_CMD_PREAD
:
1626 ret
= aio_read(&req
->common
, iocb
, false, compat
);
1628 case IOCB_CMD_PWRITE
:
1629 ret
= aio_write(&req
->common
, iocb
, false, compat
);
1631 case IOCB_CMD_PREADV
:
1632 ret
= aio_read(&req
->common
, iocb
, true, compat
);
1634 case IOCB_CMD_PWRITEV
:
1635 ret
= aio_write(&req
->common
, iocb
, true, compat
);
1638 pr_debug("invalid aio operation %d\n", iocb
->aio_lio_opcode
);
1644 if (ret
&& ret
!= -EIOCBQUEUED
)
1648 put_reqs_available(ctx
, 1);
1649 percpu_ref_put(&ctx
->reqs
);
1654 static long do_io_submit(aio_context_t ctx_id
, long nr
,
1655 struct iocb __user
*__user
*iocbpp
, bool compat
)
1660 struct blk_plug plug
;
1662 if (unlikely(nr
< 0))
1665 if (unlikely(nr
> LONG_MAX
/sizeof(*iocbpp
)))
1666 nr
= LONG_MAX
/sizeof(*iocbpp
);
1668 if (unlikely(!access_ok(VERIFY_READ
, iocbpp
, (nr
*sizeof(*iocbpp
)))))
1671 ctx
= lookup_ioctx(ctx_id
);
1672 if (unlikely(!ctx
)) {
1673 pr_debug("EINVAL: invalid context id\n");
1677 blk_start_plug(&plug
);
1680 * AKPM: should this return a partial result if some of the IOs were
1681 * successfully submitted?
1683 for (i
=0; i
<nr
; i
++) {
1684 struct iocb __user
*user_iocb
;
1687 if (unlikely(__get_user(user_iocb
, iocbpp
+ i
))) {
1692 if (unlikely(copy_from_user(&tmp
, user_iocb
, sizeof(tmp
)))) {
1697 ret
= io_submit_one(ctx
, user_iocb
, &tmp
, compat
);
1701 blk_finish_plug(&plug
);
1703 percpu_ref_put(&ctx
->users
);
1708 * Queue the nr iocbs pointed to by iocbpp for processing. Returns
1709 * the number of iocbs queued. May return -EINVAL if the aio_context
1710 * specified by ctx_id is invalid, if nr is < 0, if the iocb at
1711 * *iocbpp[0] is not properly initialized, if the operation specified
1712 * is invalid for the file descriptor in the iocb. May fail with
1713 * -EFAULT if any of the data structures point to invalid data. May
1714 * fail with -EBADF if the file descriptor specified in the first
1715 * iocb is invalid. May fail with -EAGAIN if insufficient resources
1716 * are available to queue any iocbs. Will return 0 if nr is 0. Will
1717 * fail with -ENOSYS if not implemented.
1719 SYSCALL_DEFINE3(io_submit
, aio_context_t
, ctx_id
, long, nr
,
1720 struct iocb __user
* __user
*, iocbpp
)
1722 return do_io_submit(ctx_id
, nr
, iocbpp
, 0);
1725 #ifdef CONFIG_COMPAT
1727 copy_iocb(long nr
, u32 __user
*ptr32
, struct iocb __user
* __user
*ptr64
)
1732 for (i
= 0; i
< nr
; ++i
) {
1733 if (get_user(uptr
, ptr32
+ i
))
1735 if (put_user(compat_ptr(uptr
), ptr64
+ i
))
1741 #define MAX_AIO_SUBMITS (PAGE_SIZE/sizeof(struct iocb *))
1743 COMPAT_SYSCALL_DEFINE3(io_submit
, compat_aio_context_t
, ctx_id
,
1744 int, nr
, u32 __user
*, iocb
)
1746 struct iocb __user
* __user
*iocb64
;
1749 if (unlikely(nr
< 0))
1752 if (nr
> MAX_AIO_SUBMITS
)
1753 nr
= MAX_AIO_SUBMITS
;
1755 iocb64
= compat_alloc_user_space(nr
* sizeof(*iocb64
));
1756 ret
= copy_iocb(nr
, iocb
, iocb64
);
1758 ret
= do_io_submit(ctx_id
, nr
, iocb64
, 1);
1764 * Finds a given iocb for cancellation.
1766 static struct aio_kiocb
*
1767 lookup_kiocb(struct kioctx
*ctx
, struct iocb __user
*iocb
, u32 key
)
1769 struct aio_kiocb
*kiocb
;
1771 assert_spin_locked(&ctx
->ctx_lock
);
1773 if (key
!= KIOCB_KEY
)
1776 /* TODO: use a hash or array, this sucks. */
1777 list_for_each_entry(kiocb
, &ctx
->active_reqs
, ki_list
) {
1778 if (kiocb
->ki_user_iocb
== iocb
)
1785 * Attempts to cancel an iocb previously passed to io_submit. If
1786 * the operation is successfully cancelled, the resulting event is
1787 * copied into the memory pointed to by result without being placed
1788 * into the completion queue and 0 is returned. May fail with
1789 * -EFAULT if any of the data structures pointed to are invalid.
1790 * May fail with -EINVAL if aio_context specified by ctx_id is
1791 * invalid. May fail with -EAGAIN if the iocb specified was not
1792 * cancelled. Will fail with -ENOSYS if not implemented.
1794 SYSCALL_DEFINE3(io_cancel
, aio_context_t
, ctx_id
, struct iocb __user
*, iocb
,
1795 struct io_event __user
*, result
)
1798 struct aio_kiocb
*kiocb
;
1802 ret
= get_user(key
, &iocb
->aio_key
);
1806 ctx
= lookup_ioctx(ctx_id
);
1810 spin_lock_irq(&ctx
->ctx_lock
);
1812 kiocb
= lookup_kiocb(ctx
, iocb
, key
);
1814 ret
= kiocb_cancel(kiocb
);
1818 spin_unlock_irq(&ctx
->ctx_lock
);
1822 * The result argument is no longer used - the io_event is
1823 * always delivered via the ring buffer. -EINPROGRESS indicates
1824 * cancellation is progress:
1829 percpu_ref_put(&ctx
->users
);
1834 static long do_io_getevents(aio_context_t ctx_id
,
1837 struct io_event __user
*events
,
1838 struct timespec64
*ts
)
1840 ktime_t until
= ts
? timespec64_to_ktime(*ts
) : KTIME_MAX
;
1841 struct kioctx
*ioctx
= lookup_ioctx(ctx_id
);
1844 if (likely(ioctx
)) {
1845 if (likely(min_nr
<= nr
&& min_nr
>= 0))
1846 ret
= read_events(ioctx
, min_nr
, nr
, events
, until
);
1847 percpu_ref_put(&ioctx
->users
);
1854 * Attempts to read at least min_nr events and up to nr events from
1855 * the completion queue for the aio_context specified by ctx_id. If
1856 * it succeeds, the number of read events is returned. May fail with
1857 * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
1858 * out of range, if timeout is out of range. May fail with -EFAULT
1859 * if any of the memory specified is invalid. May return 0 or
1860 * < min_nr if the timeout specified by timeout has elapsed
1861 * before sufficient events are available, where timeout == NULL
1862 * specifies an infinite timeout. Note that the timeout pointed to by
1863 * timeout is relative. Will fail with -ENOSYS if not implemented.
1865 SYSCALL_DEFINE5(io_getevents
, aio_context_t
, ctx_id
,
1868 struct io_event __user
*, events
,
1869 struct timespec __user
*, timeout
)
1871 struct timespec64 ts
;
1874 if (unlikely(get_timespec64(&ts
, timeout
)))
1878 return do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &ts
: NULL
);
1881 #ifdef CONFIG_COMPAT
1882 COMPAT_SYSCALL_DEFINE5(io_getevents
, compat_aio_context_t
, ctx_id
,
1883 compat_long_t
, min_nr
,
1885 struct io_event __user
*, events
,
1886 struct compat_timespec __user
*, timeout
)
1888 struct timespec64 t
;
1891 if (compat_get_timespec64(&t
, timeout
))
1896 return do_io_getevents(ctx_id
, min_nr
, nr
, events
, timeout
? &t
: NULL
);