1 #include <linux/kernel.h>
2 #include <linux/errno.h>
3 #include <linux/file.h>
4 #include <linux/slab.h>
6 #include <linux/io_uring.h>
12 static void io_notif_complete_tw_ext(struct io_kiocb
*notif
, bool *locked
)
14 struct io_notif_data
*nd
= io_notif_to_data(notif
);
15 struct io_ring_ctx
*ctx
= notif
->ctx
;
17 if (nd
->zc_report
&& (nd
->zc_copied
|| !nd
->zc_used
))
18 notif
->cqe
.res
|= IORING_NOTIF_USAGE_ZC_COPIED
;
20 if (nd
->account_pages
&& ctx
->user
) {
21 __io_unaccount_mem(ctx
->user
, nd
->account_pages
);
22 nd
->account_pages
= 0;
24 io_req_task_complete(notif
, locked
);
27 static void io_tx_ubuf_callback(struct sk_buff
*skb
, struct ubuf_info
*uarg
,
30 struct io_notif_data
*nd
= container_of(uarg
, struct io_notif_data
, uarg
);
31 struct io_kiocb
*notif
= cmd_to_io_kiocb(nd
);
33 if (refcount_dec_and_test(&uarg
->refcnt
))
34 io_req_task_work_add(notif
);
37 static void io_tx_ubuf_callback_ext(struct sk_buff
*skb
, struct ubuf_info
*uarg
,
40 struct io_notif_data
*nd
= container_of(uarg
, struct io_notif_data
, uarg
);
43 if (success
&& !nd
->zc_used
&& skb
)
44 WRITE_ONCE(nd
->zc_used
, true);
45 else if (!success
&& !nd
->zc_copied
)
46 WRITE_ONCE(nd
->zc_copied
, true);
48 io_tx_ubuf_callback(skb
, uarg
, success
);
51 void io_notif_set_extended(struct io_kiocb
*notif
)
53 struct io_notif_data
*nd
= io_notif_to_data(notif
);
55 if (nd
->uarg
.callback
!= io_tx_ubuf_callback_ext
) {
56 nd
->account_pages
= 0;
57 nd
->zc_report
= false;
59 nd
->zc_copied
= false;
60 nd
->uarg
.callback
= io_tx_ubuf_callback_ext
;
61 notif
->io_task_work
.func
= io_notif_complete_tw_ext
;
65 struct io_kiocb
*io_alloc_notif(struct io_ring_ctx
*ctx
)
66 __must_hold(&ctx
->uring_lock
)
68 struct io_kiocb
*notif
;
69 struct io_notif_data
*nd
;
71 if (unlikely(!io_alloc_req_refill(ctx
)))
73 notif
= io_alloc_req(ctx
);
74 notif
->opcode
= IORING_OP_NOP
;
77 notif
->task
= current
;
79 notif
->rsrc_node
= NULL
;
80 notif
->io_task_work
.func
= io_req_task_complete
;
82 nd
= io_notif_to_data(notif
);
83 nd
->uarg
.flags
= SKBFL_ZEROCOPY_FRAG
| SKBFL_DONT_ORPHAN
;
84 nd
->uarg
.callback
= io_tx_ubuf_callback
;
85 refcount_set(&nd
->uarg
.refcnt
, 1);