]>
Commit | Line | Data |
---|---|---|
771b53d0 JA |
1 | #ifndef INTERNAL_IO_WQ_H |
2 | #define INTERNAL_IO_WQ_H | |
3 | ||
4 | struct io_wq; | |
5 | ||
6 | enum { | |
7 | IO_WQ_WORK_CANCEL = 1, | |
8 | IO_WQ_WORK_HAS_MM = 2, | |
9 | IO_WQ_WORK_HASHED = 4, | |
c5def4ab | 10 | IO_WQ_WORK_UNBOUND = 32, |
b76da70f | 11 | IO_WQ_WORK_CB = 128, |
0c9d5ccd | 12 | IO_WQ_WORK_NO_CANCEL = 256, |
895e2ca0 | 13 | IO_WQ_WORK_CONCURRENT = 512, |
771b53d0 JA |
14 | |
15 | IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ | |
16 | }; | |
17 | ||
18 | enum io_wq_cancel { | |
19 | IO_WQ_CANCEL_OK, /* cancelled before started */ | |
20 | IO_WQ_CANCEL_RUNNING, /* found, running, and attempted cancelled */ | |
21 | IO_WQ_CANCEL_NOTFOUND, /* work not found */ | |
22 | }; | |
23 | ||
6206f0e1 JA |
24 | struct io_wq_work_node { |
25 | struct io_wq_work_node *next; | |
26 | }; | |
27 | ||
28 | struct io_wq_work_list { | |
29 | struct io_wq_work_node *first; | |
30 | struct io_wq_work_node *last; | |
31 | }; | |
32 | ||
33 | static inline void wq_list_add_tail(struct io_wq_work_node *node, | |
34 | struct io_wq_work_list *list) | |
35 | { | |
36 | if (!list->first) { | |
e995d512 JA |
37 | list->last = node; |
38 | WRITE_ONCE(list->first, node); | |
6206f0e1 JA |
39 | } else { |
40 | list->last->next = node; | |
41 | list->last = node; | |
42 | } | |
43 | } | |
44 | ||
45 | static inline void wq_node_del(struct io_wq_work_list *list, | |
46 | struct io_wq_work_node *node, | |
47 | struct io_wq_work_node *prev) | |
48 | { | |
49 | if (node == list->first) | |
e995d512 | 50 | WRITE_ONCE(list->first, node->next); |
6206f0e1 JA |
51 | if (node == list->last) |
52 | list->last = prev; | |
53 | if (prev) | |
54 | prev->next = node->next; | |
08bdcc35 | 55 | node->next = NULL; |
6206f0e1 JA |
56 | } |
57 | ||
58 | #define wq_list_for_each(pos, prv, head) \ | |
59 | for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) | |
60 | ||
e995d512 | 61 | #define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) |
6206f0e1 JA |
62 | #define INIT_WQ_LIST(list) do { \ |
63 | (list)->first = NULL; \ | |
64 | (list)->last = NULL; \ | |
65 | } while (0) | |
66 | ||
771b53d0 | 67 | struct io_wq_work { |
b76da70f | 68 | union { |
6206f0e1 | 69 | struct io_wq_work_node list; |
b76da70f JA |
70 | void *data; |
71 | }; | |
771b53d0 | 72 | void (*func)(struct io_wq_work **); |
fcb323cc | 73 | struct files_struct *files; |
cccf0ee8 JA |
74 | struct mm_struct *mm; |
75 | const struct cred *creds; | |
9392a27d | 76 | struct fs_struct *fs; |
6206f0e1 | 77 | unsigned flags; |
36282881 | 78 | pid_t task_pid; |
771b53d0 JA |
79 | }; |
80 | ||
2d141dd2 JA |
81 | #define INIT_IO_WORK(work, _func) \ |
82 | do { \ | |
83 | *(work) = (struct io_wq_work){ .func = _func }; \ | |
84 | } while (0) \ | |
771b53d0 | 85 | |
7d723065 JA |
86 | typedef void (get_work_fn)(struct io_wq_work *); |
87 | typedef void (put_work_fn)(struct io_wq_work *); | |
88 | ||
576a347b | 89 | struct io_wq_data { |
576a347b JA |
90 | struct user_struct *user; |
91 | ||
92 | get_work_fn *get_work; | |
93 | put_work_fn *put_work; | |
94 | }; | |
95 | ||
96 | struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); | |
eba6f5a3 | 97 | bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); |
771b53d0 JA |
98 | void io_wq_destroy(struct io_wq *wq); |
99 | ||
100 | void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work); | |
101 | void io_wq_enqueue_hashed(struct io_wq *wq, struct io_wq_work *work, void *val); | |
771b53d0 JA |
102 | |
103 | void io_wq_cancel_all(struct io_wq *wq); | |
104 | enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork); | |
36282881 | 105 | enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid); |
771b53d0 | 106 | |
62755e35 JA |
107 | typedef bool (work_cancel_fn)(struct io_wq_work *, void *); |
108 | ||
109 | enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, | |
110 | void *data); | |
111 | ||
771b53d0 JA |
112 | #if defined(CONFIG_IO_WQ) |
113 | extern void io_wq_worker_sleeping(struct task_struct *); | |
114 | extern void io_wq_worker_running(struct task_struct *); | |
115 | #else | |
116 | static inline void io_wq_worker_sleeping(struct task_struct *tsk) | |
117 | { | |
118 | } | |
119 | static inline void io_wq_worker_running(struct task_struct *tsk) | |
120 | { | |
121 | } | |
525b305d | 122 | #endif |
771b53d0 | 123 | |
525b305d JA |
124 | static inline bool io_wq_current_is_worker(void) |
125 | { | |
126 | return in_task() && (current->flags & PF_IO_WORKER); | |
127 | } | |
128 | #endif |