]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef _LINUX_ELEVATOR_H |
3 | #define _LINUX_ELEVATOR_H | |
4 | ||
4a893e83 | 5 | #include <linux/percpu.h> |
242d98f0 | 6 | #include <linux/hashtable.h> |
4a893e83 | 7 | |
9361401e DH |
8 | #ifdef CONFIG_BLOCK |
9 | ||
7e5a8794 | 10 | struct io_cq; |
d50235b7 | 11 | struct elevator_type; |
d332ce09 OS |
12 | #ifdef CONFIG_BLK_DEBUG_FS |
13 | struct blk_mq_debugfs_attr; | |
14 | #endif | |
7e5a8794 | 15 | |
34fe7c05 CH |
16 | /* |
17 | * Return values from elevator merger | |
18 | */ | |
19 | enum elv_merge { | |
20 | ELEVATOR_NO_MERGE = 0, | |
21 | ELEVATOR_FRONT_MERGE = 1, | |
22 | ELEVATOR_BACK_MERGE = 2, | |
1e739730 | 23 | ELEVATOR_DISCARD_MERGE = 3, |
34fe7c05 CH |
24 | }; |
25 | ||
26 | typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **, | |
1da177e4 LT |
27 | struct bio *); |
28 | ||
165125e1 | 29 | typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *); |
1da177e4 | 30 | |
34fe7c05 | 31 | typedef void (elevator_merged_fn) (struct request_queue *, struct request *, enum elv_merge); |
1da177e4 | 32 | |
72ef799b TE |
33 | typedef int (elevator_allow_bio_merge_fn) (struct request_queue *, |
34 | struct request *, struct bio *); | |
35 | ||
36 | typedef int (elevator_allow_rq_merge_fn) (struct request_queue *, | |
37 | struct request *, struct request *); | |
da775265 | 38 | |
812d4026 DS |
39 | typedef void (elevator_bio_merged_fn) (struct request_queue *, |
40 | struct request *, struct bio *); | |
41 | ||
165125e1 | 42 | typedef int (elevator_dispatch_fn) (struct request_queue *, int); |
1da177e4 | 43 | |
165125e1 | 44 | typedef void (elevator_add_req_fn) (struct request_queue *, struct request *); |
165125e1 JA |
45 | typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *); |
46 | typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *); | |
ef295ecf | 47 | typedef int (elevator_may_queue_fn) (struct request_queue *, unsigned int); |
1da177e4 | 48 | |
9b84cacd | 49 | typedef void (elevator_init_icq_fn) (struct io_cq *); |
7e5a8794 | 50 | typedef void (elevator_exit_icq_fn) (struct io_cq *); |
852c788f TH |
51 | typedef int (elevator_set_req_fn) (struct request_queue *, struct request *, |
52 | struct bio *, gfp_t); | |
bb37b94c | 53 | typedef void (elevator_put_req_fn) (struct request *); |
165125e1 JA |
54 | typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *); |
55 | typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *); | |
1da177e4 | 56 | |
d50235b7 JM |
57 | typedef int (elevator_init_fn) (struct request_queue *, |
58 | struct elevator_type *e); | |
b374d18a | 59 | typedef void (elevator_exit_fn) (struct elevator_queue *); |
0bb97947 | 60 | typedef void (elevator_registered_fn) (struct request_queue *); |
1da177e4 LT |
61 | |
62 | struct elevator_ops | |
63 | { | |
64 | elevator_merge_fn *elevator_merge_fn; | |
65 | elevator_merged_fn *elevator_merged_fn; | |
66 | elevator_merge_req_fn *elevator_merge_req_fn; | |
72ef799b TE |
67 | elevator_allow_bio_merge_fn *elevator_allow_bio_merge_fn; |
68 | elevator_allow_rq_merge_fn *elevator_allow_rq_merge_fn; | |
812d4026 | 69 | elevator_bio_merged_fn *elevator_bio_merged_fn; |
1da177e4 | 70 | |
8922e16c | 71 | elevator_dispatch_fn *elevator_dispatch_fn; |
1da177e4 | 72 | elevator_add_req_fn *elevator_add_req_fn; |
8922e16c | 73 | elevator_activate_req_fn *elevator_activate_req_fn; |
1da177e4 LT |
74 | elevator_deactivate_req_fn *elevator_deactivate_req_fn; |
75 | ||
1da177e4 LT |
76 | elevator_completed_req_fn *elevator_completed_req_fn; |
77 | ||
78 | elevator_request_list_fn *elevator_former_req_fn; | |
79 | elevator_request_list_fn *elevator_latter_req_fn; | |
80 | ||
f1f8cc94 TH |
81 | elevator_init_icq_fn *elevator_init_icq_fn; /* see iocontext.h */ |
82 | elevator_exit_icq_fn *elevator_exit_icq_fn; /* ditto */ | |
7e5a8794 | 83 | |
1da177e4 LT |
84 | elevator_set_req_fn *elevator_set_req_fn; |
85 | elevator_put_req_fn *elevator_put_req_fn; | |
86 | ||
87 | elevator_may_queue_fn *elevator_may_queue_fn; | |
88 | ||
89 | elevator_init_fn *elevator_init_fn; | |
90 | elevator_exit_fn *elevator_exit_fn; | |
0bb97947 | 91 | elevator_registered_fn *elevator_registered_fn; |
1da177e4 LT |
92 | }; |
93 | ||
bd166ef1 JA |
94 | struct blk_mq_alloc_data; |
95 | struct blk_mq_hw_ctx; | |
96 | ||
97 | struct elevator_mq_ops { | |
98 | int (*init_sched)(struct request_queue *, struct elevator_type *); | |
99 | void (*exit_sched)(struct elevator_queue *); | |
ee056f98 OS |
100 | int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); |
101 | void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); | |
bd166ef1 JA |
102 | |
103 | bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); | |
104 | bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *); | |
105 | int (*request_merge)(struct request_queue *q, struct request **, struct bio *); | |
34fe7c05 | 106 | void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); |
bd166ef1 | 107 | void (*requests_merged)(struct request_queue *, struct request *, struct request *); |
5bbf4e5a CH |
108 | void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); |
109 | void (*prepare_request)(struct request *, struct bio *bio); | |
7b9e9361 | 110 | void (*finish_request)(struct request *); |
bd166ef1 | 111 | void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); |
c13660a0 | 112 | struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); |
bd166ef1 | 113 | bool (*has_work)(struct blk_mq_hw_ctx *); |
ed88660a | 114 | void (*completed_request)(struct request *, u64); |
bd166ef1 JA |
115 | void (*started_request)(struct request *); |
116 | void (*requeue_request)(struct request *); | |
117 | struct request *(*former_request)(struct request_queue *, struct request *); | |
118 | struct request *(*next_request)(struct request_queue *, struct request *); | |
bd166ef1 JA |
119 | void (*init_icq)(struct io_cq *); |
120 | void (*exit_icq)(struct io_cq *); | |
121 | }; | |
122 | ||
1da177e4 LT |
123 | #define ELV_NAME_MAX (16) |
124 | ||
e572ec7e AV |
125 | struct elv_fs_entry { |
126 | struct attribute attr; | |
b374d18a JA |
127 | ssize_t (*show)(struct elevator_queue *, char *); |
128 | ssize_t (*store)(struct elevator_queue *, const char *, size_t); | |
e572ec7e AV |
129 | }; |
130 | ||
1da177e4 LT |
131 | /* |
132 | * identifies an elevator type, such as AS or deadline | |
133 | */ | |
134 | struct elevator_type | |
135 | { | |
3d3c2379 TH |
136 | /* managed by elevator core */ |
137 | struct kmem_cache *icq_cache; | |
138 | ||
139 | /* fields provided by elevator implementation */ | |
c51ca6cf JA |
140 | union { |
141 | struct elevator_ops sq; | |
bd166ef1 | 142 | struct elevator_mq_ops mq; |
c51ca6cf | 143 | } ops; |
f1f8cc94 TH |
144 | size_t icq_size; /* see iocontext.h */ |
145 | size_t icq_align; /* ditto */ | |
e572ec7e | 146 | struct elv_fs_entry *elevator_attrs; |
1da177e4 | 147 | char elevator_name[ELV_NAME_MAX]; |
8ac0d9a8 | 148 | const char *elevator_alias; |
1da177e4 | 149 | struct module *elevator_owner; |
bd166ef1 | 150 | bool uses_mq; |
d332ce09 OS |
151 | #ifdef CONFIG_BLK_DEBUG_FS |
152 | const struct blk_mq_debugfs_attr *queue_debugfs_attrs; | |
153 | const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; | |
154 | #endif | |
3d3c2379 TH |
155 | |
156 | /* managed by elevator core */ | |
9bd2bbc0 | 157 | char icq_cache_name[ELV_NAME_MAX + 6]; /* elvname + "_io_cq" */ |
22f746e2 | 158 | struct list_head list; |
1da177e4 LT |
159 | }; |
160 | ||
242d98f0 SL |
161 | #define ELV_HASH_BITS 6 |
162 | ||
70b3ea05 JA |
163 | void elv_rqhash_del(struct request_queue *q, struct request *rq); |
164 | void elv_rqhash_add(struct request_queue *q, struct request *rq); | |
165 | void elv_rqhash_reposition(struct request_queue *q, struct request *rq); | |
166 | struct request *elv_rqhash_find(struct request_queue *q, sector_t offset); | |
167 | ||
1da177e4 | 168 | /* |
54069588 | 169 | * each queue has an elevator_queue associated with it |
1da177e4 LT |
170 | */ |
171 | struct elevator_queue | |
172 | { | |
22f746e2 | 173 | struct elevator_type *type; |
1da177e4 LT |
174 | void *elevator_data; |
175 | struct kobject kobj; | |
3d1ab40f | 176 | struct mutex sysfs_lock; |
430c62fb | 177 | unsigned int registered:1; |
bd166ef1 | 178 | unsigned int uses_mq:1; |
242d98f0 | 179 | DECLARE_HASHTABLE(hash, ELV_HASH_BITS); |
1da177e4 LT |
180 | }; |
181 | ||
182 | /* | |
183 | * block elevator interface | |
184 | */ | |
165125e1 JA |
185 | extern void elv_dispatch_sort(struct request_queue *, struct request *); |
186 | extern void elv_dispatch_add_tail(struct request_queue *, struct request *); | |
7eaceacc JA |
187 | extern void elv_add_request(struct request_queue *, struct request *, int); |
188 | extern void __elv_add_request(struct request_queue *, struct request *, int); | |
34fe7c05 CH |
189 | extern enum elv_merge elv_merge(struct request_queue *, struct request **, |
190 | struct bio *); | |
165125e1 | 191 | extern void elv_merge_requests(struct request_queue *, struct request *, |
1da177e4 | 192 | struct request *); |
34fe7c05 CH |
193 | extern void elv_merged_request(struct request_queue *, struct request *, |
194 | enum elv_merge); | |
812d4026 DS |
195 | extern void elv_bio_merged(struct request_queue *q, struct request *, |
196 | struct bio *); | |
bd166ef1 | 197 | extern bool elv_attempt_insert_merge(struct request_queue *, struct request *); |
165125e1 | 198 | extern void elv_requeue_request(struct request_queue *, struct request *); |
165125e1 JA |
199 | extern struct request *elv_former_request(struct request_queue *, struct request *); |
200 | extern struct request *elv_latter_request(struct request_queue *, struct request *); | |
ef295ecf | 201 | extern int elv_may_queue(struct request_queue *, unsigned int); |
165125e1 | 202 | extern void elv_completed_request(struct request_queue *, struct request *); |
852c788f TH |
203 | extern int elv_set_request(struct request_queue *q, struct request *rq, |
204 | struct bio *bio, gfp_t gfp_mask); | |
165125e1 | 205 | extern void elv_put_request(struct request_queue *, struct request *); |
26308eab | 206 | extern void elv_drain_elevator(struct request_queue *); |
1da177e4 LT |
207 | |
208 | /* | |
209 | * io scheduler registration | |
210 | */ | |
bb813f4c | 211 | extern void __init load_default_elevator_module(void); |
e567bf71 | 212 | extern int elv_register(struct elevator_type *); |
1da177e4 LT |
213 | extern void elv_unregister(struct elevator_type *); |
214 | ||
215 | /* | |
216 | * io scheduler sysfs switching | |
217 | */ | |
165125e1 JA |
218 | extern ssize_t elv_iosched_show(struct request_queue *, char *); |
219 | extern ssize_t elv_iosched_store(struct request_queue *, const char *, size_t); | |
1da177e4 | 220 | |
72ef799b | 221 | extern bool elv_bio_merge_ok(struct request *, struct bio *); |
d50235b7 JM |
222 | extern struct elevator_queue *elevator_alloc(struct request_queue *, |
223 | struct elevator_type *); | |
1da177e4 | 224 | |
2e662b65 JA |
225 | /* |
226 | * Helper functions. | |
227 | */ | |
165125e1 JA |
228 | extern struct request *elv_rb_former_request(struct request_queue *, struct request *); |
229 | extern struct request *elv_rb_latter_request(struct request_queue *, struct request *); | |
2e662b65 JA |
230 | |
231 | /* | |
232 | * rb support functions. | |
233 | */ | |
796d5116 | 234 | extern void elv_rb_add(struct rb_root *, struct request *); |
2e662b65 JA |
235 | extern void elv_rb_del(struct rb_root *, struct request *); |
236 | extern struct request *elv_rb_find(struct rb_root *, sector_t); | |
237 | ||
1da177e4 LT |
238 | /* |
239 | * Insertion selection | |
240 | */ | |
241 | #define ELEVATOR_INSERT_FRONT 1 | |
242 | #define ELEVATOR_INSERT_BACK 2 | |
243 | #define ELEVATOR_INSERT_SORT 3 | |
797e7dbb | 244 | #define ELEVATOR_INSERT_REQUEUE 4 |
ae1b1539 | 245 | #define ELEVATOR_INSERT_FLUSH 5 |
5e84ea3a | 246 | #define ELEVATOR_INSERT_SORT_MERGE 6 |
1da177e4 LT |
247 | |
248 | /* | |
249 | * return values from elevator_may_queue_fn | |
250 | */ | |
251 | enum { | |
252 | ELV_MQUEUE_MAY, | |
253 | ELV_MQUEUE_NO, | |
254 | ELV_MQUEUE_MUST, | |
255 | }; | |
256 | ||
2e46e8b2 | 257 | #define rq_end_sector(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) |
2e662b65 | 258 | #define rb_entry_rq(node) rb_entry((node), struct request, rb_node) |
1b47f531 | 259 | |
1fbfdfcd | 260 | #define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) |
d9a74df5 | 261 | #define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist) |
1fbfdfcd | 262 | |
bb813f4c TH |
263 | #else /* CONFIG_BLOCK */ |
264 | ||
265 | static inline void load_default_elevator_module(void) { } | |
266 | ||
9361401e | 267 | #endif /* CONFIG_BLOCK */ |
1da177e4 | 268 | #endif |