]>
Commit | Line | Data |
---|---|---|
320ae51f JA |
1 | #ifndef BLK_MQ_H |
2 | #define BLK_MQ_H | |
3 | ||
4 | #include <linux/blkdev.h> | |
88459642 | 5 | #include <linux/sbitmap.h> |
6a83e74d | 6 | #include <linux/srcu.h> |
320ae51f JA |
7 | |
8 | struct blk_mq_tags; | |
f70ced09 | 9 | struct blk_flush_queue; |
320ae51f | 10 | |
320ae51f JA |
11 | struct blk_mq_hw_ctx { |
12 | struct { | |
13 | spinlock_t lock; | |
14 | struct list_head dispatch; | |
8d354f13 | 15 | unsigned long state; /* BLK_MQ_S_* flags */ |
320ae51f JA |
16 | } ____cacheline_aligned_in_smp; |
17 | ||
27489a3c | 18 | struct work_struct run_work; |
e4043dcf | 19 | cpumask_var_t cpumask; |
506e931f JA |
20 | int next_cpu; |
21 | int next_cpu_batch; | |
320ae51f JA |
22 | |
23 | unsigned long flags; /* BLK_MQ_F_* flags */ | |
24 | ||
25 | struct request_queue *queue; | |
f70ced09 | 26 | struct blk_flush_queue *fq; |
320ae51f JA |
27 | |
28 | void *driver_data; | |
29 | ||
88459642 | 30 | struct sbitmap ctx_map; |
1429d7c9 | 31 | |
4bb659b1 | 32 | struct blk_mq_ctx **ctxs; |
8d354f13 | 33 | unsigned int nr_ctx; |
4bb659b1 | 34 | |
8537b120 | 35 | atomic_t wait_index; |
320ae51f | 36 | |
320ae51f JA |
37 | struct blk_mq_tags *tags; |
38 | ||
6a83e74d BVA |
39 | struct srcu_struct queue_rq_srcu; |
40 | ||
320ae51f JA |
41 | unsigned long queued; |
42 | unsigned long run; | |
8d354f13 | 43 | #define BLK_MQ_MAX_DISPATCH_ORDER 7 |
320ae51f JA |
44 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; |
45 | ||
320ae51f | 46 | unsigned int numa_node; |
17ded320 | 47 | unsigned int queue_num; |
320ae51f | 48 | |
0d2602ca JA |
49 | atomic_t nr_active; |
50 | ||
8d354f13 JA |
51 | struct delayed_work delay_work; |
52 | ||
9467f859 | 53 | struct hlist_node cpuhp_dead; |
320ae51f | 54 | struct kobject kobj; |
05229bee | 55 | |
6e219353 | 56 | unsigned long poll_considered; |
05229bee JA |
57 | unsigned long poll_invoked; |
58 | unsigned long poll_success; | |
320ae51f JA |
59 | }; |
60 | ||
24d2f903 | 61 | struct blk_mq_tag_set { |
bdd17e75 | 62 | unsigned int *mq_map; |
320ae51f JA |
63 | struct blk_mq_ops *ops; |
64 | unsigned int nr_hw_queues; | |
e3a2b3f9 | 65 | unsigned int queue_depth; /* max hw supported */ |
320ae51f JA |
66 | unsigned int reserved_tags; |
67 | unsigned int cmd_size; /* per-request extra data */ | |
68 | int numa_node; | |
69 | unsigned int timeout; | |
70 | unsigned int flags; /* BLK_MQ_F_* */ | |
24d2f903 CH |
71 | void *driver_data; |
72 | ||
73 | struct blk_mq_tags **tags; | |
0d2602ca JA |
74 | |
75 | struct mutex tag_list_lock; | |
76 | struct list_head tag_list; | |
320ae51f JA |
77 | }; |
78 | ||
74c45052 JA |
79 | struct blk_mq_queue_data { |
80 | struct request *rq; | |
81 | struct list_head *list; | |
82 | bool last; | |
83 | }; | |
84 | ||
85 | typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); | |
0152fb6b | 86 | typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); |
320ae51f JA |
87 | typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); |
88 | typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); | |
24d2f903 CH |
89 | typedef int (init_request_fn)(void *, struct request *, unsigned int, |
90 | unsigned int, unsigned int); | |
91 | typedef void (exit_request_fn)(void *, struct request *, unsigned int, | |
92 | unsigned int); | |
486cf989 | 93 | typedef int (reinit_request_fn)(void *, struct request *); |
320ae51f | 94 | |
81481eb4 CH |
95 | typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, |
96 | bool); | |
f26cdc85 | 97 | typedef void (busy_tag_iter_fn)(struct request *, void *, bool); |
05229bee | 98 | typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int); |
da695ba2 | 99 | typedef int (map_queues_fn)(struct blk_mq_tag_set *set); |
05229bee | 100 | |
81481eb4 | 101 | |
320ae51f JA |
102 | struct blk_mq_ops { |
103 | /* | |
104 | * Queue request | |
105 | */ | |
106 | queue_rq_fn *queue_rq; | |
107 | ||
320ae51f JA |
108 | /* |
109 | * Called on request timeout | |
110 | */ | |
0152fb6b | 111 | timeout_fn *timeout; |
320ae51f | 112 | |
05229bee JA |
113 | /* |
114 | * Called to poll for completion of a specific tag. | |
115 | */ | |
116 | poll_fn *poll; | |
117 | ||
30a91cb4 CH |
118 | softirq_done_fn *complete; |
119 | ||
320ae51f JA |
120 | /* |
121 | * Called when the block layer side of a hardware queue has been | |
122 | * set up, allowing the driver to allocate/init matching structures. | |
123 | * Ditto for exit/teardown. | |
124 | */ | |
125 | init_hctx_fn *init_hctx; | |
126 | exit_hctx_fn *exit_hctx; | |
e9b267d9 CH |
127 | |
128 | /* | |
129 | * Called for every command allocated by the block layer to allow | |
130 | * the driver to set up driver specific data. | |
f70ced09 ML |
131 | * |
132 | * Tag greater than or equal to queue_depth is for setting up | |
133 | * flush request. | |
134 | * | |
e9b267d9 CH |
135 | * Ditto for exit/teardown. |
136 | */ | |
137 | init_request_fn *init_request; | |
138 | exit_request_fn *exit_request; | |
486cf989 | 139 | reinit_request_fn *reinit_request; |
da695ba2 CH |
140 | |
141 | map_queues_fn *map_queues; | |
320ae51f JA |
142 | }; |
143 | ||
144 | enum { | |
145 | BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */ | |
146 | BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */ | |
147 | BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */ | |
148 | ||
149 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, | |
8a58d1f1 JA |
150 | BLK_MQ_F_TAG_SHARED = 1 << 1, |
151 | BLK_MQ_F_SG_MERGE = 1 << 2, | |
e167dfb5 | 152 | BLK_MQ_F_DEFER_ISSUE = 1 << 4, |
1b792f2f | 153 | BLK_MQ_F_BLOCKING = 1 << 5, |
24391c0d SL |
154 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, |
155 | BLK_MQ_F_ALLOC_POLICY_BITS = 1, | |
320ae51f | 156 | |
5d12f905 | 157 | BLK_MQ_S_STOPPED = 0, |
0d2602ca | 158 | BLK_MQ_S_TAG_ACTIVE = 1, |
320ae51f | 159 | |
a4391c64 | 160 | BLK_MQ_MAX_DEPTH = 10240, |
506e931f JA |
161 | |
162 | BLK_MQ_CPU_WORK_BATCH = 8, | |
320ae51f | 163 | }; |
24391c0d SL |
164 | #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ |
165 | ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ | |
166 | ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) | |
167 | #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ | |
168 | ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ | |
169 | << BLK_MQ_F_ALLOC_POLICY_START_BIT) | |
320ae51f | 170 | |
24d2f903 | 171 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
b62c21b7 MS |
172 | struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, |
173 | struct request_queue *q); | |
b21d5b30 MB |
174 | int blk_mq_register_dev(struct device *, struct request_queue *); |
175 | void blk_mq_unregister_dev(struct device *, struct request_queue *); | |
320ae51f | 176 | |
24d2f903 CH |
177 | int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); |
178 | void blk_mq_free_tag_set(struct blk_mq_tag_set *set); | |
179 | ||
320ae51f JA |
180 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
181 | ||
feb71dae | 182 | void blk_mq_insert_request(struct request *, bool, bool, bool); |
320ae51f | 183 | void blk_mq_free_request(struct request *rq); |
7c7f2f2b | 184 | void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq); |
320ae51f | 185 | bool blk_mq_can_queue(struct blk_mq_hw_ctx *); |
6f3b0e8b CH |
186 | |
187 | enum { | |
188 | BLK_MQ_REQ_NOWAIT = (1 << 0), /* return when out of requests */ | |
189 | BLK_MQ_REQ_RESERVED = (1 << 1), /* allocate from reserved pool */ | |
190 | }; | |
191 | ||
4ce01dd1 | 192 | struct request *blk_mq_alloc_request(struct request_queue *q, int rw, |
6f3b0e8b | 193 | unsigned int flags); |
1f5bd336 ML |
194 | struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int op, |
195 | unsigned int flags, unsigned int hctx_idx); | |
0e62f51f | 196 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); |
320ae51f | 197 | |
205fb5f5 BVA |
198 | enum { |
199 | BLK_MQ_UNIQUE_TAG_BITS = 16, | |
200 | BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, | |
201 | }; | |
202 | ||
203 | u32 blk_mq_unique_tag(struct request *rq); | |
204 | ||
205 | static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) | |
206 | { | |
207 | return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; | |
208 | } | |
209 | ||
210 | static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) | |
211 | { | |
212 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; | |
213 | } | |
214 | ||
320ae51f | 215 | |
973c0191 | 216 | int blk_mq_request_started(struct request *rq); |
e2490073 | 217 | void blk_mq_start_request(struct request *rq); |
c8a446ad CH |
218 | void blk_mq_end_request(struct request *rq, int error); |
219 | void __blk_mq_end_request(struct request *rq, int error); | |
320ae51f | 220 | |
2b053aca BVA |
221 | void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); |
222 | void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, | |
223 | bool kick_requeue_list); | |
6fca6a61 | 224 | void blk_mq_kick_requeue_list(struct request_queue *q); |
2849450a | 225 | void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); |
1885b24d | 226 | void blk_mq_abort_requeue_list(struct request_queue *q); |
f4829a9b | 227 | void blk_mq_complete_request(struct request *rq, int error); |
30a91cb4 | 228 | |
fd001443 | 229 | bool blk_mq_queue_stopped(struct request_queue *q); |
320ae51f JA |
230 | void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); |
231 | void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); | |
280d45f6 | 232 | void blk_mq_stop_hw_queues(struct request_queue *q); |
2f268556 | 233 | void blk_mq_start_hw_queues(struct request_queue *q); |
ae911c5e | 234 | void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); |
1b4a3258 | 235 | void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); |
b94ec296 | 236 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); |
70f4db63 | 237 | void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
e0489487 SG |
238 | void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, |
239 | busy_tag_iter_fn *fn, void *priv); | |
c761d96b | 240 | void blk_mq_freeze_queue(struct request_queue *q); |
b4c6a028 KB |
241 | void blk_mq_unfreeze_queue(struct request_queue *q); |
242 | void blk_mq_freeze_queue_start(struct request_queue *q); | |
486cf989 | 243 | int blk_mq_reinit_tagset(struct blk_mq_tag_set *set); |
320ae51f | 244 | |
9e5a7e22 | 245 | int blk_mq_map_queues(struct blk_mq_tag_set *set); |
868f2f0b KB |
246 | void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); |
247 | ||
320ae51f JA |
248 | /* |
249 | * Driver command data is immediately after the request. So subtract request | |
2963e3f7 | 250 | * size to get back to the original request, add request size to get the PDU. |
320ae51f JA |
251 | */ |
252 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | |
253 | { | |
254 | return pdu - sizeof(struct request); | |
255 | } | |
256 | static inline void *blk_mq_rq_to_pdu(struct request *rq) | |
257 | { | |
2963e3f7 | 258 | return rq + 1; |
320ae51f JA |
259 | } |
260 | ||
320ae51f | 261 | #define queue_for_each_hw_ctx(q, hctx, i) \ |
0d0b7d42 JA |
262 | for ((i) = 0; (i) < (q)->nr_hw_queues && \ |
263 | ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) | |
320ae51f | 264 | |
320ae51f | 265 | #define hctx_for_each_ctx(hctx, ctx, i) \ |
0d0b7d42 JA |
266 | for ((i) = 0; (i) < (hctx)->nr_ctx && \ |
267 | ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) | |
320ae51f | 268 | |
320ae51f | 269 | #endif |