]>
Commit | Line | Data |
---|---|---|
320ae51f JA |
1 | #ifndef BLK_MQ_H |
2 | #define BLK_MQ_H | |
3 | ||
4 | #include <linux/blkdev.h> | |
5 | ||
6 | struct blk_mq_tags; | |
f70ced09 | 7 | struct blk_flush_queue; |
320ae51f JA |
8 | |
9 | struct blk_mq_cpu_notifier { | |
10 | struct list_head list; | |
11 | void *data; | |
e814e71b | 12 | int (*notify)(void *data, unsigned long action, unsigned int cpu); |
320ae51f JA |
13 | }; |
14 | ||
1429d7c9 | 15 | struct blk_mq_ctxmap { |
569fd0ce | 16 | unsigned int size; |
1429d7c9 JA |
17 | unsigned int bits_per_word; |
18 | struct blk_align_bitmap *map; | |
19 | }; | |
20 | ||
320ae51f JA |
21 | struct blk_mq_hw_ctx { |
22 | struct { | |
23 | spinlock_t lock; | |
24 | struct list_head dispatch; | |
25 | } ____cacheline_aligned_in_smp; | |
26 | ||
27 | unsigned long state; /* BLK_MQ_S_* flags */ | |
70f4db63 CH |
28 | struct delayed_work run_work; |
29 | struct delayed_work delay_work; | |
e4043dcf | 30 | cpumask_var_t cpumask; |
506e931f JA |
31 | int next_cpu; |
32 | int next_cpu_batch; | |
320ae51f JA |
33 | |
34 | unsigned long flags; /* BLK_MQ_F_* flags */ | |
35 | ||
36 | struct request_queue *queue; | |
f70ced09 | 37 | struct blk_flush_queue *fq; |
320ae51f JA |
38 | |
39 | void *driver_data; | |
40 | ||
1429d7c9 JA |
41 | struct blk_mq_ctxmap ctx_map; |
42 | ||
4bb659b1 JA |
43 | unsigned int nr_ctx; |
44 | struct blk_mq_ctx **ctxs; | |
45 | ||
8537b120 | 46 | atomic_t wait_index; |
320ae51f | 47 | |
320ae51f JA |
48 | struct blk_mq_tags *tags; |
49 | ||
50 | unsigned long queued; | |
51 | unsigned long run; | |
52 | #define BLK_MQ_MAX_DISPATCH_ORDER 10 | |
53 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; | |
54 | ||
320ae51f | 55 | unsigned int numa_node; |
17ded320 | 56 | unsigned int queue_num; |
320ae51f | 57 | |
0d2602ca JA |
58 | atomic_t nr_active; |
59 | ||
320ae51f JA |
60 | struct blk_mq_cpu_notifier cpu_notifier; |
61 | struct kobject kobj; | |
62 | }; | |
63 | ||
24d2f903 | 64 | struct blk_mq_tag_set { |
320ae51f JA |
65 | struct blk_mq_ops *ops; |
66 | unsigned int nr_hw_queues; | |
e3a2b3f9 | 67 | unsigned int queue_depth; /* max hw supported */ |
320ae51f JA |
68 | unsigned int reserved_tags; |
69 | unsigned int cmd_size; /* per-request extra data */ | |
70 | int numa_node; | |
71 | unsigned int timeout; | |
72 | unsigned int flags; /* BLK_MQ_F_* */ | |
24d2f903 CH |
73 | void *driver_data; |
74 | ||
75 | struct blk_mq_tags **tags; | |
0d2602ca JA |
76 | |
77 | struct mutex tag_list_lock; | |
78 | struct list_head tag_list; | |
320ae51f JA |
79 | }; |
80 | ||
74c45052 JA |
81 | struct blk_mq_queue_data { |
82 | struct request *rq; | |
83 | struct list_head *list; | |
84 | bool last; | |
85 | }; | |
86 | ||
87 | typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); | |
320ae51f | 88 | typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int); |
0152fb6b | 89 | typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); |
320ae51f JA |
90 | typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); |
91 | typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); | |
24d2f903 CH |
92 | typedef int (init_request_fn)(void *, struct request *, unsigned int, |
93 | unsigned int, unsigned int); | |
94 | typedef void (exit_request_fn)(void *, struct request *, unsigned int, | |
95 | unsigned int); | |
320ae51f | 96 | |
81481eb4 CH |
97 | typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *, |
98 | bool); | |
99 | ||
320ae51f JA |
100 | struct blk_mq_ops { |
101 | /* | |
102 | * Queue request | |
103 | */ | |
104 | queue_rq_fn *queue_rq; | |
105 | ||
106 | /* | |
107 | * Map to specific hardware queue | |
108 | */ | |
109 | map_queue_fn *map_queue; | |
110 | ||
111 | /* | |
112 | * Called on request timeout | |
113 | */ | |
0152fb6b | 114 | timeout_fn *timeout; |
320ae51f | 115 | |
30a91cb4 CH |
116 | softirq_done_fn *complete; |
117 | ||
320ae51f JA |
118 | /* |
119 | * Called when the block layer side of a hardware queue has been | |
120 | * set up, allowing the driver to allocate/init matching structures. | |
121 | * Ditto for exit/teardown. | |
122 | */ | |
123 | init_hctx_fn *init_hctx; | |
124 | exit_hctx_fn *exit_hctx; | |
e9b267d9 CH |
125 | |
126 | /* | |
127 | * Called for every command allocated by the block layer to allow | |
128 | * the driver to set up driver specific data. | |
f70ced09 ML |
129 | * |
130 | * Tag greater than or equal to queue_depth is for setting up | |
131 | * flush request. | |
132 | * | |
e9b267d9 CH |
133 | * Ditto for exit/teardown. |
134 | */ | |
135 | init_request_fn *init_request; | |
136 | exit_request_fn *exit_request; | |
320ae51f JA |
137 | }; |
138 | ||
139 | enum { | |
140 | BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */ | |
141 | BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */ | |
142 | BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */ | |
143 | ||
144 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, | |
8a58d1f1 JA |
145 | BLK_MQ_F_TAG_SHARED = 1 << 1, |
146 | BLK_MQ_F_SG_MERGE = 1 << 2, | |
147 | BLK_MQ_F_SYSFS_UP = 1 << 3, | |
e167dfb5 | 148 | BLK_MQ_F_DEFER_ISSUE = 1 << 4, |
24391c0d SL |
149 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, |
150 | BLK_MQ_F_ALLOC_POLICY_BITS = 1, | |
320ae51f | 151 | |
5d12f905 | 152 | BLK_MQ_S_STOPPED = 0, |
0d2602ca | 153 | BLK_MQ_S_TAG_ACTIVE = 1, |
320ae51f | 154 | |
a4391c64 | 155 | BLK_MQ_MAX_DEPTH = 10240, |
506e931f JA |
156 | |
157 | BLK_MQ_CPU_WORK_BATCH = 8, | |
320ae51f | 158 | }; |
24391c0d SL |
159 | #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ |
160 | ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ | |
161 | ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) | |
162 | #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ | |
163 | ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ | |
164 | << BLK_MQ_F_ALLOC_POLICY_START_BIT) | |
320ae51f | 165 | |
24d2f903 | 166 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
b62c21b7 MS |
167 | struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, |
168 | struct request_queue *q); | |
17497acb | 169 | void blk_mq_finish_init(struct request_queue *q); |
320ae51f JA |
170 | int blk_mq_register_disk(struct gendisk *); |
171 | void blk_mq_unregister_disk(struct gendisk *); | |
320ae51f | 172 | |
24d2f903 CH |
173 | int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); |
174 | void blk_mq_free_tag_set(struct blk_mq_tag_set *set); | |
175 | ||
320ae51f JA |
176 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
177 | ||
feb71dae | 178 | void blk_mq_insert_request(struct request *, bool, bool, bool); |
320ae51f | 179 | void blk_mq_free_request(struct request *rq); |
7c7f2f2b | 180 | void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq); |
320ae51f | 181 | bool blk_mq_can_queue(struct blk_mq_hw_ctx *); |
4ce01dd1 CH |
182 | struct request *blk_mq_alloc_request(struct request_queue *q, int rw, |
183 | gfp_t gfp, bool reserved); | |
0e62f51f | 184 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); |
320ae51f | 185 | |
205fb5f5 BVA |
186 | enum { |
187 | BLK_MQ_UNIQUE_TAG_BITS = 16, | |
188 | BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1, | |
189 | }; | |
190 | ||
191 | u32 blk_mq_unique_tag(struct request *rq); | |
192 | ||
193 | static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag) | |
194 | { | |
195 | return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS; | |
196 | } | |
197 | ||
198 | static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) | |
199 | { | |
200 | return unique_tag & BLK_MQ_UNIQUE_TAG_MASK; | |
201 | } | |
202 | ||
320ae51f | 203 | struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); |
f14bbe77 | 204 | struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); |
320ae51f | 205 | |
973c0191 | 206 | int blk_mq_request_started(struct request *rq); |
e2490073 | 207 | void blk_mq_start_request(struct request *rq); |
c8a446ad CH |
208 | void blk_mq_end_request(struct request *rq, int error); |
209 | void __blk_mq_end_request(struct request *rq, int error); | |
320ae51f | 210 | |
ed0791b2 | 211 | void blk_mq_requeue_request(struct request *rq); |
6fca6a61 | 212 | void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); |
c68ed59f | 213 | void blk_mq_cancel_requeue_work(struct request_queue *q); |
6fca6a61 | 214 | void blk_mq_kick_requeue_list(struct request_queue *q); |
1885b24d | 215 | void blk_mq_abort_requeue_list(struct request_queue *q); |
30a91cb4 CH |
216 | void blk_mq_complete_request(struct request *rq); |
217 | ||
320ae51f JA |
218 | void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); |
219 | void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); | |
280d45f6 | 220 | void blk_mq_stop_hw_queues(struct request_queue *q); |
2f268556 | 221 | void blk_mq_start_hw_queues(struct request_queue *q); |
1b4a3258 | 222 | void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); |
b94ec296 | 223 | void blk_mq_run_hw_queues(struct request_queue *q, bool async); |
70f4db63 | 224 | void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
81481eb4 CH |
225 | void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn, |
226 | void *priv); | |
c761d96b | 227 | void blk_mq_freeze_queue(struct request_queue *q); |
b4c6a028 KB |
228 | void blk_mq_unfreeze_queue(struct request_queue *q); |
229 | void blk_mq_freeze_queue_start(struct request_queue *q); | |
320ae51f JA |
230 | |
231 | /* | |
232 | * Driver command data is immediately after the request. So subtract request | |
2963e3f7 | 233 | * size to get back to the original request, add request size to get the PDU. |
320ae51f JA |
234 | */ |
235 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | |
236 | { | |
237 | return pdu - sizeof(struct request); | |
238 | } | |
239 | static inline void *blk_mq_rq_to_pdu(struct request *rq) | |
240 | { | |
2963e3f7 | 241 | return rq + 1; |
320ae51f JA |
242 | } |
243 | ||
320ae51f | 244 | #define queue_for_each_hw_ctx(q, hctx, i) \ |
0d0b7d42 JA |
245 | for ((i) = 0; (i) < (q)->nr_hw_queues && \ |
246 | ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) | |
320ae51f JA |
247 | |
248 | #define queue_for_each_ctx(q, ctx, i) \ | |
0d0b7d42 JA |
249 | for ((i) = 0; (i) < (q)->nr_queues && \ |
250 | ({ ctx = per_cpu_ptr((q)->queue_ctx, (i)); 1; }); (i)++) | |
320ae51f JA |
251 | |
252 | #define hctx_for_each_ctx(hctx, ctx, i) \ | |
0d0b7d42 JA |
253 | for ((i) = 0; (i) < (hctx)->nr_ctx && \ |
254 | ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) | |
320ae51f JA |
255 | |
256 | #define blk_ctx_sum(q, sum) \ | |
257 | ({ \ | |
258 | struct blk_mq_ctx *__x; \ | |
259 | unsigned int __ret = 0, __i; \ | |
260 | \ | |
261 | queue_for_each_ctx((q), __x, __i) \ | |
262 | __ret += sum; \ | |
263 | __ret; \ | |
264 | }) | |
265 | ||
266 | #endif |