]>
Commit | Line | Data |
---|---|---|
3bf2bd20 SL |
1 | /* |
2 | * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and | |
3 | * Shaohua Li <shli@fb.com> | |
4 | */ | |
f2298c04 | 5 | #include <linux/module.h> |
fc1bc354 | 6 | |
f2298c04 JA |
7 | #include <linux/moduleparam.h> |
8 | #include <linux/sched.h> | |
9 | #include <linux/fs.h> | |
10 | #include <linux/blkdev.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/slab.h> | |
13 | #include <linux/blk-mq.h> | |
14 | #include <linux/hrtimer.h> | |
b2b7e001 | 15 | #include <linux/lightnvm.h> |
3bf2bd20 | 16 | #include <linux/configfs.h> |
f2298c04 JA |
17 | |
18 | struct nullb_cmd { | |
19 | struct list_head list; | |
20 | struct llist_node ll_list; | |
21 | struct call_single_data csd; | |
22 | struct request *rq; | |
23 | struct bio *bio; | |
24 | unsigned int tag; | |
25 | struct nullb_queue *nq; | |
3c395a96 | 26 | struct hrtimer timer; |
f2298c04 JA |
27 | }; |
28 | ||
29 | struct nullb_queue { | |
30 | unsigned long *tag_map; | |
31 | wait_queue_head_t wait; | |
32 | unsigned int queue_depth; | |
2984c868 | 33 | struct nullb_device *dev; |
f2298c04 JA |
34 | |
35 | struct nullb_cmd *cmds; | |
36 | }; | |
37 | ||
3bf2bd20 SL |
38 | /* |
39 | * Status flags for nullb_device. | |
40 | * | |
41 | * CONFIGURED: Device has been configured and turned on. Cannot reconfigure. | |
42 | * UP: Device is currently on and visible in userspace. | |
43 | */ | |
44 | enum nullb_device_flags { | |
45 | NULLB_DEV_FL_CONFIGURED = 0, | |
46 | NULLB_DEV_FL_UP = 1, | |
47 | }; | |
48 | ||
2984c868 SL |
49 | struct nullb_device { |
50 | struct nullb *nullb; | |
3bf2bd20 SL |
51 | struct config_item item; |
52 | unsigned long flags; /* device flags */ | |
2984c868 SL |
53 | |
54 | unsigned long size; /* device size in MB */ | |
55 | unsigned long completion_nsec; /* time in ns to complete a request */ | |
56 | unsigned int submit_queues; /* number of submission queues */ | |
57 | unsigned int home_node; /* home node for the device */ | |
58 | unsigned int queue_mode; /* block interface */ | |
59 | unsigned int blocksize; /* block size */ | |
60 | unsigned int irqmode; /* IRQ completion handler */ | |
61 | unsigned int hw_queue_depth; /* queue depth */ | |
cedcafad | 62 | unsigned int index; /* index of the disk, only valid with a disk */ |
2984c868 SL |
63 | bool use_lightnvm; /* register as a LightNVM device */ |
64 | bool blocking; /* blocking blk-mq device */ | |
65 | bool use_per_node_hctx; /* use per-node allocation for hardware context */ | |
cedcafad | 66 | bool power; /* power on/off the device */ |
2984c868 SL |
67 | }; |
68 | ||
f2298c04 | 69 | struct nullb { |
2984c868 | 70 | struct nullb_device *dev; |
f2298c04 JA |
71 | struct list_head list; |
72 | unsigned int index; | |
73 | struct request_queue *q; | |
74 | struct gendisk *disk; | |
b0b4e09c | 75 | struct nvm_dev *ndev; |
82f402fe JA |
76 | struct blk_mq_tag_set *tag_set; |
77 | struct blk_mq_tag_set __tag_set; | |
f2298c04 JA |
78 | struct hrtimer timer; |
79 | unsigned int queue_depth; | |
80 | spinlock_t lock; | |
81 | ||
82 | struct nullb_queue *queues; | |
83 | unsigned int nr_queues; | |
b2b7e001 | 84 | char disk_name[DISK_NAME_LEN]; |
f2298c04 JA |
85 | }; |
86 | ||
87 | static LIST_HEAD(nullb_list); | |
88 | static struct mutex lock; | |
89 | static int null_major; | |
90 | static int nullb_indexes; | |
6bb9535b | 91 | static struct kmem_cache *ppa_cache; |
82f402fe | 92 | static struct blk_mq_tag_set tag_set; |
f2298c04 | 93 | |
f2298c04 JA |
94 | enum { |
95 | NULL_IRQ_NONE = 0, | |
96 | NULL_IRQ_SOFTIRQ = 1, | |
97 | NULL_IRQ_TIMER = 2, | |
ce2c350b | 98 | }; |
f2298c04 | 99 | |
ce2c350b | 100 | enum { |
f2298c04 JA |
101 | NULL_Q_BIO = 0, |
102 | NULL_Q_RQ = 1, | |
103 | NULL_Q_MQ = 2, | |
104 | }; | |
105 | ||
2984c868 SL |
106 | static int g_submit_queues = 1; |
107 | module_param_named(submit_queues, g_submit_queues, int, S_IRUGO); | |
f2298c04 JA |
108 | MODULE_PARM_DESC(submit_queues, "Number of submission queues"); |
109 | ||
2984c868 SL |
110 | static int g_home_node = NUMA_NO_NODE; |
111 | module_param_named(home_node, g_home_node, int, S_IRUGO); | |
f2298c04 JA |
112 | MODULE_PARM_DESC(home_node, "Home node for the device"); |
113 | ||
2984c868 | 114 | static int g_queue_mode = NULL_Q_MQ; |
709c8667 MB |
115 | |
116 | static int null_param_store_val(const char *str, int *val, int min, int max) | |
117 | { | |
118 | int ret, new_val; | |
119 | ||
120 | ret = kstrtoint(str, 10, &new_val); | |
121 | if (ret) | |
122 | return -EINVAL; | |
123 | ||
124 | if (new_val < min || new_val > max) | |
125 | return -EINVAL; | |
126 | ||
127 | *val = new_val; | |
128 | return 0; | |
129 | } | |
130 | ||
131 | static int null_set_queue_mode(const char *str, const struct kernel_param *kp) | |
132 | { | |
2984c868 | 133 | return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ); |
709c8667 MB |
134 | } |
135 | ||
9c27847d | 136 | static const struct kernel_param_ops null_queue_mode_param_ops = { |
709c8667 MB |
137 | .set = null_set_queue_mode, |
138 | .get = param_get_int, | |
139 | }; | |
140 | ||
2984c868 | 141 | device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, S_IRUGO); |
54ae81cd | 142 | MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)"); |
f2298c04 | 143 | |
2984c868 SL |
144 | static int g_gb = 250; |
145 | module_param_named(gb, g_gb, int, S_IRUGO); | |
f2298c04 JA |
146 | MODULE_PARM_DESC(gb, "Size in GB"); |
147 | ||
2984c868 SL |
148 | static int g_bs = 512; |
149 | module_param_named(bs, g_bs, int, S_IRUGO); | |
f2298c04 JA |
150 | MODULE_PARM_DESC(bs, "Block size (in bytes)"); |
151 | ||
82f402fe | 152 | static int nr_devices = 1; |
f2298c04 JA |
153 | module_param(nr_devices, int, S_IRUGO); |
154 | MODULE_PARM_DESC(nr_devices, "Number of devices to register"); | |
155 | ||
2984c868 SL |
156 | static bool g_use_lightnvm; |
157 | module_param_named(use_lightnvm, g_use_lightnvm, bool, S_IRUGO); | |
b2b7e001 MB |
158 | MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device"); |
159 | ||
2984c868 SL |
160 | static bool g_blocking; |
161 | module_param_named(blocking, g_blocking, bool, S_IRUGO); | |
db5bcf87 JA |
162 | MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device"); |
163 | ||
82f402fe JA |
164 | static bool shared_tags; |
165 | module_param(shared_tags, bool, S_IRUGO); | |
166 | MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq"); | |
167 | ||
2984c868 | 168 | static int g_irqmode = NULL_IRQ_SOFTIRQ; |
709c8667 MB |
169 | |
170 | static int null_set_irqmode(const char *str, const struct kernel_param *kp) | |
171 | { | |
2984c868 | 172 | return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE, |
709c8667 MB |
173 | NULL_IRQ_TIMER); |
174 | } | |
175 | ||
9c27847d | 176 | static const struct kernel_param_ops null_irqmode_param_ops = { |
709c8667 MB |
177 | .set = null_set_irqmode, |
178 | .get = param_get_int, | |
179 | }; | |
180 | ||
2984c868 | 181 | device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, S_IRUGO); |
f2298c04 JA |
182 | MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer"); |
183 | ||
2984c868 SL |
184 | static unsigned long g_completion_nsec = 10000; |
185 | module_param_named(completion_nsec, g_completion_nsec, ulong, S_IRUGO); | |
f2298c04 JA |
186 | MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns"); |
187 | ||
2984c868 SL |
188 | static int g_hw_queue_depth = 64; |
189 | module_param_named(hw_queue_depth, g_hw_queue_depth, int, S_IRUGO); | |
f2298c04 JA |
190 | MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64"); |
191 | ||
2984c868 SL |
192 | static bool g_use_per_node_hctx; |
193 | module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, S_IRUGO); | |
20005244 | 194 | MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false"); |
f2298c04 | 195 | |
3bf2bd20 SL |
196 | static struct nullb_device *null_alloc_dev(void); |
197 | static void null_free_dev(struct nullb_device *dev); | |
cedcafad SL |
198 | static void null_del_dev(struct nullb *nullb); |
199 | static int null_add_dev(struct nullb_device *dev); | |
3bf2bd20 SL |
200 | |
201 | static inline struct nullb_device *to_nullb_device(struct config_item *item) | |
202 | { | |
203 | return item ? container_of(item, struct nullb_device, item) : NULL; | |
204 | } | |
205 | ||
206 | static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page) | |
207 | { | |
208 | return snprintf(page, PAGE_SIZE, "%u\n", val); | |
209 | } | |
210 | ||
211 | static inline ssize_t nullb_device_ulong_attr_show(unsigned long val, | |
212 | char *page) | |
213 | { | |
214 | return snprintf(page, PAGE_SIZE, "%lu\n", val); | |
215 | } | |
216 | ||
217 | static inline ssize_t nullb_device_bool_attr_show(bool val, char *page) | |
218 | { | |
219 | return snprintf(page, PAGE_SIZE, "%u\n", val); | |
220 | } | |
221 | ||
222 | static ssize_t nullb_device_uint_attr_store(unsigned int *val, | |
223 | const char *page, size_t count) | |
224 | { | |
225 | unsigned int tmp; | |
226 | int result; | |
227 | ||
228 | result = kstrtouint(page, 0, &tmp); | |
229 | if (result) | |
230 | return result; | |
231 | ||
232 | *val = tmp; | |
233 | return count; | |
234 | } | |
235 | ||
236 | static ssize_t nullb_device_ulong_attr_store(unsigned long *val, | |
237 | const char *page, size_t count) | |
238 | { | |
239 | int result; | |
240 | unsigned long tmp; | |
241 | ||
242 | result = kstrtoul(page, 0, &tmp); | |
243 | if (result) | |
244 | return result; | |
245 | ||
246 | *val = tmp; | |
247 | return count; | |
248 | } | |
249 | ||
250 | static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, | |
251 | size_t count) | |
252 | { | |
253 | bool tmp; | |
254 | int result; | |
255 | ||
256 | result = kstrtobool(page, &tmp); | |
257 | if (result) | |
258 | return result; | |
259 | ||
260 | *val = tmp; | |
261 | return count; | |
262 | } | |
263 | ||
264 | /* The following macro should only be used with TYPE = {uint, ulong, bool}. */ | |
265 | #define NULLB_DEVICE_ATTR(NAME, TYPE) \ | |
266 | static ssize_t \ | |
267 | nullb_device_##NAME##_show(struct config_item *item, char *page) \ | |
268 | { \ | |
269 | return nullb_device_##TYPE##_attr_show( \ | |
270 | to_nullb_device(item)->NAME, page); \ | |
271 | } \ | |
272 | static ssize_t \ | |
273 | nullb_device_##NAME##_store(struct config_item *item, const char *page, \ | |
274 | size_t count) \ | |
275 | { \ | |
276 | if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags)) \ | |
277 | return -EBUSY; \ | |
278 | return nullb_device_##TYPE##_attr_store( \ | |
279 | &to_nullb_device(item)->NAME, page, count); \ | |
280 | } \ | |
281 | CONFIGFS_ATTR(nullb_device_, NAME); | |
282 | ||
283 | NULLB_DEVICE_ATTR(size, ulong); | |
284 | NULLB_DEVICE_ATTR(completion_nsec, ulong); | |
285 | NULLB_DEVICE_ATTR(submit_queues, uint); | |
286 | NULLB_DEVICE_ATTR(home_node, uint); | |
287 | NULLB_DEVICE_ATTR(queue_mode, uint); | |
288 | NULLB_DEVICE_ATTR(blocksize, uint); | |
289 | NULLB_DEVICE_ATTR(irqmode, uint); | |
290 | NULLB_DEVICE_ATTR(hw_queue_depth, uint); | |
cedcafad | 291 | NULLB_DEVICE_ATTR(index, uint); |
3bf2bd20 SL |
292 | NULLB_DEVICE_ATTR(use_lightnvm, bool); |
293 | NULLB_DEVICE_ATTR(blocking, bool); | |
294 | NULLB_DEVICE_ATTR(use_per_node_hctx, bool); | |
295 | ||
cedcafad SL |
296 | static ssize_t nullb_device_power_show(struct config_item *item, char *page) |
297 | { | |
298 | return nullb_device_bool_attr_show(to_nullb_device(item)->power, page); | |
299 | } | |
300 | ||
301 | static ssize_t nullb_device_power_store(struct config_item *item, | |
302 | const char *page, size_t count) | |
303 | { | |
304 | struct nullb_device *dev = to_nullb_device(item); | |
305 | bool newp = false; | |
306 | ssize_t ret; | |
307 | ||
308 | ret = nullb_device_bool_attr_store(&newp, page, count); | |
309 | if (ret < 0) | |
310 | return ret; | |
311 | ||
312 | if (!dev->power && newp) { | |
313 | if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags)) | |
314 | return count; | |
315 | if (null_add_dev(dev)) { | |
316 | clear_bit(NULLB_DEV_FL_UP, &dev->flags); | |
317 | return -ENOMEM; | |
318 | } | |
319 | ||
320 | set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); | |
321 | dev->power = newp; | |
322 | } else if (to_nullb_device(item)->power && !newp) { | |
323 | mutex_lock(&lock); | |
324 | dev->power = newp; | |
325 | null_del_dev(dev->nullb); | |
326 | mutex_unlock(&lock); | |
327 | clear_bit(NULLB_DEV_FL_UP, &dev->flags); | |
328 | } | |
329 | ||
330 | return count; | |
331 | } | |
332 | ||
333 | CONFIGFS_ATTR(nullb_device_, power); | |
334 | ||
3bf2bd20 SL |
335 | static struct configfs_attribute *nullb_device_attrs[] = { |
336 | &nullb_device_attr_size, | |
337 | &nullb_device_attr_completion_nsec, | |
338 | &nullb_device_attr_submit_queues, | |
339 | &nullb_device_attr_home_node, | |
340 | &nullb_device_attr_queue_mode, | |
341 | &nullb_device_attr_blocksize, | |
342 | &nullb_device_attr_irqmode, | |
343 | &nullb_device_attr_hw_queue_depth, | |
cedcafad | 344 | &nullb_device_attr_index, |
3bf2bd20 SL |
345 | &nullb_device_attr_use_lightnvm, |
346 | &nullb_device_attr_blocking, | |
347 | &nullb_device_attr_use_per_node_hctx, | |
cedcafad | 348 | &nullb_device_attr_power, |
3bf2bd20 SL |
349 | NULL, |
350 | }; | |
351 | ||
352 | static void nullb_device_release(struct config_item *item) | |
353 | { | |
354 | null_free_dev(to_nullb_device(item)); | |
355 | } | |
356 | ||
357 | static struct configfs_item_operations nullb_device_ops = { | |
358 | .release = nullb_device_release, | |
359 | }; | |
360 | ||
361 | static struct config_item_type nullb_device_type = { | |
362 | .ct_item_ops = &nullb_device_ops, | |
363 | .ct_attrs = nullb_device_attrs, | |
364 | .ct_owner = THIS_MODULE, | |
365 | }; | |
366 | ||
367 | static struct | |
368 | config_item *nullb_group_make_item(struct config_group *group, const char *name) | |
369 | { | |
370 | struct nullb_device *dev; | |
371 | ||
372 | dev = null_alloc_dev(); | |
373 | if (!dev) | |
374 | return ERR_PTR(-ENOMEM); | |
375 | ||
376 | config_item_init_type_name(&dev->item, name, &nullb_device_type); | |
377 | ||
378 | return &dev->item; | |
379 | } | |
380 | ||
381 | static void | |
382 | nullb_group_drop_item(struct config_group *group, struct config_item *item) | |
383 | { | |
cedcafad SL |
384 | struct nullb_device *dev = to_nullb_device(item); |
385 | ||
386 | if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) { | |
387 | mutex_lock(&lock); | |
388 | dev->power = false; | |
389 | null_del_dev(dev->nullb); | |
390 | mutex_unlock(&lock); | |
391 | } | |
392 | ||
3bf2bd20 SL |
393 | config_item_put(item); |
394 | } | |
395 | ||
396 | static ssize_t memb_group_features_show(struct config_item *item, char *page) | |
397 | { | |
398 | return snprintf(page, PAGE_SIZE, "\n"); | |
399 | } | |
400 | ||
401 | CONFIGFS_ATTR_RO(memb_group_, features); | |
402 | ||
403 | static struct configfs_attribute *nullb_group_attrs[] = { | |
404 | &memb_group_attr_features, | |
405 | NULL, | |
406 | }; | |
407 | ||
408 | static struct configfs_group_operations nullb_group_ops = { | |
409 | .make_item = nullb_group_make_item, | |
410 | .drop_item = nullb_group_drop_item, | |
411 | }; | |
412 | ||
413 | static struct config_item_type nullb_group_type = { | |
414 | .ct_group_ops = &nullb_group_ops, | |
415 | .ct_attrs = nullb_group_attrs, | |
416 | .ct_owner = THIS_MODULE, | |
417 | }; | |
418 | ||
419 | static struct configfs_subsystem nullb_subsys = { | |
420 | .su_group = { | |
421 | .cg_item = { | |
422 | .ci_namebuf = "nullb", | |
423 | .ci_type = &nullb_group_type, | |
424 | }, | |
425 | }, | |
426 | }; | |
427 | ||
2984c868 SL |
428 | static struct nullb_device *null_alloc_dev(void) |
429 | { | |
430 | struct nullb_device *dev; | |
431 | ||
432 | dev = kzalloc(sizeof(*dev), GFP_KERNEL); | |
433 | if (!dev) | |
434 | return NULL; | |
435 | dev->size = g_gb * 1024; | |
436 | dev->completion_nsec = g_completion_nsec; | |
437 | dev->submit_queues = g_submit_queues; | |
438 | dev->home_node = g_home_node; | |
439 | dev->queue_mode = g_queue_mode; | |
440 | dev->blocksize = g_bs; | |
441 | dev->irqmode = g_irqmode; | |
442 | dev->hw_queue_depth = g_hw_queue_depth; | |
443 | dev->use_lightnvm = g_use_lightnvm; | |
444 | dev->blocking = g_blocking; | |
445 | dev->use_per_node_hctx = g_use_per_node_hctx; | |
446 | return dev; | |
447 | } | |
448 | ||
449 | static void null_free_dev(struct nullb_device *dev) | |
450 | { | |
451 | kfree(dev); | |
452 | } | |
453 | ||
f2298c04 JA |
454 | static void put_tag(struct nullb_queue *nq, unsigned int tag) |
455 | { | |
456 | clear_bit_unlock(tag, nq->tag_map); | |
457 | ||
458 | if (waitqueue_active(&nq->wait)) | |
459 | wake_up(&nq->wait); | |
460 | } | |
461 | ||
462 | static unsigned int get_tag(struct nullb_queue *nq) | |
463 | { | |
464 | unsigned int tag; | |
465 | ||
466 | do { | |
467 | tag = find_first_zero_bit(nq->tag_map, nq->queue_depth); | |
468 | if (tag >= nq->queue_depth) | |
469 | return -1U; | |
470 | } while (test_and_set_bit_lock(tag, nq->tag_map)); | |
471 | ||
472 | return tag; | |
473 | } | |
474 | ||
475 | static void free_cmd(struct nullb_cmd *cmd) | |
476 | { | |
477 | put_tag(cmd->nq, cmd->tag); | |
478 | } | |
479 | ||
3c395a96 PV |
480 | static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer); |
481 | ||
f2298c04 JA |
482 | static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq) |
483 | { | |
484 | struct nullb_cmd *cmd; | |
485 | unsigned int tag; | |
486 | ||
487 | tag = get_tag(nq); | |
488 | if (tag != -1U) { | |
489 | cmd = &nq->cmds[tag]; | |
490 | cmd->tag = tag; | |
491 | cmd->nq = nq; | |
2984c868 | 492 | if (nq->dev->irqmode == NULL_IRQ_TIMER) { |
3c395a96 PV |
493 | hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, |
494 | HRTIMER_MODE_REL); | |
495 | cmd->timer.function = null_cmd_timer_expired; | |
496 | } | |
f2298c04 JA |
497 | return cmd; |
498 | } | |
499 | ||
500 | return NULL; | |
501 | } | |
502 | ||
503 | static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait) | |
504 | { | |
505 | struct nullb_cmd *cmd; | |
506 | DEFINE_WAIT(wait); | |
507 | ||
508 | cmd = __alloc_cmd(nq); | |
509 | if (cmd || !can_wait) | |
510 | return cmd; | |
511 | ||
512 | do { | |
513 | prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE); | |
514 | cmd = __alloc_cmd(nq); | |
515 | if (cmd) | |
516 | break; | |
517 | ||
518 | io_schedule(); | |
519 | } while (1); | |
520 | ||
521 | finish_wait(&nq->wait, &wait); | |
522 | return cmd; | |
523 | } | |
524 | ||
525 | static void end_cmd(struct nullb_cmd *cmd) | |
526 | { | |
cf8ecc5a | 527 | struct request_queue *q = NULL; |
2984c868 | 528 | int queue_mode = cmd->nq->dev->queue_mode; |
cf8ecc5a | 529 | |
e8271201 MK |
530 | if (cmd->rq) |
531 | q = cmd->rq->q; | |
532 | ||
ce2c350b CH |
533 | switch (queue_mode) { |
534 | case NULL_Q_MQ: | |
2a842aca | 535 | blk_mq_end_request(cmd->rq, BLK_STS_OK); |
ce2c350b CH |
536 | return; |
537 | case NULL_Q_RQ: | |
538 | INIT_LIST_HEAD(&cmd->rq->queuelist); | |
2a842aca | 539 | blk_end_request_all(cmd->rq, BLK_STS_OK); |
ce2c350b CH |
540 | break; |
541 | case NULL_Q_BIO: | |
4246a0b6 | 542 | bio_endio(cmd->bio); |
48cc661e | 543 | break; |
ce2c350b | 544 | } |
f2298c04 | 545 | |
48cc661e JA |
546 | free_cmd(cmd); |
547 | ||
cf8ecc5a | 548 | /* Restart queue if needed, as we are freeing a tag */ |
48cc661e | 549 | if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) { |
cf8ecc5a AA |
550 | unsigned long flags; |
551 | ||
552 | spin_lock_irqsave(q->queue_lock, flags); | |
48cc661e | 553 | blk_start_queue_async(q); |
cf8ecc5a | 554 | spin_unlock_irqrestore(q->queue_lock, flags); |
f2298c04 | 555 | } |
cf8ecc5a AA |
556 | } |
557 | ||
558 | static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer) | |
559 | { | |
560 | end_cmd(container_of(timer, struct nullb_cmd, timer)); | |
f2298c04 JA |
561 | |
562 | return HRTIMER_NORESTART; | |
563 | } | |
564 | ||
565 | static void null_cmd_end_timer(struct nullb_cmd *cmd) | |
566 | { | |
2984c868 | 567 | ktime_t kt = cmd->nq->dev->completion_nsec; |
f2298c04 | 568 | |
3c395a96 | 569 | hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL); |
f2298c04 JA |
570 | } |
571 | ||
572 | static void null_softirq_done_fn(struct request *rq) | |
573 | { | |
2984c868 SL |
574 | struct nullb *nullb = rq->q->queuedata; |
575 | ||
576 | if (nullb->dev->queue_mode == NULL_Q_MQ) | |
d891fa70 JA |
577 | end_cmd(blk_mq_rq_to_pdu(rq)); |
578 | else | |
579 | end_cmd(rq->special); | |
f2298c04 JA |
580 | } |
581 | ||
f2298c04 JA |
582 | static inline void null_handle_cmd(struct nullb_cmd *cmd) |
583 | { | |
584 | /* Complete IO by inline, softirq or timer */ | |
2984c868 | 585 | switch (cmd->nq->dev->irqmode) { |
f2298c04 | 586 | case NULL_IRQ_SOFTIRQ: |
2984c868 | 587 | switch (cmd->nq->dev->queue_mode) { |
ce2c350b | 588 | case NULL_Q_MQ: |
08e0029a | 589 | blk_mq_complete_request(cmd->rq); |
ce2c350b CH |
590 | break; |
591 | case NULL_Q_RQ: | |
592 | blk_complete_request(cmd->rq); | |
593 | break; | |
594 | case NULL_Q_BIO: | |
595 | /* | |
596 | * XXX: no proper submitting cpu information available. | |
597 | */ | |
598 | end_cmd(cmd); | |
599 | break; | |
600 | } | |
601 | break; | |
602 | case NULL_IRQ_NONE: | |
f2298c04 | 603 | end_cmd(cmd); |
f2298c04 JA |
604 | break; |
605 | case NULL_IRQ_TIMER: | |
606 | null_cmd_end_timer(cmd); | |
607 | break; | |
608 | } | |
609 | } | |
610 | ||
611 | static struct nullb_queue *nullb_to_queue(struct nullb *nullb) | |
612 | { | |
613 | int index = 0; | |
614 | ||
615 | if (nullb->nr_queues != 1) | |
616 | index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues); | |
617 | ||
618 | return &nullb->queues[index]; | |
619 | } | |
620 | ||
dece1635 | 621 | static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio) |
f2298c04 JA |
622 | { |
623 | struct nullb *nullb = q->queuedata; | |
624 | struct nullb_queue *nq = nullb_to_queue(nullb); | |
625 | struct nullb_cmd *cmd; | |
626 | ||
627 | cmd = alloc_cmd(nq, 1); | |
628 | cmd->bio = bio; | |
629 | ||
630 | null_handle_cmd(cmd); | |
dece1635 | 631 | return BLK_QC_T_NONE; |
f2298c04 JA |
632 | } |
633 | ||
634 | static int null_rq_prep_fn(struct request_queue *q, struct request *req) | |
635 | { | |
636 | struct nullb *nullb = q->queuedata; | |
637 | struct nullb_queue *nq = nullb_to_queue(nullb); | |
638 | struct nullb_cmd *cmd; | |
639 | ||
640 | cmd = alloc_cmd(nq, 0); | |
641 | if (cmd) { | |
642 | cmd->rq = req; | |
643 | req->special = cmd; | |
644 | return BLKPREP_OK; | |
645 | } | |
8b70f45e | 646 | blk_stop_queue(q); |
f2298c04 JA |
647 | |
648 | return BLKPREP_DEFER; | |
649 | } | |
650 | ||
651 | static void null_request_fn(struct request_queue *q) | |
652 | { | |
653 | struct request *rq; | |
654 | ||
655 | while ((rq = blk_fetch_request(q)) != NULL) { | |
656 | struct nullb_cmd *cmd = rq->special; | |
657 | ||
658 | spin_unlock_irq(q->queue_lock); | |
659 | null_handle_cmd(cmd); | |
660 | spin_lock_irq(q->queue_lock); | |
661 | } | |
662 | } | |
663 | ||
fc17b653 | 664 | static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, |
74c45052 | 665 | const struct blk_mq_queue_data *bd) |
f2298c04 | 666 | { |
74c45052 | 667 | struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); |
2984c868 | 668 | struct nullb_queue *nq = hctx->driver_data; |
f2298c04 | 669 | |
db5bcf87 JA |
670 | might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); |
671 | ||
2984c868 | 672 | if (nq->dev->irqmode == NULL_IRQ_TIMER) { |
3c395a96 PV |
673 | hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
674 | cmd->timer.function = null_cmd_timer_expired; | |
675 | } | |
74c45052 | 676 | cmd->rq = bd->rq; |
2984c868 | 677 | cmd->nq = nq; |
f2298c04 | 678 | |
74c45052 | 679 | blk_mq_start_request(bd->rq); |
e2490073 | 680 | |
f2298c04 | 681 | null_handle_cmd(cmd); |
fc17b653 | 682 | return BLK_STS_OK; |
f2298c04 JA |
683 | } |
684 | ||
f363b089 | 685 | static const struct blk_mq_ops null_mq_ops = { |
f2298c04 | 686 | .queue_rq = null_queue_rq, |
ce2c350b | 687 | .complete = null_softirq_done_fn, |
f2298c04 JA |
688 | }; |
689 | ||
de65d2d2 MB |
690 | static void cleanup_queue(struct nullb_queue *nq) |
691 | { | |
692 | kfree(nq->tag_map); | |
693 | kfree(nq->cmds); | |
694 | } | |
695 | ||
696 | static void cleanup_queues(struct nullb *nullb) | |
697 | { | |
698 | int i; | |
699 | ||
700 | for (i = 0; i < nullb->nr_queues; i++) | |
701 | cleanup_queue(&nullb->queues[i]); | |
702 | ||
703 | kfree(nullb->queues); | |
704 | } | |
705 | ||
b2b7e001 MB |
706 | #ifdef CONFIG_NVM |
707 | ||
2a842aca | 708 | static void null_lnvm_end_io(struct request *rq, blk_status_t status) |
b2b7e001 MB |
709 | { |
710 | struct nvm_rq *rqd = rq->end_io_data; | |
b2b7e001 | 711 | |
2a842aca CH |
712 | /* XXX: lighnvm core seems to expect NVM_RSP_* values here.. */ |
713 | rqd->error = status ? -EIO : 0; | |
06894efe | 714 | nvm_end_io(rqd); |
b2b7e001 MB |
715 | |
716 | blk_put_request(rq); | |
717 | } | |
718 | ||
16f26c3a | 719 | static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) |
b2b7e001 | 720 | { |
16f26c3a | 721 | struct request_queue *q = dev->q; |
b2b7e001 MB |
722 | struct request *rq; |
723 | struct bio *bio = rqd->bio; | |
724 | ||
aebf526b CH |
725 | rq = blk_mq_alloc_request(q, |
726 | op_is_write(bio_op(bio)) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); | |
b2b7e001 MB |
727 | if (IS_ERR(rq)) |
728 | return -ENOMEM; | |
729 | ||
2644a3cc | 730 | blk_init_request_from_bio(rq, bio); |
b2b7e001 MB |
731 | |
732 | rq->end_io_data = rqd; | |
733 | ||
734 | blk_execute_rq_nowait(q, NULL, rq, 0, null_lnvm_end_io); | |
735 | ||
736 | return 0; | |
737 | } | |
738 | ||
16f26c3a | 739 | static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id) |
b2b7e001 | 740 | { |
2984c868 SL |
741 | struct nullb *nullb = dev->q->queuedata; |
742 | sector_t size = (sector_t)nullb->dev->size * 1024 * 1024ULL; | |
5b40db99 | 743 | sector_t blksize; |
b2b7e001 MB |
744 | struct nvm_id_group *grp; |
745 | ||
746 | id->ver_id = 0x1; | |
747 | id->vmnt = 0; | |
bf643185 | 748 | id->cap = 0x2; |
b2b7e001 | 749 | id->dom = 0x1; |
5b40db99 MB |
750 | |
751 | id->ppaf.blk_offset = 0; | |
752 | id->ppaf.blk_len = 16; | |
753 | id->ppaf.pg_offset = 16; | |
754 | id->ppaf.pg_len = 16; | |
755 | id->ppaf.sect_offset = 32; | |
756 | id->ppaf.sect_len = 8; | |
757 | id->ppaf.pln_offset = 40; | |
758 | id->ppaf.pln_len = 8; | |
759 | id->ppaf.lun_offset = 48; | |
760 | id->ppaf.lun_len = 8; | |
761 | id->ppaf.ch_offset = 56; | |
762 | id->ppaf.ch_len = 8; | |
b2b7e001 | 763 | |
2984c868 | 764 | sector_div(size, nullb->dev->blocksize); /* convert size to pages */ |
e93d12ae | 765 | size >>= 8; /* concert size to pgs pr blk */ |
19bd6fe7 | 766 | grp = &id->grp; |
b2b7e001 | 767 | grp->mtype = 0; |
5b40db99 | 768 | grp->fmtype = 0; |
b2b7e001 | 769 | grp->num_ch = 1; |
b2b7e001 | 770 | grp->num_pg = 256; |
5b40db99 | 771 | blksize = size; |
e93d12ae | 772 | size >>= 16; |
5b40db99 | 773 | grp->num_lun = size + 1; |
e93d12ae | 774 | sector_div(blksize, grp->num_lun); |
5b40db99 MB |
775 | grp->num_blk = blksize; |
776 | grp->num_pln = 1; | |
777 | ||
2984c868 SL |
778 | grp->fpg_sz = nullb->dev->blocksize; |
779 | grp->csecs = nullb->dev->blocksize; | |
b2b7e001 MB |
780 | grp->trdt = 25000; |
781 | grp->trdm = 25000; | |
782 | grp->tprt = 500000; | |
783 | grp->tprm = 500000; | |
784 | grp->tbet = 1500000; | |
785 | grp->tbem = 1500000; | |
786 | grp->mpos = 0x010101; /* single plane rwe */ | |
2984c868 | 787 | grp->cpar = nullb->dev->hw_queue_depth; |
b2b7e001 MB |
788 | |
789 | return 0; | |
790 | } | |
791 | ||
16f26c3a | 792 | static void *null_lnvm_create_dma_pool(struct nvm_dev *dev, char *name) |
b2b7e001 MB |
793 | { |
794 | mempool_t *virtmem_pool; | |
795 | ||
6bb9535b | 796 | virtmem_pool = mempool_create_slab_pool(64, ppa_cache); |
b2b7e001 MB |
797 | if (!virtmem_pool) { |
798 | pr_err("null_blk: Unable to create virtual memory pool\n"); | |
799 | return NULL; | |
800 | } | |
801 | ||
802 | return virtmem_pool; | |
803 | } | |
804 | ||
805 | static void null_lnvm_destroy_dma_pool(void *pool) | |
806 | { | |
807 | mempool_destroy(pool); | |
808 | } | |
809 | ||
16f26c3a | 810 | static void *null_lnvm_dev_dma_alloc(struct nvm_dev *dev, void *pool, |
b2b7e001 MB |
811 | gfp_t mem_flags, dma_addr_t *dma_handler) |
812 | { | |
813 | return mempool_alloc(pool, mem_flags); | |
814 | } | |
815 | ||
816 | static void null_lnvm_dev_dma_free(void *pool, void *entry, | |
817 | dma_addr_t dma_handler) | |
818 | { | |
819 | mempool_free(entry, pool); | |
820 | } | |
821 | ||
822 | static struct nvm_dev_ops null_lnvm_dev_ops = { | |
823 | .identity = null_lnvm_id, | |
824 | .submit_io = null_lnvm_submit_io, | |
825 | ||
826 | .create_dma_pool = null_lnvm_create_dma_pool, | |
827 | .destroy_dma_pool = null_lnvm_destroy_dma_pool, | |
828 | .dev_dma_alloc = null_lnvm_dev_dma_alloc, | |
829 | .dev_dma_free = null_lnvm_dev_dma_free, | |
830 | ||
831 | /* Simulate nvme protocol restriction */ | |
832 | .max_phys_sect = 64, | |
833 | }; | |
9ae2d0aa MB |
834 | |
835 | static int null_nvm_register(struct nullb *nullb) | |
836 | { | |
b0b4e09c MB |
837 | struct nvm_dev *dev; |
838 | int rv; | |
839 | ||
840 | dev = nvm_alloc_dev(0); | |
841 | if (!dev) | |
842 | return -ENOMEM; | |
843 | ||
844 | dev->q = nullb->q; | |
845 | memcpy(dev->name, nullb->disk_name, DISK_NAME_LEN); | |
846 | dev->ops = &null_lnvm_dev_ops; | |
847 | ||
848 | rv = nvm_register(dev); | |
849 | if (rv) { | |
850 | kfree(dev); | |
851 | return rv; | |
852 | } | |
853 | nullb->ndev = dev; | |
854 | return 0; | |
9ae2d0aa MB |
855 | } |
856 | ||
857 | static void null_nvm_unregister(struct nullb *nullb) | |
858 | { | |
b0b4e09c | 859 | nvm_unregister(nullb->ndev); |
9ae2d0aa | 860 | } |
b2b7e001 | 861 | #else |
9ae2d0aa MB |
862 | static int null_nvm_register(struct nullb *nullb) |
863 | { | |
92153d30 | 864 | pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n"); |
9ae2d0aa MB |
865 | return -EINVAL; |
866 | } | |
867 | static void null_nvm_unregister(struct nullb *nullb) {} | |
b2b7e001 MB |
868 | #endif /* CONFIG_NVM */ |
869 | ||
9ae2d0aa MB |
870 | static void null_del_dev(struct nullb *nullb) |
871 | { | |
2984c868 SL |
872 | struct nullb_device *dev = nullb->dev; |
873 | ||
9ae2d0aa MB |
874 | list_del_init(&nullb->list); |
875 | ||
2984c868 | 876 | if (dev->use_lightnvm) |
9ae2d0aa MB |
877 | null_nvm_unregister(nullb); |
878 | else | |
879 | del_gendisk(nullb->disk); | |
880 | blk_cleanup_queue(nullb->q); | |
2984c868 SL |
881 | if (dev->queue_mode == NULL_Q_MQ && |
882 | nullb->tag_set == &nullb->__tag_set) | |
82f402fe | 883 | blk_mq_free_tag_set(nullb->tag_set); |
2984c868 | 884 | if (!dev->use_lightnvm) |
9ae2d0aa MB |
885 | put_disk(nullb->disk); |
886 | cleanup_queues(nullb); | |
887 | kfree(nullb); | |
2984c868 | 888 | dev->nullb = NULL; |
9ae2d0aa MB |
889 | } |
890 | ||
f2298c04 JA |
891 | static int null_open(struct block_device *bdev, fmode_t mode) |
892 | { | |
893 | return 0; | |
894 | } | |
895 | ||
896 | static void null_release(struct gendisk *disk, fmode_t mode) | |
897 | { | |
898 | } | |
899 | ||
900 | static const struct block_device_operations null_fops = { | |
901 | .owner = THIS_MODULE, | |
902 | .open = null_open, | |
903 | .release = null_release, | |
904 | }; | |
905 | ||
82f402fe JA |
906 | static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) |
907 | { | |
908 | BUG_ON(!nullb); | |
909 | BUG_ON(!nq); | |
910 | ||
911 | init_waitqueue_head(&nq->wait); | |
912 | nq->queue_depth = nullb->queue_depth; | |
2984c868 | 913 | nq->dev = nullb->dev; |
82f402fe JA |
914 | } |
915 | ||
916 | static void null_init_queues(struct nullb *nullb) | |
917 | { | |
918 | struct request_queue *q = nullb->q; | |
919 | struct blk_mq_hw_ctx *hctx; | |
920 | struct nullb_queue *nq; | |
921 | int i; | |
922 | ||
923 | queue_for_each_hw_ctx(q, hctx, i) { | |
924 | if (!hctx->nr_ctx || !hctx->tags) | |
925 | continue; | |
926 | nq = &nullb->queues[i]; | |
927 | hctx->driver_data = nq; | |
928 | null_init_queue(nullb, nq); | |
929 | nullb->nr_queues++; | |
930 | } | |
931 | } | |
932 | ||
f2298c04 JA |
933 | static int setup_commands(struct nullb_queue *nq) |
934 | { | |
935 | struct nullb_cmd *cmd; | |
936 | int i, tag_size; | |
937 | ||
938 | nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL); | |
939 | if (!nq->cmds) | |
2d263a78 | 940 | return -ENOMEM; |
f2298c04 JA |
941 | |
942 | tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG; | |
943 | nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL); | |
944 | if (!nq->tag_map) { | |
945 | kfree(nq->cmds); | |
2d263a78 | 946 | return -ENOMEM; |
f2298c04 JA |
947 | } |
948 | ||
949 | for (i = 0; i < nq->queue_depth; i++) { | |
950 | cmd = &nq->cmds[i]; | |
951 | INIT_LIST_HEAD(&cmd->list); | |
952 | cmd->ll_list.next = NULL; | |
953 | cmd->tag = -1U; | |
954 | } | |
955 | ||
956 | return 0; | |
957 | } | |
958 | ||
f2298c04 JA |
959 | static int setup_queues(struct nullb *nullb) |
960 | { | |
2984c868 SL |
961 | nullb->queues = kzalloc(nullb->dev->submit_queues * |
962 | sizeof(struct nullb_queue), GFP_KERNEL); | |
f2298c04 | 963 | if (!nullb->queues) |
2d263a78 | 964 | return -ENOMEM; |
f2298c04 JA |
965 | |
966 | nullb->nr_queues = 0; | |
2984c868 | 967 | nullb->queue_depth = nullb->dev->hw_queue_depth; |
f2298c04 | 968 | |
2d263a78 MB |
969 | return 0; |
970 | } | |
971 | ||
972 | static int init_driver_queues(struct nullb *nullb) | |
973 | { | |
974 | struct nullb_queue *nq; | |
975 | int i, ret = 0; | |
f2298c04 | 976 | |
2984c868 | 977 | for (i = 0; i < nullb->dev->submit_queues; i++) { |
f2298c04 | 978 | nq = &nullb->queues[i]; |
2d263a78 MB |
979 | |
980 | null_init_queue(nullb, nq); | |
981 | ||
982 | ret = setup_commands(nq); | |
983 | if (ret) | |
31f9690e | 984 | return ret; |
f2298c04 JA |
985 | nullb->nr_queues++; |
986 | } | |
2d263a78 | 987 | return 0; |
f2298c04 JA |
988 | } |
989 | ||
9ae2d0aa | 990 | static int null_gendisk_register(struct nullb *nullb) |
f2298c04 JA |
991 | { |
992 | struct gendisk *disk; | |
f2298c04 | 993 | sector_t size; |
9ae2d0aa | 994 | |
2984c868 | 995 | disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node); |
9ae2d0aa MB |
996 | if (!disk) |
997 | return -ENOMEM; | |
2984c868 | 998 | size = (sector_t)nullb->dev->size * 1024 * 1024ULL; |
9ae2d0aa MB |
999 | set_capacity(disk, size >> 9); |
1000 | ||
1001 | disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO; | |
1002 | disk->major = null_major; | |
1003 | disk->first_minor = nullb->index; | |
1004 | disk->fops = &null_fops; | |
1005 | disk->private_data = nullb; | |
1006 | disk->queue = nullb->q; | |
1007 | strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN); | |
1008 | ||
1009 | add_disk(disk); | |
1010 | return 0; | |
1011 | } | |
1012 | ||
2984c868 | 1013 | static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) |
82f402fe JA |
1014 | { |
1015 | set->ops = &null_mq_ops; | |
2984c868 SL |
1016 | set->nr_hw_queues = nullb ? nullb->dev->submit_queues : |
1017 | g_submit_queues; | |
1018 | set->queue_depth = nullb ? nullb->dev->hw_queue_depth : | |
1019 | g_hw_queue_depth; | |
1020 | set->numa_node = nullb ? nullb->dev->home_node : g_home_node; | |
82f402fe JA |
1021 | set->cmd_size = sizeof(struct nullb_cmd); |
1022 | set->flags = BLK_MQ_F_SHOULD_MERGE; | |
1023 | set->driver_data = NULL; | |
1024 | ||
2984c868 | 1025 | if (nullb->dev->blocking) |
82f402fe JA |
1026 | set->flags |= BLK_MQ_F_BLOCKING; |
1027 | ||
1028 | return blk_mq_alloc_tag_set(set); | |
1029 | } | |
1030 | ||
cedcafad SL |
1031 | static void null_validate_conf(struct nullb_device *dev) |
1032 | { | |
1033 | dev->blocksize = round_down(dev->blocksize, 512); | |
1034 | dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096); | |
1035 | if (dev->use_lightnvm && dev->blocksize != 4096) | |
1036 | dev->blocksize = 4096; | |
1037 | ||
1038 | if (dev->use_lightnvm && dev->queue_mode != NULL_Q_MQ) | |
1039 | dev->queue_mode = NULL_Q_MQ; | |
1040 | ||
1041 | if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) { | |
1042 | if (dev->submit_queues != nr_online_nodes) | |
1043 | dev->submit_queues = nr_online_nodes; | |
1044 | } else if (dev->submit_queues > nr_cpu_ids) | |
1045 | dev->submit_queues = nr_cpu_ids; | |
1046 | else if (dev->submit_queues == 0) | |
1047 | dev->submit_queues = 1; | |
1048 | ||
1049 | dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ); | |
1050 | dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER); | |
1051 | } | |
1052 | ||
2984c868 | 1053 | static int null_add_dev(struct nullb_device *dev) |
9ae2d0aa MB |
1054 | { |
1055 | struct nullb *nullb; | |
dc501dc0 | 1056 | int rv; |
f2298c04 | 1057 | |
cedcafad SL |
1058 | null_validate_conf(dev); |
1059 | ||
2984c868 | 1060 | nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node); |
dc501dc0 RE |
1061 | if (!nullb) { |
1062 | rv = -ENOMEM; | |
24d2f903 | 1063 | goto out; |
dc501dc0 | 1064 | } |
2984c868 SL |
1065 | nullb->dev = dev; |
1066 | dev->nullb = nullb; | |
f2298c04 JA |
1067 | |
1068 | spin_lock_init(&nullb->lock); | |
1069 | ||
dc501dc0 RE |
1070 | rv = setup_queues(nullb); |
1071 | if (rv) | |
24d2f903 | 1072 | goto out_free_nullb; |
f2298c04 | 1073 | |
2984c868 | 1074 | if (dev->queue_mode == NULL_Q_MQ) { |
82f402fe JA |
1075 | if (shared_tags) { |
1076 | nullb->tag_set = &tag_set; | |
1077 | rv = 0; | |
1078 | } else { | |
1079 | nullb->tag_set = &nullb->__tag_set; | |
2984c868 | 1080 | rv = null_init_tag_set(nullb, nullb->tag_set); |
82f402fe JA |
1081 | } |
1082 | ||
dc501dc0 | 1083 | if (rv) |
24d2f903 CH |
1084 | goto out_cleanup_queues; |
1085 | ||
82f402fe | 1086 | nullb->q = blk_mq_init_queue(nullb->tag_set); |
35b489d3 | 1087 | if (IS_ERR(nullb->q)) { |
dc501dc0 | 1088 | rv = -ENOMEM; |
24d2f903 | 1089 | goto out_cleanup_tags; |
dc501dc0 | 1090 | } |
82f402fe | 1091 | null_init_queues(nullb); |
2984c868 SL |
1092 | } else if (dev->queue_mode == NULL_Q_BIO) { |
1093 | nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node); | |
dc501dc0 RE |
1094 | if (!nullb->q) { |
1095 | rv = -ENOMEM; | |
24d2f903 | 1096 | goto out_cleanup_queues; |
dc501dc0 | 1097 | } |
f2298c04 | 1098 | blk_queue_make_request(nullb->q, null_queue_bio); |
31f9690e JK |
1099 | rv = init_driver_queues(nullb); |
1100 | if (rv) | |
1101 | goto out_cleanup_blk_queue; | |
f2298c04 | 1102 | } else { |
2984c868 SL |
1103 | nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, |
1104 | dev->home_node); | |
dc501dc0 RE |
1105 | if (!nullb->q) { |
1106 | rv = -ENOMEM; | |
24d2f903 | 1107 | goto out_cleanup_queues; |
dc501dc0 | 1108 | } |
f2298c04 | 1109 | blk_queue_prep_rq(nullb->q, null_rq_prep_fn); |
24d2f903 | 1110 | blk_queue_softirq_done(nullb->q, null_softirq_done_fn); |
31f9690e JK |
1111 | rv = init_driver_queues(nullb); |
1112 | if (rv) | |
1113 | goto out_cleanup_blk_queue; | |
f2298c04 JA |
1114 | } |
1115 | ||
f2298c04 JA |
1116 | nullb->q->queuedata = nullb; |
1117 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q); | |
b277da0a | 1118 | queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q); |
f2298c04 | 1119 | |
f2298c04 | 1120 | mutex_lock(&lock); |
f2298c04 | 1121 | nullb->index = nullb_indexes++; |
cedcafad | 1122 | dev->index = nullb->index; |
f2298c04 JA |
1123 | mutex_unlock(&lock); |
1124 | ||
2984c868 SL |
1125 | blk_queue_logical_block_size(nullb->q, dev->blocksize); |
1126 | blk_queue_physical_block_size(nullb->q, dev->blocksize); | |
f2298c04 | 1127 | |
b2b7e001 MB |
1128 | sprintf(nullb->disk_name, "nullb%d", nullb->index); |
1129 | ||
2984c868 | 1130 | if (dev->use_lightnvm) |
9ae2d0aa MB |
1131 | rv = null_nvm_register(nullb); |
1132 | else | |
1133 | rv = null_gendisk_register(nullb); | |
b2b7e001 | 1134 | |
9ae2d0aa MB |
1135 | if (rv) |
1136 | goto out_cleanup_blk_queue; | |
a514379b MB |
1137 | |
1138 | mutex_lock(&lock); | |
1139 | list_add_tail(&nullb->list, &nullb_list); | |
1140 | mutex_unlock(&lock); | |
3681c85d | 1141 | |
f2298c04 | 1142 | return 0; |
24d2f903 CH |
1143 | out_cleanup_blk_queue: |
1144 | blk_cleanup_queue(nullb->q); | |
1145 | out_cleanup_tags: | |
2984c868 | 1146 | if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) |
82f402fe | 1147 | blk_mq_free_tag_set(nullb->tag_set); |
24d2f903 CH |
1148 | out_cleanup_queues: |
1149 | cleanup_queues(nullb); | |
1150 | out_free_nullb: | |
1151 | kfree(nullb); | |
1152 | out: | |
2984c868 | 1153 | null_free_dev(dev); |
dc501dc0 | 1154 | return rv; |
f2298c04 JA |
1155 | } |
1156 | ||
1157 | static int __init null_init(void) | |
1158 | { | |
af096e22 | 1159 | int ret = 0; |
f2298c04 | 1160 | unsigned int i; |
af096e22 | 1161 | struct nullb *nullb; |
2984c868 | 1162 | struct nullb_device *dev; |
f2298c04 | 1163 | |
2984c868 | 1164 | if (g_bs > PAGE_SIZE) { |
9967d8ac R |
1165 | pr_warn("null_blk: invalid block size\n"); |
1166 | pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE); | |
2984c868 | 1167 | g_bs = PAGE_SIZE; |
9967d8ac | 1168 | } |
f2298c04 | 1169 | |
2984c868 | 1170 | if (g_use_lightnvm && g_bs != 4096) { |
6bb9535b MB |
1171 | pr_warn("null_blk: LightNVM only supports 4k block size\n"); |
1172 | pr_warn("null_blk: defaults block size to 4k\n"); | |
2984c868 | 1173 | g_bs = 4096; |
6bb9535b MB |
1174 | } |
1175 | ||
2984c868 | 1176 | if (g_use_lightnvm && g_queue_mode != NULL_Q_MQ) { |
b2b7e001 MB |
1177 | pr_warn("null_blk: LightNVM only supported for blk-mq\n"); |
1178 | pr_warn("null_blk: defaults queue mode to blk-mq\n"); | |
2984c868 | 1179 | g_queue_mode = NULL_Q_MQ; |
b2b7e001 MB |
1180 | } |
1181 | ||
2984c868 SL |
1182 | if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) { |
1183 | if (g_submit_queues != nr_online_nodes) { | |
558ab300 | 1184 | pr_warn("null_blk: submit_queues param is set to %u.\n", |
d15ee6b1 | 1185 | nr_online_nodes); |
2984c868 | 1186 | g_submit_queues = nr_online_nodes; |
fc1bc354 | 1187 | } |
2984c868 SL |
1188 | } else if (g_submit_queues > nr_cpu_ids) |
1189 | g_submit_queues = nr_cpu_ids; | |
1190 | else if (g_submit_queues <= 0) | |
1191 | g_submit_queues = 1; | |
f2298c04 | 1192 | |
2984c868 SL |
1193 | if (g_queue_mode == NULL_Q_MQ && shared_tags) { |
1194 | ret = null_init_tag_set(NULL, &tag_set); | |
db2d153d MG |
1195 | if (ret) |
1196 | return ret; | |
1197 | } | |
1198 | ||
3bf2bd20 SL |
1199 | config_group_init(&nullb_subsys.su_group); |
1200 | mutex_init(&nullb_subsys.su_mutex); | |
1201 | ||
1202 | ret = configfs_register_subsystem(&nullb_subsys); | |
1203 | if (ret) | |
1204 | goto err_tagset; | |
1205 | ||
f2298c04 JA |
1206 | mutex_init(&lock); |
1207 | ||
f2298c04 | 1208 | null_major = register_blkdev(0, "nullb"); |
db2d153d MG |
1209 | if (null_major < 0) { |
1210 | ret = null_major; | |
3bf2bd20 | 1211 | goto err_conf; |
db2d153d | 1212 | } |
f2298c04 | 1213 | |
2984c868 | 1214 | if (g_use_lightnvm) { |
6bb9535b MB |
1215 | ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64), |
1216 | 0, 0, NULL); | |
1217 | if (!ppa_cache) { | |
1218 | pr_err("null_blk: unable to create ppa cache\n"); | |
af096e22 MH |
1219 | ret = -ENOMEM; |
1220 | goto err_ppa; | |
6bb9535b MB |
1221 | } |
1222 | } | |
1223 | ||
f2298c04 | 1224 | for (i = 0; i < nr_devices; i++) { |
2984c868 SL |
1225 | dev = null_alloc_dev(); |
1226 | if (!dev) | |
1227 | goto err_dev; | |
1228 | ret = null_add_dev(dev); | |
1229 | if (ret) { | |
1230 | null_free_dev(dev); | |
af096e22 | 1231 | goto err_dev; |
2984c868 | 1232 | } |
f2298c04 JA |
1233 | } |
1234 | ||
1235 | pr_info("null: module loaded\n"); | |
1236 | return 0; | |
af096e22 MH |
1237 | |
1238 | err_dev: | |
1239 | while (!list_empty(&nullb_list)) { | |
1240 | nullb = list_entry(nullb_list.next, struct nullb, list); | |
2984c868 | 1241 | dev = nullb->dev; |
af096e22 | 1242 | null_del_dev(nullb); |
2984c868 | 1243 | null_free_dev(dev); |
af096e22 | 1244 | } |
6bb9535b | 1245 | kmem_cache_destroy(ppa_cache); |
af096e22 MH |
1246 | err_ppa: |
1247 | unregister_blkdev(null_major, "nullb"); | |
3bf2bd20 SL |
1248 | err_conf: |
1249 | configfs_unregister_subsystem(&nullb_subsys); | |
db2d153d | 1250 | err_tagset: |
2984c868 | 1251 | if (g_queue_mode == NULL_Q_MQ && shared_tags) |
db2d153d | 1252 | blk_mq_free_tag_set(&tag_set); |
af096e22 | 1253 | return ret; |
f2298c04 JA |
1254 | } |
1255 | ||
1256 | static void __exit null_exit(void) | |
1257 | { | |
1258 | struct nullb *nullb; | |
1259 | ||
3bf2bd20 SL |
1260 | configfs_unregister_subsystem(&nullb_subsys); |
1261 | ||
f2298c04 JA |
1262 | unregister_blkdev(null_major, "nullb"); |
1263 | ||
1264 | mutex_lock(&lock); | |
1265 | while (!list_empty(&nullb_list)) { | |
2984c868 SL |
1266 | struct nullb_device *dev; |
1267 | ||
f2298c04 | 1268 | nullb = list_entry(nullb_list.next, struct nullb, list); |
2984c868 | 1269 | dev = nullb->dev; |
f2298c04 | 1270 | null_del_dev(nullb); |
2984c868 | 1271 | null_free_dev(dev); |
f2298c04 JA |
1272 | } |
1273 | mutex_unlock(&lock); | |
6bb9535b | 1274 | |
2984c868 | 1275 | if (g_queue_mode == NULL_Q_MQ && shared_tags) |
82f402fe JA |
1276 | blk_mq_free_tag_set(&tag_set); |
1277 | ||
6bb9535b | 1278 | kmem_cache_destroy(ppa_cache); |
f2298c04 JA |
1279 | } |
1280 | ||
1281 | module_init(null_init); | |
1282 | module_exit(null_exit); | |
1283 | ||
1284 | MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>"); | |
1285 | MODULE_LICENSE("GPL"); |