]>
Commit | Line | Data |
---|---|---|
ae1519ec MB |
1 | /* |
2 | * Copyright (C) 2015 IT University of Copenhagen | |
3 | * Initial release: Matias Bjorling <m@bjorling.me> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public License version | |
7 | * 2 as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but | |
10 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
12 | * General Public License for more details. | |
13 | * | |
14 | * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs. | |
15 | */ | |
16 | ||
17 | #include "rrpc.h" | |
18 | ||
19 | static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache; | |
20 | static DECLARE_RWSEM(rrpc_lock); | |
21 | ||
22 | static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, | |
23 | struct nvm_rq *rqd, unsigned long flags); | |
24 | ||
25 | #define rrpc_for_each_lun(rrpc, rlun, i) \ | |
26 | for ((i) = 0, rlun = &(rrpc)->luns[0]; \ | |
27 | (i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)]) | |
28 | ||
29 | static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a) | |
30 | { | |
31 | struct rrpc_block *rblk = a->rblk; | |
32 | unsigned int pg_offset; | |
33 | ||
34 | lockdep_assert_held(&rrpc->rev_lock); | |
35 | ||
36 | if (a->addr == ADDR_EMPTY || !rblk) | |
37 | return; | |
38 | ||
39 | spin_lock(&rblk->lock); | |
40 | ||
afb18e0e | 41 | div_u64_rem(a->addr, rrpc->dev->sec_per_blk, &pg_offset); |
ae1519ec MB |
42 | WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages)); |
43 | rblk->nr_invalid_pages++; | |
44 | ||
45 | spin_unlock(&rblk->lock); | |
46 | ||
47 | rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY; | |
48 | } | |
49 | ||
50 | static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba, | |
5114e277 | 51 | unsigned int len) |
ae1519ec MB |
52 | { |
53 | sector_t i; | |
54 | ||
55 | spin_lock(&rrpc->rev_lock); | |
56 | for (i = slba; i < slba + len; i++) { | |
57 | struct rrpc_addr *gp = &rrpc->trans_map[i]; | |
58 | ||
59 | rrpc_page_invalidate(rrpc, gp); | |
60 | gp->rblk = NULL; | |
61 | } | |
62 | spin_unlock(&rrpc->rev_lock); | |
63 | } | |
64 | ||
65 | static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc, | |
66 | sector_t laddr, unsigned int pages) | |
67 | { | |
68 | struct nvm_rq *rqd; | |
69 | struct rrpc_inflight_rq *inf; | |
70 | ||
71 | rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC); | |
72 | if (!rqd) | |
73 | return ERR_PTR(-ENOMEM); | |
74 | ||
75 | inf = rrpc_get_inflight_rq(rqd); | |
76 | if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) { | |
77 | mempool_free(rqd, rrpc->rq_pool); | |
78 | return NULL; | |
79 | } | |
80 | ||
81 | return rqd; | |
82 | } | |
83 | ||
84 | static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd) | |
85 | { | |
86 | struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd); | |
87 | ||
88 | rrpc_unlock_laddr(rrpc, inf); | |
89 | ||
90 | mempool_free(rqd, rrpc->rq_pool); | |
91 | } | |
92 | ||
93 | static void rrpc_discard(struct rrpc *rrpc, struct bio *bio) | |
94 | { | |
95 | sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG; | |
96 | sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE; | |
97 | struct nvm_rq *rqd; | |
98 | ||
0de2415b | 99 | while (1) { |
ae1519ec | 100 | rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len); |
0de2415b WT |
101 | if (rqd) |
102 | break; | |
103 | ||
ae1519ec | 104 | schedule(); |
0de2415b | 105 | } |
ae1519ec MB |
106 | |
107 | if (IS_ERR(rqd)) { | |
108 | pr_err("rrpc: unable to acquire inflight IO\n"); | |
109 | bio_io_error(bio); | |
110 | return; | |
111 | } | |
112 | ||
113 | rrpc_invalidate_range(rrpc, slba, len); | |
114 | rrpc_inflight_laddr_release(rrpc, rqd); | |
115 | } | |
116 | ||
117 | static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk) | |
118 | { | |
afb18e0e | 119 | return (rblk->next_page == rrpc->dev->sec_per_blk); |
ae1519ec MB |
120 | } |
121 | ||
afb18e0e JG |
122 | /* Calculate relative addr for the given block, considering instantiated LUNs */ |
123 | static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk) | |
124 | { | |
125 | struct nvm_block *blk = rblk->parent; | |
126 | int lun_blk = blk->id % (rrpc->dev->blks_per_lun * rrpc->nr_luns); | |
127 | ||
128 | return lun_blk * rrpc->dev->sec_per_blk; | |
129 | } | |
130 | ||
131 | /* Calculate global addr for the given block */ | |
b7ceb7d5 | 132 | static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk) |
ae1519ec MB |
133 | { |
134 | struct nvm_block *blk = rblk->parent; | |
135 | ||
afb18e0e | 136 | return blk->id * rrpc->dev->sec_per_blk; |
ae1519ec MB |
137 | } |
138 | ||
b7ceb7d5 | 139 | static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr) |
ae1519ec MB |
140 | { |
141 | struct ppa_addr paddr; | |
142 | ||
143 | paddr.ppa = addr; | |
7386af27 | 144 | return linear_to_generic_addr(dev, paddr); |
ae1519ec MB |
145 | } |
146 | ||
147 | /* requires lun->lock taken */ | |
855cdd2c MB |
148 | static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk, |
149 | struct rrpc_block **cur_rblk) | |
ae1519ec MB |
150 | { |
151 | struct rrpc *rrpc = rlun->rrpc; | |
152 | ||
855cdd2c MB |
153 | if (*cur_rblk) { |
154 | spin_lock(&(*cur_rblk)->lock); | |
155 | WARN_ON(!block_is_full(rrpc, *cur_rblk)); | |
156 | spin_unlock(&(*cur_rblk)->lock); | |
ae1519ec | 157 | } |
855cdd2c | 158 | *cur_rblk = new_rblk; |
ae1519ec MB |
159 | } |
160 | ||
161 | static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, | |
162 | unsigned long flags) | |
163 | { | |
164 | struct nvm_block *blk; | |
165 | struct rrpc_block *rblk; | |
166 | ||
41285fad | 167 | blk = nvm_get_blk(rrpc->dev, rlun->parent, flags); |
ff0e498b JG |
168 | if (!blk) { |
169 | pr_err("nvm: rrpc: cannot get new block from media manager\n"); | |
ae1519ec | 170 | return NULL; |
ff0e498b | 171 | } |
ae1519ec | 172 | |
afb18e0e | 173 | rblk = rrpc_get_rblk(rlun, blk->id); |
ff0e498b | 174 | blk->priv = rblk; |
afb18e0e | 175 | bitmap_zero(rblk->invalid_pages, rrpc->dev->sec_per_blk); |
ae1519ec MB |
176 | rblk->next_page = 0; |
177 | rblk->nr_invalid_pages = 0; | |
178 | atomic_set(&rblk->data_cmnt_size, 0); | |
179 | ||
180 | return rblk; | |
181 | } | |
182 | ||
183 | static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk) | |
184 | { | |
41285fad | 185 | nvm_put_blk(rrpc->dev, rblk->parent); |
ae1519ec MB |
186 | } |
187 | ||
d3d1a438 WT |
188 | static void rrpc_put_blks(struct rrpc *rrpc) |
189 | { | |
190 | struct rrpc_lun *rlun; | |
191 | int i; | |
192 | ||
193 | for (i = 0; i < rrpc->nr_luns; i++) { | |
194 | rlun = &rrpc->luns[i]; | |
195 | if (rlun->cur) | |
196 | rrpc_put_blk(rrpc, rlun->cur); | |
197 | if (rlun->gc_cur) | |
198 | rrpc_put_blk(rrpc, rlun->gc_cur); | |
199 | } | |
200 | } | |
201 | ||
ae1519ec MB |
202 | static struct rrpc_lun *get_next_lun(struct rrpc *rrpc) |
203 | { | |
204 | int next = atomic_inc_return(&rrpc->next_lun); | |
205 | ||
206 | return &rrpc->luns[next % rrpc->nr_luns]; | |
207 | } | |
208 | ||
209 | static void rrpc_gc_kick(struct rrpc *rrpc) | |
210 | { | |
211 | struct rrpc_lun *rlun; | |
212 | unsigned int i; | |
213 | ||
214 | for (i = 0; i < rrpc->nr_luns; i++) { | |
215 | rlun = &rrpc->luns[i]; | |
216 | queue_work(rrpc->krqd_wq, &rlun->ws_gc); | |
217 | } | |
218 | } | |
219 | ||
220 | /* | |
221 | * timed GC every interval. | |
222 | */ | |
223 | static void rrpc_gc_timer(unsigned long data) | |
224 | { | |
225 | struct rrpc *rrpc = (struct rrpc *)data; | |
226 | ||
227 | rrpc_gc_kick(rrpc); | |
228 | mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10)); | |
229 | } | |
230 | ||
231 | static void rrpc_end_sync_bio(struct bio *bio) | |
232 | { | |
233 | struct completion *waiting = bio->bi_private; | |
234 | ||
235 | if (bio->bi_error) | |
236 | pr_err("nvm: gc request failed (%u).\n", bio->bi_error); | |
237 | ||
238 | complete(waiting); | |
239 | } | |
240 | ||
241 | /* | |
242 | * rrpc_move_valid_pages -- migrate live data off the block | |
243 | * @rrpc: the 'rrpc' structure | |
244 | * @block: the block from which to migrate live pages | |
245 | * | |
246 | * Description: | |
247 | * GC algorithms may call this function to migrate remaining live | |
248 | * pages off the block prior to erasing it. This function blocks | |
249 | * further execution until the operation is complete. | |
250 | */ | |
251 | static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) | |
252 | { | |
253 | struct request_queue *q = rrpc->dev->q; | |
254 | struct rrpc_rev_addr *rev; | |
255 | struct nvm_rq *rqd; | |
256 | struct bio *bio; | |
257 | struct page *page; | |
258 | int slot; | |
afb18e0e | 259 | int nr_sec_per_blk = rrpc->dev->sec_per_blk; |
b7ceb7d5 | 260 | u64 phys_addr; |
ae1519ec MB |
261 | DECLARE_COMPLETION_ONSTACK(wait); |
262 | ||
afb18e0e | 263 | if (bitmap_full(rblk->invalid_pages, nr_sec_per_blk)) |
ae1519ec MB |
264 | return 0; |
265 | ||
266 | bio = bio_alloc(GFP_NOIO, 1); | |
267 | if (!bio) { | |
268 | pr_err("nvm: could not alloc bio to gc\n"); | |
269 | return -ENOMEM; | |
270 | } | |
271 | ||
272 | page = mempool_alloc(rrpc->page_pool, GFP_NOIO); | |
16c6d048 WT |
273 | if (!page) { |
274 | bio_put(bio); | |
3bfbc6ad | 275 | return -ENOMEM; |
16c6d048 | 276 | } |
ae1519ec MB |
277 | |
278 | while ((slot = find_first_zero_bit(rblk->invalid_pages, | |
afb18e0e | 279 | nr_sec_per_blk)) < nr_sec_per_blk) { |
ae1519ec MB |
280 | |
281 | /* Lock laddr */ | |
afb18e0e | 282 | phys_addr = rblk->parent->id * nr_sec_per_blk + slot; |
ae1519ec MB |
283 | |
284 | try: | |
285 | spin_lock(&rrpc->rev_lock); | |
286 | /* Get logical address from physical to logical table */ | |
287 | rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset]; | |
288 | /* already updated by previous regular write */ | |
289 | if (rev->addr == ADDR_EMPTY) { | |
290 | spin_unlock(&rrpc->rev_lock); | |
291 | continue; | |
292 | } | |
293 | ||
294 | rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1); | |
295 | if (IS_ERR_OR_NULL(rqd)) { | |
296 | spin_unlock(&rrpc->rev_lock); | |
297 | schedule(); | |
298 | goto try; | |
299 | } | |
300 | ||
301 | spin_unlock(&rrpc->rev_lock); | |
302 | ||
303 | /* Perform read to do GC */ | |
304 | bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); | |
95fe6c1a | 305 | bio_set_op_attrs(bio, REQ_OP_READ, 0); |
ae1519ec MB |
306 | bio->bi_private = &wait; |
307 | bio->bi_end_io = rrpc_end_sync_bio; | |
308 | ||
309 | /* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */ | |
310 | bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); | |
311 | ||
312 | if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) { | |
313 | pr_err("rrpc: gc read failed.\n"); | |
314 | rrpc_inflight_laddr_release(rrpc, rqd); | |
315 | goto finished; | |
316 | } | |
317 | wait_for_completion_io(&wait); | |
2b11c1b2 WT |
318 | if (bio->bi_error) { |
319 | rrpc_inflight_laddr_release(rrpc, rqd); | |
320 | goto finished; | |
321 | } | |
ae1519ec MB |
322 | |
323 | bio_reset(bio); | |
324 | reinit_completion(&wait); | |
325 | ||
326 | bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); | |
95fe6c1a | 327 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); |
ae1519ec MB |
328 | bio->bi_private = &wait; |
329 | bio->bi_end_io = rrpc_end_sync_bio; | |
330 | ||
331 | bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); | |
332 | ||
333 | /* turn the command around and write the data back to a new | |
334 | * address | |
335 | */ | |
336 | if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) { | |
337 | pr_err("rrpc: gc write failed.\n"); | |
338 | rrpc_inflight_laddr_release(rrpc, rqd); | |
339 | goto finished; | |
340 | } | |
341 | wait_for_completion_io(&wait); | |
342 | ||
343 | rrpc_inflight_laddr_release(rrpc, rqd); | |
2b11c1b2 WT |
344 | if (bio->bi_error) |
345 | goto finished; | |
ae1519ec MB |
346 | |
347 | bio_reset(bio); | |
348 | } | |
349 | ||
350 | finished: | |
351 | mempool_free(page, rrpc->page_pool); | |
352 | bio_put(bio); | |
353 | ||
afb18e0e | 354 | if (!bitmap_full(rblk->invalid_pages, nr_sec_per_blk)) { |
ae1519ec MB |
355 | pr_err("nvm: failed to garbage collect block\n"); |
356 | return -EIO; | |
357 | } | |
358 | ||
359 | return 0; | |
360 | } | |
361 | ||
362 | static void rrpc_block_gc(struct work_struct *work) | |
363 | { | |
364 | struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc, | |
365 | ws_gc); | |
366 | struct rrpc *rrpc = gcb->rrpc; | |
367 | struct rrpc_block *rblk = gcb->rblk; | |
cca87bc9 | 368 | struct rrpc_lun *rlun = rblk->rlun; |
ae1519ec MB |
369 | struct nvm_dev *dev = rrpc->dev; |
370 | ||
d0ca798f | 371 | mempool_free(gcb, rrpc->gcb_pool); |
ae1519ec MB |
372 | pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id); |
373 | ||
374 | if (rrpc_move_valid_pages(rrpc, rblk)) | |
d0ca798f WT |
375 | goto put_back; |
376 | ||
bb314979 | 377 | if (nvm_erase_blk(dev, rblk->parent, 0)) |
d0ca798f | 378 | goto put_back; |
ae1519ec | 379 | |
ae1519ec | 380 | rrpc_put_blk(rrpc, rblk); |
d0ca798f WT |
381 | |
382 | return; | |
383 | ||
384 | put_back: | |
385 | spin_lock(&rlun->lock); | |
386 | list_add_tail(&rblk->prio, &rlun->prio_list); | |
387 | spin_unlock(&rlun->lock); | |
ae1519ec MB |
388 | } |
389 | ||
390 | /* the block with highest number of invalid pages, will be in the beginning | |
391 | * of the list | |
392 | */ | |
393 | static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra, | |
394 | struct rrpc_block *rb) | |
395 | { | |
396 | if (ra->nr_invalid_pages == rb->nr_invalid_pages) | |
397 | return ra; | |
398 | ||
399 | return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra; | |
400 | } | |
401 | ||
402 | /* linearly find the block with highest number of invalid pages | |
403 | * requires lun->lock | |
404 | */ | |
405 | static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun) | |
406 | { | |
407 | struct list_head *prio_list = &rlun->prio_list; | |
408 | struct rrpc_block *rblock, *max; | |
409 | ||
410 | BUG_ON(list_empty(prio_list)); | |
411 | ||
412 | max = list_first_entry(prio_list, struct rrpc_block, prio); | |
413 | list_for_each_entry(rblock, prio_list, prio) | |
414 | max = rblock_max_invalid(max, rblock); | |
415 | ||
416 | return max; | |
417 | } | |
418 | ||
419 | static void rrpc_lun_gc(struct work_struct *work) | |
420 | { | |
421 | struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc); | |
422 | struct rrpc *rrpc = rlun->rrpc; | |
423 | struct nvm_lun *lun = rlun->parent; | |
424 | struct rrpc_block_gc *gcb; | |
425 | unsigned int nr_blocks_need; | |
426 | ||
427 | nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE; | |
428 | ||
429 | if (nr_blocks_need < rrpc->nr_luns) | |
430 | nr_blocks_need = rrpc->nr_luns; | |
431 | ||
b262924b | 432 | spin_lock(&rlun->lock); |
ae1519ec MB |
433 | while (nr_blocks_need > lun->nr_free_blocks && |
434 | !list_empty(&rlun->prio_list)) { | |
435 | struct rrpc_block *rblock = block_prio_find_max(rlun); | |
436 | struct nvm_block *block = rblock->parent; | |
437 | ||
438 | if (!rblock->nr_invalid_pages) | |
439 | break; | |
440 | ||
b262924b WT |
441 | gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC); |
442 | if (!gcb) | |
443 | break; | |
444 | ||
ae1519ec MB |
445 | list_del_init(&rblock->prio); |
446 | ||
447 | BUG_ON(!block_is_full(rrpc, rblock)); | |
448 | ||
449 | pr_debug("rrpc: selected block '%lu' for GC\n", block->id); | |
450 | ||
ae1519ec MB |
451 | gcb->rrpc = rrpc; |
452 | gcb->rblk = rblock; | |
453 | INIT_WORK(&gcb->ws_gc, rrpc_block_gc); | |
454 | ||
455 | queue_work(rrpc->kgc_wq, &gcb->ws_gc); | |
456 | ||
457 | nr_blocks_need--; | |
458 | } | |
b262924b | 459 | spin_unlock(&rlun->lock); |
ae1519ec MB |
460 | |
461 | /* TODO: Hint that request queue can be started again */ | |
462 | } | |
463 | ||
464 | static void rrpc_gc_queue(struct work_struct *work) | |
465 | { | |
466 | struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc, | |
467 | ws_gc); | |
468 | struct rrpc *rrpc = gcb->rrpc; | |
469 | struct rrpc_block *rblk = gcb->rblk; | |
cca87bc9 | 470 | struct rrpc_lun *rlun = rblk->rlun; |
ae1519ec MB |
471 | |
472 | spin_lock(&rlun->lock); | |
473 | list_add_tail(&rblk->prio, &rlun->prio_list); | |
474 | spin_unlock(&rlun->lock); | |
475 | ||
476 | mempool_free(gcb, rrpc->gcb_pool); | |
477 | pr_debug("nvm: block '%lu' is full, allow GC (sched)\n", | |
478 | rblk->parent->id); | |
479 | } | |
480 | ||
481 | static const struct block_device_operations rrpc_fops = { | |
482 | .owner = THIS_MODULE, | |
483 | }; | |
484 | ||
485 | static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc) | |
486 | { | |
487 | unsigned int i; | |
488 | struct rrpc_lun *rlun, *max_free; | |
489 | ||
490 | if (!is_gc) | |
491 | return get_next_lun(rrpc); | |
492 | ||
493 | /* during GC, we don't care about RR, instead we want to make | |
494 | * sure that we maintain evenness between the block luns. | |
495 | */ | |
496 | max_free = &rrpc->luns[0]; | |
497 | /* prevent GC-ing lun from devouring pages of a lun with | |
498 | * little free blocks. We don't take the lock as we only need an | |
499 | * estimate. | |
500 | */ | |
501 | rrpc_for_each_lun(rrpc, rlun, i) { | |
502 | if (rlun->parent->nr_free_blocks > | |
503 | max_free->parent->nr_free_blocks) | |
504 | max_free = rlun; | |
505 | } | |
506 | ||
507 | return max_free; | |
508 | } | |
509 | ||
510 | static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr, | |
b7ceb7d5 | 511 | struct rrpc_block *rblk, u64 paddr) |
ae1519ec MB |
512 | { |
513 | struct rrpc_addr *gp; | |
514 | struct rrpc_rev_addr *rev; | |
515 | ||
4ece44af | 516 | BUG_ON(laddr >= rrpc->nr_sects); |
ae1519ec MB |
517 | |
518 | gp = &rrpc->trans_map[laddr]; | |
519 | spin_lock(&rrpc->rev_lock); | |
520 | if (gp->rblk) | |
521 | rrpc_page_invalidate(rrpc, gp); | |
522 | ||
523 | gp->addr = paddr; | |
524 | gp->rblk = rblk; | |
525 | ||
526 | rev = &rrpc->rev_trans_map[gp->addr - rrpc->poffset]; | |
527 | rev->addr = laddr; | |
528 | spin_unlock(&rrpc->rev_lock); | |
529 | ||
530 | return gp; | |
531 | } | |
532 | ||
b7ceb7d5 | 533 | static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk) |
ae1519ec | 534 | { |
b7ceb7d5 | 535 | u64 addr = ADDR_EMPTY; |
ae1519ec MB |
536 | |
537 | spin_lock(&rblk->lock); | |
538 | if (block_is_full(rrpc, rblk)) | |
539 | goto out; | |
540 | ||
541 | addr = block_to_addr(rrpc, rblk) + rblk->next_page; | |
542 | ||
543 | rblk->next_page++; | |
544 | out: | |
545 | spin_unlock(&rblk->lock); | |
546 | return addr; | |
547 | } | |
548 | ||
855cdd2c MB |
549 | /* Map logical address to a physical page. The mapping implements a round robin |
550 | * approach and allocates a page from the next lun available. | |
ae1519ec | 551 | * |
855cdd2c MB |
552 | * Returns rrpc_addr with the physical address and block. Returns NULL if no |
553 | * blocks in the next rlun are available. | |
ae1519ec MB |
554 | */ |
555 | static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr, | |
556 | int is_gc) | |
557 | { | |
558 | struct rrpc_lun *rlun; | |
855cdd2c | 559 | struct rrpc_block *rblk, **cur_rblk; |
ae1519ec | 560 | struct nvm_lun *lun; |
b7ceb7d5 | 561 | u64 paddr; |
855cdd2c | 562 | int gc_force = 0; |
ae1519ec MB |
563 | |
564 | rlun = rrpc_get_lun_rr(rrpc, is_gc); | |
565 | lun = rlun->parent; | |
566 | ||
567 | if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4) | |
568 | return NULL; | |
569 | ||
855cdd2c MB |
570 | /* |
571 | * page allocation steps: | |
572 | * 1. Try to allocate new page from current rblk | |
573 | * 2a. If succeed, proceed to map it in and return | |
574 | * 2b. If fail, first try to allocate a new block from media manger, | |
575 | * and then retry step 1. Retry until the normal block pool is | |
576 | * exhausted. | |
577 | * 3. If exhausted, and garbage collector is requesting the block, | |
578 | * go to the reserved block and retry step 1. | |
579 | * In the case that this fails as well, or it is not GC | |
580 | * requesting, report not able to retrieve a block and let the | |
581 | * caller handle further processing. | |
582 | */ | |
ae1519ec | 583 | |
855cdd2c MB |
584 | spin_lock(&rlun->lock); |
585 | cur_rblk = &rlun->cur; | |
ae1519ec MB |
586 | rblk = rlun->cur; |
587 | retry: | |
588 | paddr = rrpc_alloc_addr(rrpc, rblk); | |
589 | ||
855cdd2c MB |
590 | if (paddr != ADDR_EMPTY) |
591 | goto done; | |
ae1519ec | 592 | |
855cdd2c MB |
593 | if (!list_empty(&rlun->wblk_list)) { |
594 | new_blk: | |
595 | rblk = list_first_entry(&rlun->wblk_list, struct rrpc_block, | |
596 | prio); | |
597 | rrpc_set_lun_cur(rlun, rblk, cur_rblk); | |
598 | list_del(&rblk->prio); | |
599 | goto retry; | |
600 | } | |
601 | spin_unlock(&rlun->lock); | |
602 | ||
603 | rblk = rrpc_get_blk(rrpc, rlun, gc_force); | |
604 | if (rblk) { | |
605 | spin_lock(&rlun->lock); | |
606 | list_add_tail(&rblk->prio, &rlun->wblk_list); | |
607 | /* | |
608 | * another thread might already have added a new block, | |
609 | * Therefore, make sure that one is used, instead of the | |
610 | * one just added. | |
611 | */ | |
612 | goto new_blk; | |
613 | } | |
614 | ||
615 | if (unlikely(is_gc) && !gc_force) { | |
616 | /* retry from emergency gc block */ | |
617 | cur_rblk = &rlun->gc_cur; | |
618 | rblk = rlun->gc_cur; | |
619 | gc_force = 1; | |
620 | spin_lock(&rlun->lock); | |
621 | goto retry; | |
ae1519ec MB |
622 | } |
623 | ||
855cdd2c MB |
624 | pr_err("rrpc: failed to allocate new block\n"); |
625 | return NULL; | |
626 | done: | |
ae1519ec MB |
627 | spin_unlock(&rlun->lock); |
628 | return rrpc_update_map(rrpc, laddr, rblk, paddr); | |
ae1519ec MB |
629 | } |
630 | ||
631 | static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk) | |
632 | { | |
633 | struct rrpc_block_gc *gcb; | |
634 | ||
635 | gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC); | |
636 | if (!gcb) { | |
637 | pr_err("rrpc: unable to queue block for gc."); | |
638 | return; | |
639 | } | |
640 | ||
641 | gcb->rrpc = rrpc; | |
642 | gcb->rblk = rblk; | |
643 | ||
644 | INIT_WORK(&gcb->ws_gc, rrpc_gc_queue); | |
645 | queue_work(rrpc->kgc_wq, &gcb->ws_gc); | |
646 | } | |
647 | ||
a24ba464 JG |
648 | static void __rrpc_mark_bad_block(struct nvm_dev *dev, struct ppa_addr *ppa) |
649 | { | |
650 | nvm_mark_blk(dev, *ppa, NVM_BLK_ST_BAD); | |
651 | nvm_set_bb_tbl(dev, ppa, 1, NVM_BLK_T_GRWN_BAD); | |
652 | } | |
653 | ||
654 | static void rrpc_mark_bad_block(struct rrpc *rrpc, struct nvm_rq *rqd) | |
655 | { | |
656 | struct nvm_dev *dev = rrpc->dev; | |
657 | void *comp_bits = &rqd->ppa_status; | |
658 | struct ppa_addr ppa, prev_ppa; | |
659 | int nr_ppas = rqd->nr_ppas; | |
660 | int bit; | |
661 | ||
662 | if (rqd->nr_ppas == 1) | |
663 | __rrpc_mark_bad_block(dev, &rqd->ppa_addr); | |
664 | ||
665 | ppa_set_empty(&prev_ppa); | |
666 | bit = -1; | |
667 | while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) { | |
668 | ppa = rqd->ppa_list[bit]; | |
669 | if (ppa_cmp_blk(ppa, prev_ppa)) | |
670 | continue; | |
671 | ||
672 | __rrpc_mark_bad_block(dev, &ppa); | |
673 | } | |
674 | } | |
675 | ||
ae1519ec MB |
676 | static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd, |
677 | sector_t laddr, uint8_t npages) | |
678 | { | |
679 | struct rrpc_addr *p; | |
680 | struct rrpc_block *rblk; | |
681 | struct nvm_lun *lun; | |
682 | int cmnt_size, i; | |
683 | ||
684 | for (i = 0; i < npages; i++) { | |
685 | p = &rrpc->trans_map[laddr + i]; | |
686 | rblk = p->rblk; | |
687 | lun = rblk->parent->lun; | |
688 | ||
689 | cmnt_size = atomic_inc_return(&rblk->data_cmnt_size); | |
afb18e0e | 690 | if (unlikely(cmnt_size == rrpc->dev->sec_per_blk)) |
ae1519ec MB |
691 | rrpc_run_gc(rrpc, rblk); |
692 | } | |
693 | } | |
694 | ||
72d256ec | 695 | static void rrpc_end_io(struct nvm_rq *rqd) |
ae1519ec MB |
696 | { |
697 | struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance); | |
698 | struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); | |
6d5be959 | 699 | uint8_t npages = rqd->nr_ppas; |
ae1519ec MB |
700 | sector_t laddr = rrpc_get_laddr(rqd->bio) - npages; |
701 | ||
a24ba464 JG |
702 | if (bio_data_dir(rqd->bio) == WRITE) { |
703 | if (rqd->error == NVM_RSP_ERR_FAILWRITE) | |
704 | rrpc_mark_bad_block(rrpc, rqd); | |
705 | ||
ae1519ec | 706 | rrpc_end_io_write(rrpc, rrqd, laddr, npages); |
a24ba464 | 707 | } |
ae1519ec | 708 | |
3cd485b1 WT |
709 | bio_put(rqd->bio); |
710 | ||
ae1519ec | 711 | if (rrqd->flags & NVM_IOTYPE_GC) |
91276162 | 712 | return; |
ae1519ec MB |
713 | |
714 | rrpc_unlock_rq(rrpc, rqd); | |
ae1519ec MB |
715 | |
716 | if (npages > 1) | |
717 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | |
ae1519ec MB |
718 | |
719 | mempool_free(rqd, rrpc->rq_pool); | |
ae1519ec MB |
720 | } |
721 | ||
722 | static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, | |
723 | struct nvm_rq *rqd, unsigned long flags, int npages) | |
724 | { | |
725 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | |
726 | struct rrpc_addr *gp; | |
727 | sector_t laddr = rrpc_get_laddr(bio); | |
728 | int is_gc = flags & NVM_IOTYPE_GC; | |
729 | int i; | |
730 | ||
731 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { | |
732 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | |
733 | return NVM_IO_REQUEUE; | |
734 | } | |
735 | ||
736 | for (i = 0; i < npages; i++) { | |
737 | /* We assume that mapping occurs at 4KB granularity */ | |
4ece44af | 738 | BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects)); |
ae1519ec MB |
739 | gp = &rrpc->trans_map[laddr + i]; |
740 | ||
741 | if (gp->rblk) { | |
742 | rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, | |
743 | gp->addr); | |
744 | } else { | |
745 | BUG_ON(is_gc); | |
746 | rrpc_unlock_laddr(rrpc, r); | |
747 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, | |
748 | rqd->dma_ppa_list); | |
749 | return NVM_IO_DONE; | |
750 | } | |
751 | } | |
752 | ||
753 | rqd->opcode = NVM_OP_HBREAD; | |
754 | ||
755 | return NVM_IO_OK; | |
756 | } | |
757 | ||
758 | static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, | |
759 | unsigned long flags) | |
760 | { | |
ae1519ec MB |
761 | int is_gc = flags & NVM_IOTYPE_GC; |
762 | sector_t laddr = rrpc_get_laddr(bio); | |
763 | struct rrpc_addr *gp; | |
764 | ||
765 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) | |
766 | return NVM_IO_REQUEUE; | |
767 | ||
4ece44af | 768 | BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects)); |
ae1519ec MB |
769 | gp = &rrpc->trans_map[laddr]; |
770 | ||
771 | if (gp->rblk) { | |
772 | rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr); | |
773 | } else { | |
774 | BUG_ON(is_gc); | |
775 | rrpc_unlock_rq(rrpc, rqd); | |
776 | return NVM_IO_DONE; | |
777 | } | |
778 | ||
779 | rqd->opcode = NVM_OP_HBREAD; | |
ae1519ec MB |
780 | |
781 | return NVM_IO_OK; | |
782 | } | |
783 | ||
784 | static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio, | |
785 | struct nvm_rq *rqd, unsigned long flags, int npages) | |
786 | { | |
787 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | |
788 | struct rrpc_addr *p; | |
789 | sector_t laddr = rrpc_get_laddr(bio); | |
790 | int is_gc = flags & NVM_IOTYPE_GC; | |
791 | int i; | |
792 | ||
793 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { | |
794 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | |
795 | return NVM_IO_REQUEUE; | |
796 | } | |
797 | ||
798 | for (i = 0; i < npages; i++) { | |
799 | /* We assume that mapping occurs at 4KB granularity */ | |
800 | p = rrpc_map_page(rrpc, laddr + i, is_gc); | |
801 | if (!p) { | |
802 | BUG_ON(is_gc); | |
803 | rrpc_unlock_laddr(rrpc, r); | |
804 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, | |
805 | rqd->dma_ppa_list); | |
806 | rrpc_gc_kick(rrpc); | |
807 | return NVM_IO_REQUEUE; | |
808 | } | |
809 | ||
810 | rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, | |
811 | p->addr); | |
812 | } | |
813 | ||
814 | rqd->opcode = NVM_OP_HBWRITE; | |
815 | ||
816 | return NVM_IO_OK; | |
817 | } | |
818 | ||
819 | static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, | |
820 | struct nvm_rq *rqd, unsigned long flags) | |
821 | { | |
ae1519ec MB |
822 | struct rrpc_addr *p; |
823 | int is_gc = flags & NVM_IOTYPE_GC; | |
824 | sector_t laddr = rrpc_get_laddr(bio); | |
825 | ||
826 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) | |
827 | return NVM_IO_REQUEUE; | |
828 | ||
829 | p = rrpc_map_page(rrpc, laddr, is_gc); | |
830 | if (!p) { | |
831 | BUG_ON(is_gc); | |
832 | rrpc_unlock_rq(rrpc, rqd); | |
833 | rrpc_gc_kick(rrpc); | |
834 | return NVM_IO_REQUEUE; | |
835 | } | |
836 | ||
837 | rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr); | |
838 | rqd->opcode = NVM_OP_HBWRITE; | |
ae1519ec MB |
839 | |
840 | return NVM_IO_OK; | |
841 | } | |
842 | ||
843 | static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio, | |
844 | struct nvm_rq *rqd, unsigned long flags, uint8_t npages) | |
845 | { | |
846 | if (npages > 1) { | |
847 | rqd->ppa_list = nvm_dev_dma_alloc(rrpc->dev, GFP_KERNEL, | |
848 | &rqd->dma_ppa_list); | |
849 | if (!rqd->ppa_list) { | |
850 | pr_err("rrpc: not able to allocate ppa list\n"); | |
851 | return NVM_IO_ERR; | |
852 | } | |
853 | ||
70246286 | 854 | if (bio_op(bio) == REQ_OP_WRITE) |
ae1519ec MB |
855 | return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags, |
856 | npages); | |
857 | ||
858 | return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages); | |
859 | } | |
860 | ||
70246286 | 861 | if (bio_op(bio) == REQ_OP_WRITE) |
ae1519ec MB |
862 | return rrpc_write_rq(rrpc, bio, rqd, flags); |
863 | ||
864 | return rrpc_read_rq(rrpc, bio, rqd, flags); | |
865 | } | |
866 | ||
867 | static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, | |
868 | struct nvm_rq *rqd, unsigned long flags) | |
869 | { | |
870 | int err; | |
871 | struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd); | |
872 | uint8_t nr_pages = rrpc_get_pages(bio); | |
873 | int bio_size = bio_sectors(bio) << 9; | |
874 | ||
875 | if (bio_size < rrpc->dev->sec_size) | |
876 | return NVM_IO_ERR; | |
877 | else if (bio_size > rrpc->dev->max_rq_size) | |
878 | return NVM_IO_ERR; | |
879 | ||
880 | err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages); | |
881 | if (err) | |
882 | return err; | |
883 | ||
884 | bio_get(bio); | |
885 | rqd->bio = bio; | |
886 | rqd->ins = &rrpc->instance; | |
6d5be959 | 887 | rqd->nr_ppas = nr_pages; |
ae1519ec MB |
888 | rrq->flags = flags; |
889 | ||
890 | err = nvm_submit_io(rrpc->dev, rqd); | |
891 | if (err) { | |
892 | pr_err("rrpc: I/O submission failed: %d\n", err); | |
3cd485b1 | 893 | bio_put(bio); |
c27278bd WT |
894 | if (!(flags & NVM_IOTYPE_GC)) { |
895 | rrpc_unlock_rq(rrpc, rqd); | |
6d5be959 | 896 | if (rqd->nr_ppas > 1) |
c27278bd WT |
897 | nvm_dev_dma_free(rrpc->dev, |
898 | rqd->ppa_list, rqd->dma_ppa_list); | |
899 | } | |
ae1519ec MB |
900 | return NVM_IO_ERR; |
901 | } | |
902 | ||
903 | return NVM_IO_OK; | |
904 | } | |
905 | ||
dece1635 | 906 | static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio) |
ae1519ec MB |
907 | { |
908 | struct rrpc *rrpc = q->queuedata; | |
909 | struct nvm_rq *rqd; | |
910 | int err; | |
911 | ||
f0b01b6a JG |
912 | blk_queue_split(q, &bio, q->bio_split); |
913 | ||
95fe6c1a | 914 | if (bio_op(bio) == REQ_OP_DISCARD) { |
ae1519ec | 915 | rrpc_discard(rrpc, bio); |
dece1635 | 916 | return BLK_QC_T_NONE; |
ae1519ec MB |
917 | } |
918 | ||
919 | rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL); | |
920 | if (!rqd) { | |
921 | pr_err_ratelimited("rrpc: not able to queue bio."); | |
922 | bio_io_error(bio); | |
dece1635 | 923 | return BLK_QC_T_NONE; |
ae1519ec MB |
924 | } |
925 | memset(rqd, 0, sizeof(struct nvm_rq)); | |
926 | ||
927 | err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE); | |
928 | switch (err) { | |
929 | case NVM_IO_OK: | |
dece1635 | 930 | return BLK_QC_T_NONE; |
ae1519ec MB |
931 | case NVM_IO_ERR: |
932 | bio_io_error(bio); | |
933 | break; | |
934 | case NVM_IO_DONE: | |
935 | bio_endio(bio); | |
936 | break; | |
937 | case NVM_IO_REQUEUE: | |
938 | spin_lock(&rrpc->bio_lock); | |
939 | bio_list_add(&rrpc->requeue_bios, bio); | |
940 | spin_unlock(&rrpc->bio_lock); | |
941 | queue_work(rrpc->kgc_wq, &rrpc->ws_requeue); | |
942 | break; | |
943 | } | |
944 | ||
945 | mempool_free(rqd, rrpc->rq_pool); | |
dece1635 | 946 | return BLK_QC_T_NONE; |
ae1519ec MB |
947 | } |
948 | ||
949 | static void rrpc_requeue(struct work_struct *work) | |
950 | { | |
951 | struct rrpc *rrpc = container_of(work, struct rrpc, ws_requeue); | |
952 | struct bio_list bios; | |
953 | struct bio *bio; | |
954 | ||
955 | bio_list_init(&bios); | |
956 | ||
957 | spin_lock(&rrpc->bio_lock); | |
958 | bio_list_merge(&bios, &rrpc->requeue_bios); | |
959 | bio_list_init(&rrpc->requeue_bios); | |
960 | spin_unlock(&rrpc->bio_lock); | |
961 | ||
962 | while ((bio = bio_list_pop(&bios))) | |
963 | rrpc_make_rq(rrpc->disk->queue, bio); | |
964 | } | |
965 | ||
966 | static void rrpc_gc_free(struct rrpc *rrpc) | |
967 | { | |
ae1519ec MB |
968 | if (rrpc->krqd_wq) |
969 | destroy_workqueue(rrpc->krqd_wq); | |
970 | ||
971 | if (rrpc->kgc_wq) | |
972 | destroy_workqueue(rrpc->kgc_wq); | |
ae1519ec MB |
973 | } |
974 | ||
975 | static int rrpc_gc_init(struct rrpc *rrpc) | |
976 | { | |
977 | rrpc->krqd_wq = alloc_workqueue("rrpc-lun", WQ_MEM_RECLAIM|WQ_UNBOUND, | |
978 | rrpc->nr_luns); | |
979 | if (!rrpc->krqd_wq) | |
980 | return -ENOMEM; | |
981 | ||
982 | rrpc->kgc_wq = alloc_workqueue("rrpc-bg", WQ_MEM_RECLAIM, 1); | |
983 | if (!rrpc->kgc_wq) | |
984 | return -ENOMEM; | |
985 | ||
986 | setup_timer(&rrpc->gc_timer, rrpc_gc_timer, (unsigned long)rrpc); | |
987 | ||
988 | return 0; | |
989 | } | |
990 | ||
991 | static void rrpc_map_free(struct rrpc *rrpc) | |
992 | { | |
993 | vfree(rrpc->rev_trans_map); | |
994 | vfree(rrpc->trans_map); | |
995 | } | |
996 | ||
997 | static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private) | |
998 | { | |
999 | struct rrpc *rrpc = (struct rrpc *)private; | |
1000 | struct nvm_dev *dev = rrpc->dev; | |
1001 | struct rrpc_addr *addr = rrpc->trans_map + slba; | |
1002 | struct rrpc_rev_addr *raddr = rrpc->rev_trans_map; | |
ae1519ec MB |
1003 | u64 elba = slba + nlb; |
1004 | u64 i; | |
1005 | ||
4ece44af | 1006 | if (unlikely(elba > dev->total_secs)) { |
ae1519ec MB |
1007 | pr_err("nvm: L2P data from device is out of bounds!\n"); |
1008 | return -EINVAL; | |
1009 | } | |
1010 | ||
1011 | for (i = 0; i < nlb; i++) { | |
1012 | u64 pba = le64_to_cpu(entries[i]); | |
afb18e0e | 1013 | unsigned int mod; |
ae1519ec MB |
1014 | /* LNVM treats address-spaces as silos, LBA and PBA are |
1015 | * equally large and zero-indexed. | |
1016 | */ | |
4ece44af | 1017 | if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) { |
ae1519ec MB |
1018 | pr_err("nvm: L2P data entry is out of bounds!\n"); |
1019 | return -EINVAL; | |
1020 | } | |
1021 | ||
1022 | /* Address zero is a special one. The first page on a disk is | |
1023 | * protected. As it often holds internal device boot | |
1024 | * information. | |
1025 | */ | |
1026 | if (!pba) | |
1027 | continue; | |
1028 | ||
afb18e0e JG |
1029 | div_u64_rem(pba, rrpc->nr_sects, &mod); |
1030 | ||
ae1519ec | 1031 | addr[i].addr = pba; |
afb18e0e | 1032 | raddr[mod].addr = slba + i; |
ae1519ec MB |
1033 | } |
1034 | ||
1035 | return 0; | |
1036 | } | |
1037 | ||
1038 | static int rrpc_map_init(struct rrpc *rrpc) | |
1039 | { | |
1040 | struct nvm_dev *dev = rrpc->dev; | |
1041 | sector_t i; | |
1042 | int ret; | |
1043 | ||
4ece44af | 1044 | rrpc->trans_map = vzalloc(sizeof(struct rrpc_addr) * rrpc->nr_sects); |
ae1519ec MB |
1045 | if (!rrpc->trans_map) |
1046 | return -ENOMEM; | |
1047 | ||
1048 | rrpc->rev_trans_map = vmalloc(sizeof(struct rrpc_rev_addr) | |
4ece44af | 1049 | * rrpc->nr_sects); |
ae1519ec MB |
1050 | if (!rrpc->rev_trans_map) |
1051 | return -ENOMEM; | |
1052 | ||
4ece44af | 1053 | for (i = 0; i < rrpc->nr_sects; i++) { |
ae1519ec MB |
1054 | struct rrpc_addr *p = &rrpc->trans_map[i]; |
1055 | struct rrpc_rev_addr *r = &rrpc->rev_trans_map[i]; | |
1056 | ||
1057 | p->addr = ADDR_EMPTY; | |
1058 | r->addr = ADDR_EMPTY; | |
1059 | } | |
1060 | ||
1061 | if (!dev->ops->get_l2p_tbl) | |
1062 | return 0; | |
1063 | ||
1064 | /* Bring up the mapping table from device */ | |
909049a7 WT |
1065 | ret = dev->ops->get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects, |
1066 | rrpc_l2p_update, rrpc); | |
ae1519ec MB |
1067 | if (ret) { |
1068 | pr_err("nvm: rrpc: could not read L2P table.\n"); | |
1069 | return -EINVAL; | |
1070 | } | |
1071 | ||
1072 | return 0; | |
1073 | } | |
1074 | ||
ae1519ec MB |
1075 | /* Minimum pages needed within a lun */ |
1076 | #define PAGE_POOL_SIZE 16 | |
1077 | #define ADDR_POOL_SIZE 64 | |
1078 | ||
1079 | static int rrpc_core_init(struct rrpc *rrpc) | |
1080 | { | |
1081 | down_write(&rrpc_lock); | |
1082 | if (!rrpc_gcb_cache) { | |
1083 | rrpc_gcb_cache = kmem_cache_create("rrpc_gcb", | |
1084 | sizeof(struct rrpc_block_gc), 0, 0, NULL); | |
1085 | if (!rrpc_gcb_cache) { | |
1086 | up_write(&rrpc_lock); | |
1087 | return -ENOMEM; | |
1088 | } | |
1089 | ||
1090 | rrpc_rq_cache = kmem_cache_create("rrpc_rq", | |
1091 | sizeof(struct nvm_rq) + sizeof(struct rrpc_rq), | |
1092 | 0, 0, NULL); | |
1093 | if (!rrpc_rq_cache) { | |
1094 | kmem_cache_destroy(rrpc_gcb_cache); | |
1095 | up_write(&rrpc_lock); | |
1096 | return -ENOMEM; | |
1097 | } | |
1098 | } | |
1099 | up_write(&rrpc_lock); | |
1100 | ||
1101 | rrpc->page_pool = mempool_create_page_pool(PAGE_POOL_SIZE, 0); | |
1102 | if (!rrpc->page_pool) | |
1103 | return -ENOMEM; | |
1104 | ||
1105 | rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->nr_luns, | |
1106 | rrpc_gcb_cache); | |
1107 | if (!rrpc->gcb_pool) | |
1108 | return -ENOMEM; | |
1109 | ||
1110 | rrpc->rq_pool = mempool_create_slab_pool(64, rrpc_rq_cache); | |
1111 | if (!rrpc->rq_pool) | |
1112 | return -ENOMEM; | |
1113 | ||
1114 | spin_lock_init(&rrpc->inflights.lock); | |
1115 | INIT_LIST_HEAD(&rrpc->inflights.reqs); | |
1116 | ||
1117 | return 0; | |
1118 | } | |
1119 | ||
1120 | static void rrpc_core_free(struct rrpc *rrpc) | |
1121 | { | |
1122 | mempool_destroy(rrpc->page_pool); | |
1123 | mempool_destroy(rrpc->gcb_pool); | |
1124 | mempool_destroy(rrpc->rq_pool); | |
1125 | } | |
1126 | ||
1127 | static void rrpc_luns_free(struct rrpc *rrpc) | |
1128 | { | |
da1e2849 WT |
1129 | struct nvm_lun *lun; |
1130 | struct rrpc_lun *rlun; | |
1131 | int i; | |
1132 | ||
1133 | if (!rrpc->luns) | |
1134 | return; | |
1135 | ||
1136 | for (i = 0; i < rrpc->nr_luns; i++) { | |
1137 | rlun = &rrpc->luns[i]; | |
1138 | lun = rlun->parent; | |
1139 | if (!lun) | |
1140 | break; | |
da1e2849 WT |
1141 | vfree(rlun->blocks); |
1142 | } | |
1143 | ||
ae1519ec MB |
1144 | kfree(rrpc->luns); |
1145 | } | |
1146 | ||
1147 | static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) | |
1148 | { | |
1149 | struct nvm_dev *dev = rrpc->dev; | |
1150 | struct rrpc_lun *rlun; | |
da1e2849 | 1151 | int i, j, ret = -EINVAL; |
ae1519ec | 1152 | |
afb18e0e | 1153 | if (dev->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) { |
4b79beb4 WT |
1154 | pr_err("rrpc: number of pages per block too high."); |
1155 | return -EINVAL; | |
1156 | } | |
1157 | ||
ae1519ec MB |
1158 | spin_lock_init(&rrpc->rev_lock); |
1159 | ||
1160 | rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun), | |
1161 | GFP_KERNEL); | |
1162 | if (!rrpc->luns) | |
1163 | return -ENOMEM; | |
1164 | ||
1165 | /* 1:1 mapping */ | |
1166 | for (i = 0; i < rrpc->nr_luns; i++) { | |
da1e2849 WT |
1167 | int lunid = lun_begin + i; |
1168 | struct nvm_lun *lun; | |
ff0e498b | 1169 | |
da1e2849 WT |
1170 | lun = dev->mt->get_lun(dev, lunid); |
1171 | if (!lun) | |
1172 | goto err; | |
ae1519ec | 1173 | |
da1e2849 WT |
1174 | rlun = &rrpc->luns[i]; |
1175 | rlun->parent = lun; | |
ae1519ec MB |
1176 | rlun->blocks = vzalloc(sizeof(struct rrpc_block) * |
1177 | rrpc->dev->blks_per_lun); | |
da1e2849 WT |
1178 | if (!rlun->blocks) { |
1179 | ret = -ENOMEM; | |
ae1519ec | 1180 | goto err; |
da1e2849 | 1181 | } |
ae1519ec MB |
1182 | |
1183 | for (j = 0; j < rrpc->dev->blks_per_lun; j++) { | |
1184 | struct rrpc_block *rblk = &rlun->blocks[j]; | |
1185 | struct nvm_block *blk = &lun->blocks[j]; | |
1186 | ||
1187 | rblk->parent = blk; | |
d7a64d27 | 1188 | rblk->rlun = rlun; |
ae1519ec MB |
1189 | INIT_LIST_HEAD(&rblk->prio); |
1190 | spin_lock_init(&rblk->lock); | |
1191 | } | |
da1e2849 WT |
1192 | |
1193 | rlun->rrpc = rrpc; | |
1194 | INIT_LIST_HEAD(&rlun->prio_list); | |
855cdd2c | 1195 | INIT_LIST_HEAD(&rlun->wblk_list); |
da1e2849 WT |
1196 | |
1197 | INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); | |
1198 | spin_lock_init(&rlun->lock); | |
ae1519ec MB |
1199 | } |
1200 | ||
1201 | return 0; | |
1202 | err: | |
da1e2849 | 1203 | return ret; |
ae1519ec MB |
1204 | } |
1205 | ||
4c9dacb8 WT |
1206 | /* returns 0 on success and stores the beginning address in *begin */ |
1207 | static int rrpc_area_init(struct rrpc *rrpc, sector_t *begin) | |
1208 | { | |
1209 | struct nvm_dev *dev = rrpc->dev; | |
1210 | struct nvmm_type *mt = dev->mt; | |
1211 | sector_t size = rrpc->nr_sects * dev->sec_size; | |
909049a7 | 1212 | int ret; |
4c9dacb8 WT |
1213 | |
1214 | size >>= 9; | |
1215 | ||
909049a7 WT |
1216 | ret = mt->get_area(dev, begin, size); |
1217 | if (!ret) | |
1218 | *begin >>= (ilog2(dev->sec_size) - 9); | |
1219 | ||
1220 | return ret; | |
4c9dacb8 WT |
1221 | } |
1222 | ||
1223 | static void rrpc_area_free(struct rrpc *rrpc) | |
1224 | { | |
1225 | struct nvm_dev *dev = rrpc->dev; | |
1226 | struct nvmm_type *mt = dev->mt; | |
909049a7 | 1227 | sector_t begin = rrpc->soffset << (ilog2(dev->sec_size) - 9); |
4c9dacb8 | 1228 | |
909049a7 | 1229 | mt->put_area(dev, begin); |
4c9dacb8 WT |
1230 | } |
1231 | ||
ae1519ec MB |
1232 | static void rrpc_free(struct rrpc *rrpc) |
1233 | { | |
1234 | rrpc_gc_free(rrpc); | |
1235 | rrpc_map_free(rrpc); | |
1236 | rrpc_core_free(rrpc); | |
1237 | rrpc_luns_free(rrpc); | |
4c9dacb8 | 1238 | rrpc_area_free(rrpc); |
ae1519ec MB |
1239 | |
1240 | kfree(rrpc); | |
1241 | } | |
1242 | ||
1243 | static void rrpc_exit(void *private) | |
1244 | { | |
1245 | struct rrpc *rrpc = private; | |
1246 | ||
1247 | del_timer(&rrpc->gc_timer); | |
1248 | ||
1249 | flush_workqueue(rrpc->krqd_wq); | |
1250 | flush_workqueue(rrpc->kgc_wq); | |
1251 | ||
1252 | rrpc_free(rrpc); | |
1253 | } | |
1254 | ||
1255 | static sector_t rrpc_capacity(void *private) | |
1256 | { | |
1257 | struct rrpc *rrpc = private; | |
1258 | struct nvm_dev *dev = rrpc->dev; | |
1259 | sector_t reserved, provisioned; | |
1260 | ||
1261 | /* cur, gc, and two emergency blocks for each lun */ | |
116f7d4a | 1262 | reserved = rrpc->nr_luns * dev->sec_per_blk * 4; |
4ece44af | 1263 | provisioned = rrpc->nr_sects - reserved; |
ae1519ec | 1264 | |
4ece44af | 1265 | if (reserved > rrpc->nr_sects) { |
ae1519ec MB |
1266 | pr_err("rrpc: not enough space available to expose storage.\n"); |
1267 | return 0; | |
1268 | } | |
1269 | ||
1270 | sector_div(provisioned, 10); | |
1271 | return provisioned * 9 * NR_PHY_IN_LOG; | |
1272 | } | |
1273 | ||
1274 | /* | |
1275 | * Looks up the logical address from reverse trans map and check if its valid by | |
1276 | * comparing the logical to physical address with the physical address. | |
1277 | * Returns 0 on free, otherwise 1 if in use | |
1278 | */ | |
1279 | static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk) | |
1280 | { | |
1281 | struct nvm_dev *dev = rrpc->dev; | |
1282 | int offset; | |
1283 | struct rrpc_addr *laddr; | |
afb18e0e | 1284 | u64 bpaddr, paddr, pladdr; |
ae1519ec | 1285 | |
afb18e0e JG |
1286 | bpaddr = block_to_rel_addr(rrpc, rblk); |
1287 | for (offset = 0; offset < dev->sec_per_blk; offset++) { | |
1288 | paddr = bpaddr + offset; | |
ae1519ec MB |
1289 | |
1290 | pladdr = rrpc->rev_trans_map[paddr].addr; | |
1291 | if (pladdr == ADDR_EMPTY) | |
1292 | continue; | |
1293 | ||
1294 | laddr = &rrpc->trans_map[pladdr]; | |
1295 | ||
1296 | if (paddr == laddr->addr) { | |
1297 | laddr->rblk = rblk; | |
1298 | } else { | |
1299 | set_bit(offset, rblk->invalid_pages); | |
1300 | rblk->nr_invalid_pages++; | |
1301 | } | |
1302 | } | |
1303 | } | |
1304 | ||
1305 | static int rrpc_blocks_init(struct rrpc *rrpc) | |
1306 | { | |
1307 | struct rrpc_lun *rlun; | |
1308 | struct rrpc_block *rblk; | |
1309 | int lun_iter, blk_iter; | |
1310 | ||
1311 | for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) { | |
1312 | rlun = &rrpc->luns[lun_iter]; | |
1313 | ||
1314 | for (blk_iter = 0; blk_iter < rrpc->dev->blks_per_lun; | |
1315 | blk_iter++) { | |
1316 | rblk = &rlun->blocks[blk_iter]; | |
1317 | rrpc_block_map_update(rrpc, rblk); | |
1318 | } | |
1319 | } | |
1320 | ||
1321 | return 0; | |
1322 | } | |
1323 | ||
1324 | static int rrpc_luns_configure(struct rrpc *rrpc) | |
1325 | { | |
1326 | struct rrpc_lun *rlun; | |
1327 | struct rrpc_block *rblk; | |
1328 | int i; | |
1329 | ||
1330 | for (i = 0; i < rrpc->nr_luns; i++) { | |
1331 | rlun = &rrpc->luns[i]; | |
1332 | ||
1333 | rblk = rrpc_get_blk(rrpc, rlun, 0); | |
1334 | if (!rblk) | |
d3d1a438 | 1335 | goto err; |
855cdd2c | 1336 | rrpc_set_lun_cur(rlun, rblk, &rlun->cur); |
ae1519ec MB |
1337 | |
1338 | /* Emergency gc block */ | |
1339 | rblk = rrpc_get_blk(rrpc, rlun, 1); | |
1340 | if (!rblk) | |
d3d1a438 | 1341 | goto err; |
855cdd2c | 1342 | rrpc_set_lun_cur(rlun, rblk, &rlun->gc_cur); |
ae1519ec MB |
1343 | } |
1344 | ||
1345 | return 0; | |
d3d1a438 WT |
1346 | err: |
1347 | rrpc_put_blks(rrpc); | |
1348 | return -EINVAL; | |
ae1519ec MB |
1349 | } |
1350 | ||
1351 | static struct nvm_tgt_type tt_rrpc; | |
1352 | ||
1353 | static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk, | |
1354 | int lun_begin, int lun_end) | |
1355 | { | |
1356 | struct request_queue *bqueue = dev->q; | |
1357 | struct request_queue *tqueue = tdisk->queue; | |
1358 | struct rrpc *rrpc; | |
4c9dacb8 | 1359 | sector_t soffset; |
ae1519ec MB |
1360 | int ret; |
1361 | ||
1362 | if (!(dev->identity.dom & NVM_RSP_L2P)) { | |
1363 | pr_err("nvm: rrpc: device does not support l2p (%x)\n", | |
1364 | dev->identity.dom); | |
1365 | return ERR_PTR(-EINVAL); | |
1366 | } | |
1367 | ||
1368 | rrpc = kzalloc(sizeof(struct rrpc), GFP_KERNEL); | |
1369 | if (!rrpc) | |
1370 | return ERR_PTR(-ENOMEM); | |
1371 | ||
1372 | rrpc->instance.tt = &tt_rrpc; | |
1373 | rrpc->dev = dev; | |
1374 | rrpc->disk = tdisk; | |
1375 | ||
1376 | bio_list_init(&rrpc->requeue_bios); | |
1377 | spin_lock_init(&rrpc->bio_lock); | |
1378 | INIT_WORK(&rrpc->ws_requeue, rrpc_requeue); | |
1379 | ||
1380 | rrpc->nr_luns = lun_end - lun_begin + 1; | |
66e3d07f | 1381 | rrpc->nr_sects = (unsigned long long)dev->sec_per_lun * rrpc->nr_luns; |
ae1519ec MB |
1382 | |
1383 | /* simple round-robin strategy */ | |
1384 | atomic_set(&rrpc->next_lun, -1); | |
1385 | ||
4c9dacb8 WT |
1386 | ret = rrpc_area_init(rrpc, &soffset); |
1387 | if (ret < 0) { | |
1388 | pr_err("nvm: rrpc: could not initialize area\n"); | |
1389 | return ERR_PTR(ret); | |
1390 | } | |
1391 | rrpc->soffset = soffset; | |
1392 | ||
ae1519ec MB |
1393 | ret = rrpc_luns_init(rrpc, lun_begin, lun_end); |
1394 | if (ret) { | |
1395 | pr_err("nvm: rrpc: could not initialize luns\n"); | |
1396 | goto err; | |
1397 | } | |
1398 | ||
1399 | rrpc->poffset = dev->sec_per_lun * lun_begin; | |
ae1519ec MB |
1400 | |
1401 | ret = rrpc_core_init(rrpc); | |
1402 | if (ret) { | |
1403 | pr_err("nvm: rrpc: could not initialize core\n"); | |
1404 | goto err; | |
1405 | } | |
1406 | ||
1407 | ret = rrpc_map_init(rrpc); | |
1408 | if (ret) { | |
1409 | pr_err("nvm: rrpc: could not initialize maps\n"); | |
1410 | goto err; | |
1411 | } | |
1412 | ||
1413 | ret = rrpc_blocks_init(rrpc); | |
1414 | if (ret) { | |
1415 | pr_err("nvm: rrpc: could not initialize state for blocks\n"); | |
1416 | goto err; | |
1417 | } | |
1418 | ||
1419 | ret = rrpc_luns_configure(rrpc); | |
1420 | if (ret) { | |
1421 | pr_err("nvm: rrpc: not enough blocks available in LUNs.\n"); | |
1422 | goto err; | |
1423 | } | |
1424 | ||
1425 | ret = rrpc_gc_init(rrpc); | |
1426 | if (ret) { | |
1427 | pr_err("nvm: rrpc: could not initialize gc\n"); | |
1428 | goto err; | |
1429 | } | |
1430 | ||
1431 | /* inherit the size from the underlying device */ | |
1432 | blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue)); | |
1433 | blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue)); | |
1434 | ||
1435 | pr_info("nvm: rrpc initialized with %u luns and %llu pages.\n", | |
4ece44af | 1436 | rrpc->nr_luns, (unsigned long long)rrpc->nr_sects); |
ae1519ec MB |
1437 | |
1438 | mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10)); | |
1439 | ||
1440 | return rrpc; | |
1441 | err: | |
1442 | rrpc_free(rrpc); | |
1443 | return ERR_PTR(ret); | |
1444 | } | |
1445 | ||
1446 | /* round robin, page-based FTL, and cost-based GC */ | |
1447 | static struct nvm_tgt_type tt_rrpc = { | |
1448 | .name = "rrpc", | |
1449 | .version = {1, 0, 0}, | |
1450 | ||
1451 | .make_rq = rrpc_make_rq, | |
1452 | .capacity = rrpc_capacity, | |
1453 | .end_io = rrpc_end_io, | |
1454 | ||
1455 | .init = rrpc_init, | |
1456 | .exit = rrpc_exit, | |
1457 | }; | |
1458 | ||
1459 | static int __init rrpc_module_init(void) | |
1460 | { | |
6063fe39 | 1461 | return nvm_register_tgt_type(&tt_rrpc); |
ae1519ec MB |
1462 | } |
1463 | ||
1464 | static void rrpc_module_exit(void) | |
1465 | { | |
6063fe39 | 1466 | nvm_unregister_tgt_type(&tt_rrpc); |
ae1519ec MB |
1467 | } |
1468 | ||
1469 | module_init(rrpc_module_init); | |
1470 | module_exit(rrpc_module_exit); | |
1471 | MODULE_LICENSE("GPL v2"); | |
1472 | MODULE_DESCRIPTION("Block-Device Target for Open-Channel SSDs"); |