]>
Commit | Line | Data |
---|---|---|
a4bd217b JG |
1 | /* |
2 | * Copyright (C) 2016 CNEX Labs | |
3 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> | |
4 | * | |
5 | * Based upon the circular ringbuffer. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License version | |
9 | * 2 as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, but | |
12 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * General Public License for more details. | |
15 | * | |
16 | * pblk-rb.c - pblk's write buffer | |
17 | */ | |
18 | ||
19 | #include <linux/circ_buf.h> | |
20 | ||
21 | #include "pblk.h" | |
22 | ||
23 | static DECLARE_RWSEM(pblk_rb_lock); | |
24 | ||
25 | void pblk_rb_data_free(struct pblk_rb *rb) | |
26 | { | |
27 | struct pblk_rb_pages *p, *t; | |
28 | ||
29 | down_write(&pblk_rb_lock); | |
30 | list_for_each_entry_safe(p, t, &rb->pages, list) { | |
31 | free_pages((unsigned long)page_address(p->pages), p->order); | |
32 | list_del(&p->list); | |
33 | kfree(p); | |
34 | } | |
35 | up_write(&pblk_rb_lock); | |
36 | } | |
37 | ||
38 | /* | |
39 | * Initialize ring buffer. The data and metadata buffers must be previously | |
40 | * allocated and their size must be a power of two | |
41 | * (Documentation/circular-buffers.txt) | |
42 | */ | |
43 | int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base, | |
44 | unsigned int power_size, unsigned int power_seg_sz) | |
45 | { | |
46 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
47 | unsigned int init_entry = 0; | |
48 | unsigned int alloc_order = power_size; | |
49 | unsigned int max_order = MAX_ORDER - 1; | |
50 | unsigned int order, iter; | |
51 | ||
52 | down_write(&pblk_rb_lock); | |
53 | rb->entries = rb_entry_base; | |
54 | rb->seg_size = (1 << power_seg_sz); | |
55 | rb->nr_entries = (1 << power_size); | |
56 | rb->mem = rb->subm = rb->sync = rb->l2p_update = 0; | |
57 | rb->sync_point = EMPTY_ENTRY; | |
58 | ||
59 | spin_lock_init(&rb->w_lock); | |
60 | spin_lock_init(&rb->s_lock); | |
61 | ||
62 | INIT_LIST_HEAD(&rb->pages); | |
63 | ||
64 | if (alloc_order >= max_order) { | |
65 | order = max_order; | |
66 | iter = (1 << (alloc_order - max_order)); | |
67 | } else { | |
68 | order = alloc_order; | |
69 | iter = 1; | |
70 | } | |
71 | ||
72 | do { | |
73 | struct pblk_rb_entry *entry; | |
74 | struct pblk_rb_pages *page_set; | |
75 | void *kaddr; | |
76 | unsigned long set_size; | |
77 | int i; | |
78 | ||
79 | page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL); | |
80 | if (!page_set) { | |
81 | up_write(&pblk_rb_lock); | |
82 | return -ENOMEM; | |
83 | } | |
84 | ||
85 | page_set->order = order; | |
86 | page_set->pages = alloc_pages(GFP_KERNEL, order); | |
87 | if (!page_set->pages) { | |
88 | kfree(page_set); | |
89 | pblk_rb_data_free(rb); | |
90 | up_write(&pblk_rb_lock); | |
91 | return -ENOMEM; | |
92 | } | |
93 | kaddr = page_address(page_set->pages); | |
94 | ||
95 | entry = &rb->entries[init_entry]; | |
96 | entry->data = kaddr; | |
97 | entry->cacheline = pblk_cacheline_to_addr(init_entry++); | |
98 | entry->w_ctx.flags = PBLK_WRITABLE_ENTRY; | |
99 | ||
100 | set_size = (1 << order); | |
101 | for (i = 1; i < set_size; i++) { | |
102 | entry = &rb->entries[init_entry]; | |
103 | entry->cacheline = pblk_cacheline_to_addr(init_entry++); | |
104 | entry->data = kaddr + (i * rb->seg_size); | |
105 | entry->w_ctx.flags = PBLK_WRITABLE_ENTRY; | |
106 | bio_list_init(&entry->w_ctx.bios); | |
107 | } | |
108 | ||
109 | list_add_tail(&page_set->list, &rb->pages); | |
110 | iter--; | |
111 | } while (iter > 0); | |
112 | up_write(&pblk_rb_lock); | |
113 | ||
114 | #ifdef CONFIG_NVM_DEBUG | |
115 | atomic_set(&rb->inflight_sync_point, 0); | |
116 | #endif | |
117 | ||
118 | /* | |
119 | * Initialize rate-limiter, which controls access to the write buffer | |
120 | * but user and GC I/O | |
121 | */ | |
122 | pblk_rl_init(&pblk->rl, rb->nr_entries); | |
123 | ||
124 | return 0; | |
125 | } | |
126 | ||
127 | /* | |
128 | * pblk_rb_calculate_size -- calculate the size of the write buffer | |
129 | */ | |
130 | unsigned int pblk_rb_calculate_size(unsigned int nr_entries) | |
131 | { | |
132 | /* Alloc a write buffer that can at least fit 128 entries */ | |
133 | return (1 << max(get_count_order(nr_entries), 7)); | |
134 | } | |
135 | ||
136 | void *pblk_rb_entries_ref(struct pblk_rb *rb) | |
137 | { | |
138 | return rb->entries; | |
139 | } | |
140 | ||
141 | static void clean_wctx(struct pblk_w_ctx *w_ctx) | |
142 | { | |
143 | int flags; | |
144 | ||
a4bd217b | 145 | flags = READ_ONCE(w_ctx->flags); |
3a902b07 JG |
146 | WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY), |
147 | "pblk: overwriting unsubmitted data\n"); | |
a4bd217b JG |
148 | |
149 | /* Release flags on context. Protect from writes and reads */ | |
150 | smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY); | |
151 | pblk_ppa_set_empty(&w_ctx->ppa); | |
07698466 | 152 | w_ctx->lba = ADDR_EMPTY; |
a4bd217b JG |
153 | } |
154 | ||
155 | #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size) | |
156 | #define pblk_rb_ring_space(rb, head, tail, size) \ | |
157 | (CIRC_SPACE(head, tail, size)) | |
158 | ||
159 | /* | |
160 | * Buffer space is calculated with respect to the back pointer signaling | |
161 | * synchronized entries to the media. | |
162 | */ | |
163 | static unsigned int pblk_rb_space(struct pblk_rb *rb) | |
164 | { | |
165 | unsigned int mem = READ_ONCE(rb->mem); | |
166 | unsigned int sync = READ_ONCE(rb->sync); | |
167 | ||
168 | return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries); | |
169 | } | |
170 | ||
171 | /* | |
172 | * Buffer count is calculated with respect to the submission entry signaling the | |
173 | * entries that are available to send to the media | |
174 | */ | |
175 | unsigned int pblk_rb_read_count(struct pblk_rb *rb) | |
176 | { | |
177 | unsigned int mem = READ_ONCE(rb->mem); | |
178 | unsigned int subm = READ_ONCE(rb->subm); | |
179 | ||
180 | return pblk_rb_ring_count(mem, subm, rb->nr_entries); | |
181 | } | |
182 | ||
ee8d5c1a JG |
183 | unsigned int pblk_rb_sync_count(struct pblk_rb *rb) |
184 | { | |
185 | unsigned int mem = READ_ONCE(rb->mem); | |
186 | unsigned int sync = READ_ONCE(rb->sync); | |
187 | ||
188 | return pblk_rb_ring_count(mem, sync, rb->nr_entries); | |
189 | } | |
190 | ||
a4bd217b JG |
191 | unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries) |
192 | { | |
193 | unsigned int subm; | |
194 | ||
195 | subm = READ_ONCE(rb->subm); | |
196 | /* Commit read means updating submission pointer */ | |
197 | smp_store_release(&rb->subm, | |
198 | (subm + nr_entries) & (rb->nr_entries - 1)); | |
199 | ||
200 | return subm; | |
201 | } | |
202 | ||
05ed3447 | 203 | static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update) |
a4bd217b JG |
204 | { |
205 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
206 | struct pblk_line *line; | |
207 | struct pblk_rb_entry *entry; | |
208 | struct pblk_w_ctx *w_ctx; | |
b20ba1bc | 209 | unsigned int user_io = 0, gc_io = 0; |
a4bd217b | 210 | unsigned int i; |
b20ba1bc | 211 | int flags; |
a4bd217b JG |
212 | |
213 | for (i = 0; i < to_update; i++) { | |
05ed3447 | 214 | entry = &rb->entries[rb->l2p_update]; |
a4bd217b JG |
215 | w_ctx = &entry->w_ctx; |
216 | ||
b20ba1bc JG |
217 | flags = READ_ONCE(entry->w_ctx.flags); |
218 | if (flags & PBLK_IOTYPE_USER) | |
219 | user_io++; | |
220 | else if (flags & PBLK_IOTYPE_GC) | |
221 | gc_io++; | |
222 | else | |
223 | WARN(1, "pblk: unknown IO type\n"); | |
224 | ||
a4bd217b JG |
225 | pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa, |
226 | entry->cacheline); | |
227 | ||
228 | line = &pblk->lines[pblk_tgt_ppa_to_line(w_ctx->ppa)]; | |
229 | kref_put(&line->ref, pblk_line_put); | |
230 | clean_wctx(w_ctx); | |
05ed3447 | 231 | rb->l2p_update = (rb->l2p_update + 1) & (rb->nr_entries - 1); |
a4bd217b JG |
232 | } |
233 | ||
b20ba1bc JG |
234 | pblk_rl_out(&pblk->rl, user_io, gc_io); |
235 | ||
a4bd217b JG |
236 | return 0; |
237 | } | |
238 | ||
239 | /* | |
240 | * When we move the l2p_update pointer, we update the l2p table - lookups will | |
241 | * point to the physical address instead of to the cacheline in the write buffer | |
242 | * from this moment on. | |
243 | */ | |
244 | static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries, | |
245 | unsigned int mem, unsigned int sync) | |
246 | { | |
247 | unsigned int space, count; | |
248 | int ret = 0; | |
249 | ||
250 | lockdep_assert_held(&rb->w_lock); | |
251 | ||
252 | /* Update l2p only as buffer entries are being overwritten */ | |
253 | space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries); | |
254 | if (space > nr_entries) | |
255 | goto out; | |
256 | ||
257 | count = nr_entries - space; | |
258 | /* l2p_update used exclusively under rb->w_lock */ | |
05ed3447 | 259 | ret = __pblk_rb_update_l2p(rb, count); |
a4bd217b JG |
260 | |
261 | out: | |
262 | return ret; | |
263 | } | |
264 | ||
265 | /* | |
266 | * Update the l2p entry for all sectors stored on the write buffer. This means | |
267 | * that all future lookups to the l2p table will point to a device address, not | |
268 | * to the cacheline in the write buffer. | |
269 | */ | |
270 | void pblk_rb_sync_l2p(struct pblk_rb *rb) | |
271 | { | |
272 | unsigned int sync; | |
273 | unsigned int to_update; | |
274 | ||
275 | spin_lock(&rb->w_lock); | |
276 | ||
277 | /* Protect from reads and writes */ | |
278 | sync = smp_load_acquire(&rb->sync); | |
279 | ||
280 | to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries); | |
05ed3447 | 281 | __pblk_rb_update_l2p(rb, to_update); |
a4bd217b JG |
282 | |
283 | spin_unlock(&rb->w_lock); | |
284 | } | |
285 | ||
286 | /* | |
287 | * Write @nr_entries to ring buffer from @data buffer if there is enough space. | |
288 | * Typically, 4KB data chunks coming from a bio will be copied to the ring | |
289 | * buffer, thus the write will fail if not all incoming data can be copied. | |
290 | * | |
291 | */ | |
292 | static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data, | |
293 | struct pblk_w_ctx w_ctx, | |
294 | struct pblk_rb_entry *entry) | |
295 | { | |
296 | memcpy(entry->data, data, rb->seg_size); | |
297 | ||
298 | entry->w_ctx.lba = w_ctx.lba; | |
299 | entry->w_ctx.ppa = w_ctx.ppa; | |
300 | } | |
301 | ||
302 | void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data, | |
303 | struct pblk_w_ctx w_ctx, unsigned int ring_pos) | |
304 | { | |
305 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
306 | struct pblk_rb_entry *entry; | |
307 | int flags; | |
308 | ||
309 | entry = &rb->entries[ring_pos]; | |
310 | flags = READ_ONCE(entry->w_ctx.flags); | |
311 | #ifdef CONFIG_NVM_DEBUG | |
312 | /* Caller must guarantee that the entry is free */ | |
313 | BUG_ON(!(flags & PBLK_WRITABLE_ENTRY)); | |
314 | #endif | |
315 | ||
316 | __pblk_rb_write_entry(rb, data, w_ctx, entry); | |
317 | ||
318 | pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline); | |
319 | flags = w_ctx.flags | PBLK_WRITTEN_DATA; | |
320 | ||
321 | /* Release flags on write context. Protect from writes */ | |
322 | smp_store_release(&entry->w_ctx.flags, flags); | |
323 | } | |
324 | ||
325 | void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data, | |
d340121e JG |
326 | struct pblk_w_ctx w_ctx, struct pblk_line *line, |
327 | u64 paddr, unsigned int ring_pos) | |
a4bd217b JG |
328 | { |
329 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
330 | struct pblk_rb_entry *entry; | |
331 | int flags; | |
332 | ||
333 | entry = &rb->entries[ring_pos]; | |
334 | flags = READ_ONCE(entry->w_ctx.flags); | |
335 | #ifdef CONFIG_NVM_DEBUG | |
336 | /* Caller must guarantee that the entry is free */ | |
337 | BUG_ON(!(flags & PBLK_WRITABLE_ENTRY)); | |
338 | #endif | |
339 | ||
340 | __pblk_rb_write_entry(rb, data, w_ctx, entry); | |
341 | ||
d340121e | 342 | if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr)) |
a4bd217b JG |
343 | entry->w_ctx.lba = ADDR_EMPTY; |
344 | ||
345 | flags = w_ctx.flags | PBLK_WRITTEN_DATA; | |
346 | ||
347 | /* Release flags on write context. Protect from writes */ | |
348 | smp_store_release(&entry->w_ctx.flags, flags); | |
349 | } | |
350 | ||
351 | static int pblk_rb_sync_point_set(struct pblk_rb *rb, struct bio *bio, | |
352 | unsigned int pos) | |
353 | { | |
354 | struct pblk_rb_entry *entry; | |
355 | unsigned int subm, sync_point; | |
a4bd217b JG |
356 | |
357 | subm = READ_ONCE(rb->subm); | |
358 | ||
359 | #ifdef CONFIG_NVM_DEBUG | |
360 | atomic_inc(&rb->inflight_sync_point); | |
361 | #endif | |
362 | ||
363 | if (pos == subm) | |
364 | return 0; | |
365 | ||
366 | sync_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1); | |
367 | entry = &rb->entries[sync_point]; | |
368 | ||
a4bd217b JG |
369 | /* Protect syncs */ |
370 | smp_store_release(&rb->sync_point, sync_point); | |
371 | ||
588726d3 JG |
372 | if (!bio) |
373 | return 0; | |
374 | ||
a4bd217b JG |
375 | spin_lock_irq(&rb->s_lock); |
376 | bio_list_add(&entry->w_ctx.bios, bio); | |
377 | spin_unlock_irq(&rb->s_lock); | |
378 | ||
379 | return 1; | |
380 | } | |
381 | ||
382 | static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, | |
383 | unsigned int *pos) | |
384 | { | |
385 | unsigned int mem; | |
386 | unsigned int sync; | |
387 | ||
388 | sync = READ_ONCE(rb->sync); | |
389 | mem = READ_ONCE(rb->mem); | |
390 | ||
391 | if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < nr_entries) | |
392 | return 0; | |
393 | ||
394 | if (pblk_rb_update_l2p(rb, nr_entries, mem, sync)) | |
395 | return 0; | |
396 | ||
397 | *pos = mem; | |
398 | ||
399 | return 1; | |
400 | } | |
401 | ||
402 | static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, | |
403 | unsigned int *pos) | |
404 | { | |
405 | if (!__pblk_rb_may_write(rb, nr_entries, pos)) | |
406 | return 0; | |
407 | ||
408 | /* Protect from read count */ | |
409 | smp_store_release(&rb->mem, (*pos + nr_entries) & (rb->nr_entries - 1)); | |
410 | return 1; | |
411 | } | |
412 | ||
588726d3 JG |
413 | void pblk_rb_flush(struct pblk_rb *rb) |
414 | { | |
415 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
416 | unsigned int mem = READ_ONCE(rb->mem); | |
417 | ||
418 | if (pblk_rb_sync_point_set(rb, NULL, mem)) | |
419 | return; | |
420 | ||
421 | pblk_write_should_kick(pblk); | |
422 | } | |
423 | ||
a4bd217b JG |
424 | static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries, |
425 | unsigned int *pos, struct bio *bio, | |
426 | int *io_ret) | |
427 | { | |
428 | unsigned int mem; | |
429 | ||
430 | if (!__pblk_rb_may_write(rb, nr_entries, pos)) | |
431 | return 0; | |
432 | ||
433 | mem = (*pos + nr_entries) & (rb->nr_entries - 1); | |
434 | *io_ret = NVM_IO_DONE; | |
435 | ||
436 | if (bio->bi_opf & REQ_PREFLUSH) { | |
437 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
438 | ||
439 | #ifdef CONFIG_NVM_DEBUG | |
440 | atomic_long_inc(&pblk->nr_flush); | |
441 | #endif | |
442 | if (pblk_rb_sync_point_set(&pblk->rwb, bio, mem)) | |
443 | *io_ret = NVM_IO_OK; | |
444 | } | |
445 | ||
446 | /* Protect from read count */ | |
447 | smp_store_release(&rb->mem, mem); | |
6ca2f71f | 448 | |
a4bd217b JG |
449 | return 1; |
450 | } | |
451 | ||
452 | /* | |
453 | * Atomically check that (i) there is space on the write buffer for the | |
454 | * incoming I/O, and (ii) the current I/O type has enough budget in the write | |
455 | * buffer (rate-limiter). | |
456 | */ | |
457 | int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio, | |
458 | unsigned int nr_entries, unsigned int *pos) | |
459 | { | |
460 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
588726d3 | 461 | int io_ret; |
a4bd217b JG |
462 | |
463 | spin_lock(&rb->w_lock); | |
588726d3 JG |
464 | io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries); |
465 | if (io_ret) { | |
a4bd217b | 466 | spin_unlock(&rb->w_lock); |
588726d3 | 467 | return io_ret; |
a4bd217b JG |
468 | } |
469 | ||
588726d3 | 470 | if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) { |
a4bd217b JG |
471 | spin_unlock(&rb->w_lock); |
472 | return NVM_IO_REQUEUE; | |
473 | } | |
474 | ||
475 | pblk_rl_user_in(&pblk->rl, nr_entries); | |
476 | spin_unlock(&rb->w_lock); | |
477 | ||
588726d3 | 478 | return io_ret; |
a4bd217b JG |
479 | } |
480 | ||
481 | /* | |
482 | * Look at pblk_rb_may_write_user comment | |
483 | */ | |
484 | int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries, | |
485 | unsigned int *pos) | |
486 | { | |
487 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
488 | ||
489 | spin_lock(&rb->w_lock); | |
490 | if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) { | |
491 | spin_unlock(&rb->w_lock); | |
492 | return 0; | |
493 | } | |
494 | ||
495 | if (!pblk_rb_may_write(rb, nr_entries, pos)) { | |
496 | spin_unlock(&rb->w_lock); | |
497 | return 0; | |
498 | } | |
499 | ||
500 | pblk_rl_gc_in(&pblk->rl, nr_entries); | |
501 | spin_unlock(&rb->w_lock); | |
502 | ||
503 | return 1; | |
504 | } | |
505 | ||
506 | /* | |
507 | * The caller of this function must ensure that the backpointer will not | |
508 | * overwrite the entries passed on the list. | |
509 | */ | |
510 | unsigned int pblk_rb_read_to_bio_list(struct pblk_rb *rb, struct bio *bio, | |
511 | struct list_head *list, | |
512 | unsigned int max) | |
513 | { | |
514 | struct pblk_rb_entry *entry, *tentry; | |
515 | struct page *page; | |
516 | unsigned int read = 0; | |
517 | int ret; | |
518 | ||
519 | list_for_each_entry_safe(entry, tentry, list, index) { | |
520 | if (read > max) { | |
521 | pr_err("pblk: too many entries on list\n"); | |
522 | goto out; | |
523 | } | |
524 | ||
525 | page = virt_to_page(entry->data); | |
526 | if (!page) { | |
527 | pr_err("pblk: could not allocate write bio page\n"); | |
528 | goto out; | |
529 | } | |
530 | ||
531 | ret = bio_add_page(bio, page, rb->seg_size, 0); | |
532 | if (ret != rb->seg_size) { | |
533 | pr_err("pblk: could not add page to write bio\n"); | |
534 | goto out; | |
535 | } | |
536 | ||
537 | list_del(&entry->index); | |
538 | read++; | |
539 | } | |
540 | ||
541 | out: | |
542 | return read; | |
543 | } | |
544 | ||
545 | /* | |
546 | * Read available entries on rb and add them to the given bio. To avoid a memory | |
547 | * copy, a page reference to the write buffer is used to be added to the bio. | |
548 | * | |
549 | * This function is used by the write thread to form the write bio that will | |
550 | * persist data on the write buffer to the media. | |
551 | */ | |
d624f371 | 552 | unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd, |
875d94f3 JG |
553 | unsigned int pos, unsigned int nr_entries, |
554 | unsigned int count) | |
a4bd217b JG |
555 | { |
556 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
d624f371 JG |
557 | struct request_queue *q = pblk->dev->q; |
558 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); | |
875d94f3 | 559 | struct bio *bio = rqd->bio; |
a4bd217b JG |
560 | struct pblk_rb_entry *entry; |
561 | struct page *page; | |
d624f371 | 562 | unsigned int pad = 0, to_read = nr_entries; |
a4bd217b JG |
563 | unsigned int i; |
564 | int flags; | |
a4bd217b JG |
565 | |
566 | if (count < nr_entries) { | |
567 | pad = nr_entries - count; | |
568 | to_read = count; | |
569 | } | |
570 | ||
571 | c_ctx->sentry = pos; | |
572 | c_ctx->nr_valid = to_read; | |
573 | c_ctx->nr_padded = pad; | |
574 | ||
575 | for (i = 0; i < to_read; i++) { | |
576 | entry = &rb->entries[pos]; | |
577 | ||
578 | /* A write has been allowed into the buffer, but data is still | |
579 | * being copied to it. It is ok to busy wait. | |
580 | */ | |
581 | try: | |
582 | flags = READ_ONCE(entry->w_ctx.flags); | |
10888129 JG |
583 | if (!(flags & PBLK_WRITTEN_DATA)) { |
584 | io_schedule(); | |
a4bd217b | 585 | goto try; |
10888129 | 586 | } |
a4bd217b | 587 | |
a4bd217b JG |
588 | page = virt_to_page(entry->data); |
589 | if (!page) { | |
590 | pr_err("pblk: could not allocate write bio page\n"); | |
591 | flags &= ~PBLK_WRITTEN_DATA; | |
592 | flags |= PBLK_SUBMITTED_ENTRY; | |
593 | /* Release flags on context. Protect from writes */ | |
594 | smp_store_release(&entry->w_ctx.flags, flags); | |
d624f371 | 595 | return NVM_IO_ERR; |
a4bd217b JG |
596 | } |
597 | ||
d624f371 JG |
598 | if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) != |
599 | rb->seg_size) { | |
a4bd217b JG |
600 | pr_err("pblk: could not add page to write bio\n"); |
601 | flags &= ~PBLK_WRITTEN_DATA; | |
602 | flags |= PBLK_SUBMITTED_ENTRY; | |
603 | /* Release flags on context. Protect from writes */ | |
604 | smp_store_release(&entry->w_ctx.flags, flags); | |
d624f371 | 605 | return NVM_IO_ERR; |
a4bd217b JG |
606 | } |
607 | ||
608 | if (flags & PBLK_FLUSH_ENTRY) { | |
609 | unsigned int sync_point; | |
610 | ||
611 | sync_point = READ_ONCE(rb->sync_point); | |
612 | if (sync_point == pos) { | |
613 | /* Protect syncs */ | |
614 | smp_store_release(&rb->sync_point, EMPTY_ENTRY); | |
615 | } | |
616 | ||
617 | flags &= ~PBLK_FLUSH_ENTRY; | |
618 | #ifdef CONFIG_NVM_DEBUG | |
619 | atomic_dec(&rb->inflight_sync_point); | |
620 | #endif | |
621 | } | |
622 | ||
623 | flags &= ~PBLK_WRITTEN_DATA; | |
624 | flags |= PBLK_SUBMITTED_ENTRY; | |
625 | ||
626 | /* Release flags on context. Protect from writes */ | |
627 | smp_store_release(&entry->w_ctx.flags, flags); | |
628 | ||
629 | pos = (pos + 1) & (rb->nr_entries - 1); | |
630 | } | |
631 | ||
d624f371 JG |
632 | if (pad) { |
633 | if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) { | |
634 | pr_err("pblk: could not pad page in write bio\n"); | |
635 | return NVM_IO_ERR; | |
636 | } | |
637 | } | |
638 | ||
a4bd217b JG |
639 | #ifdef CONFIG_NVM_DEBUG |
640 | atomic_long_add(pad, &((struct pblk *) | |
641 | (container_of(rb, struct pblk, rwb)))->padded_writes); | |
642 | #endif | |
d624f371 JG |
643 | |
644 | return NVM_IO_OK; | |
a4bd217b JG |
645 | } |
646 | ||
647 | /* | |
648 | * Copy to bio only if the lba matches the one on the given cache entry. | |
649 | * Otherwise, it means that the entry has been overwritten, and the bio should | |
650 | * be directed to disk. | |
651 | */ | |
652 | int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba, | |
75cb8e93 | 653 | struct ppa_addr ppa, int bio_iter, bool advanced_bio) |
a4bd217b | 654 | { |
07698466 | 655 | struct pblk *pblk = container_of(rb, struct pblk, rwb); |
a4bd217b JG |
656 | struct pblk_rb_entry *entry; |
657 | struct pblk_w_ctx *w_ctx; | |
07698466 JG |
658 | struct ppa_addr l2p_ppa; |
659 | u64 pos = pblk_addr_to_cacheline(ppa); | |
a4bd217b JG |
660 | void *data; |
661 | int flags; | |
662 | int ret = 1; | |
663 | ||
a4bd217b JG |
664 | |
665 | #ifdef CONFIG_NVM_DEBUG | |
666 | /* Caller must ensure that the access will not cause an overflow */ | |
667 | BUG_ON(pos >= rb->nr_entries); | |
668 | #endif | |
669 | entry = &rb->entries[pos]; | |
670 | w_ctx = &entry->w_ctx; | |
671 | flags = READ_ONCE(w_ctx->flags); | |
672 | ||
07698466 JG |
673 | spin_lock(&rb->w_lock); |
674 | spin_lock(&pblk->trans_lock); | |
675 | l2p_ppa = pblk_trans_map_get(pblk, lba); | |
676 | spin_unlock(&pblk->trans_lock); | |
677 | ||
a4bd217b | 678 | /* Check if the entry has been overwritten or is scheduled to be */ |
07698466 JG |
679 | if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba || |
680 | flags & PBLK_WRITABLE_ENTRY) { | |
a4bd217b JG |
681 | ret = 0; |
682 | goto out; | |
683 | } | |
684 | ||
685 | /* Only advance the bio if it hasn't been advanced already. If advanced, | |
686 | * this bio is at least a partial bio (i.e., it has partially been | |
687 | * filled with data from the cache). If part of the data resides on the | |
688 | * media, we will read later on | |
689 | */ | |
75cb8e93 | 690 | if (unlikely(!advanced_bio)) |
a4bd217b JG |
691 | bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE); |
692 | ||
693 | data = bio_data(bio); | |
694 | memcpy(data, entry->data, rb->seg_size); | |
695 | ||
696 | out: | |
697 | spin_unlock(&rb->w_lock); | |
698 | return ret; | |
699 | } | |
700 | ||
701 | struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos) | |
702 | { | |
703 | unsigned int entry = pos & (rb->nr_entries - 1); | |
704 | ||
705 | return &rb->entries[entry].w_ctx; | |
706 | } | |
707 | ||
708 | unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags) | |
709 | __acquires(&rb->s_lock) | |
710 | { | |
711 | if (flags) | |
712 | spin_lock_irqsave(&rb->s_lock, *flags); | |
713 | else | |
714 | spin_lock_irq(&rb->s_lock); | |
715 | ||
716 | return rb->sync; | |
717 | } | |
718 | ||
719 | void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags) | |
720 | __releases(&rb->s_lock) | |
721 | { | |
722 | lockdep_assert_held(&rb->s_lock); | |
723 | ||
724 | if (flags) | |
725 | spin_unlock_irqrestore(&rb->s_lock, *flags); | |
726 | else | |
727 | spin_unlock_irq(&rb->s_lock); | |
728 | } | |
729 | ||
730 | unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries) | |
731 | { | |
732 | unsigned int sync; | |
733 | unsigned int i; | |
734 | ||
735 | lockdep_assert_held(&rb->s_lock); | |
736 | ||
737 | sync = READ_ONCE(rb->sync); | |
738 | ||
739 | for (i = 0; i < nr_entries; i++) | |
740 | sync = (sync + 1) & (rb->nr_entries - 1); | |
741 | ||
742 | /* Protect from counts */ | |
743 | smp_store_release(&rb->sync, sync); | |
744 | ||
745 | return sync; | |
746 | } | |
747 | ||
748 | unsigned int pblk_rb_sync_point_count(struct pblk_rb *rb) | |
749 | { | |
750 | unsigned int subm, sync_point; | |
751 | unsigned int count; | |
752 | ||
753 | /* Protect syncs */ | |
754 | sync_point = smp_load_acquire(&rb->sync_point); | |
755 | if (sync_point == EMPTY_ENTRY) | |
756 | return 0; | |
757 | ||
758 | subm = READ_ONCE(rb->subm); | |
759 | ||
760 | /* The sync point itself counts as a sector to sync */ | |
761 | count = pblk_rb_ring_count(sync_point, subm, rb->nr_entries) + 1; | |
762 | ||
763 | return count; | |
764 | } | |
765 | ||
766 | /* | |
767 | * Scan from the current position of the sync pointer to find the entry that | |
768 | * corresponds to the given ppa. This is necessary since write requests can be | |
769 | * completed out of order. The assumption is that the ppa is close to the sync | |
770 | * pointer thus the search will not take long. | |
771 | * | |
772 | * The caller of this function must guarantee that the sync pointer will no | |
773 | * reach the entry while it is using the metadata associated with it. With this | |
774 | * assumption in mind, there is no need to take the sync lock. | |
775 | */ | |
776 | struct pblk_rb_entry *pblk_rb_sync_scan_entry(struct pblk_rb *rb, | |
777 | struct ppa_addr *ppa) | |
778 | { | |
779 | unsigned int sync, subm, count; | |
780 | unsigned int i; | |
781 | ||
782 | sync = READ_ONCE(rb->sync); | |
783 | subm = READ_ONCE(rb->subm); | |
784 | count = pblk_rb_ring_count(subm, sync, rb->nr_entries); | |
785 | ||
786 | for (i = 0; i < count; i++) | |
787 | sync = (sync + 1) & (rb->nr_entries - 1); | |
788 | ||
789 | return NULL; | |
790 | } | |
791 | ||
792 | int pblk_rb_tear_down_check(struct pblk_rb *rb) | |
793 | { | |
794 | struct pblk_rb_entry *entry; | |
795 | int i; | |
796 | int ret = 0; | |
797 | ||
798 | spin_lock(&rb->w_lock); | |
799 | spin_lock_irq(&rb->s_lock); | |
800 | ||
801 | if ((rb->mem == rb->subm) && (rb->subm == rb->sync) && | |
802 | (rb->sync == rb->l2p_update) && | |
803 | (rb->sync_point == EMPTY_ENTRY)) { | |
804 | goto out; | |
805 | } | |
806 | ||
807 | if (!rb->entries) { | |
808 | ret = 1; | |
809 | goto out; | |
810 | } | |
811 | ||
812 | for (i = 0; i < rb->nr_entries; i++) { | |
813 | entry = &rb->entries[i]; | |
814 | ||
815 | if (!entry->data) { | |
816 | ret = 1; | |
817 | goto out; | |
818 | } | |
819 | } | |
820 | ||
821 | out: | |
822 | spin_unlock(&rb->w_lock); | |
823 | spin_unlock_irq(&rb->s_lock); | |
824 | ||
825 | return ret; | |
826 | } | |
827 | ||
828 | unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos) | |
829 | { | |
830 | return (pos & (rb->nr_entries - 1)); | |
831 | } | |
832 | ||
833 | int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos) | |
834 | { | |
835 | return (pos >= rb->nr_entries); | |
836 | } | |
837 | ||
838 | ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf) | |
839 | { | |
840 | struct pblk *pblk = container_of(rb, struct pblk, rwb); | |
841 | struct pblk_c_ctx *c; | |
842 | ssize_t offset; | |
843 | int queued_entries = 0; | |
844 | ||
845 | spin_lock_irq(&rb->s_lock); | |
846 | list_for_each_entry(c, &pblk->compl_list, list) | |
847 | queued_entries++; | |
848 | spin_unlock_irq(&rb->s_lock); | |
849 | ||
850 | if (rb->sync_point != EMPTY_ENTRY) | |
851 | offset = scnprintf(buf, PAGE_SIZE, | |
852 | "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n", | |
853 | rb->nr_entries, | |
854 | rb->mem, | |
855 | rb->subm, | |
856 | rb->sync, | |
857 | rb->l2p_update, | |
858 | #ifdef CONFIG_NVM_DEBUG | |
859 | atomic_read(&rb->inflight_sync_point), | |
860 | #else | |
861 | 0, | |
862 | #endif | |
863 | rb->sync_point, | |
864 | pblk_rb_read_count(rb), | |
865 | pblk_rb_space(rb), | |
866 | pblk_rb_sync_point_count(rb), | |
867 | queued_entries); | |
868 | else | |
869 | offset = scnprintf(buf, PAGE_SIZE, | |
870 | "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n", | |
871 | rb->nr_entries, | |
872 | rb->mem, | |
873 | rb->subm, | |
874 | rb->sync, | |
875 | rb->l2p_update, | |
876 | #ifdef CONFIG_NVM_DEBUG | |
877 | atomic_read(&rb->inflight_sync_point), | |
878 | #else | |
879 | 0, | |
880 | #endif | |
881 | pblk_rb_read_count(rb), | |
882 | pblk_rb_space(rb), | |
883 | pblk_rb_sync_point_count(rb), | |
884 | queued_entries); | |
885 | ||
886 | return offset; | |
887 | } |