]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/lightnvm/pblk-rb.c
2 * Copyright (C) 2016 CNEX Labs
3 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
5 * Based upon the circular ringbuffer.
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * pblk-rb.c - pblk's write buffer
19 #include <linux/circ_buf.h>
23 static DECLARE_RWSEM(pblk_rb_lock
);
25 void pblk_rb_data_free(struct pblk_rb
*rb
)
27 struct pblk_rb_pages
*p
, *t
;
29 down_write(&pblk_rb_lock
);
30 list_for_each_entry_safe(p
, t
, &rb
->pages
, list
) {
31 free_pages((unsigned long)page_address(p
->pages
), p
->order
);
35 up_write(&pblk_rb_lock
);
39 * Initialize ring buffer. The data and metadata buffers must be previously
40 * allocated and their size must be a power of two
41 * (Documentation/circular-buffers.txt)
43 int pblk_rb_init(struct pblk_rb
*rb
, struct pblk_rb_entry
*rb_entry_base
,
44 unsigned int power_size
, unsigned int power_seg_sz
)
46 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
47 unsigned int init_entry
= 0;
48 unsigned int alloc_order
= power_size
;
49 unsigned int max_order
= MAX_ORDER
- 1;
50 unsigned int order
, iter
;
52 down_write(&pblk_rb_lock
);
53 rb
->entries
= rb_entry_base
;
54 rb
->seg_size
= (1 << power_seg_sz
);
55 rb
->nr_entries
= (1 << power_size
);
56 rb
->mem
= rb
->subm
= rb
->sync
= rb
->l2p_update
= 0;
57 rb
->sync_point
= EMPTY_ENTRY
;
59 spin_lock_init(&rb
->w_lock
);
60 spin_lock_init(&rb
->s_lock
);
62 INIT_LIST_HEAD(&rb
->pages
);
64 if (alloc_order
>= max_order
) {
66 iter
= (1 << (alloc_order
- max_order
));
73 struct pblk_rb_entry
*entry
;
74 struct pblk_rb_pages
*page_set
;
76 unsigned long set_size
;
79 page_set
= kmalloc(sizeof(struct pblk_rb_pages
), GFP_KERNEL
);
81 up_write(&pblk_rb_lock
);
85 page_set
->order
= order
;
86 page_set
->pages
= alloc_pages(GFP_KERNEL
, order
);
87 if (!page_set
->pages
) {
89 pblk_rb_data_free(rb
);
90 up_write(&pblk_rb_lock
);
93 kaddr
= page_address(page_set
->pages
);
95 entry
= &rb
->entries
[init_entry
];
97 entry
->cacheline
= pblk_cacheline_to_addr(init_entry
++);
98 entry
->w_ctx
.flags
= PBLK_WRITABLE_ENTRY
;
100 set_size
= (1 << order
);
101 for (i
= 1; i
< set_size
; i
++) {
102 entry
= &rb
->entries
[init_entry
];
103 entry
->cacheline
= pblk_cacheline_to_addr(init_entry
++);
104 entry
->data
= kaddr
+ (i
* rb
->seg_size
);
105 entry
->w_ctx
.flags
= PBLK_WRITABLE_ENTRY
;
106 bio_list_init(&entry
->w_ctx
.bios
);
109 list_add_tail(&page_set
->list
, &rb
->pages
);
112 up_write(&pblk_rb_lock
);
114 #ifdef CONFIG_NVM_DEBUG
115 atomic_set(&rb
->inflight_sync_point
, 0);
119 * Initialize rate-limiter, which controls access to the write buffer
120 * but user and GC I/O
122 pblk_rl_init(&pblk
->rl
, rb
->nr_entries
);
128 * pblk_rb_calculate_size -- calculate the size of the write buffer
130 unsigned int pblk_rb_calculate_size(unsigned int nr_entries
)
132 /* Alloc a write buffer that can at least fit 128 entries */
133 return (1 << max(get_count_order(nr_entries
), 7));
136 void *pblk_rb_entries_ref(struct pblk_rb
*rb
)
141 static void clean_wctx(struct pblk_w_ctx
*w_ctx
)
145 flags
= READ_ONCE(w_ctx
->flags
);
146 WARN_ONCE(!(flags
& PBLK_SUBMITTED_ENTRY
),
147 "pblk: overwriting unsubmitted data\n");
149 /* Release flags on context. Protect from writes and reads */
150 smp_store_release(&w_ctx
->flags
, PBLK_WRITABLE_ENTRY
);
151 pblk_ppa_set_empty(&w_ctx
->ppa
);
152 w_ctx
->lba
= ADDR_EMPTY
;
155 #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
156 #define pblk_rb_ring_space(rb, head, tail, size) \
157 (CIRC_SPACE(head, tail, size))
160 * Buffer space is calculated with respect to the back pointer signaling
161 * synchronized entries to the media.
163 static unsigned int pblk_rb_space(struct pblk_rb
*rb
)
165 unsigned int mem
= READ_ONCE(rb
->mem
);
166 unsigned int sync
= READ_ONCE(rb
->sync
);
168 return pblk_rb_ring_space(rb
, mem
, sync
, rb
->nr_entries
);
172 * Buffer count is calculated with respect to the submission entry signaling the
173 * entries that are available to send to the media
175 unsigned int pblk_rb_read_count(struct pblk_rb
*rb
)
177 unsigned int mem
= READ_ONCE(rb
->mem
);
178 unsigned int subm
= READ_ONCE(rb
->subm
);
180 return pblk_rb_ring_count(mem
, subm
, rb
->nr_entries
);
183 unsigned int pblk_rb_sync_count(struct pblk_rb
*rb
)
185 unsigned int mem
= READ_ONCE(rb
->mem
);
186 unsigned int sync
= READ_ONCE(rb
->sync
);
188 return pblk_rb_ring_count(mem
, sync
, rb
->nr_entries
);
191 unsigned int pblk_rb_read_commit(struct pblk_rb
*rb
, unsigned int nr_entries
)
195 subm
= READ_ONCE(rb
->subm
);
196 /* Commit read means updating submission pointer */
197 smp_store_release(&rb
->subm
,
198 (subm
+ nr_entries
) & (rb
->nr_entries
- 1));
203 static int __pblk_rb_update_l2p(struct pblk_rb
*rb
, unsigned int to_update
)
205 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
206 struct pblk_line
*line
;
207 struct pblk_rb_entry
*entry
;
208 struct pblk_w_ctx
*w_ctx
;
209 unsigned int user_io
= 0, gc_io
= 0;
213 for (i
= 0; i
< to_update
; i
++) {
214 entry
= &rb
->entries
[rb
->l2p_update
];
215 w_ctx
= &entry
->w_ctx
;
217 flags
= READ_ONCE(entry
->w_ctx
.flags
);
218 if (flags
& PBLK_IOTYPE_USER
)
220 else if (flags
& PBLK_IOTYPE_GC
)
223 WARN(1, "pblk: unknown IO type\n");
225 pblk_update_map_dev(pblk
, w_ctx
->lba
, w_ctx
->ppa
,
228 line
= &pblk
->lines
[pblk_tgt_ppa_to_line(w_ctx
->ppa
)];
229 kref_put(&line
->ref
, pblk_line_put
);
231 rb
->l2p_update
= (rb
->l2p_update
+ 1) & (rb
->nr_entries
- 1);
234 pblk_rl_out(&pblk
->rl
, user_io
, gc_io
);
240 * When we move the l2p_update pointer, we update the l2p table - lookups will
241 * point to the physical address instead of to the cacheline in the write buffer
242 * from this moment on.
244 static int pblk_rb_update_l2p(struct pblk_rb
*rb
, unsigned int nr_entries
,
245 unsigned int mem
, unsigned int sync
)
247 unsigned int space
, count
;
250 lockdep_assert_held(&rb
->w_lock
);
252 /* Update l2p only as buffer entries are being overwritten */
253 space
= pblk_rb_ring_space(rb
, mem
, rb
->l2p_update
, rb
->nr_entries
);
254 if (space
> nr_entries
)
257 count
= nr_entries
- space
;
258 /* l2p_update used exclusively under rb->w_lock */
259 ret
= __pblk_rb_update_l2p(rb
, count
);
266 * Update the l2p entry for all sectors stored on the write buffer. This means
267 * that all future lookups to the l2p table will point to a device address, not
268 * to the cacheline in the write buffer.
270 void pblk_rb_sync_l2p(struct pblk_rb
*rb
)
273 unsigned int to_update
;
275 spin_lock(&rb
->w_lock
);
277 /* Protect from reads and writes */
278 sync
= smp_load_acquire(&rb
->sync
);
280 to_update
= pblk_rb_ring_count(sync
, rb
->l2p_update
, rb
->nr_entries
);
281 __pblk_rb_update_l2p(rb
, to_update
);
283 spin_unlock(&rb
->w_lock
);
287 * Write @nr_entries to ring buffer from @data buffer if there is enough space.
288 * Typically, 4KB data chunks coming from a bio will be copied to the ring
289 * buffer, thus the write will fail if not all incoming data can be copied.
292 static void __pblk_rb_write_entry(struct pblk_rb
*rb
, void *data
,
293 struct pblk_w_ctx w_ctx
,
294 struct pblk_rb_entry
*entry
)
296 memcpy(entry
->data
, data
, rb
->seg_size
);
298 entry
->w_ctx
.lba
= w_ctx
.lba
;
299 entry
->w_ctx
.ppa
= w_ctx
.ppa
;
302 void pblk_rb_write_entry_user(struct pblk_rb
*rb
, void *data
,
303 struct pblk_w_ctx w_ctx
, unsigned int ring_pos
)
305 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
306 struct pblk_rb_entry
*entry
;
309 entry
= &rb
->entries
[ring_pos
];
310 flags
= READ_ONCE(entry
->w_ctx
.flags
);
311 #ifdef CONFIG_NVM_DEBUG
312 /* Caller must guarantee that the entry is free */
313 BUG_ON(!(flags
& PBLK_WRITABLE_ENTRY
));
316 __pblk_rb_write_entry(rb
, data
, w_ctx
, entry
);
318 pblk_update_map_cache(pblk
, w_ctx
.lba
, entry
->cacheline
);
319 flags
= w_ctx
.flags
| PBLK_WRITTEN_DATA
;
321 /* Release flags on write context. Protect from writes */
322 smp_store_release(&entry
->w_ctx
.flags
, flags
);
325 void pblk_rb_write_entry_gc(struct pblk_rb
*rb
, void *data
,
326 struct pblk_w_ctx w_ctx
, struct pblk_line
*line
,
327 u64 paddr
, unsigned int ring_pos
)
329 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
330 struct pblk_rb_entry
*entry
;
333 entry
= &rb
->entries
[ring_pos
];
334 flags
= READ_ONCE(entry
->w_ctx
.flags
);
335 #ifdef CONFIG_NVM_DEBUG
336 /* Caller must guarantee that the entry is free */
337 BUG_ON(!(flags
& PBLK_WRITABLE_ENTRY
));
340 __pblk_rb_write_entry(rb
, data
, w_ctx
, entry
);
342 if (!pblk_update_map_gc(pblk
, w_ctx
.lba
, entry
->cacheline
, line
, paddr
))
343 entry
->w_ctx
.lba
= ADDR_EMPTY
;
345 flags
= w_ctx
.flags
| PBLK_WRITTEN_DATA
;
347 /* Release flags on write context. Protect from writes */
348 smp_store_release(&entry
->w_ctx
.flags
, flags
);
351 static int pblk_rb_sync_point_set(struct pblk_rb
*rb
, struct bio
*bio
,
354 struct pblk_rb_entry
*entry
;
355 unsigned int subm
, sync_point
;
357 subm
= READ_ONCE(rb
->subm
);
359 #ifdef CONFIG_NVM_DEBUG
360 atomic_inc(&rb
->inflight_sync_point
);
366 sync_point
= (pos
== 0) ? (rb
->nr_entries
- 1) : (pos
- 1);
367 entry
= &rb
->entries
[sync_point
];
370 smp_store_release(&rb
->sync_point
, sync_point
);
375 spin_lock_irq(&rb
->s_lock
);
376 bio_list_add(&entry
->w_ctx
.bios
, bio
);
377 spin_unlock_irq(&rb
->s_lock
);
382 static int __pblk_rb_may_write(struct pblk_rb
*rb
, unsigned int nr_entries
,
388 sync
= READ_ONCE(rb
->sync
);
389 mem
= READ_ONCE(rb
->mem
);
391 if (pblk_rb_ring_space(rb
, mem
, sync
, rb
->nr_entries
) < nr_entries
)
394 if (pblk_rb_update_l2p(rb
, nr_entries
, mem
, sync
))
402 static int pblk_rb_may_write(struct pblk_rb
*rb
, unsigned int nr_entries
,
405 if (!__pblk_rb_may_write(rb
, nr_entries
, pos
))
408 /* Protect from read count */
409 smp_store_release(&rb
->mem
, (*pos
+ nr_entries
) & (rb
->nr_entries
- 1));
413 void pblk_rb_flush(struct pblk_rb
*rb
)
415 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
416 unsigned int mem
= READ_ONCE(rb
->mem
);
418 if (pblk_rb_sync_point_set(rb
, NULL
, mem
))
421 pblk_write_should_kick(pblk
);
424 static int pblk_rb_may_write_flush(struct pblk_rb
*rb
, unsigned int nr_entries
,
425 unsigned int *pos
, struct bio
*bio
,
430 if (!__pblk_rb_may_write(rb
, nr_entries
, pos
))
433 mem
= (*pos
+ nr_entries
) & (rb
->nr_entries
- 1);
434 *io_ret
= NVM_IO_DONE
;
436 if (bio
->bi_opf
& REQ_PREFLUSH
) {
437 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
439 #ifdef CONFIG_NVM_DEBUG
440 atomic_long_inc(&pblk
->nr_flush
);
442 if (pblk_rb_sync_point_set(&pblk
->rwb
, bio
, mem
))
446 /* Protect from read count */
447 smp_store_release(&rb
->mem
, mem
);
453 * Atomically check that (i) there is space on the write buffer for the
454 * incoming I/O, and (ii) the current I/O type has enough budget in the write
455 * buffer (rate-limiter).
457 int pblk_rb_may_write_user(struct pblk_rb
*rb
, struct bio
*bio
,
458 unsigned int nr_entries
, unsigned int *pos
)
460 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
463 spin_lock(&rb
->w_lock
);
464 io_ret
= pblk_rl_user_may_insert(&pblk
->rl
, nr_entries
);
466 spin_unlock(&rb
->w_lock
);
470 if (!pblk_rb_may_write_flush(rb
, nr_entries
, pos
, bio
, &io_ret
)) {
471 spin_unlock(&rb
->w_lock
);
472 return NVM_IO_REQUEUE
;
475 pblk_rl_user_in(&pblk
->rl
, nr_entries
);
476 spin_unlock(&rb
->w_lock
);
482 * Look at pblk_rb_may_write_user comment
484 int pblk_rb_may_write_gc(struct pblk_rb
*rb
, unsigned int nr_entries
,
487 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
489 spin_lock(&rb
->w_lock
);
490 if (!pblk_rl_gc_may_insert(&pblk
->rl
, nr_entries
)) {
491 spin_unlock(&rb
->w_lock
);
495 if (!pblk_rb_may_write(rb
, nr_entries
, pos
)) {
496 spin_unlock(&rb
->w_lock
);
500 pblk_rl_gc_in(&pblk
->rl
, nr_entries
);
501 spin_unlock(&rb
->w_lock
);
507 * The caller of this function must ensure that the backpointer will not
508 * overwrite the entries passed on the list.
510 unsigned int pblk_rb_read_to_bio_list(struct pblk_rb
*rb
, struct bio
*bio
,
511 struct list_head
*list
,
514 struct pblk_rb_entry
*entry
, *tentry
;
516 unsigned int read
= 0;
519 list_for_each_entry_safe(entry
, tentry
, list
, index
) {
521 pr_err("pblk: too many entries on list\n");
525 page
= virt_to_page(entry
->data
);
527 pr_err("pblk: could not allocate write bio page\n");
531 ret
= bio_add_page(bio
, page
, rb
->seg_size
, 0);
532 if (ret
!= rb
->seg_size
) {
533 pr_err("pblk: could not add page to write bio\n");
537 list_del(&entry
->index
);
546 * Read available entries on rb and add them to the given bio. To avoid a memory
547 * copy, a page reference to the write buffer is used to be added to the bio.
549 * This function is used by the write thread to form the write bio that will
550 * persist data on the write buffer to the media.
552 unsigned int pblk_rb_read_to_bio(struct pblk_rb
*rb
, struct nvm_rq
*rqd
,
553 unsigned int pos
, unsigned int nr_entries
,
556 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
557 struct request_queue
*q
= pblk
->dev
->q
;
558 struct pblk_c_ctx
*c_ctx
= nvm_rq_to_pdu(rqd
);
559 struct bio
*bio
= rqd
->bio
;
560 struct pblk_rb_entry
*entry
;
562 unsigned int pad
= 0, to_read
= nr_entries
;
566 if (count
< nr_entries
) {
567 pad
= nr_entries
- count
;
572 c_ctx
->nr_valid
= to_read
;
573 c_ctx
->nr_padded
= pad
;
575 for (i
= 0; i
< to_read
; i
++) {
576 entry
= &rb
->entries
[pos
];
578 /* A write has been allowed into the buffer, but data is still
579 * being copied to it. It is ok to busy wait.
582 flags
= READ_ONCE(entry
->w_ctx
.flags
);
583 if (!(flags
& PBLK_WRITTEN_DATA
)) {
588 page
= virt_to_page(entry
->data
);
590 pr_err("pblk: could not allocate write bio page\n");
591 flags
&= ~PBLK_WRITTEN_DATA
;
592 flags
|= PBLK_SUBMITTED_ENTRY
;
593 /* Release flags on context. Protect from writes */
594 smp_store_release(&entry
->w_ctx
.flags
, flags
);
598 if (bio_add_pc_page(q
, bio
, page
, rb
->seg_size
, 0) !=
600 pr_err("pblk: could not add page to write bio\n");
601 flags
&= ~PBLK_WRITTEN_DATA
;
602 flags
|= PBLK_SUBMITTED_ENTRY
;
603 /* Release flags on context. Protect from writes */
604 smp_store_release(&entry
->w_ctx
.flags
, flags
);
608 if (flags
& PBLK_FLUSH_ENTRY
) {
609 unsigned int sync_point
;
611 sync_point
= READ_ONCE(rb
->sync_point
);
612 if (sync_point
== pos
) {
614 smp_store_release(&rb
->sync_point
, EMPTY_ENTRY
);
617 flags
&= ~PBLK_FLUSH_ENTRY
;
618 #ifdef CONFIG_NVM_DEBUG
619 atomic_dec(&rb
->inflight_sync_point
);
623 flags
&= ~PBLK_WRITTEN_DATA
;
624 flags
|= PBLK_SUBMITTED_ENTRY
;
626 /* Release flags on context. Protect from writes */
627 smp_store_release(&entry
->w_ctx
.flags
, flags
);
629 pos
= (pos
+ 1) & (rb
->nr_entries
- 1);
633 if (pblk_bio_add_pages(pblk
, bio
, GFP_KERNEL
, pad
)) {
634 pr_err("pblk: could not pad page in write bio\n");
639 #ifdef CONFIG_NVM_DEBUG
640 atomic_long_add(pad
, &((struct pblk
*)
641 (container_of(rb
, struct pblk
, rwb
)))->padded_writes
);
648 * Copy to bio only if the lba matches the one on the given cache entry.
649 * Otherwise, it means that the entry has been overwritten, and the bio should
650 * be directed to disk.
652 int pblk_rb_copy_to_bio(struct pblk_rb
*rb
, struct bio
*bio
, sector_t lba
,
653 struct ppa_addr ppa
, int bio_iter
, bool advanced_bio
)
655 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
656 struct pblk_rb_entry
*entry
;
657 struct pblk_w_ctx
*w_ctx
;
658 struct ppa_addr l2p_ppa
;
659 u64 pos
= pblk_addr_to_cacheline(ppa
);
665 #ifdef CONFIG_NVM_DEBUG
666 /* Caller must ensure that the access will not cause an overflow */
667 BUG_ON(pos
>= rb
->nr_entries
);
669 entry
= &rb
->entries
[pos
];
670 w_ctx
= &entry
->w_ctx
;
671 flags
= READ_ONCE(w_ctx
->flags
);
673 spin_lock(&rb
->w_lock
);
674 spin_lock(&pblk
->trans_lock
);
675 l2p_ppa
= pblk_trans_map_get(pblk
, lba
);
676 spin_unlock(&pblk
->trans_lock
);
678 /* Check if the entry has been overwritten or is scheduled to be */
679 if (!pblk_ppa_comp(l2p_ppa
, ppa
) || w_ctx
->lba
!= lba
||
680 flags
& PBLK_WRITABLE_ENTRY
) {
685 /* Only advance the bio if it hasn't been advanced already. If advanced,
686 * this bio is at least a partial bio (i.e., it has partially been
687 * filled with data from the cache). If part of the data resides on the
688 * media, we will read later on
690 if (unlikely(!advanced_bio
))
691 bio_advance(bio
, bio_iter
* PBLK_EXPOSED_PAGE_SIZE
);
693 data
= bio_data(bio
);
694 memcpy(data
, entry
->data
, rb
->seg_size
);
697 spin_unlock(&rb
->w_lock
);
701 struct pblk_w_ctx
*pblk_rb_w_ctx(struct pblk_rb
*rb
, unsigned int pos
)
703 unsigned int entry
= pos
& (rb
->nr_entries
- 1);
705 return &rb
->entries
[entry
].w_ctx
;
708 unsigned int pblk_rb_sync_init(struct pblk_rb
*rb
, unsigned long *flags
)
709 __acquires(&rb
->s_lock
)
712 spin_lock_irqsave(&rb
->s_lock
, *flags
);
714 spin_lock_irq(&rb
->s_lock
);
719 void pblk_rb_sync_end(struct pblk_rb
*rb
, unsigned long *flags
)
720 __releases(&rb
->s_lock
)
722 lockdep_assert_held(&rb
->s_lock
);
725 spin_unlock_irqrestore(&rb
->s_lock
, *flags
);
727 spin_unlock_irq(&rb
->s_lock
);
730 unsigned int pblk_rb_sync_advance(struct pblk_rb
*rb
, unsigned int nr_entries
)
735 lockdep_assert_held(&rb
->s_lock
);
737 sync
= READ_ONCE(rb
->sync
);
739 for (i
= 0; i
< nr_entries
; i
++)
740 sync
= (sync
+ 1) & (rb
->nr_entries
- 1);
742 /* Protect from counts */
743 smp_store_release(&rb
->sync
, sync
);
748 unsigned int pblk_rb_sync_point_count(struct pblk_rb
*rb
)
750 unsigned int subm
, sync_point
;
754 sync_point
= smp_load_acquire(&rb
->sync_point
);
755 if (sync_point
== EMPTY_ENTRY
)
758 subm
= READ_ONCE(rb
->subm
);
760 /* The sync point itself counts as a sector to sync */
761 count
= pblk_rb_ring_count(sync_point
, subm
, rb
->nr_entries
) + 1;
767 * Scan from the current position of the sync pointer to find the entry that
768 * corresponds to the given ppa. This is necessary since write requests can be
769 * completed out of order. The assumption is that the ppa is close to the sync
770 * pointer thus the search will not take long.
772 * The caller of this function must guarantee that the sync pointer will no
773 * reach the entry while it is using the metadata associated with it. With this
774 * assumption in mind, there is no need to take the sync lock.
776 struct pblk_rb_entry
*pblk_rb_sync_scan_entry(struct pblk_rb
*rb
,
777 struct ppa_addr
*ppa
)
779 unsigned int sync
, subm
, count
;
782 sync
= READ_ONCE(rb
->sync
);
783 subm
= READ_ONCE(rb
->subm
);
784 count
= pblk_rb_ring_count(subm
, sync
, rb
->nr_entries
);
786 for (i
= 0; i
< count
; i
++)
787 sync
= (sync
+ 1) & (rb
->nr_entries
- 1);
792 int pblk_rb_tear_down_check(struct pblk_rb
*rb
)
794 struct pblk_rb_entry
*entry
;
798 spin_lock(&rb
->w_lock
);
799 spin_lock_irq(&rb
->s_lock
);
801 if ((rb
->mem
== rb
->subm
) && (rb
->subm
== rb
->sync
) &&
802 (rb
->sync
== rb
->l2p_update
) &&
803 (rb
->sync_point
== EMPTY_ENTRY
)) {
812 for (i
= 0; i
< rb
->nr_entries
; i
++) {
813 entry
= &rb
->entries
[i
];
822 spin_unlock(&rb
->w_lock
);
823 spin_unlock_irq(&rb
->s_lock
);
828 unsigned int pblk_rb_wrap_pos(struct pblk_rb
*rb
, unsigned int pos
)
830 return (pos
& (rb
->nr_entries
- 1));
833 int pblk_rb_pos_oob(struct pblk_rb
*rb
, u64 pos
)
835 return (pos
>= rb
->nr_entries
);
838 ssize_t
pblk_rb_sysfs(struct pblk_rb
*rb
, char *buf
)
840 struct pblk
*pblk
= container_of(rb
, struct pblk
, rwb
);
841 struct pblk_c_ctx
*c
;
843 int queued_entries
= 0;
845 spin_lock_irq(&rb
->s_lock
);
846 list_for_each_entry(c
, &pblk
->compl_list
, list
)
848 spin_unlock_irq(&rb
->s_lock
);
850 if (rb
->sync_point
!= EMPTY_ENTRY
)
851 offset
= scnprintf(buf
, PAGE_SIZE
,
852 "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
858 #ifdef CONFIG_NVM_DEBUG
859 atomic_read(&rb
->inflight_sync_point
),
864 pblk_rb_read_count(rb
),
866 pblk_rb_sync_point_count(rb
),
869 offset
= scnprintf(buf
, PAGE_SIZE
,
870 "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
876 #ifdef CONFIG_NVM_DEBUG
877 atomic_read(&rb
->inflight_sync_point
),
881 pblk_rb_read_count(rb
),
883 pblk_rb_sync_point_count(rb
),