]>
git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - block/blk-flush.c
2 * Functions to sequence FLUSH and FUA writes.
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/blkdev.h>
12 /* FLUSH/FUA sequences */
14 QUEUE_FSEQ_STARTED
= (1 << 0), /* flushing in progress */
15 QUEUE_FSEQ_PREFLUSH
= (1 << 1), /* pre-flushing in progress */
16 QUEUE_FSEQ_DATA
= (1 << 2), /* data write in progress */
17 QUEUE_FSEQ_POSTFLUSH
= (1 << 3), /* post-flushing in progress */
18 QUEUE_FSEQ_DONE
= (1 << 4),
21 static struct request
*queue_next_fseq(struct request_queue
*q
);
23 unsigned blk_flush_cur_seq(struct request_queue
*q
)
27 return 1 << ffz(q
->flush_seq
);
30 static struct request
*blk_flush_complete_seq(struct request_queue
*q
,
31 unsigned seq
, int error
)
33 struct request
*next_rq
= NULL
;
35 if (error
&& !q
->flush_err
)
38 BUG_ON(q
->flush_seq
& seq
);
41 if (blk_flush_cur_seq(q
) != QUEUE_FSEQ_DONE
) {
42 /* not complete yet, queue the next flush sequence */
43 next_rq
= queue_next_fseq(q
);
45 /* complete this flush request */
46 __blk_end_request_all(q
->orig_flush_rq
, q
->flush_err
);
47 q
->orig_flush_rq
= NULL
;
50 /* dispatch the next flush if there's one */
51 if (!list_empty(&q
->pending_flushes
)) {
52 next_rq
= list_entry_rq(q
->pending_flushes
.next
);
53 list_move(&next_rq
->queuelist
, &q
->queue_head
);
59 static void blk_flush_complete_seq_end_io(struct request_queue
*q
,
60 unsigned seq
, int error
)
62 bool was_empty
= elv_queue_empty(q
);
63 struct request
*next_rq
;
65 next_rq
= blk_flush_complete_seq(q
, seq
, error
);
68 * Moving a request silently to empty queue_head may stall the
69 * queue. Kick the queue in those cases.
71 if (was_empty
&& next_rq
)
75 static void pre_flush_end_io(struct request
*rq
, int error
)
77 elv_completed_request(rq
->q
, rq
);
78 blk_flush_complete_seq_end_io(rq
->q
, QUEUE_FSEQ_PREFLUSH
, error
);
81 static void flush_data_end_io(struct request
*rq
, int error
)
83 elv_completed_request(rq
->q
, rq
);
84 blk_flush_complete_seq_end_io(rq
->q
, QUEUE_FSEQ_DATA
, error
);
87 static void post_flush_end_io(struct request
*rq
, int error
)
89 elv_completed_request(rq
->q
, rq
);
90 blk_flush_complete_seq_end_io(rq
->q
, QUEUE_FSEQ_POSTFLUSH
, error
);
93 static void init_flush_request(struct request
*rq
, struct gendisk
*disk
)
95 rq
->cmd_type
= REQ_TYPE_FS
;
96 rq
->cmd_flags
= WRITE_FLUSH
;
100 static struct request
*queue_next_fseq(struct request_queue
*q
)
102 struct request
*orig_rq
= q
->orig_flush_rq
;
103 struct request
*rq
= &q
->flush_rq
;
107 switch (blk_flush_cur_seq(q
)) {
108 case QUEUE_FSEQ_PREFLUSH
:
109 init_flush_request(rq
, orig_rq
->rq_disk
);
110 rq
->end_io
= pre_flush_end_io
;
112 case QUEUE_FSEQ_DATA
:
113 init_request_from_bio(rq
, orig_rq
->bio
);
115 * orig_rq->rq_disk may be different from
116 * bio->bi_bdev->bd_disk if orig_rq got here through
117 * remapping drivers. Make sure rq->rq_disk points
118 * to the same one as orig_rq.
120 rq
->rq_disk
= orig_rq
->rq_disk
;
121 rq
->cmd_flags
&= ~(REQ_FLUSH
| REQ_FUA
);
122 rq
->cmd_flags
|= orig_rq
->cmd_flags
& (REQ_FLUSH
| REQ_FUA
);
123 rq
->end_io
= flush_data_end_io
;
125 case QUEUE_FSEQ_POSTFLUSH
:
126 init_flush_request(rq
, orig_rq
->rq_disk
);
127 rq
->end_io
= post_flush_end_io
;
133 rq
->cmd_flags
|= REQ_FLUSH_SEQ
;
134 elv_insert(q
, rq
, ELEVATOR_INSERT_FRONT
);
138 struct request
*blk_do_flush(struct request_queue
*q
, struct request
*rq
)
140 unsigned int fflags
= q
->flush_flags
; /* may change, cache it */
141 bool has_flush
= fflags
& REQ_FLUSH
, has_fua
= fflags
& REQ_FUA
;
142 bool do_preflush
= has_flush
&& (rq
->cmd_flags
& REQ_FLUSH
);
143 bool do_postflush
= has_flush
&& !has_fua
&& (rq
->cmd_flags
& REQ_FUA
);
147 * Special case. If there's data but flush is not necessary,
148 * the request can be issued directly.
150 * Flush w/o data should be able to be issued directly too but
151 * currently some drivers assume that rq->bio contains
152 * non-zero data if it isn't NULL and empty FLUSH requests
153 * getting here usually have bio's without data.
155 if (blk_rq_sectors(rq
) && !do_preflush
&& !do_postflush
) {
156 rq
->cmd_flags
&= ~REQ_FLUSH
;
158 rq
->cmd_flags
&= ~REQ_FUA
;
163 * Sequenced flushes can't be processed in parallel. If
164 * another one is already in progress, queue for later
168 list_move_tail(&rq
->queuelist
, &q
->pending_flushes
);
173 * Start a new flush sequence
176 q
->flush_seq
|= QUEUE_FSEQ_STARTED
;
178 /* adjust FLUSH/FUA of the original request and stash it away */
179 rq
->cmd_flags
&= ~REQ_FLUSH
;
181 rq
->cmd_flags
&= ~REQ_FUA
;
182 blk_dequeue_request(rq
);
183 q
->orig_flush_rq
= rq
;
185 /* skip unneded sequences and return the first one */
187 skip
|= QUEUE_FSEQ_PREFLUSH
;
188 if (!blk_rq_sectors(rq
))
189 skip
|= QUEUE_FSEQ_DATA
;
191 skip
|= QUEUE_FSEQ_POSTFLUSH
;
192 return blk_flush_complete_seq(q
, skip
, 0);
195 static void bio_end_flush(struct bio
*bio
, int err
)
198 clear_bit(BIO_UPTODATE
, &bio
->bi_flags
);
200 complete(bio
->bi_private
);
205 * blkdev_issue_flush - queue a flush
206 * @bdev: blockdev to issue flush for
207 * @gfp_mask: memory allocation flags (for bio_alloc)
208 * @error_sector: error sector
211 * Issue a flush for the block device in question. Caller can supply
212 * room for storing the error offset in case of a flush error, if they
213 * wish to. If WAIT flag is not passed then caller may check only what
214 * request was pushed in some internal queue for later handling.
216 int blkdev_issue_flush(struct block_device
*bdev
, gfp_t gfp_mask
,
217 sector_t
*error_sector
)
219 DECLARE_COMPLETION_ONSTACK(wait
);
220 struct request_queue
*q
;
224 if (bdev
->bd_disk
== NULL
)
227 q
= bdev_get_queue(bdev
);
232 * some block devices may not have their queue correctly set up here
233 * (e.g. loop device without a backing file) and so issuing a flush
234 * here will panic. Ensure there is a request function before issuing
237 if (!q
->make_request_fn
)
240 bio
= bio_alloc(gfp_mask
, 0);
241 bio
->bi_end_io
= bio_end_flush
;
243 bio
->bi_private
= &wait
;
246 submit_bio(WRITE_FLUSH
, bio
);
247 wait_for_completion(&wait
);
250 * The driver must store the error location in ->bi_sector, if
251 * it supports it. For non-stacked drivers, this should be
252 * copied from blk_rq_pos(rq).
255 *error_sector
= bio
->bi_sector
;
257 if (!bio_flagged(bio
, BIO_UPTODATE
))
263 EXPORT_SYMBOL(blkdev_issue_flush
);