]>
Commit | Line | Data |
---|---|---|
86db1e29 | 1 | /* |
4fed947c | 2 | * Functions to sequence FLUSH and FUA writes. |
86db1e29 JA |
3 | */ |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/bio.h> | |
7 | #include <linux/blkdev.h> | |
5a0e3ad6 | 8 | #include <linux/gfp.h> |
86db1e29 JA |
9 | |
10 | #include "blk.h" | |
11 | ||
4fed947c TH |
12 | /* FLUSH/FUA sequences */ |
13 | enum { | |
14 | QUEUE_FSEQ_STARTED = (1 << 0), /* flushing in progress */ | |
15 | QUEUE_FSEQ_PREFLUSH = (1 << 1), /* pre-flushing in progress */ | |
16 | QUEUE_FSEQ_DATA = (1 << 2), /* data write in progress */ | |
17 | QUEUE_FSEQ_POSTFLUSH = (1 << 3), /* post-flushing in progress */ | |
18 | QUEUE_FSEQ_DONE = (1 << 4), | |
19 | }; | |
20 | ||
dd4c133f | 21 | static struct request *queue_next_fseq(struct request_queue *q); |
28e7d184 | 22 | |
dd4c133f | 23 | unsigned blk_flush_cur_seq(struct request_queue *q) |
86db1e29 | 24 | { |
dd4c133f | 25 | if (!q->flush_seq) |
86db1e29 | 26 | return 0; |
dd4c133f | 27 | return 1 << ffz(q->flush_seq); |
86db1e29 JA |
28 | } |
29 | ||
dd4c133f TH |
30 | static struct request *blk_flush_complete_seq(struct request_queue *q, |
31 | unsigned seq, int error) | |
86db1e29 | 32 | { |
28e7d184 | 33 | struct request *next_rq = NULL; |
86db1e29 | 34 | |
dd4c133f TH |
35 | if (error && !q->flush_err) |
36 | q->flush_err = error; | |
86db1e29 | 37 | |
dd4c133f TH |
38 | BUG_ON(q->flush_seq & seq); |
39 | q->flush_seq |= seq; | |
86db1e29 | 40 | |
dd4c133f TH |
41 | if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { |
42 | /* not complete yet, queue the next flush sequence */ | |
43 | next_rq = queue_next_fseq(q); | |
28e7d184 | 44 | } else { |
dd4c133f TH |
45 | /* complete this flush request */ |
46 | __blk_end_request_all(q->orig_flush_rq, q->flush_err); | |
47 | q->orig_flush_rq = NULL; | |
48 | q->flush_seq = 0; | |
49 | ||
50 | /* dispatch the next flush if there's one */ | |
51 | if (!list_empty(&q->pending_flushes)) { | |
52 | next_rq = list_entry_rq(q->pending_flushes.next); | |
28e7d184 TH |
53 | list_move(&next_rq->queuelist, &q->queue_head); |
54 | } | |
55 | } | |
56 | return next_rq; | |
86db1e29 JA |
57 | } |
58 | ||
47f70d5a TH |
59 | static void blk_flush_complete_seq_end_io(struct request_queue *q, |
60 | unsigned seq, int error) | |
61 | { | |
62 | bool was_empty = elv_queue_empty(q); | |
63 | struct request *next_rq; | |
64 | ||
65 | next_rq = blk_flush_complete_seq(q, seq, error); | |
66 | ||
67 | /* | |
68 | * Moving a request silently to empty queue_head may stall the | |
69 | * queue. Kick the queue in those cases. | |
70 | */ | |
71 | if (was_empty && next_rq) | |
72 | __blk_run_queue(q); | |
73 | } | |
74 | ||
86db1e29 JA |
75 | static void pre_flush_end_io(struct request *rq, int error) |
76 | { | |
77 | elv_completed_request(rq->q, rq); | |
47f70d5a | 78 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_PREFLUSH, error); |
86db1e29 JA |
79 | } |
80 | ||
dd4c133f | 81 | static void flush_data_end_io(struct request *rq, int error) |
86db1e29 JA |
82 | { |
83 | elv_completed_request(rq->q, rq); | |
47f70d5a | 84 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_DATA, error); |
86db1e29 JA |
85 | } |
86 | ||
87 | static void post_flush_end_io(struct request *rq, int error) | |
88 | { | |
89 | elv_completed_request(rq->q, rq); | |
47f70d5a | 90 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_POSTFLUSH, error); |
86db1e29 JA |
91 | } |
92 | ||
cde4c406 | 93 | static void init_flush_request(struct request *rq, struct gendisk *disk) |
86db1e29 | 94 | { |
28e18d01 | 95 | rq->cmd_type = REQ_TYPE_FS; |
337238be | 96 | rq->cmd_flags = WRITE_FLUSH; |
cde4c406 | 97 | rq->rq_disk = disk; |
86db1e29 JA |
98 | } |
99 | ||
dd4c133f | 100 | static struct request *queue_next_fseq(struct request_queue *q) |
86db1e29 | 101 | { |
4fed947c | 102 | struct request *orig_rq = q->orig_flush_rq; |
dd4c133f | 103 | struct request *rq = &q->flush_rq; |
86db1e29 | 104 | |
cde4c406 CH |
105 | blk_rq_init(q, rq); |
106 | ||
dd4c133f TH |
107 | switch (blk_flush_cur_seq(q)) { |
108 | case QUEUE_FSEQ_PREFLUSH: | |
cde4c406 CH |
109 | init_flush_request(rq, orig_rq->rq_disk); |
110 | rq->end_io = pre_flush_end_io; | |
28e7d184 | 111 | break; |
dd4c133f | 112 | case QUEUE_FSEQ_DATA: |
4fed947c | 113 | init_request_from_bio(rq, orig_rq->bio); |
09d60c70 TH |
114 | /* |
115 | * orig_rq->rq_disk may be different from | |
116 | * bio->bi_bdev->bd_disk if orig_rq got here through | |
117 | * remapping drivers. Make sure rq->rq_disk points | |
118 | * to the same one as orig_rq. | |
119 | */ | |
120 | rq->rq_disk = orig_rq->rq_disk; | |
4fed947c TH |
121 | rq->cmd_flags &= ~(REQ_FLUSH | REQ_FUA); |
122 | rq->cmd_flags |= orig_rq->cmd_flags & (REQ_FLUSH | REQ_FUA); | |
dd4c133f | 123 | rq->end_io = flush_data_end_io; |
28e7d184 | 124 | break; |
dd4c133f | 125 | case QUEUE_FSEQ_POSTFLUSH: |
cde4c406 CH |
126 | init_flush_request(rq, orig_rq->rq_disk); |
127 | rq->end_io = post_flush_end_io; | |
28e7d184 | 128 | break; |
28e7d184 TH |
129 | default: |
130 | BUG(); | |
131 | } | |
cde4c406 CH |
132 | |
133 | elv_insert(q, rq, ELEVATOR_INSERT_FRONT); | |
dd831006 | 134 | return rq; |
86db1e29 JA |
135 | } |
136 | ||
dd4c133f | 137 | struct request *blk_do_flush(struct request_queue *q, struct request *rq) |
86db1e29 | 138 | { |
4fed947c TH |
139 | unsigned int fflags = q->flush_flags; /* may change, cache it */ |
140 | bool has_flush = fflags & REQ_FLUSH, has_fua = fflags & REQ_FUA; | |
141 | bool do_preflush = has_flush && (rq->cmd_flags & REQ_FLUSH); | |
142 | bool do_postflush = has_flush && !has_fua && (rq->cmd_flags & REQ_FUA); | |
28e7d184 TH |
143 | unsigned skip = 0; |
144 | ||
4fed947c TH |
145 | /* |
146 | * Special case. If there's data but flush is not necessary, | |
147 | * the request can be issued directly. | |
148 | * | |
149 | * Flush w/o data should be able to be issued directly too but | |
150 | * currently some drivers assume that rq->bio contains | |
151 | * non-zero data if it isn't NULL and empty FLUSH requests | |
152 | * getting here usually have bio's without data. | |
153 | */ | |
154 | if (blk_rq_sectors(rq) && !do_preflush && !do_postflush) { | |
155 | rq->cmd_flags &= ~REQ_FLUSH; | |
156 | if (!has_fua) | |
157 | rq->cmd_flags &= ~REQ_FUA; | |
28e7d184 | 158 | return rq; |
4fed947c | 159 | } |
28e7d184 | 160 | |
4fed947c TH |
161 | /* |
162 | * Sequenced flushes can't be processed in parallel. If | |
163 | * another one is already in progress, queue for later | |
164 | * processing. | |
165 | */ | |
dd4c133f | 166 | if (q->flush_seq) { |
dd4c133f | 167 | list_move_tail(&rq->queuelist, &q->pending_flushes); |
28e7d184 TH |
168 | return NULL; |
169 | } | |
170 | ||
86db1e29 | 171 | /* |
dd4c133f | 172 | * Start a new flush sequence |
86db1e29 | 173 | */ |
dd4c133f | 174 | q->flush_err = 0; |
dd4c133f | 175 | q->flush_seq |= QUEUE_FSEQ_STARTED; |
86db1e29 | 176 | |
4fed947c TH |
177 | /* adjust FLUSH/FUA of the original request and stash it away */ |
178 | rq->cmd_flags &= ~REQ_FLUSH; | |
179 | if (!has_fua) | |
180 | rq->cmd_flags &= ~REQ_FUA; | |
28e7d184 | 181 | blk_dequeue_request(rq); |
dd4c133f | 182 | q->orig_flush_rq = rq; |
86db1e29 | 183 | |
4fed947c TH |
184 | /* skip unneded sequences and return the first one */ |
185 | if (!do_preflush) | |
dd4c133f | 186 | skip |= QUEUE_FSEQ_PREFLUSH; |
4fed947c | 187 | if (!blk_rq_sectors(rq)) |
dd4c133f | 188 | skip |= QUEUE_FSEQ_DATA; |
4fed947c | 189 | if (!do_postflush) |
dd4c133f | 190 | skip |= QUEUE_FSEQ_POSTFLUSH; |
dd4c133f | 191 | return blk_flush_complete_seq(q, skip, 0); |
86db1e29 JA |
192 | } |
193 | ||
194 | static void bio_end_empty_barrier(struct bio *bio, int err) | |
195 | { | |
cc66b451 JA |
196 | if (err) { |
197 | if (err == -EOPNOTSUPP) | |
198 | set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); | |
86db1e29 | 199 | clear_bit(BIO_UPTODATE, &bio->bi_flags); |
cc66b451 | 200 | } |
f17e232e DM |
201 | if (bio->bi_private) |
202 | complete(bio->bi_private); | |
203 | bio_put(bio); | |
86db1e29 JA |
204 | } |
205 | ||
206 | /** | |
207 | * blkdev_issue_flush - queue a flush | |
208 | * @bdev: blockdev to issue flush for | |
fbd9b09a | 209 | * @gfp_mask: memory allocation flags (for bio_alloc) |
86db1e29 | 210 | * @error_sector: error sector |
fbd9b09a | 211 | * @flags: BLKDEV_IFL_* flags to control behaviour |
86db1e29 JA |
212 | * |
213 | * Description: | |
214 | * Issue a flush for the block device in question. Caller can supply | |
215 | * room for storing the error offset in case of a flush error, if they | |
f17e232e DM |
216 | * wish to. If WAIT flag is not passed then caller may check only what |
217 | * request was pushed in some internal queue for later handling. | |
86db1e29 | 218 | */ |
fbd9b09a DM |
219 | int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, |
220 | sector_t *error_sector, unsigned long flags) | |
86db1e29 JA |
221 | { |
222 | DECLARE_COMPLETION_ONSTACK(wait); | |
223 | struct request_queue *q; | |
224 | struct bio *bio; | |
fbd9b09a | 225 | int ret = 0; |
86db1e29 JA |
226 | |
227 | if (bdev->bd_disk == NULL) | |
228 | return -ENXIO; | |
229 | ||
230 | q = bdev_get_queue(bdev); | |
231 | if (!q) | |
232 | return -ENXIO; | |
233 | ||
f10d9f61 DC |
234 | /* |
235 | * some block devices may not have their queue correctly set up here | |
236 | * (e.g. loop device without a backing file) and so issuing a flush | |
237 | * here will panic. Ensure there is a request function before issuing | |
238 | * the barrier. | |
239 | */ | |
240 | if (!q->make_request_fn) | |
241 | return -ENXIO; | |
242 | ||
fbd9b09a | 243 | bio = bio_alloc(gfp_mask, 0); |
86db1e29 | 244 | bio->bi_end_io = bio_end_empty_barrier; |
86db1e29 | 245 | bio->bi_bdev = bdev; |
f17e232e DM |
246 | if (test_bit(BLKDEV_WAIT, &flags)) |
247 | bio->bi_private = &wait; | |
86db1e29 | 248 | |
f17e232e DM |
249 | bio_get(bio); |
250 | submit_bio(WRITE_BARRIER, bio); | |
251 | if (test_bit(BLKDEV_WAIT, &flags)) { | |
252 | wait_for_completion(&wait); | |
253 | /* | |
254 | * The driver must store the error location in ->bi_sector, if | |
255 | * it supports it. For non-stacked drivers, this should be | |
256 | * copied from blk_rq_pos(rq). | |
257 | */ | |
258 | if (error_sector) | |
259 | *error_sector = bio->bi_sector; | |
260 | } | |
86db1e29 | 261 | |
cc66b451 JA |
262 | if (bio_flagged(bio, BIO_EOPNOTSUPP)) |
263 | ret = -EOPNOTSUPP; | |
264 | else if (!bio_flagged(bio, BIO_UPTODATE)) | |
86db1e29 JA |
265 | ret = -EIO; |
266 | ||
267 | bio_put(bio); | |
268 | return ret; | |
269 | } | |
86db1e29 | 270 | EXPORT_SYMBOL(blkdev_issue_flush); |