]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame_incremental - include/linux/blkdev.h
projects / mirror_ubuntu-eoan-kernel.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
block: consolidate struct request timestamp fields
[mirror_ubuntu-eoan-kernel.git] / include / linux / blkdev.h
... / ...
CommitLineData
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_BLKDEV_H
3#define _LINUX_BLKDEV_H
4
5#include <linux/sched.h>
6#include <linux/sched/clock.h>
7
8#ifdef CONFIG_BLOCK
9
10#include <linux/major.h>
11#include <linux/genhd.h>
12#include <linux/list.h>
13#include <linux/llist.h>
14#include <linux/timer.h>
15#include <linux/workqueue.h>
16#include <linux/pagemap.h>
17#include <linux/backing-dev-defs.h>
18#include <linux/wait.h>
19#include <linux/mempool.h>
20#include <linux/pfn.h>
21#include <linux/bio.h>
22#include <linux/stringify.h>
23#include <linux/gfp.h>
24#include <linux/bsg.h>
25#include <linux/smp.h>
26#include <linux/rcupdate.h>
27#include <linux/percpu-refcount.h>
28#include <linux/scatterlist.h>
29#include <linux/blkzoned.h>
30#include <linux/seqlock.h>
31#include <linux/u64_stats_sync.h>
32
33struct module;
34struct scsi_ioctl_command;
35
36struct request_queue;
37struct elevator_queue;
38struct blk_trace;
39struct request;
40struct sg_io_hdr;
41struct bsg_job;
42struct blkcg_gq;
43struct blk_flush_queue;
44struct pr_ops;
45struct rq_wb;
46struct blk_queue_stats;
47struct blk_stat_callback;
48
49#define BLKDEV_MIN_RQ 4
50#define BLKDEV_MAX_RQ 128 /* Default maximum */
51
52/* Must be consistent with blk_mq_poll_stats_bkt() */
53#define BLK_MQ_POLL_STATS_BKTS 16
54
55/*
56 * Maximum number of blkcg policies allowed to be registered concurrently.
57 * Defined here to simplify include dependency.
58 */
59#define BLKCG_MAX_POLS 3
60
61typedef void (rq_end_io_fn)(struct request *, blk_status_t);
62
63#define BLK_RL_SYNCFULL (1U << 0)
64#define BLK_RL_ASYNCFULL (1U << 1)
65
66struct request_list {
67 struct request_queue *q; /* the queue this rl belongs to */
68#ifdef CONFIG_BLK_CGROUP
69 struct blkcg_gq *blkg; /* blkg this request pool belongs to */
70#endif
71 /*
72 * count[], starved[], and wait[] are indexed by
73 * BLK_RW_SYNC/BLK_RW_ASYNC
74 */
75 int count[2];
76 int starved[2];
77 mempool_t *rq_pool;
78 wait_queue_head_t wait[2];
79 unsigned int flags;
80};
81
82/*
83 * request flags */
84typedef __u32 __bitwise req_flags_t;
85
86/* elevator knows about this request */
87#define RQF_SORTED ((__force req_flags_t)(1 << 0))
88/* drive already may have started this one */
89#define RQF_STARTED ((__force req_flags_t)(1 << 1))
90/* uses tagged queueing */
91#define RQF_QUEUED ((__force req_flags_t)(1 << 2))
92/* may not be passed by ioscheduler */
93#define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
94/* request for flush sequence */
95#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
96/* merge of different types, fail separately */
97#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
98/* track inflight for MQ */
99#define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
100/* don't call prep for this one */
101#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
102/* set for "ide_preempt" requests and also for requests for which the SCSI
103 "quiesce" state must be ignored. */
104#define RQF_PREEMPT ((__force req_flags_t)(1 << 8))
105/* contains copies of user pages */
106#define RQF_COPY_USER ((__force req_flags_t)(1 << 9))
107/* vaguely specified driver internal error. Ignored by the block layer */
108#define RQF_FAILED ((__force req_flags_t)(1 << 10))
109/* don't warn about errors */
110#define RQF_QUIET ((__force req_flags_t)(1 << 11))
111/* elevator private data attached */
112#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
113/* account I/O stat */
114#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
115/* request came from our alloc pool */
116#define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
117/* runtime pm request */
118#define RQF_PM ((__force req_flags_t)(1 << 15))
119/* on IO scheduler merge hash */
120#define RQF_HASHED ((__force req_flags_t)(1 << 16))
121/* IO stats tracking on */
122#define RQF_STATS ((__force req_flags_t)(1 << 17))
123/* Look at ->special_vec for the actual data payload instead of the
124 bio chain. */
125#define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
126/* The per-zone write lock is held for this request */
127#define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
128/* timeout is expired */
129#define RQF_MQ_TIMEOUT_EXPIRED ((__force req_flags_t)(1 << 20))
130/* already slept for hybrid poll */
131#define RQF_MQ_POLL_SLEPT ((__force req_flags_t)(1 << 21))
132
133/* flags that prevent us from merging requests: */
134#define RQF_NOMERGE_FLAGS \
135 (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
136
137/*
138 * Try to put the fields that are referenced together in the same cacheline.
139 *
140 * If you modify this structure, make sure to update blk_rq_init() and
141 * especially blk_mq_rq_ctx_init() to take care of the added fields.
142 */
143struct request {
144 struct request_queue *q;
145 struct blk_mq_ctx *mq_ctx;
146
147 int cpu;
148 unsigned int cmd_flags; /* op and common flags */
149 req_flags_t rq_flags;
150
151 int internal_tag;
152
153 /* the following two fields are internal, NEVER access directly */
154 unsigned int __data_len; /* total data len */
155 int tag;
156 sector_t __sector; /* sector cursor */
157
158 struct bio *bio;
159 struct bio *biotail;
160
161 struct list_head queuelist;
162
163 /*
164 * The hash is used inside the scheduler, and killed once the
165 * request reaches the dispatch list. The ipi_list is only used
166 * to queue the request for softirq completion, which is long
167 * after the request has been unhashed (and even removed from
168 * the dispatch list).
169 */
170 union {
171 struct hlist_node hash; /* merge hash */
172 struct list_head ipi_list;
173 };
174
175 /*
176 * The rb_node is only used inside the io scheduler, requests
177 * are pruned when moved to the dispatch queue. So let the
178 * completion_data share space with the rb_node.
179 */
180 union {
181 struct rb_node rb_node; /* sort/lookup */
182 struct bio_vec special_vec;
183 void *completion_data;
184 int error_count; /* for legacy drivers, don't use */
185 };
186
187 /*
188 * Three pointers are available for the IO schedulers, if they need
189 * more they have to dynamically allocate it. Flush requests are
190 * never put on the IO scheduler. So let the flush fields share
191 * space with the elevator data.
192 */
193 union {
194 struct {
195 struct io_cq *icq;
196 void *priv[2];
197 } elv;
198
199 struct {
200 unsigned int seq;
201 struct list_head list;
202 rq_end_io_fn *saved_end_io;
203 } flush;
204 };
205
206 struct gendisk *rq_disk;
207 struct hd_struct *part;
208 /* Time that I/O was submitted to the kernel. */
209 u64 start_time_ns;
210 /* Time that I/O was submitted to the device. */
211 u64 io_start_time_ns;
212
213#ifdef CONFIG_BLK_WBT
214 unsigned short wbt_flags;
215#endif
216#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
217 unsigned short throtl_size;
218#endif
219
220 /*
221 * Number of scatter-gather DMA addr+len pairs after
222 * physical address coalescing is performed.
223 */
224 unsigned short nr_phys_segments;
225
226#if defined(CONFIG_BLK_DEV_INTEGRITY)
227 unsigned short nr_integrity_segments;
228#endif
229
230 unsigned short write_hint;
231 unsigned short ioprio;
232
233 unsigned int timeout;
234
235 void *special; /* opaque pointer available for LLD use */
236
237 unsigned int extra_len; /* length of alignment and padding */
238
239 /*
240 * On blk-mq, the lower bits of ->gstate (generation number and
241 * state) carry the MQ_RQ_* state value and the upper bits the
242 * generation number which is monotonically incremented and used to
243 * distinguish the reuse instances.
244 *
245 * ->gstate_seq allows updates to ->gstate and other fields
246 * (currently ->deadline) during request start to be read
247 * atomically from the timeout path, so that it can operate on a
248 * coherent set of information.
249 */
250 seqcount_t gstate_seq;
251 u64 gstate;
252
253 /*
254 * ->aborted_gstate is used by the timeout to claim a specific
255 * recycle instance of this request. See blk_mq_timeout_work().
256 */
257 struct u64_stats_sync aborted_gstate_sync;
258 u64 aborted_gstate;
259
260 /* access through blk_rq_set_deadline, blk_rq_deadline */
261 unsigned long __deadline;
262
263 struct list_head timeout_list;
264
265 union {
266 struct __call_single_data csd;
267 u64 fifo_time;
268 };
269
270 /*
271 * completion callback.
272 */
273 rq_end_io_fn *end_io;
274 void *end_io_data;
275
276 /* for bidi */
277 struct request *next_rq;
278
279#ifdef CONFIG_BLK_CGROUP
280 struct request_list *rl; /* rl this rq is alloced from */
281#endif
282};
283
284static inline bool blk_op_is_scsi(unsigned int op)
285{
286 return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
287}
288
289static inline bool blk_op_is_private(unsigned int op)
290{
291 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
292}
293
294static inline bool blk_rq_is_scsi(struct request *rq)
295{
296 return blk_op_is_scsi(req_op(rq));
297}
298
299static inline bool blk_rq_is_private(struct request *rq)
300{
301 return blk_op_is_private(req_op(rq));
302}
303
304static inline bool blk_rq_is_passthrough(struct request *rq)
305{
306 return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
307}
308
309static inline bool bio_is_passthrough(struct bio *bio)
310{
311 unsigned op = bio_op(bio);
312
313 return blk_op_is_scsi(op) || blk_op_is_private(op);
314}
315
316static inline unsigned short req_get_ioprio(struct request *req)
317{
318 return req->ioprio;
319}
320
321#include <linux/elevator.h>
322
323struct blk_queue_ctx;
324
325typedef void (request_fn_proc) (struct request_queue *q);
326typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
327typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
328typedef int (prep_rq_fn) (struct request_queue *, struct request *);
329typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
330
331struct bio_vec;
332typedef void (softirq_done_fn)(struct request *);
333typedef int (dma_drain_needed_fn)(struct request *);
334typedef int (lld_busy_fn) (struct request_queue *q);
335typedef int (bsg_job_fn) (struct bsg_job *);
336typedef int (init_rq_fn)(struct request_queue *, struct request *, gfp_t);
337typedef void (exit_rq_fn)(struct request_queue *, struct request *);
338
339enum blk_eh_timer_return {
340 BLK_EH_NOT_HANDLED,
341 BLK_EH_HANDLED,
342 BLK_EH_RESET_TIMER,
343};
344
345typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
346
347enum blk_queue_state {
348 Queue_down,
349 Queue_up,
350};
351
352struct blk_queue_tag {
353 struct request **tag_index; /* map of busy tags */
354 unsigned long *tag_map; /* bit map of free/busy tags */
355 int max_depth; /* what we will send to device */
356 int real_max_depth; /* what the array can hold */
357 atomic_t refcnt; /* map can be shared */
358 int alloc_policy; /* tag allocation policy */
359 int next_tag; /* next tag */
360};
361#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
362#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
363
364#define BLK_SCSI_MAX_CMDS (256)
365#define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
366
367/*
368 * Zoned block device models (zoned limit).
369 */
370enum blk_zoned_model {
371 BLK_ZONED_NONE, /* Regular block device */
372 BLK_ZONED_HA, /* Host-aware zoned block device */
373 BLK_ZONED_HM, /* Host-managed zoned block device */
374};
375
376struct queue_limits {
377 unsigned long bounce_pfn;
378 unsigned long seg_boundary_mask;
379 unsigned long virt_boundary_mask;
380
381 unsigned int max_hw_sectors;
382 unsigned int max_dev_sectors;
383 unsigned int chunk_sectors;
384 unsigned int max_sectors;
385 unsigned int max_segment_size;
386 unsigned int physical_block_size;
387 unsigned int alignment_offset;
388 unsigned int io_min;
389 unsigned int io_opt;
390 unsigned int max_discard_sectors;
391 unsigned int max_hw_discard_sectors;
392 unsigned int max_write_same_sectors;
393 unsigned int max_write_zeroes_sectors;
394 unsigned int discard_granularity;
395 unsigned int discard_alignment;
396
397 unsigned short logical_block_size;
398 unsigned short max_segments;
399 unsigned short max_integrity_segments;
400 unsigned short max_discard_segments;
401
402 unsigned char misaligned;
403 unsigned char discard_misaligned;
404 unsigned char cluster;
405 unsigned char raid_partial_stripes_expensive;
406 enum blk_zoned_model zoned;
407};
408
409#ifdef CONFIG_BLK_DEV_ZONED
410
411struct blk_zone_report_hdr {
412 unsigned int nr_zones;
413 u8 padding[60];
414};
415
416extern int blkdev_report_zones(struct block_device *bdev,
417 sector_t sector, struct blk_zone *zones,
418 unsigned int *nr_zones, gfp_t gfp_mask);
419extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
420 sector_t nr_sectors, gfp_t gfp_mask);
421
422extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
423 unsigned int cmd, unsigned long arg);
424extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
425 unsigned int cmd, unsigned long arg);
426
427#else /* CONFIG_BLK_DEV_ZONED */
428
429static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
430 fmode_t mode, unsigned int cmd,
431 unsigned long arg)
432{
433 return -ENOTTY;
434}
435
436static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
437 fmode_t mode, unsigned int cmd,
438 unsigned long arg)
439{
440 return -ENOTTY;
441}
442
443#endif /* CONFIG_BLK_DEV_ZONED */
444
445struct request_queue {
446 /*
447 * Together with queue_head for cacheline sharing
448 */
449 struct list_head queue_head;
450 struct request *last_merge;
451 struct elevator_queue *elevator;
452 int nr_rqs[2]; /* # allocated [a]sync rqs */
453 int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */
454
455 atomic_t shared_hctx_restart;
456
457 struct blk_queue_stats *stats;
458 struct rq_wb *rq_wb;
459
460 /*
461 * If blkcg is not used, @q->root_rl serves all requests. If blkcg
462 * is used, root blkg allocates from @q->root_rl and all other
463 * blkgs from their own blkg->rl. Which one to use should be
464 * determined using bio_request_list().
465 */
466 struct request_list root_rl;
467
468 request_fn_proc *request_fn;
469 make_request_fn *make_request_fn;
470 poll_q_fn *poll_fn;
471 prep_rq_fn *prep_rq_fn;
472 unprep_rq_fn *unprep_rq_fn;
473 softirq_done_fn *softirq_done_fn;
474 rq_timed_out_fn *rq_timed_out_fn;
475 dma_drain_needed_fn *dma_drain_needed;
476 lld_busy_fn *lld_busy_fn;
477 /* Called just after a request is allocated */
478 init_rq_fn *init_rq_fn;
479 /* Called just before a request is freed */
480 exit_rq_fn *exit_rq_fn;
481 /* Called from inside blk_get_request() */
482 void (*initialize_rq_fn)(struct request *rq);
483
484 const struct blk_mq_ops *mq_ops;
485
486 unsigned int *mq_map;
487
488 /* sw queues */
489 struct blk_mq_ctx __percpu *queue_ctx;
490 unsigned int nr_queues;
491
492 unsigned int queue_depth;
493
494 /* hw dispatch queues */
495 struct blk_mq_hw_ctx **queue_hw_ctx;
496 unsigned int nr_hw_queues;
497
498 /*
499 * Dispatch queue sorting
500 */
501 sector_t end_sector;
502 struct request *boundary_rq;
503
504 /*
505 * Delayed queue handling
506 */
507 struct delayed_work delay_work;
508
509 struct backing_dev_info *backing_dev_info;
510
511 /*
512 * The queue owner gets to use this for whatever they like.
513 * ll_rw_blk doesn't touch it.
514 */
515 void *queuedata;
516
517 /*
518 * various queue flags, see QUEUE_* below
519 */
520 unsigned long queue_flags;
521
522 /*
523 * ida allocated id for this queue. Used to index queues from
524 * ioctx.
525 */
526 int id;
527
528 /*
529 * queue needs bounce pages for pages above this limit
530 */
531 gfp_t bounce_gfp;
532
533 /*
534 * protects queue structures from reentrancy. ->__queue_lock should
535 * _never_ be used directly, it is queue private. always use
536 * ->queue_lock.
537 */
538 spinlock_t __queue_lock;
539 spinlock_t *queue_lock;
540
541 /*
542 * queue kobject
543 */
544 struct kobject kobj;
545
546 /*
547 * mq queue kobject
548 */
549 struct kobject mq_kobj;
550
551#ifdef CONFIG_BLK_DEV_INTEGRITY
552 struct blk_integrity integrity;
553#endif /* CONFIG_BLK_DEV_INTEGRITY */
554
555#ifdef CONFIG_PM
556 struct device *dev;
557 int rpm_status;
558 unsigned int nr_pending;
559#endif
560
561 /*
562 * queue settings
563 */
564 unsigned long nr_requests; /* Max # of requests */
565 unsigned int nr_congestion_on;
566 unsigned int nr_congestion_off;
567 unsigned int nr_batching;
568
569 unsigned int dma_drain_size;
570 void *dma_drain_buffer;
571 unsigned int dma_pad_mask;
572 unsigned int dma_alignment;
573
574 struct blk_queue_tag *queue_tags;
575 struct list_head tag_busy_list;
576
577 unsigned int nr_sorted;
578 unsigned int in_flight[2];
579
580 /*
581 * Number of active block driver functions for which blk_drain_queue()
582 * must wait. Must be incremented around functions that unlock the
583 * queue_lock internally, e.g. scsi_request_fn().
584 */
585 unsigned int request_fn_active;
586
587 unsigned int rq_timeout;
588 int poll_nsec;
589
590 struct blk_stat_callback *poll_cb;
591 struct blk_rq_stat poll_stat[BLK_MQ_POLL_STATS_BKTS];
592
593 struct timer_list timeout;
594 struct work_struct timeout_work;
595 struct list_head timeout_list;
596
597 struct list_head icq_list;
598#ifdef CONFIG_BLK_CGROUP
599 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
600 struct blkcg_gq *root_blkg;
601 struct list_head blkg_list;
602#endif
603
604 struct queue_limits limits;
605
606 /*
607 * Zoned block device information for request dispatch control.
608 * nr_zones is the total number of zones of the device. This is always
609 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
610 * bits which indicates if a zone is conventional (bit clear) or
611 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
612 * bits which indicates if a zone is write locked, that is, if a write
613 * request targeting the zone was dispatched. All three fields are
614 * initialized by the low level device driver (e.g. scsi/sd.c).
615 * Stacking drivers (device mappers) may or may not initialize
616 * these fields.
617 *
618 * Reads of this information must be protected with blk_queue_enter() /
619 * blk_queue_exit(). Modifying this information is only allowed while
620 * no requests are being processed. See also blk_mq_freeze_queue() and
621 * blk_mq_unfreeze_queue().
622 */
623 unsigned int nr_zones;
624 unsigned long *seq_zones_bitmap;
625 unsigned long *seq_zones_wlock;
626
627 /*
628 * sg stuff
629 */
630 unsigned int sg_timeout;
631 unsigned int sg_reserved_size;
632 int node;
633#ifdef CONFIG_BLK_DEV_IO_TRACE
634 struct blk_trace *blk_trace;
635 struct mutex blk_trace_mutex;
636#endif
637 /*
638 * for flush operations
639 */
640 struct blk_flush_queue *fq;
641
642 struct list_head requeue_list;
643 spinlock_t requeue_lock;
644 struct delayed_work requeue_work;
645
646 struct mutex sysfs_lock;
647
648 int bypass_depth;
649 atomic_t mq_freeze_depth;
650
651#if defined(CONFIG_BLK_DEV_BSG)
652 bsg_job_fn *bsg_job_fn;
653 struct bsg_class_device bsg_dev;
654#endif
655
656#ifdef CONFIG_BLK_DEV_THROTTLING
657 /* Throttle data */
658 struct throtl_data *td;
659#endif
660 struct rcu_head rcu_head;
661 wait_queue_head_t mq_freeze_wq;
662 struct percpu_ref q_usage_counter;
663 struct list_head all_q_node;
664
665 struct blk_mq_tag_set *tag_set;
666 struct list_head tag_set_list;
667 struct bio_set *bio_split;
668
669#ifdef CONFIG_BLK_DEBUG_FS
670 struct dentry *debugfs_dir;
671 struct dentry *sched_debugfs_dir;
672#endif
673
674 bool mq_sysfs_init_done;
675
676 size_t cmd_size;
677 void *rq_alloc_data;
678
679 struct work_struct release_work;
680
681#define BLK_MAX_WRITE_HINTS 5
682 u64 write_hints[BLK_MAX_WRITE_HINTS];
683};
684
685#define QUEUE_FLAG_QUEUED 0 /* uses generic tag queueing */
686#define QUEUE_FLAG_STOPPED 1 /* queue is stopped */
687#define QUEUE_FLAG_DYING 2 /* queue being torn down */
688#define QUEUE_FLAG_BYPASS 3 /* act as dumb FIFO queue */
689#define QUEUE_FLAG_BIDI 4 /* queue supports bidi requests */
690#define QUEUE_FLAG_NOMERGES 5 /* disable merge attempts */
691#define QUEUE_FLAG_SAME_COMP 6 /* complete on same CPU-group */
692#define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */
693#define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */
694#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
695#define QUEUE_FLAG_IO_STAT 10 /* do IO stats */
696#define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */
697#define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */
698#define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */
699#define QUEUE_FLAG_SECERASE 14 /* supports secure erase */
700#define QUEUE_FLAG_SAME_FORCE 15 /* force complete on same CPU */
701#define QUEUE_FLAG_DEAD 16 /* queue tear-down finished */
702#define QUEUE_FLAG_INIT_DONE 17 /* queue is initialized */
703#define QUEUE_FLAG_NO_SG_MERGE 18 /* don't attempt to merge SG segments*/
704#define QUEUE_FLAG_POLL 19 /* IO polling enabled if set */
705#define QUEUE_FLAG_WC 20 /* Write back caching */
706#define QUEUE_FLAG_FUA 21 /* device supports FUA writes */
707#define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */
708#define QUEUE_FLAG_DAX 23 /* device supports DAX */
709#define QUEUE_FLAG_STATS 24 /* track rq completion times */
710#define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */
711#define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */
712#define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */
713#define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */
714#define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */
715
716#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
717 (1 << QUEUE_FLAG_SAME_COMP) | \
718 (1 << QUEUE_FLAG_ADD_RANDOM))
719
720#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
721 (1 << QUEUE_FLAG_SAME_COMP) | \
722 (1 << QUEUE_FLAG_POLL))
723
724void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
725void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
726bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
727bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
728
729#define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
730#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
731#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
732#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
733#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
734#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
735#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
736#define blk_queue_noxmerges(q) \
737 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
738#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
739#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
740#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
741#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
742#define blk_queue_secure_erase(q) \
743 (test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
744#define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
745#define blk_queue_scsi_passthrough(q) \
746 test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
747
748#define blk_noretry_request(rq) \
749 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
750 REQ_FAILFAST_DRIVER))
751#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
752#define blk_queue_preempt_only(q) \
753 test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)
754#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
755
756extern int blk_set_preempt_only(struct request_queue *q);
757extern void blk_clear_preempt_only(struct request_queue *q);
758
759static inline int queue_in_flight(struct request_queue *q)
760{
761 return q->in_flight[0] + q->in_flight[1];
762}
763
764static inline bool blk_account_rq(struct request *rq)
765{
766 return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
767}
768
769#define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
770#define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
771/* rq->queuelist of dequeued request must be list_empty() */
772#define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
773
774#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
775
776#define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
777
778/*
779 * Driver can handle struct request, if it either has an old style
780 * request_fn defined, or is blk-mq based.
781 */
782static inline bool queue_is_rq_based(struct request_queue *q)
783{
784 return q->request_fn || q->mq_ops;
785}
786
787static inline unsigned int blk_queue_cluster(struct request_queue *q)
788{
789 return q->limits.cluster;
790}
791
792static inline enum blk_zoned_model
793blk_queue_zoned_model(struct request_queue *q)
794{
795 return q->limits.zoned;
796}
797
798static inline bool blk_queue_is_zoned(struct request_queue *q)
799{
800 switch (blk_queue_zoned_model(q)) {
801 case BLK_ZONED_HA:
802 case BLK_ZONED_HM:
803 return true;
804 default:
805 return false;
806 }
807}
808
809static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
810{
811 return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
812}
813
814static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
815{
816 return q->nr_zones;
817}
818
819static inline unsigned int blk_queue_zone_no(struct request_queue *q,
820 sector_t sector)
821{
822 if (!blk_queue_is_zoned(q))
823 return 0;
824 return sector >> ilog2(q->limits.chunk_sectors);
825}
826
827static inline bool blk_queue_zone_is_seq(struct request_queue *q,
828 sector_t sector)
829{
830 if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
831 return false;
832 return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
833}
834
835static inline bool rq_is_sync(struct request *rq)
836{
837 return op_is_sync(rq->cmd_flags);
838}
839
840static inline bool blk_rl_full(struct request_list *rl, bool sync)
841{
842 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
843
844 return rl->flags & flag;
845}
846
847static inline void blk_set_rl_full(struct request_list *rl, bool sync)
848{
849 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
850
851 rl->flags |= flag;
852}
853
854static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
855{
856 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
857
858 rl->flags &= ~flag;
859}
860
861static inline bool rq_mergeable(struct request *rq)
862{
863 if (blk_rq_is_passthrough(rq))
864 return false;
865
866 if (req_op(rq) == REQ_OP_FLUSH)
867 return false;
868
869 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
870 return false;
871
872 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
873 return false;
874 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
875 return false;
876
877 return true;
878}
879
880static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
881{
882 if (bio_page(a) == bio_page(b) &&
883 bio_offset(a) == bio_offset(b))
884 return true;
885
886 return false;
887}
888
889static inline unsigned int blk_queue_depth(struct request_queue *q)
890{
891 if (q->queue_depth)
892 return q->queue_depth;
893
894 return q->nr_requests;
895}
896
897/*
898 * q->prep_rq_fn return values
899 */
900enum {
901 BLKPREP_OK, /* serve it */
902 BLKPREP_KILL, /* fatal error, kill, return -EIO */
903 BLKPREP_DEFER, /* leave on queue */
904 BLKPREP_INVALID, /* invalid command, kill, return -EREMOTEIO */
905};
906
907extern unsigned long blk_max_low_pfn, blk_max_pfn;
908
909/*
910 * standard bounce addresses:
911 *
912 * BLK_BOUNCE_HIGH : bounce all highmem pages
913 * BLK_BOUNCE_ANY : don't bounce anything
914 * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary
915 */
916
917#if BITS_PER_LONG == 32
918#define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
919#else
920#define BLK_BOUNCE_HIGH -1ULL
921#endif
922#define BLK_BOUNCE_ANY (-1ULL)
923#define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
924
925/*
926 * default timeout for SG_IO if none specified
927 */
928#define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
929#define BLK_MIN_SG_TIMEOUT (7 * HZ)
930
931struct rq_map_data {
932 struct page **pages;
933 int page_order;
934 int nr_entries;
935 unsigned long offset;
936 int null_mapped;
937 int from_user;
938};
939
940struct req_iterator {
941 struct bvec_iter iter;
942 struct bio *bio;
943};
944
945/* This should not be used directly - use rq_for_each_segment */
946#define for_each_bio(_bio) \
947 for (; _bio; _bio = _bio->bi_next)
948#define __rq_for_each_bio(_bio, rq) \
949 if ((rq->bio)) \
950 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
951
952#define rq_for_each_segment(bvl, _rq, _iter) \
953 __rq_for_each_bio(_iter.bio, _rq) \
954 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
955
956#define rq_iter_last(bvec, _iter) \
957 (_iter.bio->bi_next == NULL && \
958 bio_iter_last(bvec, _iter.iter))
959
960#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
961# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
962#endif
963#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
964extern void rq_flush_dcache_pages(struct request *rq);
965#else
966static inline void rq_flush_dcache_pages(struct request *rq)
967{
968}
969#endif
970
971extern int blk_register_queue(struct gendisk *disk);
972extern void blk_unregister_queue(struct gendisk *disk);
973extern blk_qc_t generic_make_request(struct bio *bio);
974extern blk_qc_t direct_make_request(struct bio *bio);
975extern void blk_rq_init(struct request_queue *q, struct request *rq);
976extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
977extern void blk_put_request(struct request *);
978extern void __blk_put_request(struct request_queue *, struct request *);
979extern struct request *blk_get_request_flags(struct request_queue *,
980 unsigned int op,
981 blk_mq_req_flags_t flags);
982extern struct request *blk_get_request(struct request_queue *, unsigned int op,
983 gfp_t gfp_mask);
984extern void blk_requeue_request(struct request_queue *, struct request *);
985extern int blk_lld_busy(struct request_queue *q);
986extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
987 struct bio_set *bs, gfp_t gfp_mask,
988 int (*bio_ctr)(struct bio *, struct bio *, void *),
989 void *data);
990extern void blk_rq_unprep_clone(struct request *rq);
991extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
992 struct request *rq);
993extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
994extern void blk_delay_queue(struct request_queue *, unsigned long);
995extern void blk_queue_split(struct request_queue *, struct bio **);
996extern void blk_recount_segments(struct request_queue *, struct bio *);
997extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
998extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
999 unsigned int, void __user *);
1000extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
1001 unsigned int, void __user *);
1002extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
1003 struct scsi_ioctl_command __user *);
1004
1005extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
1006extern void blk_queue_exit(struct request_queue *q);
1007extern void blk_start_queue(struct request_queue *q);
1008extern void blk_start_queue_async(struct request_queue *q);
1009extern void blk_stop_queue(struct request_queue *q);
1010extern void blk_sync_queue(struct request_queue *q);
1011extern void __blk_stop_queue(struct request_queue *q);
1012extern void __blk_run_queue(struct request_queue *q);
1013extern void __blk_run_queue_uncond(struct request_queue *q);
1014extern void blk_run_queue(struct request_queue *);
1015extern void blk_run_queue_async(struct request_queue *q);
1016extern int blk_rq_map_user(struct request_queue *, struct request *,
1017 struct rq_map_data *, void __user *, unsigned long,
1018 gfp_t);
1019extern int blk_rq_unmap_user(struct bio *);
1020extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
1021extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
1022 struct rq_map_data *, const struct iov_iter *,
1023 gfp_t);
1024extern void blk_execute_rq(struct request_queue *, struct gendisk *,
1025 struct request *, int);
1026extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
1027 struct request *, int, rq_end_io_fn *);
1028
1029int blk_status_to_errno(blk_status_t status);
1030blk_status_t errno_to_blk_status(int errno);
1031
1032bool blk_poll(struct request_queue *q, blk_qc_t cookie);
1033
1034static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
1035{
1036 return bdev->bd_disk->queue; /* this is never NULL */
1037}
1038
1039/*
1040 * The basic unit of block I/O is a sector. It is used in a number of contexts
1041 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
1042 * bytes. Variables of type sector_t represent an offset or size that is a
1043 * multiple of 512 bytes. Hence these two constants.
1044 */
1045#ifndef SECTOR_SHIFT
1046#define SECTOR_SHIFT 9
1047#endif
1048#ifndef SECTOR_SIZE
1049#define SECTOR_SIZE (1 << SECTOR_SHIFT)
1050#endif
1051
1052/*
1053 * blk_rq_pos() : the current sector
1054 * blk_rq_bytes() : bytes left in the entire request
1055 * blk_rq_cur_bytes() : bytes left in the current segment
1056 * blk_rq_err_bytes() : bytes left till the next error boundary
1057 * blk_rq_sectors() : sectors left in the entire request
1058 * blk_rq_cur_sectors() : sectors left in the current segment
1059 */
1060static inline sector_t blk_rq_pos(const struct request *rq)
1061{
1062 return rq->__sector;
1063}
1064
1065static inline unsigned int blk_rq_bytes(const struct request *rq)
1066{
1067 return rq->__data_len;
1068}
1069
1070static inline int blk_rq_cur_bytes(const struct request *rq)
1071{
1072 return rq->bio ? bio_cur_bytes(rq->bio) : 0;
1073}
1074
1075extern unsigned int blk_rq_err_bytes(const struct request *rq);
1076
1077static inline unsigned int blk_rq_sectors(const struct request *rq)
1078{
1079 return blk_rq_bytes(rq) >> SECTOR_SHIFT;
1080}
1081
1082static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
1083{
1084 return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
1085}
1086
1087static inline unsigned int blk_rq_zone_no(struct request *rq)
1088{
1089 return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
1090}
1091
1092static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
1093{
1094 return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
1095}
1096
1097/*
1098 * Some commands like WRITE SAME have a payload or data transfer size which
1099 * is different from the size of the request. Any driver that supports such
1100 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
1101 * calculate the data transfer size.
1102 */
1103static inline unsigned int blk_rq_payload_bytes(struct request *rq)
1104{
1105 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1106 return rq->special_vec.bv_len;
1107 return blk_rq_bytes(rq);
1108}
1109
1110static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1111 int op)
1112{
1113 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
1114 return min(q->limits.max_discard_sectors,
1115 UINT_MAX >> SECTOR_SHIFT);
1116
1117 if (unlikely(op == REQ_OP_WRITE_SAME))
1118 return q->limits.max_write_same_sectors;
1119
1120 if (unlikely(op == REQ_OP_WRITE_ZEROES))
1121 return q->limits.max_write_zeroes_sectors;
1122
1123 return q->limits.max_sectors;
1124}
1125
1126/*
1127 * Return maximum size of a request at given offset. Only valid for
1128 * file system requests.
1129 */
1130static inline unsigned int blk_max_size_offset(struct request_queue *q,
1131 sector_t offset)
1132{
1133 if (!q->limits.chunk_sectors)
1134 return q->limits.max_sectors;
1135
1136 return q->limits.chunk_sectors -
1137 (offset & (q->limits.chunk_sectors - 1));
1138}
1139
1140static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
1141 sector_t offset)
1142{
1143 struct request_queue *q = rq->q;
1144
1145 if (blk_rq_is_passthrough(rq))
1146 return q->limits.max_hw_sectors;
1147
1148 if (!q->limits.chunk_sectors ||
1149 req_op(rq) == REQ_OP_DISCARD ||
1150 req_op(rq) == REQ_OP_SECURE_ERASE)
1151 return blk_queue_get_max_sectors(q, req_op(rq));
1152
1153 return min(blk_max_size_offset(q, offset),
1154 blk_queue_get_max_sectors(q, req_op(rq)));
1155}
1156
1157static inline unsigned int blk_rq_count_bios(struct request *rq)
1158{
1159 unsigned int nr_bios = 0;
1160 struct bio *bio;
1161
1162 __rq_for_each_bio(bio, rq)
1163 nr_bios++;
1164
1165 return nr_bios;
1166}
1167
1168/*
1169 * Request issue related functions.
1170 */
1171extern struct request *blk_peek_request(struct request_queue *q);
1172extern void blk_start_request(struct request *rq);
1173extern struct request *blk_fetch_request(struct request_queue *q);
1174
1175void blk_steal_bios(struct bio_list *list, struct request *rq);
1176
1177/*
1178 * Request completion related functions.
1179 *
1180 * blk_update_request() completes given number of bytes and updates
1181 * the request without completing it.
1182 *
1183 * blk_end_request() and friends. __blk_end_request() must be called
1184 * with the request queue spinlock acquired.
1185 *
1186 * Several drivers define their own end_request and call
1187 * blk_end_request() for parts of the original function.
1188 * This prevents code duplication in drivers.
1189 */
1190extern bool blk_update_request(struct request *rq, blk_status_t error,
1191 unsigned int nr_bytes);
1192extern void blk_finish_request(struct request *rq, blk_status_t error);
1193extern bool blk_end_request(struct request *rq, blk_status_t error,
1194 unsigned int nr_bytes);
1195extern void blk_end_request_all(struct request *rq, blk_status_t error);
1196extern bool __blk_end_request(struct request *rq, blk_status_t error,
1197 unsigned int nr_bytes);
1198extern void __blk_end_request_all(struct request *rq, blk_status_t error);
1199extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1200
1201extern void blk_complete_request(struct request *);
1202extern void __blk_complete_request(struct request *);
1203extern void blk_abort_request(struct request *);
1204extern void blk_unprep_request(struct request *);
1205
1206/*
1207 * Access functions for manipulating queue properties
1208 */
1209extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1210 spinlock_t *lock, int node_id);
1211extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
1212extern int blk_init_allocated_queue(struct request_queue *);
1213extern void blk_cleanup_queue(struct request_queue *);
1214extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
1215extern void blk_queue_bounce_limit(struct request_queue *, u64);
1216extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1217extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1218extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1219extern void blk_queue_max_discard_segments(struct request_queue *,
1220 unsigned short);
1221extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1222extern void blk_queue_max_discard_sectors(struct request_queue *q,
1223 unsigned int max_discard_sectors);
1224extern void blk_queue_max_write_same_sectors(struct request_queue *q,
1225 unsigned int max_write_same_sectors);
1226extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
1227 unsigned int max_write_same_sectors);
1228extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
1229extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1230extern void blk_queue_alignment_offset(struct request_queue *q,
1231 unsigned int alignment);
1232extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1233extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1234extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1235extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1236extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1237extern void blk_set_default_limits(struct queue_limits *lim);
1238extern void blk_set_stacking_limits(struct queue_limits *lim);
1239extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1240 sector_t offset);
1241extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
1242 sector_t offset);
1243extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
1244 sector_t offset);
1245extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
1246extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
1247extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1248extern int blk_queue_dma_drain(struct request_queue *q,
1249 dma_drain_needed_fn *dma_drain_needed,
1250 void *buf, unsigned int size);
1251extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
1252extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1253extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1254extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1255extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1256extern void blk_queue_dma_alignment(struct request_queue *, int);
1257extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1258extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
1259extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
1260extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1261extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1262extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
1263
1264/*
1265 * Number of physical segments as sent to the device.
1266 *
1267 * Normally this is the number of discontiguous data segments sent by the
1268 * submitter. But for data-less command like discard we might have no
1269 * actual data segments submitted, but the driver might have to add it's
1270 * own special payload. In that case we still return 1 here so that this
1271 * special payload will be mapped.
1272 */
1273static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
1274{
1275 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1276 return 1;
1277 return rq->nr_phys_segments;
1278}
1279
1280/*
1281 * Number of discard segments (or ranges) the driver needs to fill in.
1282 * Each discard bio merged into a request is counted as one segment.
1283 */
1284static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
1285{
1286 return max_t(unsigned short, rq->nr_phys_segments, 1);
1287}
1288
1289extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
1290extern void blk_dump_rq_flags(struct request *, char *);
1291extern long nr_blockdev_pages(void);
1292
1293bool __must_check blk_get_queue(struct request_queue *);
1294struct request_queue *blk_alloc_queue(gfp_t);
1295struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1296 spinlock_t *lock);
1297extern void blk_put_queue(struct request_queue *);
1298extern void blk_set_queue_dying(struct request_queue *);
1299
1300/*
1301 * block layer runtime pm functions
1302 */
1303#ifdef CONFIG_PM
1304extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
1305extern int blk_pre_runtime_suspend(struct request_queue *q);
1306extern void blk_post_runtime_suspend(struct request_queue *q, int err);
1307extern void blk_pre_runtime_resume(struct request_queue *q);
1308extern void blk_post_runtime_resume(struct request_queue *q, int err);
1309extern void blk_set_runtime_active(struct request_queue *q);
1310#else
1311static inline void blk_pm_runtime_init(struct request_queue *q,
1312 struct device *dev) {}
1313static inline int blk_pre_runtime_suspend(struct request_queue *q)
1314{
1315 return -ENOSYS;
1316}
1317static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
1318static inline void blk_pre_runtime_resume(struct request_queue *q) {}
1319static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1320static inline void blk_set_runtime_active(struct request_queue *q) {}
1321#endif
1322
1323/*
1324 * blk_plug permits building a queue of related requests by holding the I/O
1325 * fragments for a short period. This allows merging of sequential requests
1326 * into single larger request. As the requests are moved from a per-task list to
1327 * the device's request_queue in a batch, this results in improved scalability
1328 * as the lock contention for request_queue lock is reduced.
1329 *
1330 * It is ok not to disable preemption when adding the request to the plug list
1331 * or when attempting a merge, because blk_schedule_flush_list() will only flush
1332 * the plug list when the task sleeps by itself. For details, please see
1333 * schedule() where blk_schedule_flush_plug() is called.
1334 */
1335struct blk_plug {
1336 struct list_head list; /* requests */
1337 struct list_head mq_list; /* blk-mq requests */
1338 struct list_head cb_list; /* md requires an unplug callback */
1339};
1340#define BLK_MAX_REQUEST_COUNT 16
1341#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1342
1343struct blk_plug_cb;
1344typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1345struct blk_plug_cb {
1346 struct list_head list;
1347 blk_plug_cb_fn callback;
1348 void *data;
1349};
1350extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1351 void *data, int size);
1352extern void blk_start_plug(struct blk_plug *);
1353extern void blk_finish_plug(struct blk_plug *);
1354extern void blk_flush_plug_list(struct blk_plug *, bool);
1355
1356static inline void blk_flush_plug(struct task_struct *tsk)
1357{
1358 struct blk_plug *plug = tsk->plug;
1359
1360 if (plug)
1361 blk_flush_plug_list(plug, false);
1362}
1363
1364static inline void blk_schedule_flush_plug(struct task_struct *tsk)
1365{
1366 struct blk_plug *plug = tsk->plug;
1367
1368 if (plug)
1369 blk_flush_plug_list(plug, true);
1370}
1371
1372static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1373{
1374 struct blk_plug *plug = tsk->plug;
1375
1376 return plug &&
1377 (!list_empty(&plug->list) ||
1378 !list_empty(&plug->mq_list) ||
1379 !list_empty(&plug->cb_list));
1380}
1381
1382/*
1383 * tag stuff
1384 */
1385extern int blk_queue_start_tag(struct request_queue *, struct request *);
1386extern struct request *blk_queue_find_tag(struct request_queue *, int);
1387extern void blk_queue_end_tag(struct request_queue *, struct request *);
1388extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
1389extern void blk_queue_free_tags(struct request_queue *);
1390extern int blk_queue_resize_tags(struct request_queue *, int);
1391extern void blk_queue_invalidate_tags(struct request_queue *);
1392extern struct blk_queue_tag *blk_init_tags(int, int);
1393extern void blk_free_tags(struct blk_queue_tag *);
1394
1395static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
1396 int tag)
1397{
1398 if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
1399 return NULL;
1400 return bqt->tag_index[tag];
1401}
1402
1403extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
1404extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1405 sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1406
1407#define BLKDEV_DISCARD_SECURE (1 << 0) /* issue a secure erase */
1408
1409extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1410 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1411extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1412 sector_t nr_sects, gfp_t gfp_mask, int flags,
1413 struct bio **biop);
1414
1415#define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1416#define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1417
1418extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1419 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1420 unsigned flags);
1421extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1422 sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1423
1424static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1425 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1426{
1427 return blkdev_issue_discard(sb->s_bdev,
1428 block << (sb->s_blocksize_bits -
1429 SECTOR_SHIFT),
1430 nr_blocks << (sb->s_blocksize_bits -
1431 SECTOR_SHIFT),
1432 gfp_mask, flags);
1433}
1434static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1435 sector_t nr_blocks, gfp_t gfp_mask)
1436{
1437 return blkdev_issue_zeroout(sb->s_bdev,
1438 block << (sb->s_blocksize_bits -
1439 SECTOR_SHIFT),
1440 nr_blocks << (sb->s_blocksize_bits -
1441 SECTOR_SHIFT),
1442 gfp_mask, 0);
1443}
1444
1445extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1446
1447enum blk_default_limits {
1448 BLK_MAX_SEGMENTS = 128,
1449 BLK_SAFE_MAX_SECTORS = 255,
1450 BLK_DEF_MAX_SECTORS = 2560,
1451 BLK_MAX_SEGMENT_SIZE = 65536,
1452 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1453};
1454
1455#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
1456
1457static inline unsigned long queue_segment_boundary(struct request_queue *q)
1458{
1459 return q->limits.seg_boundary_mask;
1460}
1461
1462static inline unsigned long queue_virt_boundary(struct request_queue *q)
1463{
1464 return q->limits.virt_boundary_mask;
1465}
1466
1467static inline unsigned int queue_max_sectors(struct request_queue *q)
1468{
1469 return q->limits.max_sectors;
1470}
1471
1472static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
1473{
1474 return q->limits.max_hw_sectors;
1475}
1476
1477static inline unsigned short queue_max_segments(struct request_queue *q)
1478{
1479 return q->limits.max_segments;
1480}
1481
1482static inline unsigned short queue_max_discard_segments(struct request_queue *q)
1483{
1484 return q->limits.max_discard_segments;
1485}
1486
1487static inline unsigned int queue_max_segment_size(struct request_queue *q)
1488{
1489 return q->limits.max_segment_size;
1490}
1491
1492static inline unsigned short queue_logical_block_size(struct request_queue *q)
1493{
1494 int retval = 512;
1495
1496 if (q && q->limits.logical_block_size)
1497 retval = q->limits.logical_block_size;
1498
1499 return retval;
1500}
1501
1502static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
1503{
1504 return queue_logical_block_size(bdev_get_queue(bdev));
1505}
1506
1507static inline unsigned int queue_physical_block_size(struct request_queue *q)
1508{
1509 return q->limits.physical_block_size;
1510}
1511
1512static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1513{
1514 return queue_physical_block_size(bdev_get_queue(bdev));
1515}
1516
1517static inline unsigned int queue_io_min(struct request_queue *q)
1518{
1519 return q->limits.io_min;
1520}
1521
1522static inline int bdev_io_min(struct block_device *bdev)
1523{
1524 return queue_io_min(bdev_get_queue(bdev));
1525}
1526
1527static inline unsigned int queue_io_opt(struct request_queue *q)
1528{
1529 return q->limits.io_opt;
1530}
1531
1532static inline int bdev_io_opt(struct block_device *bdev)
1533{
1534 return queue_io_opt(bdev_get_queue(bdev));
1535}
1536
1537static inline int queue_alignment_offset(struct request_queue *q)
1538{
1539 if (q->limits.misaligned)
1540 return -1;
1541
1542 return q->limits.alignment_offset;
1543}
1544
1545static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1546{
1547 unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1548 unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
1549 << SECTOR_SHIFT;
1550
1551 return (granularity + lim->alignment_offset - alignment) % granularity;
1552}
1553
1554static inline int bdev_alignment_offset(struct block_device *bdev)
1555{
1556 struct request_queue *q = bdev_get_queue(bdev);
1557
1558 if (q->limits.misaligned)
1559 return -1;
1560
1561 if (bdev != bdev->bd_contains)
1562 return bdev->bd_part->alignment_offset;
1563
1564 return q->limits.alignment_offset;
1565}
1566
1567static inline int queue_discard_alignment(struct request_queue *q)
1568{
1569 if (q->limits.discard_misaligned)
1570 return -1;
1571
1572 return q->limits.discard_alignment;
1573}
1574
1575static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1576{
1577 unsigned int alignment, granularity, offset;
1578
1579 if (!lim->max_discard_sectors)
1580 return 0;
1581
1582 /* Why are these in bytes, not sectors? */
1583 alignment = lim->discard_alignment >> SECTOR_SHIFT;
1584 granularity = lim->discard_granularity >> SECTOR_SHIFT;
1585 if (!granularity)
1586 return 0;
1587
1588 /* Offset of the partition start in 'granularity' sectors */
1589 offset = sector_div(sector, granularity);
1590
1591 /* And why do we do this modulus *again* in blkdev_issue_discard()? */
1592 offset = (granularity + alignment - offset) % granularity;
1593
1594 /* Turn it back into bytes, gaah */
1595 return offset << SECTOR_SHIFT;
1596}
1597
1598static inline int bdev_discard_alignment(struct block_device *bdev)
1599{
1600 struct request_queue *q = bdev_get_queue(bdev);
1601
1602 if (bdev != bdev->bd_contains)
1603 return bdev->bd_part->discard_alignment;
1604
1605 return q->limits.discard_alignment;
1606}
1607
1608static inline unsigned int bdev_write_same(struct block_device *bdev)
1609{
1610 struct request_queue *q = bdev_get_queue(bdev);
1611
1612 if (q)
1613 return q->limits.max_write_same_sectors;
1614
1615 return 0;
1616}
1617
1618static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1619{
1620 struct request_queue *q = bdev_get_queue(bdev);
1621
1622 if (q)
1623 return q->limits.max_write_zeroes_sectors;
1624
1625 return 0;
1626}
1627
1628static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
1629{
1630 struct request_queue *q = bdev_get_queue(bdev);
1631
1632 if (q)
1633 return blk_queue_zoned_model(q);
1634
1635 return BLK_ZONED_NONE;
1636}
1637
1638static inline bool bdev_is_zoned(struct block_device *bdev)
1639{
1640 struct request_queue *q = bdev_get_queue(bdev);
1641
1642 if (q)
1643 return blk_queue_is_zoned(q);
1644
1645 return false;
1646}
1647
1648static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
1649{
1650 struct request_queue *q = bdev_get_queue(bdev);
1651
1652 if (q)
1653 return blk_queue_zone_sectors(q);
1654 return 0;
1655}
1656
1657static inline unsigned int bdev_nr_zones(struct block_device *bdev)
1658{
1659 struct request_queue *q = bdev_get_queue(bdev);
1660
1661 if (q)
1662 return blk_queue_nr_zones(q);
1663 return 0;
1664}
1665
1666static inline int queue_dma_alignment(struct request_queue *q)
1667{
1668 return q ? q->dma_alignment : 511;
1669}
1670
1671static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1672 unsigned int len)
1673{
1674 unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1675 return !(addr & alignment) && !(len & alignment);
1676}
1677
1678/* assumes size > 256 */
1679static inline unsigned int blksize_bits(unsigned int size)
1680{
1681 unsigned int bits = 8;
1682 do {
1683 bits++;
1684 size >>= 1;
1685 } while (size > 256);
1686 return bits;
1687}
1688
1689static inline unsigned int block_size(struct block_device *bdev)
1690{
1691 return bdev->bd_block_size;
1692}
1693
1694static inline bool queue_flush_queueable(struct request_queue *q)
1695{
1696 return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
1697}
1698
1699typedef struct {struct page *v;} Sector;
1700
1701unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
1702
1703static inline void put_dev_sector(Sector p)
1704{
1705 put_page(p.v);
1706}
1707
1708static inline bool __bvec_gap_to_prev(struct request_queue *q,
1709 struct bio_vec *bprv, unsigned int offset)
1710{
1711 return offset ||
1712 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1713}
1714
1715/*
1716 * Check if adding a bio_vec after bprv with offset would create a gap in
1717 * the SG list. Most drivers don't care about this, but some do.
1718 */
1719static inline bool bvec_gap_to_prev(struct request_queue *q,
1720 struct bio_vec *bprv, unsigned int offset)
1721{
1722 if (!queue_virt_boundary(q))
1723 return false;
1724 return __bvec_gap_to_prev(q, bprv, offset);
1725}
1726
1727/*
1728 * Check if the two bvecs from two bios can be merged to one segment.
1729 * If yes, no need to check gap between the two bios since the 1st bio
1730 * and the 1st bvec in the 2nd bio can be handled in one segment.
1731 */
1732static inline bool bios_segs_mergeable(struct request_queue *q,
1733 struct bio *prev, struct bio_vec *prev_last_bv,
1734 struct bio_vec *next_first_bv)
1735{
1736 if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
1737 return false;
1738 if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
1739 return false;
1740 if (prev->bi_seg_back_size + next_first_bv->bv_len >
1741 queue_max_segment_size(q))
1742 return false;
1743 return true;
1744}
1745
1746static inline bool bio_will_gap(struct request_queue *q,
1747 struct request *prev_rq,
1748 struct bio *prev,
1749 struct bio *next)
1750{
1751 if (bio_has_data(prev) && queue_virt_boundary(q)) {
1752 struct bio_vec pb, nb;
1753
1754 /*
1755 * don't merge if the 1st bio starts with non-zero
1756 * offset, otherwise it is quite difficult to respect
1757 * sg gap limit. We work hard to merge a huge number of small
1758 * single bios in case of mkfs.
1759 */
1760 if (prev_rq)
1761 bio_get_first_bvec(prev_rq->bio, &pb);
1762 else
1763 bio_get_first_bvec(prev, &pb);
1764 if (pb.bv_offset)
1765 return true;
1766
1767 /*
1768 * We don't need to worry about the situation that the
1769 * merged segment ends in unaligned virt boundary:
1770 *
1771 * - if 'pb' ends aligned, the merged segment ends aligned
1772 * - if 'pb' ends unaligned, the next bio must include
1773 * one single bvec of 'nb', otherwise the 'nb' can't
1774 * merge with 'pb'
1775 */
1776 bio_get_last_bvec(prev, &pb);
1777 bio_get_first_bvec(next, &nb);
1778
1779 if (!bios_segs_mergeable(q, prev, &pb, &nb))
1780 return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1781 }
1782
1783 return false;
1784}
1785
1786static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
1787{
1788 return bio_will_gap(req->q, req, req->biotail, bio);
1789}
1790
1791static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
1792{
1793 return bio_will_gap(req->q, NULL, bio, req->bio);
1794}
1795
1796int kblockd_schedule_work(struct work_struct *work);
1797int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1798int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1799
1800#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1801 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1802#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1803 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1804
1805#if defined(CONFIG_BLK_DEV_INTEGRITY)
1806
1807enum blk_integrity_flags {
1808 BLK_INTEGRITY_VERIFY = 1 << 0,
1809 BLK_INTEGRITY_GENERATE = 1 << 1,
1810 BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2,
1811 BLK_INTEGRITY_IP_CHECKSUM = 1 << 3,
1812};
1813
1814struct blk_integrity_iter {
1815 void *prot_buf;
1816 void *data_buf;
1817 sector_t seed;
1818 unsigned int data_size;
1819 unsigned short interval;
1820 const char *disk_name;
1821};
1822
1823typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1824
1825struct blk_integrity_profile {
1826 integrity_processing_fn *generate_fn;
1827 integrity_processing_fn *verify_fn;
1828 const char *name;
1829};
1830
1831extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1832extern void blk_integrity_unregister(struct gendisk *);
1833extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1834extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
1835 struct scatterlist *);
1836extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1837extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
1838 struct request *);
1839extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
1840 struct bio *);
1841
1842static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1843{
1844 struct blk_integrity *bi = &disk->queue->integrity;
1845
1846 if (!bi->profile)
1847 return NULL;
1848
1849 return bi;
1850}
1851
1852static inline
1853struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1854{
1855 return blk_get_integrity(bdev->bd_disk);
1856}
1857
1858static inline bool blk_integrity_rq(struct request *rq)
1859{
1860 return rq->cmd_flags & REQ_INTEGRITY;
1861}
1862
1863static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1864 unsigned int segs)
1865{
1866 q->limits.max_integrity_segments = segs;
1867}
1868
1869static inline unsigned short
1870queue_max_integrity_segments(struct request_queue *q)
1871{
1872 return q->limits.max_integrity_segments;
1873}
1874
1875static inline bool integrity_req_gap_back_merge(struct request *req,
1876 struct bio *next)
1877{
1878 struct bio_integrity_payload *bip = bio_integrity(req->bio);
1879 struct bio_integrity_payload *bip_next = bio_integrity(next);
1880
1881 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1882 bip_next->bip_vec[0].bv_offset);
1883}
1884
1885static inline bool integrity_req_gap_front_merge(struct request *req,
1886 struct bio *bio)
1887{
1888 struct bio_integrity_payload *bip = bio_integrity(bio);
1889 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
1890
1891 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1892 bip_next->bip_vec[0].bv_offset);
1893}
1894
1895#else /* CONFIG_BLK_DEV_INTEGRITY */
1896
1897struct bio;
1898struct block_device;
1899struct gendisk;
1900struct blk_integrity;
1901
1902static inline int blk_integrity_rq(struct request *rq)
1903{
1904 return 0;
1905}
1906static inline int blk_rq_count_integrity_sg(struct request_queue *q,
1907 struct bio *b)
1908{
1909 return 0;
1910}
1911static inline int blk_rq_map_integrity_sg(struct request_queue *q,
1912 struct bio *b,
1913 struct scatterlist *s)
1914{
1915 return 0;
1916}
1917static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
1918{
1919 return NULL;
1920}
1921static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1922{
1923 return NULL;
1924}
1925static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
1926{
1927 return 0;
1928}
1929static inline void blk_integrity_register(struct gendisk *d,
1930 struct blk_integrity *b)
1931{
1932}
1933static inline void blk_integrity_unregister(struct gendisk *d)
1934{
1935}
1936static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1937 unsigned int segs)
1938{
1939}
1940static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
1941{
1942 return 0;
1943}
1944static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1945 struct request *r1,
1946 struct request *r2)
1947{
1948 return true;
1949}
1950static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1951 struct request *r,
1952 struct bio *b)
1953{
1954 return true;
1955}
1956
1957static inline bool integrity_req_gap_back_merge(struct request *req,
1958 struct bio *next)
1959{
1960 return false;
1961}
1962static inline bool integrity_req_gap_front_merge(struct request *req,
1963 struct bio *bio)
1964{
1965 return false;
1966}
1967
1968#endif /* CONFIG_BLK_DEV_INTEGRITY */
1969
1970struct block_device_operations {
1971 int (*open) (struct block_device *, fmode_t);
1972 void (*release) (struct gendisk *, fmode_t);
1973 int (*rw_page)(struct block_device *, sector_t, struct page *, bool);
1974 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1975 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1976 unsigned int (*check_events) (struct gendisk *disk,
1977 unsigned int clearing);
1978 /* ->media_changed() is DEPRECATED, use ->check_events() instead */
1979 int (*media_changed) (struct gendisk *);
1980 void (*unlock_native_capacity) (struct gendisk *);
1981 int (*revalidate_disk) (struct gendisk *);
1982 int (*getgeo)(struct block_device *, struct hd_geometry *);
1983 /* this callback is with swap_lock and sometimes page table lock held */
1984 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1985 struct module *owner;
1986 const struct pr_ops *pr_ops;
1987};
1988
1989extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
1990 unsigned long);
1991extern int bdev_read_page(struct block_device *, sector_t, struct page *);
1992extern int bdev_write_page(struct block_device *, sector_t, struct page *,
1993 struct writeback_control *);
1994
1995#ifdef CONFIG_BLK_DEV_ZONED
1996bool blk_req_needs_zone_write_lock(struct request *rq);
1997void __blk_req_zone_write_lock(struct request *rq);
1998void __blk_req_zone_write_unlock(struct request *rq);
1999
2000static inline void blk_req_zone_write_lock(struct request *rq)
2001{
2002 if (blk_req_needs_zone_write_lock(rq))
2003 __blk_req_zone_write_lock(rq);
2004}
2005
2006static inline void blk_req_zone_write_unlock(struct request *rq)
2007{
2008 if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
2009 __blk_req_zone_write_unlock(rq);
2010}
2011
2012static inline bool blk_req_zone_is_write_locked(struct request *rq)
2013{
2014 return rq->q->seq_zones_wlock &&
2015 test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
2016}
2017
2018static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
2019{
2020 if (!blk_req_needs_zone_write_lock(rq))
2021 return true;
2022 return !blk_req_zone_is_write_locked(rq);
2023}
2024#else
2025static inline bool blk_req_needs_zone_write_lock(struct request *rq)
2026{
2027 return false;
2028}
2029
2030static inline void blk_req_zone_write_lock(struct request *rq)
2031{
2032}
2033
2034static inline void blk_req_zone_write_unlock(struct request *rq)
2035{
2036}
2037static inline bool blk_req_zone_is_write_locked(struct request *rq)
2038{
2039 return false;
2040}
2041
2042static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
2043{
2044 return true;
2045}
2046#endif /* CONFIG_BLK_DEV_ZONED */
2047
2048#else /* CONFIG_BLOCK */
2049
2050struct block_device;
2051
2052/*
2053 * stubs for when the block layer is configured out
2054 */
2055#define buffer_heads_over_limit 0
2056
2057static inline long nr_blockdev_pages(void)
2058{
2059 return 0;
2060}
2061
2062struct blk_plug {
2063};
2064
2065static inline void blk_start_plug(struct blk_plug *plug)
2066{
2067}
2068
2069static inline void blk_finish_plug(struct blk_plug *plug)
2070{
2071}
2072
2073static inline void blk_flush_plug(struct task_struct *task)
2074{
2075}
2076
2077static inline void blk_schedule_flush_plug(struct task_struct *task)
2078{
2079}
2080
2081
2082static inline bool blk_needs_flush_plug(struct task_struct *tsk)
2083{
2084 return false;
2085}
2086
2087static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
2088 sector_t *error_sector)
2089{
2090 return 0;
2091}
2092
2093#endif /* CONFIG_BLOCK */
2094
2095#endif
Ubuntu Eoan Linux kernel mirror