1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_BLKDEV_H
3 #define _LINUX_BLKDEV_H
5 #include <linux/sched.h>
6 #include <linux/sched/clock.h>
10 #include <linux/major.h>
11 #include <linux/genhd.h>
12 #include <linux/list.h>
13 #include <linux/llist.h>
14 #include <linux/timer.h>
15 #include <linux/workqueue.h>
16 #include <linux/pagemap.h>
17 #include <linux/backing-dev-defs.h>
18 #include <linux/wait.h>
19 #include <linux/mempool.h>
20 #include <linux/pfn.h>
21 #include <linux/bio.h>
22 #include <linux/stringify.h>
23 #include <linux/gfp.h>
24 #include <linux/bsg.h>
25 #include <linux/smp.h>
26 #include <linux/rcupdate.h>
27 #include <linux/percpu-refcount.h>
28 #include <linux/scatterlist.h>
29 #include <linux/blkzoned.h>
30 #include <linux/seqlock.h>
31 #include <linux/u64_stats_sync.h>
34 struct scsi_ioctl_command
;
37 struct elevator_queue
;
43 struct blk_flush_queue
;
46 struct blk_queue_stats
;
47 struct blk_stat_callback
;
49 #define BLKDEV_MIN_RQ 4
50 #define BLKDEV_MAX_RQ 128 /* Default maximum */
52 /* Must be consistent with blk_mq_poll_stats_bkt() */
53 #define BLK_MQ_POLL_STATS_BKTS 16
56 * Maximum number of blkcg policies allowed to be registered concurrently.
57 * Defined here to simplify include dependency.
59 #define BLKCG_MAX_POLS 3
61 typedef void (rq_end_io_fn
)(struct request
*, blk_status_t
);
63 #define BLK_RL_SYNCFULL (1U << 0)
64 #define BLK_RL_ASYNCFULL (1U << 1)
67 struct request_queue
*q
; /* the queue this rl belongs to */
68 #ifdef CONFIG_BLK_CGROUP
69 struct blkcg_gq
*blkg
; /* blkg this request pool belongs to */
72 * count[], starved[], and wait[] are indexed by
73 * BLK_RW_SYNC/BLK_RW_ASYNC
78 wait_queue_head_t wait
[2];
84 typedef __u32 __bitwise req_flags_t
;
86 /* elevator knows about this request */
87 #define RQF_SORTED ((__force req_flags_t)(1 << 0))
88 /* drive already may have started this one */
89 #define RQF_STARTED ((__force req_flags_t)(1 << 1))
90 /* uses tagged queueing */
91 #define RQF_QUEUED ((__force req_flags_t)(1 << 2))
92 /* may not be passed by ioscheduler */
93 #define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
94 /* request for flush sequence */
95 #define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
96 /* merge of different types, fail separately */
97 #define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
98 /* track inflight for MQ */
99 #define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
100 /* don't call prep for this one */
101 #define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
102 /* set for "ide_preempt" requests and also for requests for which the SCSI
103 "quiesce" state must be ignored. */
104 #define RQF_PREEMPT ((__force req_flags_t)(1 << 8))
105 /* contains copies of user pages */
106 #define RQF_COPY_USER ((__force req_flags_t)(1 << 9))
107 /* vaguely specified driver internal error. Ignored by the block layer */
108 #define RQF_FAILED ((__force req_flags_t)(1 << 10))
109 /* don't warn about errors */
110 #define RQF_QUIET ((__force req_flags_t)(1 << 11))
111 /* elevator private data attached */
112 #define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
113 /* account I/O stat */
114 #define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
115 /* request came from our alloc pool */
116 #define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
117 /* runtime pm request */
118 #define RQF_PM ((__force req_flags_t)(1 << 15))
119 /* on IO scheduler merge hash */
120 #define RQF_HASHED ((__force req_flags_t)(1 << 16))
121 /* IO stats tracking on */
122 #define RQF_STATS ((__force req_flags_t)(1 << 17))
123 /* Look at ->special_vec for the actual data payload instead of the
125 #define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
126 /* The per-zone write lock is held for this request */
127 #define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
128 /* already slept for hybrid poll */
129 #define RQF_MQ_POLL_SLEPT ((__force req_flags_t)(1 << 20))
130 /* ->timeout has been called, don't expire again */
131 #define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))
133 /* flags that prevent us from merging requests: */
134 #define RQF_NOMERGE_FLAGS \
135 (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
138 * Request state for blk-mq.
147 * Try to put the fields that are referenced together in the same cacheline.
149 * If you modify this structure, make sure to update blk_rq_init() and
150 * especially blk_mq_rq_ctx_init() to take care of the added fields.
153 struct request_queue
*q
;
154 struct blk_mq_ctx
*mq_ctx
;
157 unsigned int cmd_flags
; /* op and common flags */
158 req_flags_t rq_flags
;
162 /* the following two fields are internal, NEVER access directly */
163 unsigned int __data_len
; /* total data len */
165 sector_t __sector
; /* sector cursor */
170 struct list_head queuelist
;
173 * The hash is used inside the scheduler, and killed once the
174 * request reaches the dispatch list. The ipi_list is only used
175 * to queue the request for softirq completion, which is long
176 * after the request has been unhashed (and even removed from
177 * the dispatch list).
180 struct hlist_node hash
; /* merge hash */
181 struct list_head ipi_list
;
185 * The rb_node is only used inside the io scheduler, requests
186 * are pruned when moved to the dispatch queue. So let the
187 * completion_data share space with the rb_node.
190 struct rb_node rb_node
; /* sort/lookup */
191 struct bio_vec special_vec
;
192 void *completion_data
;
193 int error_count
; /* for legacy drivers, don't use */
197 * Three pointers are available for the IO schedulers, if they need
198 * more they have to dynamically allocate it. Flush requests are
199 * never put on the IO scheduler. So let the flush fields share
200 * space with the elevator data.
210 struct list_head list
;
211 rq_end_io_fn
*saved_end_io
;
215 struct gendisk
*rq_disk
;
216 struct hd_struct
*part
;
217 /* Time that I/O was submitted to the kernel. */
219 /* Time that I/O was submitted to the device. */
220 u64 io_start_time_ns
;
222 #ifdef CONFIG_BLK_WBT
223 unsigned short wbt_flags
;
225 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
226 unsigned short throtl_size
;
230 * Number of scatter-gather DMA addr+len pairs after
231 * physical address coalescing is performed.
233 unsigned short nr_phys_segments
;
235 #if defined(CONFIG_BLK_DEV_INTEGRITY)
236 unsigned short nr_integrity_segments
;
239 unsigned short write_hint
;
240 unsigned short ioprio
;
242 void *special
; /* opaque pointer available for LLD use */
244 unsigned int extra_len
; /* length of alignment and padding */
246 enum mq_rq_state state
;
249 unsigned int timeout
;
251 /* access through blk_rq_set_deadline, blk_rq_deadline */
252 unsigned long __deadline
;
254 struct list_head timeout_list
;
257 struct __call_single_data csd
;
262 * completion callback.
264 rq_end_io_fn
*end_io
;
268 struct request
*next_rq
;
270 #ifdef CONFIG_BLK_CGROUP
271 struct request_list
*rl
; /* rl this rq is alloced from */
275 static inline bool blk_op_is_scsi(unsigned int op
)
277 return op
== REQ_OP_SCSI_IN
|| op
== REQ_OP_SCSI_OUT
;
280 static inline bool blk_op_is_private(unsigned int op
)
282 return op
== REQ_OP_DRV_IN
|| op
== REQ_OP_DRV_OUT
;
285 static inline bool blk_rq_is_scsi(struct request
*rq
)
287 return blk_op_is_scsi(req_op(rq
));
290 static inline bool blk_rq_is_private(struct request
*rq
)
292 return blk_op_is_private(req_op(rq
));
295 static inline bool blk_rq_is_passthrough(struct request
*rq
)
297 return blk_rq_is_scsi(rq
) || blk_rq_is_private(rq
);
300 static inline bool bio_is_passthrough(struct bio
*bio
)
302 unsigned op
= bio_op(bio
);
304 return blk_op_is_scsi(op
) || blk_op_is_private(op
);
307 static inline unsigned short req_get_ioprio(struct request
*req
)
312 #include <linux/elevator.h>
314 struct blk_queue_ctx
;
316 typedef void (request_fn_proc
) (struct request_queue
*q
);
317 typedef blk_qc_t (make_request_fn
) (struct request_queue
*q
, struct bio
*bio
);
318 typedef bool (poll_q_fn
) (struct request_queue
*q
, blk_qc_t
);
319 typedef int (prep_rq_fn
) (struct request_queue
*, struct request
*);
320 typedef void (unprep_rq_fn
) (struct request_queue
*, struct request
*);
323 typedef void (softirq_done_fn
)(struct request
*);
324 typedef int (dma_drain_needed_fn
)(struct request
*);
325 typedef int (lld_busy_fn
) (struct request_queue
*q
);
326 typedef int (bsg_job_fn
) (struct bsg_job
*);
327 typedef int (init_rq_fn
)(struct request_queue
*, struct request
*, gfp_t
);
328 typedef void (exit_rq_fn
)(struct request_queue
*, struct request
*);
330 enum blk_eh_timer_return
{
331 BLK_EH_DONE
, /* drivers has completed the command */
332 BLK_EH_RESET_TIMER
, /* reset timer and try again */
335 typedef enum blk_eh_timer_return (rq_timed_out_fn
)(struct request
*);
337 enum blk_queue_state
{
342 struct blk_queue_tag
{
343 struct request
**tag_index
; /* map of busy tags */
344 unsigned long *tag_map
; /* bit map of free/busy tags */
345 int max_depth
; /* what we will send to device */
346 int real_max_depth
; /* what the array can hold */
347 atomic_t refcnt
; /* map can be shared */
348 int alloc_policy
; /* tag allocation policy */
349 int next_tag
; /* next tag */
351 #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
352 #define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
354 #define BLK_SCSI_MAX_CMDS (256)
355 #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
358 * Zoned block device models (zoned limit).
360 enum blk_zoned_model
{
361 BLK_ZONED_NONE
, /* Regular block device */
362 BLK_ZONED_HA
, /* Host-aware zoned block device */
363 BLK_ZONED_HM
, /* Host-managed zoned block device */
366 struct queue_limits
{
367 unsigned long bounce_pfn
;
368 unsigned long seg_boundary_mask
;
369 unsigned long virt_boundary_mask
;
371 unsigned int max_hw_sectors
;
372 unsigned int max_dev_sectors
;
373 unsigned int chunk_sectors
;
374 unsigned int max_sectors
;
375 unsigned int max_segment_size
;
376 unsigned int physical_block_size
;
377 unsigned int alignment_offset
;
380 unsigned int max_discard_sectors
;
381 unsigned int max_hw_discard_sectors
;
382 unsigned int max_write_same_sectors
;
383 unsigned int max_write_zeroes_sectors
;
384 unsigned int discard_granularity
;
385 unsigned int discard_alignment
;
387 unsigned short logical_block_size
;
388 unsigned short max_segments
;
389 unsigned short max_integrity_segments
;
390 unsigned short max_discard_segments
;
392 unsigned char misaligned
;
393 unsigned char discard_misaligned
;
394 unsigned char cluster
;
395 unsigned char raid_partial_stripes_expensive
;
396 enum blk_zoned_model zoned
;
399 #ifdef CONFIG_BLK_DEV_ZONED
401 struct blk_zone_report_hdr
{
402 unsigned int nr_zones
;
406 extern int blkdev_report_zones(struct block_device
*bdev
,
407 sector_t sector
, struct blk_zone
*zones
,
408 unsigned int *nr_zones
, gfp_t gfp_mask
);
409 extern int blkdev_reset_zones(struct block_device
*bdev
, sector_t sectors
,
410 sector_t nr_sectors
, gfp_t gfp_mask
);
412 extern int blkdev_report_zones_ioctl(struct block_device
*bdev
, fmode_t mode
,
413 unsigned int cmd
, unsigned long arg
);
414 extern int blkdev_reset_zones_ioctl(struct block_device
*bdev
, fmode_t mode
,
415 unsigned int cmd
, unsigned long arg
);
417 #else /* CONFIG_BLK_DEV_ZONED */
419 static inline int blkdev_report_zones_ioctl(struct block_device
*bdev
,
420 fmode_t mode
, unsigned int cmd
,
426 static inline int blkdev_reset_zones_ioctl(struct block_device
*bdev
,
427 fmode_t mode
, unsigned int cmd
,
433 #endif /* CONFIG_BLK_DEV_ZONED */
435 struct request_queue
{
437 * Together with queue_head for cacheline sharing
439 struct list_head queue_head
;
440 struct request
*last_merge
;
441 struct elevator_queue
*elevator
;
442 int nr_rqs
[2]; /* # allocated [a]sync rqs */
443 int nr_rqs_elvpriv
; /* # allocated rqs w/ elvpriv */
445 atomic_t shared_hctx_restart
;
447 struct blk_queue_stats
*stats
;
451 * If blkcg is not used, @q->root_rl serves all requests. If blkcg
452 * is used, root blkg allocates from @q->root_rl and all other
453 * blkgs from their own blkg->rl. Which one to use should be
454 * determined using bio_request_list().
456 struct request_list root_rl
;
458 request_fn_proc
*request_fn
;
459 make_request_fn
*make_request_fn
;
461 prep_rq_fn
*prep_rq_fn
;
462 unprep_rq_fn
*unprep_rq_fn
;
463 softirq_done_fn
*softirq_done_fn
;
464 rq_timed_out_fn
*rq_timed_out_fn
;
465 dma_drain_needed_fn
*dma_drain_needed
;
466 lld_busy_fn
*lld_busy_fn
;
467 /* Called just after a request is allocated */
468 init_rq_fn
*init_rq_fn
;
469 /* Called just before a request is freed */
470 exit_rq_fn
*exit_rq_fn
;
471 /* Called from inside blk_get_request() */
472 void (*initialize_rq_fn
)(struct request
*rq
);
474 const struct blk_mq_ops
*mq_ops
;
476 unsigned int *mq_map
;
479 struct blk_mq_ctx __percpu
*queue_ctx
;
480 unsigned int nr_queues
;
482 unsigned int queue_depth
;
484 /* hw dispatch queues */
485 struct blk_mq_hw_ctx
**queue_hw_ctx
;
486 unsigned int nr_hw_queues
;
489 * Dispatch queue sorting
492 struct request
*boundary_rq
;
495 * Delayed queue handling
497 struct delayed_work delay_work
;
499 struct backing_dev_info
*backing_dev_info
;
502 * The queue owner gets to use this for whatever they like.
503 * ll_rw_blk doesn't touch it.
508 * various queue flags, see QUEUE_* below
510 unsigned long queue_flags
;
513 * ida allocated id for this queue. Used to index queues from
519 * queue needs bounce pages for pages above this limit
524 * protects queue structures from reentrancy. ->__queue_lock should
525 * _never_ be used directly, it is queue private. always use
528 spinlock_t __queue_lock
;
529 spinlock_t
*queue_lock
;
539 struct kobject mq_kobj
;
541 #ifdef CONFIG_BLK_DEV_INTEGRITY
542 struct blk_integrity integrity
;
543 #endif /* CONFIG_BLK_DEV_INTEGRITY */
548 unsigned int nr_pending
;
554 unsigned long nr_requests
; /* Max # of requests */
555 unsigned int nr_congestion_on
;
556 unsigned int nr_congestion_off
;
557 unsigned int nr_batching
;
559 unsigned int dma_drain_size
;
560 void *dma_drain_buffer
;
561 unsigned int dma_pad_mask
;
562 unsigned int dma_alignment
;
564 struct blk_queue_tag
*queue_tags
;
566 unsigned int nr_sorted
;
567 unsigned int in_flight
[2];
570 * Number of active block driver functions for which blk_drain_queue()
571 * must wait. Must be incremented around functions that unlock the
572 * queue_lock internally, e.g. scsi_request_fn().
574 unsigned int request_fn_active
;
576 unsigned int rq_timeout
;
579 struct blk_stat_callback
*poll_cb
;
580 struct blk_rq_stat poll_stat
[BLK_MQ_POLL_STATS_BKTS
];
582 struct timer_list timeout
;
583 struct work_struct timeout_work
;
584 struct list_head timeout_list
;
586 struct list_head icq_list
;
587 #ifdef CONFIG_BLK_CGROUP
588 DECLARE_BITMAP (blkcg_pols
, BLKCG_MAX_POLS
);
589 struct blkcg_gq
*root_blkg
;
590 struct list_head blkg_list
;
593 struct queue_limits limits
;
595 #ifdef CONFIG_BLK_DEV_ZONED
597 * Zoned block device information for request dispatch control.
598 * nr_zones is the total number of zones of the device. This is always
599 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
600 * bits which indicates if a zone is conventional (bit clear) or
601 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
602 * bits which indicates if a zone is write locked, that is, if a write
603 * request targeting the zone was dispatched. All three fields are
604 * initialized by the low level device driver (e.g. scsi/sd.c).
605 * Stacking drivers (device mappers) may or may not initialize
608 * Reads of this information must be protected with blk_queue_enter() /
609 * blk_queue_exit(). Modifying this information is only allowed while
610 * no requests are being processed. See also blk_mq_freeze_queue() and
611 * blk_mq_unfreeze_queue().
613 unsigned int nr_zones
;
614 unsigned long *seq_zones_bitmap
;
615 unsigned long *seq_zones_wlock
;
616 #endif /* CONFIG_BLK_DEV_ZONED */
621 unsigned int sg_timeout
;
622 unsigned int sg_reserved_size
;
624 #ifdef CONFIG_BLK_DEV_IO_TRACE
625 struct blk_trace
*blk_trace
;
626 struct mutex blk_trace_mutex
;
629 * for flush operations
631 struct blk_flush_queue
*fq
;
633 struct list_head requeue_list
;
634 spinlock_t requeue_lock
;
635 struct delayed_work requeue_work
;
637 struct mutex sysfs_lock
;
640 atomic_t mq_freeze_depth
;
642 #if defined(CONFIG_BLK_DEV_BSG)
643 bsg_job_fn
*bsg_job_fn
;
644 struct bsg_class_device bsg_dev
;
647 #ifdef CONFIG_BLK_DEV_THROTTLING
649 struct throtl_data
*td
;
651 struct rcu_head rcu_head
;
652 wait_queue_head_t mq_freeze_wq
;
653 struct percpu_ref q_usage_counter
;
654 struct list_head all_q_node
;
656 struct blk_mq_tag_set
*tag_set
;
657 struct list_head tag_set_list
;
658 struct bio_set bio_split
;
660 #ifdef CONFIG_BLK_DEBUG_FS
661 struct dentry
*debugfs_dir
;
662 struct dentry
*sched_debugfs_dir
;
665 bool mq_sysfs_init_done
;
670 struct work_struct release_work
;
672 #define BLK_MAX_WRITE_HINTS 5
673 u64 write_hints
[BLK_MAX_WRITE_HINTS
];
676 #define QUEUE_FLAG_QUEUED 0 /* uses generic tag queueing */
677 #define QUEUE_FLAG_STOPPED 1 /* queue is stopped */
678 #define QUEUE_FLAG_DYING 2 /* queue being torn down */
679 #define QUEUE_FLAG_BYPASS 3 /* act as dumb FIFO queue */
680 #define QUEUE_FLAG_BIDI 4 /* queue supports bidi requests */
681 #define QUEUE_FLAG_NOMERGES 5 /* disable merge attempts */
682 #define QUEUE_FLAG_SAME_COMP 6 /* complete on same CPU-group */
683 #define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */
684 #define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */
685 #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
686 #define QUEUE_FLAG_IO_STAT 10 /* do IO stats */
687 #define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */
688 #define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */
689 #define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */
690 #define QUEUE_FLAG_SECERASE 14 /* supports secure erase */
691 #define QUEUE_FLAG_SAME_FORCE 15 /* force complete on same CPU */
692 #define QUEUE_FLAG_DEAD 16 /* queue tear-down finished */
693 #define QUEUE_FLAG_INIT_DONE 17 /* queue is initialized */
694 #define QUEUE_FLAG_NO_SG_MERGE 18 /* don't attempt to merge SG segments*/
695 #define QUEUE_FLAG_POLL 19 /* IO polling enabled if set */
696 #define QUEUE_FLAG_WC 20 /* Write back caching */
697 #define QUEUE_FLAG_FUA 21 /* device supports FUA writes */
698 #define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */
699 #define QUEUE_FLAG_DAX 23 /* device supports DAX */
700 #define QUEUE_FLAG_STATS 24 /* track rq completion times */
701 #define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */
702 #define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */
703 #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */
704 #define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */
705 #define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */
707 #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
708 (1 << QUEUE_FLAG_SAME_COMP) | \
709 (1 << QUEUE_FLAG_ADD_RANDOM))
711 #define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
712 (1 << QUEUE_FLAG_SAME_COMP) | \
713 (1 << QUEUE_FLAG_POLL))
715 void blk_queue_flag_set(unsigned int flag
, struct request_queue
*q
);
716 void blk_queue_flag_clear(unsigned int flag
, struct request_queue
*q
);
717 bool blk_queue_flag_test_and_set(unsigned int flag
, struct request_queue
*q
);
718 bool blk_queue_flag_test_and_clear(unsigned int flag
, struct request_queue
*q
);
720 #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
721 #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
722 #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
723 #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
724 #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
725 #define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
726 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
727 #define blk_queue_noxmerges(q) \
728 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
729 #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
730 #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
731 #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
732 #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
733 #define blk_queue_secure_erase(q) \
734 (test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
735 #define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
736 #define blk_queue_scsi_passthrough(q) \
737 test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
739 #define blk_noretry_request(rq) \
740 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
741 REQ_FAILFAST_DRIVER))
742 #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
743 #define blk_queue_preempt_only(q) \
744 test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)
745 #define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
747 extern int blk_set_preempt_only(struct request_queue
*q
);
748 extern void blk_clear_preempt_only(struct request_queue
*q
);
750 static inline int queue_in_flight(struct request_queue
*q
)
752 return q
->in_flight
[0] + q
->in_flight
[1];
755 static inline bool blk_account_rq(struct request
*rq
)
757 return (rq
->rq_flags
& RQF_STARTED
) && !blk_rq_is_passthrough(rq
);
760 #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
761 #define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
762 /* rq->queuelist of dequeued request must be list_empty() */
763 #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
765 #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
767 #define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
770 * Driver can handle struct request, if it either has an old style
771 * request_fn defined, or is blk-mq based.
773 static inline bool queue_is_rq_based(struct request_queue
*q
)
775 return q
->request_fn
|| q
->mq_ops
;
778 static inline unsigned int blk_queue_cluster(struct request_queue
*q
)
780 return q
->limits
.cluster
;
783 static inline enum blk_zoned_model
784 blk_queue_zoned_model(struct request_queue
*q
)
786 return q
->limits
.zoned
;
789 static inline bool blk_queue_is_zoned(struct request_queue
*q
)
791 switch (blk_queue_zoned_model(q
)) {
800 static inline unsigned int blk_queue_zone_sectors(struct request_queue
*q
)
802 return blk_queue_is_zoned(q
) ? q
->limits
.chunk_sectors
: 0;
805 #ifdef CONFIG_BLK_DEV_ZONED
806 static inline unsigned int blk_queue_zone_no(struct request_queue
*q
,
809 if (!blk_queue_is_zoned(q
))
811 return sector
>> ilog2(q
->limits
.chunk_sectors
);
814 static inline bool blk_queue_zone_is_seq(struct request_queue
*q
,
817 if (!blk_queue_is_zoned(q
) || !q
->seq_zones_bitmap
)
819 return test_bit(blk_queue_zone_no(q
, sector
), q
->seq_zones_bitmap
);
821 #endif /* CONFIG_BLK_DEV_ZONED */
823 static inline bool rq_is_sync(struct request
*rq
)
825 return op_is_sync(rq
->cmd_flags
);
828 static inline bool blk_rl_full(struct request_list
*rl
, bool sync
)
830 unsigned int flag
= sync
? BLK_RL_SYNCFULL
: BLK_RL_ASYNCFULL
;
832 return rl
->flags
& flag
;
835 static inline void blk_set_rl_full(struct request_list
*rl
, bool sync
)
837 unsigned int flag
= sync
? BLK_RL_SYNCFULL
: BLK_RL_ASYNCFULL
;
842 static inline void blk_clear_rl_full(struct request_list
*rl
, bool sync
)
844 unsigned int flag
= sync
? BLK_RL_SYNCFULL
: BLK_RL_ASYNCFULL
;
849 static inline bool rq_mergeable(struct request
*rq
)
851 if (blk_rq_is_passthrough(rq
))
854 if (req_op(rq
) == REQ_OP_FLUSH
)
857 if (req_op(rq
) == REQ_OP_WRITE_ZEROES
)
860 if (rq
->cmd_flags
& REQ_NOMERGE_FLAGS
)
862 if (rq
->rq_flags
& RQF_NOMERGE_FLAGS
)
868 static inline bool blk_write_same_mergeable(struct bio
*a
, struct bio
*b
)
870 if (bio_page(a
) == bio_page(b
) &&
871 bio_offset(a
) == bio_offset(b
))
877 static inline unsigned int blk_queue_depth(struct request_queue
*q
)
880 return q
->queue_depth
;
882 return q
->nr_requests
;
886 * q->prep_rq_fn return values
889 BLKPREP_OK
, /* serve it */
890 BLKPREP_KILL
, /* fatal error, kill, return -EIO */
891 BLKPREP_DEFER
, /* leave on queue */
892 BLKPREP_INVALID
, /* invalid command, kill, return -EREMOTEIO */
895 extern unsigned long blk_max_low_pfn
, blk_max_pfn
;
898 * standard bounce addresses:
900 * BLK_BOUNCE_HIGH : bounce all highmem pages
901 * BLK_BOUNCE_ANY : don't bounce anything
902 * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary
905 #if BITS_PER_LONG == 32
906 #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
908 #define BLK_BOUNCE_HIGH -1ULL
910 #define BLK_BOUNCE_ANY (-1ULL)
911 #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
914 * default timeout for SG_IO if none specified
916 #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
917 #define BLK_MIN_SG_TIMEOUT (7 * HZ)
923 unsigned long offset
;
928 struct req_iterator
{
929 struct bvec_iter iter
;
933 /* This should not be used directly - use rq_for_each_segment */
934 #define for_each_bio(_bio) \
935 for (; _bio; _bio = _bio->bi_next)
936 #define __rq_for_each_bio(_bio, rq) \
938 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
940 #define rq_for_each_segment(bvl, _rq, _iter) \
941 __rq_for_each_bio(_iter.bio, _rq) \
942 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
944 #define rq_iter_last(bvec, _iter) \
945 (_iter.bio->bi_next == NULL && \
946 bio_iter_last(bvec, _iter.iter))
948 #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
949 # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
951 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
952 extern void rq_flush_dcache_pages(struct request
*rq
);
954 static inline void rq_flush_dcache_pages(struct request
*rq
)
959 extern int blk_register_queue(struct gendisk
*disk
);
960 extern void blk_unregister_queue(struct gendisk
*disk
);
961 extern blk_qc_t
generic_make_request(struct bio
*bio
);
962 extern blk_qc_t
direct_make_request(struct bio
*bio
);
963 extern void blk_rq_init(struct request_queue
*q
, struct request
*rq
);
964 extern void blk_init_request_from_bio(struct request
*req
, struct bio
*bio
);
965 extern void blk_put_request(struct request
*);
966 extern void __blk_put_request(struct request_queue
*, struct request
*);
967 extern struct request
*blk_get_request(struct request_queue
*, unsigned int op
,
968 blk_mq_req_flags_t flags
);
969 extern void blk_requeue_request(struct request_queue
*, struct request
*);
970 extern int blk_lld_busy(struct request_queue
*q
);
971 extern int blk_rq_prep_clone(struct request
*rq
, struct request
*rq_src
,
972 struct bio_set
*bs
, gfp_t gfp_mask
,
973 int (*bio_ctr
)(struct bio
*, struct bio
*, void *),
975 extern void blk_rq_unprep_clone(struct request
*rq
);
976 extern blk_status_t
blk_insert_cloned_request(struct request_queue
*q
,
978 extern int blk_rq_append_bio(struct request
*rq
, struct bio
**bio
);
979 extern void blk_delay_queue(struct request_queue
*, unsigned long);
980 extern void blk_queue_split(struct request_queue
*, struct bio
**);
981 extern void blk_recount_segments(struct request_queue
*, struct bio
*);
982 extern int scsi_verify_blk_ioctl(struct block_device
*, unsigned int);
983 extern int scsi_cmd_blk_ioctl(struct block_device
*, fmode_t
,
984 unsigned int, void __user
*);
985 extern int scsi_cmd_ioctl(struct request_queue
*, struct gendisk
*, fmode_t
,
986 unsigned int, void __user
*);
987 extern int sg_scsi_ioctl(struct request_queue
*, struct gendisk
*, fmode_t
,
988 struct scsi_ioctl_command __user
*);
990 extern int blk_queue_enter(struct request_queue
*q
, blk_mq_req_flags_t flags
);
991 extern void blk_queue_exit(struct request_queue
*q
);
992 extern void blk_start_queue(struct request_queue
*q
);
993 extern void blk_start_queue_async(struct request_queue
*q
);
994 extern void blk_stop_queue(struct request_queue
*q
);
995 extern void blk_sync_queue(struct request_queue
*q
);
996 extern void __blk_stop_queue(struct request_queue
*q
);
997 extern void __blk_run_queue(struct request_queue
*q
);
998 extern void __blk_run_queue_uncond(struct request_queue
*q
);
999 extern void blk_run_queue(struct request_queue
*);
1000 extern void blk_run_queue_async(struct request_queue
*q
);
1001 extern int blk_rq_map_user(struct request_queue
*, struct request
*,
1002 struct rq_map_data
*, void __user
*, unsigned long,
1004 extern int blk_rq_unmap_user(struct bio
*);
1005 extern int blk_rq_map_kern(struct request_queue
*, struct request
*, void *, unsigned int, gfp_t
);
1006 extern int blk_rq_map_user_iov(struct request_queue
*, struct request
*,
1007 struct rq_map_data
*, const struct iov_iter
*,
1009 extern void blk_execute_rq(struct request_queue
*, struct gendisk
*,
1010 struct request
*, int);
1011 extern void blk_execute_rq_nowait(struct request_queue
*, struct gendisk
*,
1012 struct request
*, int, rq_end_io_fn
*);
1014 int blk_status_to_errno(blk_status_t status
);
1015 blk_status_t
errno_to_blk_status(int errno
);
1017 bool blk_poll(struct request_queue
*q
, blk_qc_t cookie
);
1019 static inline struct request_queue
*bdev_get_queue(struct block_device
*bdev
)
1021 return bdev
->bd_disk
->queue
; /* this is never NULL */
1025 * The basic unit of block I/O is a sector. It is used in a number of contexts
1026 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
1027 * bytes. Variables of type sector_t represent an offset or size that is a
1028 * multiple of 512 bytes. Hence these two constants.
1030 #ifndef SECTOR_SHIFT
1031 #define SECTOR_SHIFT 9
1034 #define SECTOR_SIZE (1 << SECTOR_SHIFT)
1038 * blk_rq_pos() : the current sector
1039 * blk_rq_bytes() : bytes left in the entire request
1040 * blk_rq_cur_bytes() : bytes left in the current segment
1041 * blk_rq_err_bytes() : bytes left till the next error boundary
1042 * blk_rq_sectors() : sectors left in the entire request
1043 * blk_rq_cur_sectors() : sectors left in the current segment
1045 static inline sector_t
blk_rq_pos(const struct request
*rq
)
1047 return rq
->__sector
;
1050 static inline unsigned int blk_rq_bytes(const struct request
*rq
)
1052 return rq
->__data_len
;
1055 static inline int blk_rq_cur_bytes(const struct request
*rq
)
1057 return rq
->bio
? bio_cur_bytes(rq
->bio
) : 0;
1060 extern unsigned int blk_rq_err_bytes(const struct request
*rq
);
1062 static inline unsigned int blk_rq_sectors(const struct request
*rq
)
1064 return blk_rq_bytes(rq
) >> SECTOR_SHIFT
;
1067 static inline unsigned int blk_rq_cur_sectors(const struct request
*rq
)
1069 return blk_rq_cur_bytes(rq
) >> SECTOR_SHIFT
;
1072 #ifdef CONFIG_BLK_DEV_ZONED
1073 static inline unsigned int blk_rq_zone_no(struct request
*rq
)
1075 return blk_queue_zone_no(rq
->q
, blk_rq_pos(rq
));
1078 static inline unsigned int blk_rq_zone_is_seq(struct request
*rq
)
1080 return blk_queue_zone_is_seq(rq
->q
, blk_rq_pos(rq
));
1082 #endif /* CONFIG_BLK_DEV_ZONED */
1085 * Some commands like WRITE SAME have a payload or data transfer size which
1086 * is different from the size of the request. Any driver that supports such
1087 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
1088 * calculate the data transfer size.
1090 static inline unsigned int blk_rq_payload_bytes(struct request
*rq
)
1092 if (rq
->rq_flags
& RQF_SPECIAL_PAYLOAD
)
1093 return rq
->special_vec
.bv_len
;
1094 return blk_rq_bytes(rq
);
1097 static inline unsigned int blk_queue_get_max_sectors(struct request_queue
*q
,
1100 if (unlikely(op
== REQ_OP_DISCARD
|| op
== REQ_OP_SECURE_ERASE
))
1101 return min(q
->limits
.max_discard_sectors
,
1102 UINT_MAX
>> SECTOR_SHIFT
);
1104 if (unlikely(op
== REQ_OP_WRITE_SAME
))
1105 return q
->limits
.max_write_same_sectors
;
1107 if (unlikely(op
== REQ_OP_WRITE_ZEROES
))
1108 return q
->limits
.max_write_zeroes_sectors
;
1110 return q
->limits
.max_sectors
;
1114 * Return maximum size of a request at given offset. Only valid for
1115 * file system requests.
1117 static inline unsigned int blk_max_size_offset(struct request_queue
*q
,
1120 if (!q
->limits
.chunk_sectors
)
1121 return q
->limits
.max_sectors
;
1123 return min(q
->limits
.max_sectors
, (unsigned int)(q
->limits
.chunk_sectors
-
1124 (offset
& (q
->limits
.chunk_sectors
- 1))));
1127 static inline unsigned int blk_rq_get_max_sectors(struct request
*rq
,
1130 struct request_queue
*q
= rq
->q
;
1132 if (blk_rq_is_passthrough(rq
))
1133 return q
->limits
.max_hw_sectors
;
1135 if (!q
->limits
.chunk_sectors
||
1136 req_op(rq
) == REQ_OP_DISCARD
||
1137 req_op(rq
) == REQ_OP_SECURE_ERASE
)
1138 return blk_queue_get_max_sectors(q
, req_op(rq
));
1140 return min(blk_max_size_offset(q
, offset
),
1141 blk_queue_get_max_sectors(q
, req_op(rq
)));
1144 static inline unsigned int blk_rq_count_bios(struct request
*rq
)
1146 unsigned int nr_bios
= 0;
1149 __rq_for_each_bio(bio
, rq
)
1156 * Request issue related functions.
1158 extern struct request
*blk_peek_request(struct request_queue
*q
);
1159 extern void blk_start_request(struct request
*rq
);
1160 extern struct request
*blk_fetch_request(struct request_queue
*q
);
1162 void blk_steal_bios(struct bio_list
*list
, struct request
*rq
);
1165 * Request completion related functions.
1167 * blk_update_request() completes given number of bytes and updates
1168 * the request without completing it.
1170 * blk_end_request() and friends. __blk_end_request() must be called
1171 * with the request queue spinlock acquired.
1173 * Several drivers define their own end_request and call
1174 * blk_end_request() for parts of the original function.
1175 * This prevents code duplication in drivers.
1177 extern bool blk_update_request(struct request
*rq
, blk_status_t error
,
1178 unsigned int nr_bytes
);
1179 extern void blk_finish_request(struct request
*rq
, blk_status_t error
);
1180 extern bool blk_end_request(struct request
*rq
, blk_status_t error
,
1181 unsigned int nr_bytes
);
1182 extern void blk_end_request_all(struct request
*rq
, blk_status_t error
);
1183 extern bool __blk_end_request(struct request
*rq
, blk_status_t error
,
1184 unsigned int nr_bytes
);
1185 extern void __blk_end_request_all(struct request
*rq
, blk_status_t error
);
1186 extern bool __blk_end_request_cur(struct request
*rq
, blk_status_t error
);
1188 extern void blk_complete_request(struct request
*);
1189 extern void __blk_complete_request(struct request
*);
1190 extern void blk_abort_request(struct request
*);
1191 extern void blk_unprep_request(struct request
*);
1194 * Access functions for manipulating queue properties
1196 extern struct request_queue
*blk_init_queue_node(request_fn_proc
*rfn
,
1197 spinlock_t
*lock
, int node_id
);
1198 extern struct request_queue
*blk_init_queue(request_fn_proc
*, spinlock_t
*);
1199 extern int blk_init_allocated_queue(struct request_queue
*);
1200 extern void blk_cleanup_queue(struct request_queue
*);
1201 extern void blk_queue_make_request(struct request_queue
*, make_request_fn
*);
1202 extern void blk_queue_bounce_limit(struct request_queue
*, u64
);
1203 extern void blk_queue_max_hw_sectors(struct request_queue
*, unsigned int);
1204 extern void blk_queue_chunk_sectors(struct request_queue
*, unsigned int);
1205 extern void blk_queue_max_segments(struct request_queue
*, unsigned short);
1206 extern void blk_queue_max_discard_segments(struct request_queue
*,
1208 extern void blk_queue_max_segment_size(struct request_queue
*, unsigned int);
1209 extern void blk_queue_max_discard_sectors(struct request_queue
*q
,
1210 unsigned int max_discard_sectors
);
1211 extern void blk_queue_max_write_same_sectors(struct request_queue
*q
,
1212 unsigned int max_write_same_sectors
);
1213 extern void blk_queue_max_write_zeroes_sectors(struct request_queue
*q
,
1214 unsigned int max_write_same_sectors
);
1215 extern void blk_queue_logical_block_size(struct request_queue
*, unsigned short);
1216 extern void blk_queue_physical_block_size(struct request_queue
*, unsigned int);
1217 extern void blk_queue_alignment_offset(struct request_queue
*q
,
1218 unsigned int alignment
);
1219 extern void blk_limits_io_min(struct queue_limits
*limits
, unsigned int min
);
1220 extern void blk_queue_io_min(struct request_queue
*q
, unsigned int min
);
1221 extern void blk_limits_io_opt(struct queue_limits
*limits
, unsigned int opt
);
1222 extern void blk_queue_io_opt(struct request_queue
*q
, unsigned int opt
);
1223 extern void blk_set_queue_depth(struct request_queue
*q
, unsigned int depth
);
1224 extern void blk_set_default_limits(struct queue_limits
*lim
);
1225 extern void blk_set_stacking_limits(struct queue_limits
*lim
);
1226 extern int blk_stack_limits(struct queue_limits
*t
, struct queue_limits
*b
,
1228 extern int bdev_stack_limits(struct queue_limits
*t
, struct block_device
*bdev
,
1230 extern void disk_stack_limits(struct gendisk
*disk
, struct block_device
*bdev
,
1232 extern void blk_queue_stack_limits(struct request_queue
*t
, struct request_queue
*b
);
1233 extern void blk_queue_dma_pad(struct request_queue
*, unsigned int);
1234 extern void blk_queue_update_dma_pad(struct request_queue
*, unsigned int);
1235 extern int blk_queue_dma_drain(struct request_queue
*q
,
1236 dma_drain_needed_fn
*dma_drain_needed
,
1237 void *buf
, unsigned int size
);
1238 extern void blk_queue_lld_busy(struct request_queue
*q
, lld_busy_fn
*fn
);
1239 extern void blk_queue_segment_boundary(struct request_queue
*, unsigned long);
1240 extern void blk_queue_virt_boundary(struct request_queue
*, unsigned long);
1241 extern void blk_queue_prep_rq(struct request_queue
*, prep_rq_fn
*pfn
);
1242 extern void blk_queue_unprep_rq(struct request_queue
*, unprep_rq_fn
*ufn
);
1243 extern void blk_queue_dma_alignment(struct request_queue
*, int);
1244 extern void blk_queue_update_dma_alignment(struct request_queue
*, int);
1245 extern void blk_queue_softirq_done(struct request_queue
*, softirq_done_fn
*);
1246 extern void blk_queue_rq_timed_out(struct request_queue
*, rq_timed_out_fn
*);
1247 extern void blk_queue_rq_timeout(struct request_queue
*, unsigned int);
1248 extern void blk_queue_flush_queueable(struct request_queue
*q
, bool queueable
);
1249 extern void blk_queue_write_cache(struct request_queue
*q
, bool enabled
, bool fua
);
1252 * Number of physical segments as sent to the device.
1254 * Normally this is the number of discontiguous data segments sent by the
1255 * submitter. But for data-less command like discard we might have no
1256 * actual data segments submitted, but the driver might have to add it's
1257 * own special payload. In that case we still return 1 here so that this
1258 * special payload will be mapped.
1260 static inline unsigned short blk_rq_nr_phys_segments(struct request
*rq
)
1262 if (rq
->rq_flags
& RQF_SPECIAL_PAYLOAD
)
1264 return rq
->nr_phys_segments
;
1268 * Number of discard segments (or ranges) the driver needs to fill in.
1269 * Each discard bio merged into a request is counted as one segment.
1271 static inline unsigned short blk_rq_nr_discard_segments(struct request
*rq
)
1273 return max_t(unsigned short, rq
->nr_phys_segments
, 1);
1276 extern int blk_rq_map_sg(struct request_queue
*, struct request
*, struct scatterlist
*);
1277 extern void blk_dump_rq_flags(struct request
*, char *);
1278 extern long nr_blockdev_pages(void);
1280 bool __must_check
blk_get_queue(struct request_queue
*);
1281 struct request_queue
*blk_alloc_queue(gfp_t
);
1282 struct request_queue
*blk_alloc_queue_node(gfp_t gfp_mask
, int node_id
,
1284 extern void blk_put_queue(struct request_queue
*);
1285 extern void blk_set_queue_dying(struct request_queue
*);
1288 * block layer runtime pm functions
1291 extern void blk_pm_runtime_init(struct request_queue
*q
, struct device
*dev
);
1292 extern int blk_pre_runtime_suspend(struct request_queue
*q
);
1293 extern void blk_post_runtime_suspend(struct request_queue
*q
, int err
);
1294 extern void blk_pre_runtime_resume(struct request_queue
*q
);
1295 extern void blk_post_runtime_resume(struct request_queue
*q
, int err
);
1296 extern void blk_set_runtime_active(struct request_queue
*q
);
1298 static inline void blk_pm_runtime_init(struct request_queue
*q
,
1299 struct device
*dev
) {}
1300 static inline int blk_pre_runtime_suspend(struct request_queue
*q
)
1304 static inline void blk_post_runtime_suspend(struct request_queue
*q
, int err
) {}
1305 static inline void blk_pre_runtime_resume(struct request_queue
*q
) {}
1306 static inline void blk_post_runtime_resume(struct request_queue
*q
, int err
) {}
1307 static inline void blk_set_runtime_active(struct request_queue
*q
) {}
1311 * blk_plug permits building a queue of related requests by holding the I/O
1312 * fragments for a short period. This allows merging of sequential requests
1313 * into single larger request. As the requests are moved from a per-task list to
1314 * the device's request_queue in a batch, this results in improved scalability
1315 * as the lock contention for request_queue lock is reduced.
1317 * It is ok not to disable preemption when adding the request to the plug list
1318 * or when attempting a merge, because blk_schedule_flush_list() will only flush
1319 * the plug list when the task sleeps by itself. For details, please see
1320 * schedule() where blk_schedule_flush_plug() is called.
1323 struct list_head list
; /* requests */
1324 struct list_head mq_list
; /* blk-mq requests */
1325 struct list_head cb_list
; /* md requires an unplug callback */
1327 #define BLK_MAX_REQUEST_COUNT 16
1328 #define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1331 typedef void (*blk_plug_cb_fn
)(struct blk_plug_cb
*, bool);
1332 struct blk_plug_cb
{
1333 struct list_head list
;
1334 blk_plug_cb_fn callback
;
1337 extern struct blk_plug_cb
*blk_check_plugged(blk_plug_cb_fn unplug
,
1338 void *data
, int size
);
1339 extern void blk_start_plug(struct blk_plug
*);
1340 extern void blk_finish_plug(struct blk_plug
*);
1341 extern void blk_flush_plug_list(struct blk_plug
*, bool);
1343 static inline void blk_flush_plug(struct task_struct
*tsk
)
1345 struct blk_plug
*plug
= tsk
->plug
;
1348 blk_flush_plug_list(plug
, false);
1351 static inline void blk_schedule_flush_plug(struct task_struct
*tsk
)
1353 struct blk_plug
*plug
= tsk
->plug
;
1356 blk_flush_plug_list(plug
, true);
1359 static inline bool blk_needs_flush_plug(struct task_struct
*tsk
)
1361 struct blk_plug
*plug
= tsk
->plug
;
1364 (!list_empty(&plug
->list
) ||
1365 !list_empty(&plug
->mq_list
) ||
1366 !list_empty(&plug
->cb_list
));
1372 extern int blk_queue_start_tag(struct request_queue
*, struct request
*);
1373 extern struct request
*blk_queue_find_tag(struct request_queue
*, int);
1374 extern void blk_queue_end_tag(struct request_queue
*, struct request
*);
1375 extern int blk_queue_init_tags(struct request_queue
*, int, struct blk_queue_tag
*, int);
1376 extern void blk_queue_free_tags(struct request_queue
*);
1377 extern int blk_queue_resize_tags(struct request_queue
*, int);
1378 extern struct blk_queue_tag
*blk_init_tags(int, int);
1379 extern void blk_free_tags(struct blk_queue_tag
*);
1381 static inline struct request
*blk_map_queue_find_tag(struct blk_queue_tag
*bqt
,
1384 if (unlikely(bqt
== NULL
|| tag
>= bqt
->real_max_depth
))
1386 return bqt
->tag_index
[tag
];
1389 extern int blkdev_issue_flush(struct block_device
*, gfp_t
, sector_t
*);
1390 extern int blkdev_issue_write_same(struct block_device
*bdev
, sector_t sector
,
1391 sector_t nr_sects
, gfp_t gfp_mask
, struct page
*page
);
1393 #define BLKDEV_DISCARD_SECURE (1 << 0) /* issue a secure erase */
1395 extern int blkdev_issue_discard(struct block_device
*bdev
, sector_t sector
,
1396 sector_t nr_sects
, gfp_t gfp_mask
, unsigned long flags
);
1397 extern int __blkdev_issue_discard(struct block_device
*bdev
, sector_t sector
,
1398 sector_t nr_sects
, gfp_t gfp_mask
, int flags
,
1401 #define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1402 #define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1404 extern int __blkdev_issue_zeroout(struct block_device
*bdev
, sector_t sector
,
1405 sector_t nr_sects
, gfp_t gfp_mask
, struct bio
**biop
,
1407 extern int blkdev_issue_zeroout(struct block_device
*bdev
, sector_t sector
,
1408 sector_t nr_sects
, gfp_t gfp_mask
, unsigned flags
);
1410 static inline int sb_issue_discard(struct super_block
*sb
, sector_t block
,
1411 sector_t nr_blocks
, gfp_t gfp_mask
, unsigned long flags
)
1413 return blkdev_issue_discard(sb
->s_bdev
,
1414 block
<< (sb
->s_blocksize_bits
-
1416 nr_blocks
<< (sb
->s_blocksize_bits
-
1420 static inline int sb_issue_zeroout(struct super_block
*sb
, sector_t block
,
1421 sector_t nr_blocks
, gfp_t gfp_mask
)
1423 return blkdev_issue_zeroout(sb
->s_bdev
,
1424 block
<< (sb
->s_blocksize_bits
-
1426 nr_blocks
<< (sb
->s_blocksize_bits
-
1431 extern int blk_verify_command(unsigned char *cmd
, fmode_t mode
);
1433 enum blk_default_limits
{
1434 BLK_MAX_SEGMENTS
= 128,
1435 BLK_SAFE_MAX_SECTORS
= 255,
1436 BLK_DEF_MAX_SECTORS
= 2560,
1437 BLK_MAX_SEGMENT_SIZE
= 65536,
1438 BLK_SEG_BOUNDARY_MASK
= 0xFFFFFFFFUL
,
1441 #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
1443 static inline unsigned long queue_segment_boundary(struct request_queue
*q
)
1445 return q
->limits
.seg_boundary_mask
;
1448 static inline unsigned long queue_virt_boundary(struct request_queue
*q
)
1450 return q
->limits
.virt_boundary_mask
;
1453 static inline unsigned int queue_max_sectors(struct request_queue
*q
)
1455 return q
->limits
.max_sectors
;
1458 static inline unsigned int queue_max_hw_sectors(struct request_queue
*q
)
1460 return q
->limits
.max_hw_sectors
;
1463 static inline unsigned short queue_max_segments(struct request_queue
*q
)
1465 return q
->limits
.max_segments
;
1468 static inline unsigned short queue_max_discard_segments(struct request_queue
*q
)
1470 return q
->limits
.max_discard_segments
;
1473 static inline unsigned int queue_max_segment_size(struct request_queue
*q
)
1475 return q
->limits
.max_segment_size
;
1478 static inline unsigned short queue_logical_block_size(struct request_queue
*q
)
1482 if (q
&& q
->limits
.logical_block_size
)
1483 retval
= q
->limits
.logical_block_size
;
1488 static inline unsigned short bdev_logical_block_size(struct block_device
*bdev
)
1490 return queue_logical_block_size(bdev_get_queue(bdev
));
1493 static inline unsigned int queue_physical_block_size(struct request_queue
*q
)
1495 return q
->limits
.physical_block_size
;
1498 static inline unsigned int bdev_physical_block_size(struct block_device
*bdev
)
1500 return queue_physical_block_size(bdev_get_queue(bdev
));
1503 static inline unsigned int queue_io_min(struct request_queue
*q
)
1505 return q
->limits
.io_min
;
1508 static inline int bdev_io_min(struct block_device
*bdev
)
1510 return queue_io_min(bdev_get_queue(bdev
));
1513 static inline unsigned int queue_io_opt(struct request_queue
*q
)
1515 return q
->limits
.io_opt
;
1518 static inline int bdev_io_opt(struct block_device
*bdev
)
1520 return queue_io_opt(bdev_get_queue(bdev
));
1523 static inline int queue_alignment_offset(struct request_queue
*q
)
1525 if (q
->limits
.misaligned
)
1528 return q
->limits
.alignment_offset
;
1531 static inline int queue_limit_alignment_offset(struct queue_limits
*lim
, sector_t sector
)
1533 unsigned int granularity
= max(lim
->physical_block_size
, lim
->io_min
);
1534 unsigned int alignment
= sector_div(sector
, granularity
>> SECTOR_SHIFT
)
1537 return (granularity
+ lim
->alignment_offset
- alignment
) % granularity
;
1540 static inline int bdev_alignment_offset(struct block_device
*bdev
)
1542 struct request_queue
*q
= bdev_get_queue(bdev
);
1544 if (q
->limits
.misaligned
)
1547 if (bdev
!= bdev
->bd_contains
)
1548 return bdev
->bd_part
->alignment_offset
;
1550 return q
->limits
.alignment_offset
;
1553 static inline int queue_discard_alignment(struct request_queue
*q
)
1555 if (q
->limits
.discard_misaligned
)
1558 return q
->limits
.discard_alignment
;
1561 static inline int queue_limit_discard_alignment(struct queue_limits
*lim
, sector_t sector
)
1563 unsigned int alignment
, granularity
, offset
;
1565 if (!lim
->max_discard_sectors
)
1568 /* Why are these in bytes, not sectors? */
1569 alignment
= lim
->discard_alignment
>> SECTOR_SHIFT
;
1570 granularity
= lim
->discard_granularity
>> SECTOR_SHIFT
;
1574 /* Offset of the partition start in 'granularity' sectors */
1575 offset
= sector_div(sector
, granularity
);
1577 /* And why do we do this modulus *again* in blkdev_issue_discard()? */
1578 offset
= (granularity
+ alignment
- offset
) % granularity
;
1580 /* Turn it back into bytes, gaah */
1581 return offset
<< SECTOR_SHIFT
;
1584 static inline int bdev_discard_alignment(struct block_device
*bdev
)
1586 struct request_queue
*q
= bdev_get_queue(bdev
);
1588 if (bdev
!= bdev
->bd_contains
)
1589 return bdev
->bd_part
->discard_alignment
;
1591 return q
->limits
.discard_alignment
;
1594 static inline unsigned int bdev_write_same(struct block_device
*bdev
)
1596 struct request_queue
*q
= bdev_get_queue(bdev
);
1599 return q
->limits
.max_write_same_sectors
;
1604 static inline unsigned int bdev_write_zeroes_sectors(struct block_device
*bdev
)
1606 struct request_queue
*q
= bdev_get_queue(bdev
);
1609 return q
->limits
.max_write_zeroes_sectors
;
1614 static inline enum blk_zoned_model
bdev_zoned_model(struct block_device
*bdev
)
1616 struct request_queue
*q
= bdev_get_queue(bdev
);
1619 return blk_queue_zoned_model(q
);
1621 return BLK_ZONED_NONE
;
1624 static inline bool bdev_is_zoned(struct block_device
*bdev
)
1626 struct request_queue
*q
= bdev_get_queue(bdev
);
1629 return blk_queue_is_zoned(q
);
1634 static inline unsigned int bdev_zone_sectors(struct block_device
*bdev
)
1636 struct request_queue
*q
= bdev_get_queue(bdev
);
1639 return blk_queue_zone_sectors(q
);
1643 static inline int queue_dma_alignment(struct request_queue
*q
)
1645 return q
? q
->dma_alignment
: 511;
1648 static inline int blk_rq_aligned(struct request_queue
*q
, unsigned long addr
,
1651 unsigned int alignment
= queue_dma_alignment(q
) | q
->dma_pad_mask
;
1652 return !(addr
& alignment
) && !(len
& alignment
);
1655 /* assumes size > 256 */
1656 static inline unsigned int blksize_bits(unsigned int size
)
1658 unsigned int bits
= 8;
1662 } while (size
> 256);
1666 static inline unsigned int block_size(struct block_device
*bdev
)
1668 return bdev
->bd_block_size
;
1671 static inline bool queue_flush_queueable(struct request_queue
*q
)
1673 return !test_bit(QUEUE_FLAG_FLUSH_NQ
, &q
->queue_flags
);
1676 typedef struct {struct page
*v
;} Sector
;
1678 unsigned char *read_dev_sector(struct block_device
*, sector_t
, Sector
*);
1680 static inline void put_dev_sector(Sector p
)
1685 static inline bool __bvec_gap_to_prev(struct request_queue
*q
,
1686 struct bio_vec
*bprv
, unsigned int offset
)
1689 ((bprv
->bv_offset
+ bprv
->bv_len
) & queue_virt_boundary(q
));
1693 * Check if adding a bio_vec after bprv with offset would create a gap in
1694 * the SG list. Most drivers don't care about this, but some do.
1696 static inline bool bvec_gap_to_prev(struct request_queue
*q
,
1697 struct bio_vec
*bprv
, unsigned int offset
)
1699 if (!queue_virt_boundary(q
))
1701 return __bvec_gap_to_prev(q
, bprv
, offset
);
1705 * Check if the two bvecs from two bios can be merged to one segment.
1706 * If yes, no need to check gap between the two bios since the 1st bio
1707 * and the 1st bvec in the 2nd bio can be handled in one segment.
1709 static inline bool bios_segs_mergeable(struct request_queue
*q
,
1710 struct bio
*prev
, struct bio_vec
*prev_last_bv
,
1711 struct bio_vec
*next_first_bv
)
1713 if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv
, next_first_bv
))
1715 if (!BIOVEC_SEG_BOUNDARY(q
, prev_last_bv
, next_first_bv
))
1717 if (prev
->bi_seg_back_size
+ next_first_bv
->bv_len
>
1718 queue_max_segment_size(q
))
1723 static inline bool bio_will_gap(struct request_queue
*q
,
1724 struct request
*prev_rq
,
1728 if (bio_has_data(prev
) && queue_virt_boundary(q
)) {
1729 struct bio_vec pb
, nb
;
1732 * don't merge if the 1st bio starts with non-zero
1733 * offset, otherwise it is quite difficult to respect
1734 * sg gap limit. We work hard to merge a huge number of small
1735 * single bios in case of mkfs.
1738 bio_get_first_bvec(prev_rq
->bio
, &pb
);
1740 bio_get_first_bvec(prev
, &pb
);
1745 * We don't need to worry about the situation that the
1746 * merged segment ends in unaligned virt boundary:
1748 * - if 'pb' ends aligned, the merged segment ends aligned
1749 * - if 'pb' ends unaligned, the next bio must include
1750 * one single bvec of 'nb', otherwise the 'nb' can't
1753 bio_get_last_bvec(prev
, &pb
);
1754 bio_get_first_bvec(next
, &nb
);
1756 if (!bios_segs_mergeable(q
, prev
, &pb
, &nb
))
1757 return __bvec_gap_to_prev(q
, &pb
, nb
.bv_offset
);
1763 static inline bool req_gap_back_merge(struct request
*req
, struct bio
*bio
)
1765 return bio_will_gap(req
->q
, req
, req
->biotail
, bio
);
1768 static inline bool req_gap_front_merge(struct request
*req
, struct bio
*bio
)
1770 return bio_will_gap(req
->q
, NULL
, bio
, req
->bio
);
1773 int kblockd_schedule_work(struct work_struct
*work
);
1774 int kblockd_schedule_work_on(int cpu
, struct work_struct
*work
);
1775 int kblockd_mod_delayed_work_on(int cpu
, struct delayed_work
*dwork
, unsigned long delay
);
1777 #define MODULE_ALIAS_BLOCKDEV(major,minor) \
1778 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1779 #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1780 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1782 #if defined(CONFIG_BLK_DEV_INTEGRITY)
1784 enum blk_integrity_flags
{
1785 BLK_INTEGRITY_VERIFY
= 1 << 0,
1786 BLK_INTEGRITY_GENERATE
= 1 << 1,
1787 BLK_INTEGRITY_DEVICE_CAPABLE
= 1 << 2,
1788 BLK_INTEGRITY_IP_CHECKSUM
= 1 << 3,
1791 struct blk_integrity_iter
{
1795 unsigned int data_size
;
1796 unsigned short interval
;
1797 const char *disk_name
;
1800 typedef blk_status_t (integrity_processing_fn
) (struct blk_integrity_iter
*);
1802 struct blk_integrity_profile
{
1803 integrity_processing_fn
*generate_fn
;
1804 integrity_processing_fn
*verify_fn
;
1808 extern void blk_integrity_register(struct gendisk
*, struct blk_integrity
*);
1809 extern void blk_integrity_unregister(struct gendisk
*);
1810 extern int blk_integrity_compare(struct gendisk
*, struct gendisk
*);
1811 extern int blk_rq_map_integrity_sg(struct request_queue
*, struct bio
*,
1812 struct scatterlist
*);
1813 extern int blk_rq_count_integrity_sg(struct request_queue
*, struct bio
*);
1814 extern bool blk_integrity_merge_rq(struct request_queue
*, struct request
*,
1816 extern bool blk_integrity_merge_bio(struct request_queue
*, struct request
*,
1819 static inline struct blk_integrity
*blk_get_integrity(struct gendisk
*disk
)
1821 struct blk_integrity
*bi
= &disk
->queue
->integrity
;
1830 struct blk_integrity
*bdev_get_integrity(struct block_device
*bdev
)
1832 return blk_get_integrity(bdev
->bd_disk
);
1835 static inline bool blk_integrity_rq(struct request
*rq
)
1837 return rq
->cmd_flags
& REQ_INTEGRITY
;
1840 static inline void blk_queue_max_integrity_segments(struct request_queue
*q
,
1843 q
->limits
.max_integrity_segments
= segs
;
1846 static inline unsigned short
1847 queue_max_integrity_segments(struct request_queue
*q
)
1849 return q
->limits
.max_integrity_segments
;
1852 static inline bool integrity_req_gap_back_merge(struct request
*req
,
1855 struct bio_integrity_payload
*bip
= bio_integrity(req
->bio
);
1856 struct bio_integrity_payload
*bip_next
= bio_integrity(next
);
1858 return bvec_gap_to_prev(req
->q
, &bip
->bip_vec
[bip
->bip_vcnt
- 1],
1859 bip_next
->bip_vec
[0].bv_offset
);
1862 static inline bool integrity_req_gap_front_merge(struct request
*req
,
1865 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
1866 struct bio_integrity_payload
*bip_next
= bio_integrity(req
->bio
);
1868 return bvec_gap_to_prev(req
->q
, &bip
->bip_vec
[bip
->bip_vcnt
- 1],
1869 bip_next
->bip_vec
[0].bv_offset
);
1872 #else /* CONFIG_BLK_DEV_INTEGRITY */
1875 struct block_device
;
1877 struct blk_integrity
;
1879 static inline int blk_integrity_rq(struct request
*rq
)
1883 static inline int blk_rq_count_integrity_sg(struct request_queue
*q
,
1888 static inline int blk_rq_map_integrity_sg(struct request_queue
*q
,
1890 struct scatterlist
*s
)
1894 static inline struct blk_integrity
*bdev_get_integrity(struct block_device
*b
)
1898 static inline struct blk_integrity
*blk_get_integrity(struct gendisk
*disk
)
1902 static inline int blk_integrity_compare(struct gendisk
*a
, struct gendisk
*b
)
1906 static inline void blk_integrity_register(struct gendisk
*d
,
1907 struct blk_integrity
*b
)
1910 static inline void blk_integrity_unregister(struct gendisk
*d
)
1913 static inline void blk_queue_max_integrity_segments(struct request_queue
*q
,
1917 static inline unsigned short queue_max_integrity_segments(struct request_queue
*q
)
1921 static inline bool blk_integrity_merge_rq(struct request_queue
*rq
,
1927 static inline bool blk_integrity_merge_bio(struct request_queue
*rq
,
1934 static inline bool integrity_req_gap_back_merge(struct request
*req
,
1939 static inline bool integrity_req_gap_front_merge(struct request
*req
,
1945 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1947 struct block_device_operations
{
1948 int (*open
) (struct block_device
*, fmode_t
);
1949 void (*release
) (struct gendisk
*, fmode_t
);
1950 int (*rw_page
)(struct block_device
*, sector_t
, struct page
*, bool);
1951 int (*ioctl
) (struct block_device
*, fmode_t
, unsigned, unsigned long);
1952 int (*compat_ioctl
) (struct block_device
*, fmode_t
, unsigned, unsigned long);
1953 unsigned int (*check_events
) (struct gendisk
*disk
,
1954 unsigned int clearing
);
1955 /* ->media_changed() is DEPRECATED, use ->check_events() instead */
1956 int (*media_changed
) (struct gendisk
*);
1957 void (*unlock_native_capacity
) (struct gendisk
*);
1958 int (*revalidate_disk
) (struct gendisk
*);
1959 int (*getgeo
)(struct block_device
*, struct hd_geometry
*);
1960 /* this callback is with swap_lock and sometimes page table lock held */
1961 void (*swap_slot_free_notify
) (struct block_device
*, unsigned long);
1962 struct module
*owner
;
1963 const struct pr_ops
*pr_ops
;
1966 extern int __blkdev_driver_ioctl(struct block_device
*, fmode_t
, unsigned int,
1968 extern int bdev_read_page(struct block_device
*, sector_t
, struct page
*);
1969 extern int bdev_write_page(struct block_device
*, sector_t
, struct page
*,
1970 struct writeback_control
*);
1972 #ifdef CONFIG_BLK_DEV_ZONED
1973 bool blk_req_needs_zone_write_lock(struct request
*rq
);
1974 void __blk_req_zone_write_lock(struct request
*rq
);
1975 void __blk_req_zone_write_unlock(struct request
*rq
);
1977 static inline void blk_req_zone_write_lock(struct request
*rq
)
1979 if (blk_req_needs_zone_write_lock(rq
))
1980 __blk_req_zone_write_lock(rq
);
1983 static inline void blk_req_zone_write_unlock(struct request
*rq
)
1985 if (rq
->rq_flags
& RQF_ZONE_WRITE_LOCKED
)
1986 __blk_req_zone_write_unlock(rq
);
1989 static inline bool blk_req_zone_is_write_locked(struct request
*rq
)
1991 return rq
->q
->seq_zones_wlock
&&
1992 test_bit(blk_rq_zone_no(rq
), rq
->q
->seq_zones_wlock
);
1995 static inline bool blk_req_can_dispatch_to_zone(struct request
*rq
)
1997 if (!blk_req_needs_zone_write_lock(rq
))
1999 return !blk_req_zone_is_write_locked(rq
);
2002 static inline bool blk_req_needs_zone_write_lock(struct request
*rq
)
2007 static inline void blk_req_zone_write_lock(struct request
*rq
)
2011 static inline void blk_req_zone_write_unlock(struct request
*rq
)
2014 static inline bool blk_req_zone_is_write_locked(struct request
*rq
)
2019 static inline bool blk_req_can_dispatch_to_zone(struct request
*rq
)
2023 #endif /* CONFIG_BLK_DEV_ZONED */
2025 #else /* CONFIG_BLOCK */
2027 struct block_device
;
2030 * stubs for when the block layer is configured out
2032 #define buffer_heads_over_limit 0
2034 static inline long nr_blockdev_pages(void)
2042 static inline void blk_start_plug(struct blk_plug
*plug
)
2046 static inline void blk_finish_plug(struct blk_plug
*plug
)
2050 static inline void blk_flush_plug(struct task_struct
*task
)
2054 static inline void blk_schedule_flush_plug(struct task_struct
*task
)
2059 static inline bool blk_needs_flush_plug(struct task_struct
*tsk
)
2064 static inline int blkdev_issue_flush(struct block_device
*bdev
, gfp_t gfp_mask
,
2065 sector_t
*error_sector
)
2070 #endif /* CONFIG_BLOCK */