block/blk-stat.h

   1 #ifndef BLK_STAT_H
   2 #define BLK_STAT_H
   3
   4 #include <linux/kernel.h>
   5 #include <linux/blkdev.h>
   6 #include <linux/ktime.h>
   7 #include <linux/rcupdate.h>
   8 #include <linux/timer.h>
   9
  10 /*
  11  * from upper:
  12  * 3 bits: reserved for other usage
  13  * 12 bits: size
  14  * 49 bits: time
  15  */
  16 #define BLK_STAT_RES_BITS       3
  17 #define BLK_STAT_SIZE_BITS      12
  18 #define BLK_STAT_RES_SHIFT      (64 - BLK_STAT_RES_BITS)
  19 #define BLK_STAT_SIZE_SHIFT     (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS)
  20 #define BLK_STAT_TIME_MASK      ((1ULL << BLK_STAT_SIZE_SHIFT) - 1)
  21 #define BLK_STAT_SIZE_MASK      \
  22         (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT)
  23 #define BLK_STAT_RES_MASK       (~((1ULL << BLK_STAT_RES_SHIFT) - 1))
  24
  25 /**
  26  * struct blk_stat_callback - Block statistics callback.
  27  *
  28  * A &struct blk_stat_callback is associated with a &struct request_queue. While
  29  * @timer is active, that queue's request completion latencies are sorted into
  30  * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the
  31  * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked.
  32  */
  33 struct blk_stat_callback {
  34         /*
  35          * @list: RCU list of callbacks for a &struct request_queue.
  36          */
  37         struct list_head list;
  38
  39         /**
  40          * @timer: Timer for the next callback invocation.
  41          */
  42         struct timer_list timer;
  43
  44         /**
  45          * @cpu_stat: Per-cpu statistics buckets.
  46          */
  47         struct blk_rq_stat __percpu *cpu_stat;
  48
  49         /**
  50          * @bucket_fn: Given a request, returns which statistics bucket it
  51          * should be accounted under.
  52          */
  53         unsigned int (*bucket_fn)(const struct request *);
  54
  55         /**
  56          * @buckets: Number of statistics buckets.
  57          */
  58         unsigned int buckets;
  59
  60         /**
  61          * @stat: Array of statistics buckets.
  62          */
  63         struct blk_rq_stat *stat;
  64
  65         /**
  66          * @fn: Callback function.
  67          */
  68         void (*timer_fn)(struct blk_stat_callback *);
  69
  70         /**
  71          * @data: Private pointer for the user.
  72          */
  73         void *data;
  74
  75         struct rcu_head rcu;
  76 };
  77
  78 struct blk_queue_stats *blk_alloc_queue_stats(void);
  79 void blk_free_queue_stats(struct blk_queue_stats *);
  80
  81 void blk_stat_add(struct request *);
  82
  83 static inline u64 __blk_stat_time(u64 time)
  84 {
  85         return time & BLK_STAT_TIME_MASK;
  86 }
  87
  88 static inline u64 blk_stat_time(struct blk_issue_stat *stat)
  89 {
  90         return __blk_stat_time(stat->stat);
  91 }
  92
  93 static inline sector_t blk_capped_size(sector_t size)
  94 {
  95         return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1);
  96 }
  97
  98 static inline sector_t blk_stat_size(struct blk_issue_stat *stat)
  99 {
 100         return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT;
 101 }
 102
 103 static inline void blk_stat_set_issue(struct blk_issue_stat *stat,
 104         sector_t size)
 105 {
 106         stat->stat = (stat->stat & BLK_STAT_RES_MASK) |
 107                 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) |
 108                 (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT);
 109 }
 110
 111 /* record time/size info in request but not add a callback */
 112 void blk_stat_enable_accounting(struct request_queue *q);
 113
 114 /*
 115  * blk_stat_rq_ddir() - Bucket callback function for the request data direction.
 116  * @rq: Request.
 117  *
 118  * This is the same as rq_data_dir() but as a function so it can be used as
 119  * @bucket_fn for blk_stat_alloc_callback().
 120  *
 121  * Return: Data direction of the request, either READ or WRITE.
 122  */
 123 unsigned int blk_stat_rq_ddir(const struct request *rq);
 124
 125 /**
 126  * blk_stat_alloc_callback() - Allocate a block statistics callback.
 127  * @timer_fn: Timer callback function.
 128  * @bucket_fn: Bucket callback function.
 129  * @buckets: Number of statistics buckets.
 130  * @data: Value for the @data field of the &struct blk_stat_callback.
 131  *
 132  * See &struct blk_stat_callback for details on the callback functions.
 133  *
 134  * Return: &struct blk_stat_callback on success or NULL on ENOMEM.
 135  */
 136 struct blk_stat_callback *
 137 blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
 138                         unsigned int (*bucket_fn)(const struct request *),
 139                         unsigned int buckets, void *data);
 140
 141 /**
 142  * blk_stat_add_callback() - Add a block statistics callback to be run on a
 143  * request queue.
 144  * @q: The request queue.
 145  * @cb: The callback.
 146  *
 147  * Note that a single &struct blk_stat_callback can only be added to a single
 148  * &struct request_queue.
 149  */
 150 void blk_stat_add_callback(struct request_queue *q,
 151                            struct blk_stat_callback *cb);
 152
 153 /**
 154  * blk_stat_remove_callback() - Remove a block statistics callback from a
 155  * request queue.
 156  * @q: The request queue.
 157  * @cb: The callback.
 158  *
 159  * When this returns, the callback is not running on any CPUs and will not be
 160  * called again unless readded.
 161  */
 162 void blk_stat_remove_callback(struct request_queue *q,
 163                               struct blk_stat_callback *cb);
 164
 165 /**
 166  * blk_stat_free_callback() - Free a block statistics callback.
 167  * @cb: The callback.
 168  *
 169  * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must
 170  * not be associated with a request queue. I.e., if it was previously added with
 171  * blk_stat_add_callback(), it must also have been removed since then with
 172  * blk_stat_remove_callback().
 173  */
 174 void blk_stat_free_callback(struct blk_stat_callback *cb);
 175
 176 /**
 177  * blk_stat_is_active() - Check if a block statistics callback is currently
 178  * gathering statistics.
 179  * @cb: The callback.
 180  */
 181 static inline bool blk_stat_is_active(struct blk_stat_callback *cb)
 182 {
 183         return timer_pending(&cb->timer);
 184 }
 185
 186 /**
 187  * blk_stat_activate_nsecs() - Gather block statistics during a time window in
 188  * nanoseconds.
 189  * @cb: The callback.
 190  * @nsecs: Number of nanoseconds to gather statistics for.
 191  *
 192  * The timer callback will be called when the window expires.
 193  */
 194 static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb,
 195                                            u64 nsecs)
 196 {
 197         mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs));
 198 }
 199
 200 /**
 201  * blk_stat_activate_msecs() - Gather block statistics during a time window in
 202  * milliseconds.
 203  * @cb: The callback.
 204  * @msecs: Number of milliseconds to gather statistics for.
 205  *
 206  * The timer callback will be called when the window expires.
 207  */
 208 static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
 209                                            unsigned int msecs)
 210 {
 211         mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs));
 212 }
 213
 214 #endif