]>
Commit | Line | Data |
---|---|---|
5e605b64 JA |
1 | /* |
2 | * Functions related to interrupt-poll handling in the block layer. This | |
3 | * is similar to NAPI for network devices. | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/module.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/bio.h> | |
5e605b64 JA |
9 | #include <linux/interrupt.h> |
10 | #include <linux/cpu.h> | |
511cbce2 | 11 | #include <linux/irq_poll.h> |
5e605b64 JA |
12 | #include <linux/delay.h> |
13 | ||
511cbce2 | 14 | static unsigned int irq_poll_budget __read_mostly = 256; |
37867ae7 | 15 | |
5e605b64 JA |
16 | static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); |
17 | ||
18 | /** | |
511cbce2 | 19 | * irq_poll_sched - Schedule a run of the iopoll handler |
5e605b64 JA |
20 | * @iop: The parent iopoll structure |
21 | * | |
22 | * Description: | |
511cbce2 | 23 | * Add this irq_poll structure to the pending poll list and trigger the |
ea51190c | 24 | * raise of the blk iopoll softirq. |
5e605b64 | 25 | **/ |
511cbce2 | 26 | void irq_poll_sched(struct irq_poll *iop) |
5e605b64 JA |
27 | { |
28 | unsigned long flags; | |
29 | ||
ea51190c CH |
30 | if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) |
31 | return; | |
32 | if (!test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) | |
33 | return; | |
34 | ||
5e605b64 | 35 | local_irq_save(flags); |
170d800a | 36 | list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2 | 37 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
38 | local_irq_restore(flags); |
39 | } | |
511cbce2 | 40 | EXPORT_SYMBOL(irq_poll_sched); |
5e605b64 JA |
41 | |
42 | /** | |
511cbce2 | 43 | * __irq_poll_complete - Mark this @iop as un-polled again |
5e605b64 JA |
44 | * @iop: The parent iopoll structure |
45 | * | |
46 | * Description: | |
511cbce2 | 47 | * See irq_poll_complete(). This function must be called with interrupts |
1badcfbd | 48 | * disabled. |
5e605b64 | 49 | **/ |
511cbce2 | 50 | void __irq_poll_complete(struct irq_poll *iop) |
5e605b64 JA |
51 | { |
52 | list_del(&iop->list); | |
4e857c58 | 53 | smp_mb__before_atomic(); |
511cbce2 | 54 | clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64 | 55 | } |
511cbce2 | 56 | EXPORT_SYMBOL(__irq_poll_complete); |
5e605b64 JA |
57 | |
58 | /** | |
511cbce2 | 59 | * irq_poll_complete - Mark this @iop as un-polled again |
5e605b64 JA |
60 | * @iop: The parent iopoll structure |
61 | * | |
62 | * Description: | |
1badcfbd JA |
63 | * If a driver consumes less than the assigned budget in its run of the |
64 | * iopoll handler, it'll end the polled mode by calling this function. The | |
ea51190c | 65 | * iopoll handler will not be invoked again before irq_poll_sched() |
1badcfbd | 66 | * is called. |
5e605b64 | 67 | **/ |
511cbce2 | 68 | void irq_poll_complete(struct irq_poll *iop) |
5e605b64 JA |
69 | { |
70 | unsigned long flags; | |
71 | ||
72 | local_irq_save(flags); | |
511cbce2 | 73 | __irq_poll_complete(iop); |
5e605b64 JA |
74 | local_irq_restore(flags); |
75 | } | |
511cbce2 | 76 | EXPORT_SYMBOL(irq_poll_complete); |
5e605b64 | 77 | |
511cbce2 | 78 | static void irq_poll_softirq(struct softirq_action *h) |
5e605b64 | 79 | { |
170d800a | 80 | struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); |
511cbce2 | 81 | int rearm = 0, budget = irq_poll_budget; |
5e605b64 | 82 | unsigned long start_time = jiffies; |
5e605b64 JA |
83 | |
84 | local_irq_disable(); | |
85 | ||
86 | while (!list_empty(list)) { | |
511cbce2 | 87 | struct irq_poll *iop; |
5e605b64 JA |
88 | int work, weight; |
89 | ||
90 | /* | |
91 | * If softirq window is exhausted then punt. | |
92 | */ | |
93 | if (budget <= 0 || time_after(jiffies, start_time)) { | |
94 | rearm = 1; | |
95 | break; | |
96 | } | |
97 | ||
98 | local_irq_enable(); | |
99 | ||
100 | /* Even though interrupts have been re-enabled, this | |
101 | * access is safe because interrupts can only add new | |
102 | * entries to the tail of this list, and only ->poll() | |
103 | * calls can remove this head entry from the list. | |
104 | */ | |
511cbce2 | 105 | iop = list_entry(list->next, struct irq_poll, list); |
5e605b64 JA |
106 | |
107 | weight = iop->weight; | |
108 | work = 0; | |
511cbce2 | 109 | if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64 JA |
110 | work = iop->poll(iop, weight); |
111 | ||
112 | budget -= work; | |
113 | ||
114 | local_irq_disable(); | |
115 | ||
fca51d64 JA |
116 | /* |
117 | * Drivers must not modify the iopoll state, if they | |
118 | * consume their assigned weight (or more, some drivers can't | |
119 | * easily just stop processing, they have to complete an | |
120 | * entire mask of commands).In such cases this code | |
121 | * still "owns" the iopoll instance and therefore can | |
5e605b64 JA |
122 | * move the instance around on the list at-will. |
123 | */ | |
124 | if (work >= weight) { | |
511cbce2 CH |
125 | if (irq_poll_disable_pending(iop)) |
126 | __irq_poll_complete(iop); | |
5e605b64 JA |
127 | else |
128 | list_move_tail(&iop->list, list); | |
129 | } | |
130 | } | |
131 | ||
132 | if (rearm) | |
511cbce2 | 133 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
134 | |
135 | local_irq_enable(); | |
136 | } | |
137 | ||
138 | /** | |
511cbce2 | 139 | * irq_poll_disable - Disable iopoll on this @iop |
5e605b64 JA |
140 | * @iop: The parent iopoll structure |
141 | * | |
142 | * Description: | |
143 | * Disable io polling and wait for any pending callbacks to have completed. | |
144 | **/ | |
511cbce2 | 145 | void irq_poll_disable(struct irq_poll *iop) |
5e605b64 | 146 | { |
511cbce2 CH |
147 | set_bit(IRQ_POLL_F_DISABLE, &iop->state); |
148 | while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) | |
5e605b64 | 149 | msleep(1); |
511cbce2 | 150 | clear_bit(IRQ_POLL_F_DISABLE, &iop->state); |
5e605b64 | 151 | } |
511cbce2 | 152 | EXPORT_SYMBOL(irq_poll_disable); |
5e605b64 JA |
153 | |
154 | /** | |
511cbce2 | 155 | * irq_poll_enable - Enable iopoll on this @iop |
5e605b64 JA |
156 | * @iop: The parent iopoll structure |
157 | * | |
158 | * Description: | |
1badcfbd JA |
159 | * Enable iopoll on this @iop. Note that the handler run will not be |
160 | * scheduled, it will only mark it as active. | |
5e605b64 | 161 | **/ |
511cbce2 | 162 | void irq_poll_enable(struct irq_poll *iop) |
5e605b64 | 163 | { |
511cbce2 | 164 | BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); |
4e857c58 | 165 | smp_mb__before_atomic(); |
511cbce2 | 166 | clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64 | 167 | } |
511cbce2 | 168 | EXPORT_SYMBOL(irq_poll_enable); |
5e605b64 JA |
169 | |
170 | /** | |
511cbce2 | 171 | * irq_poll_init - Initialize this @iop |
5e605b64 JA |
172 | * @iop: The parent iopoll structure |
173 | * @weight: The default weight (or command completion budget) | |
174 | * @poll_fn: The handler to invoke | |
175 | * | |
176 | * Description: | |
78d0264e | 177 | * Initialize and enable this irq_poll structure. |
5e605b64 | 178 | **/ |
511cbce2 | 179 | void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) |
5e605b64 JA |
180 | { |
181 | memset(iop, 0, sizeof(*iop)); | |
182 | INIT_LIST_HEAD(&iop->list); | |
183 | iop->weight = weight; | |
184 | iop->poll = poll_fn; | |
5e605b64 | 185 | } |
511cbce2 | 186 | EXPORT_SYMBOL(irq_poll_init); |
5e605b64 | 187 | |
511cbce2 | 188 | static int irq_poll_cpu_notify(struct notifier_block *self, |
0b776b06 | 189 | unsigned long action, void *hcpu) |
5e605b64 JA |
190 | { |
191 | /* | |
192 | * If a CPU goes away, splice its entries to the current CPU | |
193 | * and trigger a run of the softirq | |
194 | */ | |
195 | if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { | |
196 | int cpu = (unsigned long) hcpu; | |
197 | ||
198 | local_irq_disable(); | |
199 | list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), | |
170d800a | 200 | this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2 | 201 | __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64 JA |
202 | local_irq_enable(); |
203 | } | |
204 | ||
205 | return NOTIFY_OK; | |
206 | } | |
207 | ||
511cbce2 CH |
208 | static struct notifier_block irq_poll_cpu_notifier = { |
209 | .notifier_call = irq_poll_cpu_notify, | |
5e605b64 JA |
210 | }; |
211 | ||
511cbce2 | 212 | static __init int irq_poll_setup(void) |
5e605b64 JA |
213 | { |
214 | int i; | |
215 | ||
216 | for_each_possible_cpu(i) | |
217 | INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); | |
218 | ||
511cbce2 CH |
219 | open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq); |
220 | register_hotcpu_notifier(&irq_poll_cpu_notifier); | |
5e605b64 JA |
221 | return 0; |
222 | } | |
511cbce2 | 223 | subsys_initcall(irq_poll_setup); |