]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - block/blk-ioc.c
2 * Functions related to io context handling
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/init.h>
8 #include <linux/blkdev.h>
9 #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
10 #include <linux/slab.h>
15 * For io context allocations
17 static struct kmem_cache
*iocontext_cachep
;
20 * get_io_context - increment reference count to io_context
21 * @ioc: io_context to get
23 * Increment reference count to @ioc.
25 void get_io_context(struct io_context
*ioc
)
27 BUG_ON(atomic_long_read(&ioc
->refcount
) <= 0);
28 atomic_long_inc(&ioc
->refcount
);
30 EXPORT_SYMBOL(get_io_context
);
33 * Releasing ioc may nest into another put_io_context() leading to nested
34 * fast path release. As the ioc's can't be the same, this is okay but
35 * makes lockdep whine. Keep track of nesting and use it as subclass.
38 #define ioc_release_depth(q) ((q) ? (q)->ioc_release_depth : 0)
39 #define ioc_release_depth_inc(q) (q)->ioc_release_depth++
40 #define ioc_release_depth_dec(q) (q)->ioc_release_depth--
42 #define ioc_release_depth(q) 0
43 #define ioc_release_depth_inc(q) do { } while (0)
44 #define ioc_release_depth_dec(q) do { } while (0)
48 * Slow path for ioc release in put_io_context(). Performs double-lock
49 * dancing to unlink all cic's and then frees ioc.
51 static void ioc_release_fn(struct work_struct
*work
)
53 struct io_context
*ioc
= container_of(work
, struct io_context
,
55 struct request_queue
*last_q
= NULL
;
57 spin_lock_irq(&ioc
->lock
);
59 while (!hlist_empty(&ioc
->cic_list
)) {
60 struct cfq_io_context
*cic
= hlist_entry(ioc
->cic_list
.first
,
61 struct cfq_io_context
,
63 struct request_queue
*this_q
= cic
->q
;
65 if (this_q
!= last_q
) {
67 * Need to switch to @this_q. Once we release
68 * @ioc->lock, it can go away along with @cic.
71 __blk_get_queue(this_q
);
74 * blk_put_queue() might sleep thanks to kobject
75 * idiocy. Always release both locks, put and
79 spin_unlock(last_q
->queue_lock
);
80 spin_unlock_irq(&ioc
->lock
);
81 blk_put_queue(last_q
);
83 spin_unlock_irq(&ioc
->lock
);
87 spin_lock_irq(this_q
->queue_lock
);
88 spin_lock(&ioc
->lock
);
91 ioc_release_depth_inc(this_q
);
94 ioc_release_depth_dec(this_q
);
98 spin_unlock(last_q
->queue_lock
);
99 spin_unlock_irq(&ioc
->lock
);
100 blk_put_queue(last_q
);
102 spin_unlock_irq(&ioc
->lock
);
105 kmem_cache_free(iocontext_cachep
, ioc
);
109 * put_io_context - put a reference of io_context
110 * @ioc: io_context to put
111 * @locked_q: request_queue the caller is holding queue_lock of (hint)
113 * Decrement reference count of @ioc and release it if the count reaches
114 * zero. If the caller is holding queue_lock of a queue, it can indicate
115 * that with @locked_q. This is an optimization hint and the caller is
116 * allowed to pass in %NULL even when it's holding a queue_lock.
118 void put_io_context(struct io_context
*ioc
, struct request_queue
*locked_q
)
120 struct request_queue
*last_q
= locked_q
;
126 BUG_ON(atomic_long_read(&ioc
->refcount
) <= 0);
128 lockdep_assert_held(locked_q
->queue_lock
);
130 if (!atomic_long_dec_and_test(&ioc
->refcount
))
134 * Destroy @ioc. This is a bit messy because cic's are chained
135 * from both ioc and queue, and ioc->lock nests inside queue_lock.
136 * The inner ioc->lock should be held to walk our cic_list and then
137 * for each cic the outer matching queue_lock should be grabbed.
138 * ie. We need to do reverse-order double lock dancing.
140 * Another twist is that we are often called with one of the
141 * matching queue_locks held as indicated by @locked_q, which
142 * prevents performing double-lock dance for other queues.
144 * So, we do it in two stages. The fast path uses the queue_lock
145 * the caller is holding and, if other queues need to be accessed,
146 * uses trylock to avoid introducing locking dependency. This can
147 * handle most cases, especially if @ioc was performing IO on only
150 * If trylock doesn't cut it, we defer to @ioc->release_work which
151 * can do all the double-locking dancing.
153 spin_lock_irqsave_nested(&ioc
->lock
, flags
,
154 ioc_release_depth(locked_q
));
156 while (!hlist_empty(&ioc
->cic_list
)) {
157 struct cfq_io_context
*cic
= hlist_entry(ioc
->cic_list
.first
,
158 struct cfq_io_context
,
160 struct request_queue
*this_q
= cic
->q
;
162 if (this_q
!= last_q
) {
163 if (last_q
&& last_q
!= locked_q
)
164 spin_unlock(last_q
->queue_lock
);
167 if (!spin_trylock(this_q
->queue_lock
))
172 ioc_release_depth_inc(this_q
);
175 ioc_release_depth_dec(this_q
);
178 if (last_q
&& last_q
!= locked_q
)
179 spin_unlock(last_q
->queue_lock
);
181 spin_unlock_irqrestore(&ioc
->lock
, flags
);
183 /* if no cic's left, we're done; otherwise, kick release_work */
184 if (hlist_empty(&ioc
->cic_list
))
185 kmem_cache_free(iocontext_cachep
, ioc
);
187 schedule_work(&ioc
->release_work
);
189 EXPORT_SYMBOL(put_io_context
);
191 /* Called by the exiting task */
192 void exit_io_context(struct task_struct
*task
)
194 struct io_context
*ioc
;
196 /* PF_EXITING prevents new io_context from being attached to @task */
197 WARN_ON_ONCE(!(current
->flags
& PF_EXITING
));
200 ioc
= task
->io_context
;
201 task
->io_context
= NULL
;
204 atomic_dec(&ioc
->nr_tasks
);
205 put_io_context(ioc
, NULL
);
208 static struct io_context
*create_task_io_context(struct task_struct
*task
,
209 gfp_t gfp_flags
, int node
,
212 struct io_context
*ioc
;
214 ioc
= kmem_cache_alloc_node(iocontext_cachep
, gfp_flags
| __GFP_ZERO
,
220 atomic_long_set(&ioc
->refcount
, 1);
221 atomic_set(&ioc
->nr_tasks
, 1);
222 spin_lock_init(&ioc
->lock
);
223 INIT_RADIX_TREE(&ioc
->radix_root
, GFP_ATOMIC
| __GFP_HIGH
);
224 INIT_HLIST_HEAD(&ioc
->cic_list
);
225 INIT_WORK(&ioc
->release_work
, ioc_release_fn
);
227 /* try to install, somebody might already have beaten us to it */
230 if (!task
->io_context
&& !(task
->flags
& PF_EXITING
)) {
231 task
->io_context
= ioc
;
233 kmem_cache_free(iocontext_cachep
, ioc
);
234 ioc
= task
->io_context
;
245 * current_io_context - get io_context of %current
246 * @gfp_flags: allocation flags, used if allocation is necessary
247 * @node: allocation node, used if allocation is necessary
249 * Return io_context of %current. If it doesn't exist, it is created with
250 * @gfp_flags and @node. The returned io_context does NOT have its
251 * reference count incremented. Because io_context is exited only on task
252 * exit, %current can be sure that the returned io_context is valid and
253 * alive as long as it is executing.
255 struct io_context
*current_io_context(gfp_t gfp_flags
, int node
)
257 might_sleep_if(gfp_flags
& __GFP_WAIT
);
259 if (current
->io_context
)
260 return current
->io_context
;
262 return create_task_io_context(current
, gfp_flags
, node
, false);
264 EXPORT_SYMBOL(current_io_context
);
267 * get_task_io_context - get io_context of a task
268 * @task: task of interest
269 * @gfp_flags: allocation flags, used if allocation is necessary
270 * @node: allocation node, used if allocation is necessary
272 * Return io_context of @task. If it doesn't exist, it is created with
273 * @gfp_flags and @node. The returned io_context has its reference count
276 * This function always goes through task_lock() and it's better to use
277 * current_io_context() + get_io_context() for %current.
279 struct io_context
*get_task_io_context(struct task_struct
*task
,
280 gfp_t gfp_flags
, int node
)
282 struct io_context
*ioc
;
284 might_sleep_if(gfp_flags
& __GFP_WAIT
);
287 ioc
= task
->io_context
;
295 return create_task_io_context(task
, gfp_flags
, node
, true);
297 EXPORT_SYMBOL(get_task_io_context
);
299 void ioc_set_changed(struct io_context
*ioc
, int which
)
301 struct cfq_io_context
*cic
;
302 struct hlist_node
*n
;
304 hlist_for_each_entry(cic
, n
, &ioc
->cic_list
, cic_list
)
305 set_bit(which
, &cic
->changed
);
309 * ioc_ioprio_changed - notify ioprio change
310 * @ioc: io_context of interest
311 * @ioprio: new ioprio
313 * @ioc's ioprio has changed to @ioprio. Set %CIC_IOPRIO_CHANGED for all
314 * cic's. iosched is responsible for checking the bit and applying it on
315 * request issue path.
317 void ioc_ioprio_changed(struct io_context
*ioc
, int ioprio
)
321 spin_lock_irqsave(&ioc
->lock
, flags
);
322 ioc
->ioprio
= ioprio
;
323 ioc_set_changed(ioc
, CIC_IOPRIO_CHANGED
);
324 spin_unlock_irqrestore(&ioc
->lock
, flags
);
328 * ioc_cgroup_changed - notify cgroup change
329 * @ioc: io_context of interest
331 * @ioc's cgroup has changed. Set %CIC_CGROUP_CHANGED for all cic's.
332 * iosched is responsible for checking the bit and applying it on request
335 void ioc_cgroup_changed(struct io_context
*ioc
)
339 spin_lock_irqsave(&ioc
->lock
, flags
);
340 ioc_set_changed(ioc
, CIC_CGROUP_CHANGED
);
341 spin_unlock_irqrestore(&ioc
->lock
, flags
);
344 static int __init
blk_ioc_init(void)
346 iocontext_cachep
= kmem_cache_create("blkdev_ioc",
347 sizeof(struct io_context
), 0, SLAB_PANIC
, NULL
);
350 subsys_initcall(blk_ioc_init
);