2 * async.c: Asynchronous function calls for boot performance
4 * (C) Copyright 2009 Intel Corporation
5 * Author: Arjan van de Ven <arjan@linux.intel.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
16 Goals and Theory of Operation
18 The primary goal of this feature is to reduce the kernel boot time,
19 by doing various independent hardware delays and discovery operations
20 decoupled and not strictly serialized.
22 More specifically, the asynchronous function call concept allows
23 certain operations (primarily during system boot) to happen
24 asynchronously, out of order, while these operations still
25 have their externally visible parts happen sequentially and in-order.
26 (not unlike how out-of-order CPUs retire their instructions in order)
28 Key to the asynchronous function call implementation is the concept of
29 a "sequence cookie" (which, although it has an abstracted type, can be
30 thought of as a monotonically incrementing number).
32 The async core will assign each scheduled event such a sequence cookie and
33 pass this to the called functions.
35 The asynchronously called function should before doing a globally visible
36 operation, such as registering device numbers, call the
37 async_synchronize_cookie() function and pass in its own cookie. The
38 async_synchronize_cookie() function will make sure that all asynchronous
39 operations that were scheduled prior to the operation corresponding with the
40 cookie have completed.
42 Subsystem/driver initialization code that scheduled asynchronous probe
43 functions, but which shares global resources with other drivers/subsystems
44 that do not use the asynchronous call feature, need to do a full
45 synchronization with the async_synchronize_full() function, before returning
46 from their init function. This is to maintain strict ordering between the
47 asynchronous and synchronous parts of the kernel.
51 #include <linux/async.h>
52 #include <linux/module.h>
53 #include <linux/wait.h>
54 #include <linux/sched.h>
55 #include <linux/init.h>
56 #include <linux/kthread.h>
57 #include <linux/delay.h>
58 #include <asm/atomic.h>
60 static async_cookie_t next_cookie
= 1;
62 #define MAX_THREADS 256
63 #define MAX_WORK 32768
65 static LIST_HEAD(async_pending
);
66 static LIST_HEAD(async_running
);
67 static DEFINE_SPINLOCK(async_lock
);
69 static int async_enabled
= 0;
72 struct list_head list
;
73 async_cookie_t cookie
;
76 struct list_head
*running
;
79 static DECLARE_WAIT_QUEUE_HEAD(async_done
);
80 static DECLARE_WAIT_QUEUE_HEAD(async_new
);
82 static atomic_t entry_count
;
83 static atomic_t thread_count
;
85 extern int initcall_debug
;
89 * MUST be called with the lock held!
91 static async_cookie_t
__lowest_in_progress(struct list_head
*running
)
93 struct async_entry
*entry
;
94 if (!list_empty(running
)) {
95 entry
= list_first_entry(running
,
96 struct async_entry
, list
);
98 } else if (!list_empty(&async_pending
)) {
99 entry
= list_first_entry(&async_pending
,
100 struct async_entry
, list
);
101 return entry
->cookie
;
103 /* nothing in progress... next_cookie is "infinity" */
109 static async_cookie_t
lowest_in_progress(struct list_head
*running
)
114 spin_lock_irqsave(&async_lock
, flags
);
115 ret
= __lowest_in_progress(running
);
116 spin_unlock_irqrestore(&async_lock
, flags
);
120 * pick the first pending entry and run it
122 static void run_one_entry(void)
125 struct async_entry
*entry
;
126 ktime_t calltime
, delta
, rettime
;
128 /* 1) pick one task from the pending queue */
130 spin_lock_irqsave(&async_lock
, flags
);
131 if (list_empty(&async_pending
))
133 entry
= list_first_entry(&async_pending
, struct async_entry
, list
);
135 /* 2) move it to the running queue */
136 list_del(&entry
->list
);
137 list_add_tail(&entry
->list
, entry
->running
);
138 spin_unlock_irqrestore(&async_lock
, flags
);
140 /* 3) run it (and print duration)*/
141 if (initcall_debug
&& system_state
== SYSTEM_BOOTING
) {
142 printk("calling %lli_%pF @ %i\n", (long long)entry
->cookie
,
143 entry
->func
, task_pid_nr(current
));
144 calltime
= ktime_get();
146 entry
->func(entry
->data
, entry
->cookie
);
147 if (initcall_debug
&& system_state
== SYSTEM_BOOTING
) {
148 rettime
= ktime_get();
149 delta
= ktime_sub(rettime
, calltime
);
150 printk("initcall %lli_%pF returned 0 after %lld usecs\n",
151 (long long)entry
->cookie
,
153 (long long)ktime_to_ns(delta
) >> 10);
156 /* 4) remove it from the running queue */
157 spin_lock_irqsave(&async_lock
, flags
);
158 list_del(&entry
->list
);
160 /* 5) free the entry */
162 atomic_dec(&entry_count
);
164 spin_unlock_irqrestore(&async_lock
, flags
);
166 /* 6) wake up any waiters. */
167 wake_up(&async_done
);
171 spin_unlock_irqrestore(&async_lock
, flags
);
175 static async_cookie_t
__async_schedule(async_func_ptr
*ptr
, void *data
, struct list_head
*running
)
177 struct async_entry
*entry
;
179 async_cookie_t newcookie
;
182 /* allow irq-off callers */
183 entry
= kzalloc(sizeof(struct async_entry
), GFP_ATOMIC
);
186 * If we're out of memory or if there's too much work
187 * pending already, we execute synchronously.
189 if (!async_enabled
|| !entry
|| atomic_read(&entry_count
) > MAX_WORK
) {
191 spin_lock_irqsave(&async_lock
, flags
);
192 newcookie
= next_cookie
++;
193 spin_unlock_irqrestore(&async_lock
, flags
);
195 /* low on memory.. run synchronously */
196 ptr(data
, newcookie
);
201 entry
->running
= running
;
203 spin_lock_irqsave(&async_lock
, flags
);
204 newcookie
= entry
->cookie
= next_cookie
++;
205 list_add_tail(&entry
->list
, &async_pending
);
206 atomic_inc(&entry_count
);
207 spin_unlock_irqrestore(&async_lock
, flags
);
212 async_cookie_t
async_schedule(async_func_ptr
*ptr
, void *data
)
214 return __async_schedule(ptr
, data
, &async_running
);
216 EXPORT_SYMBOL_GPL(async_schedule
);
218 async_cookie_t
async_schedule_special(async_func_ptr
*ptr
, void *data
, struct list_head
*running
)
220 return __async_schedule(ptr
, data
, running
);
222 EXPORT_SYMBOL_GPL(async_schedule_special
);
224 void async_synchronize_full(void)
227 async_synchronize_cookie(next_cookie
);
228 } while (!list_empty(&async_running
) || !list_empty(&async_pending
));
230 EXPORT_SYMBOL_GPL(async_synchronize_full
);
232 void async_synchronize_full_special(struct list_head
*list
)
234 async_synchronize_cookie_special(next_cookie
, list
);
236 EXPORT_SYMBOL_GPL(async_synchronize_full_special
);
238 void async_synchronize_cookie_special(async_cookie_t cookie
, struct list_head
*running
)
240 ktime_t starttime
, delta
, endtime
;
242 if (initcall_debug
&& system_state
== SYSTEM_BOOTING
) {
243 printk("async_waiting @ %i\n", task_pid_nr(current
));
244 starttime
= ktime_get();
247 wait_event(async_done
, lowest_in_progress(running
) >= cookie
);
249 if (initcall_debug
&& system_state
== SYSTEM_BOOTING
) {
250 endtime
= ktime_get();
251 delta
= ktime_sub(endtime
, starttime
);
253 printk("async_continuing @ %i after %lli usec\n",
254 task_pid_nr(current
),
255 (long long)ktime_to_ns(delta
) >> 10);
258 EXPORT_SYMBOL_GPL(async_synchronize_cookie_special
);
260 void async_synchronize_cookie(async_cookie_t cookie
)
262 async_synchronize_cookie_special(cookie
, &async_running
);
264 EXPORT_SYMBOL_GPL(async_synchronize_cookie
);
267 static int async_thread(void *unused
)
269 DECLARE_WAITQUEUE(wq
, current
);
270 add_wait_queue(&async_new
, &wq
);
272 while (!kthread_should_stop()) {
274 set_current_state(TASK_INTERRUPTIBLE
);
276 * check the list head without lock.. false positives
277 * are dealt with inside run_one_entry() while holding
281 if (!list_empty(&async_pending
))
284 ret
= schedule_timeout(HZ
);
288 * we timed out, this means we as thread are redundant.
289 * we sign off and die, but we to avoid any races there
290 * is a last-straw check to see if work snuck in.
292 atomic_dec(&thread_count
);
293 wmb(); /* manager must see our departure first */
294 if (list_empty(&async_pending
))
297 * woops work came in between us timing out and us
298 * signing off; we need to stay alive and keep working.
300 atomic_inc(&thread_count
);
303 remove_wait_queue(&async_new
, &wq
);
308 static int async_manager_thread(void *unused
)
310 DECLARE_WAITQUEUE(wq
, current
);
311 add_wait_queue(&async_new
, &wq
);
313 while (!kthread_should_stop()) {
316 set_current_state(TASK_INTERRUPTIBLE
);
318 tc
= atomic_read(&thread_count
);
320 ec
= atomic_read(&entry_count
);
322 while (tc
< ec
&& tc
< MAX_THREADS
) {
323 if (IS_ERR(kthread_run(async_thread
, NULL
, "async/%i",
328 atomic_inc(&thread_count
);
334 remove_wait_queue(&async_new
, &wq
);
339 static int __init
async_init(void)
342 if (IS_ERR(kthread_run(async_manager_thread
, NULL
,
348 static int __init
setup_async(char *str
)
354 __setup("fastboot", setup_async
);
357 core_initcall(async_init
);