]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> | |
3 | * | |
4 | * Provides a framework for enqueueing and running callbacks from hardirq | |
5 | * context. The enqueueing is NMI-safe. | |
6 | */ | |
7 | ||
8 | #include <linux/bug.h> | |
9 | #include <linux/kernel.h> | |
10 | #include <linux/export.h> | |
11 | #include <linux/irq_work.h> | |
12 | #include <linux/percpu.h> | |
13 | #include <linux/hardirq.h> | |
14 | #include <linux/irqflags.h> | |
15 | #include <asm/processor.h> | |
16 | ||
17 | /* | |
18 | * An entry can be in one of four states: | |
19 | * | |
20 | * free NULL, 0 -> {claimed} : free to be used | |
21 | * claimed NULL, 3 -> {pending} : claimed to be enqueued | |
22 | * pending next, 3 -> {busy} : queued, pending callback | |
23 | * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed | |
24 | */ | |
25 | ||
26 | #define IRQ_WORK_PENDING 1UL | |
27 | #define IRQ_WORK_BUSY 2UL | |
28 | #define IRQ_WORK_FLAGS 3UL | |
29 | ||
30 | static DEFINE_PER_CPU(struct llist_head, irq_work_list); | |
31 | ||
32 | /* | |
33 | * Claim the entry so that no one else will poke at it. | |
34 | */ | |
35 | static bool irq_work_claim(struct irq_work *work) | |
36 | { | |
37 | unsigned long flags, nflags; | |
38 | ||
39 | for (;;) { | |
40 | flags = work->flags; | |
41 | if (flags & IRQ_WORK_PENDING) | |
42 | return false; | |
43 | nflags = flags | IRQ_WORK_FLAGS; | |
44 | if (cmpxchg(&work->flags, flags, nflags) == flags) | |
45 | break; | |
46 | cpu_relax(); | |
47 | } | |
48 | ||
49 | return true; | |
50 | } | |
51 | ||
52 | void __weak arch_irq_work_raise(void) | |
53 | { | |
54 | /* | |
55 | * Lame architectures will get the timer tick callback | |
56 | */ | |
57 | } | |
58 | ||
59 | /* | |
60 | * Queue the entry and raise the IPI if needed. | |
61 | */ | |
62 | static void __irq_work_queue(struct irq_work *work) | |
63 | { | |
64 | bool empty; | |
65 | ||
66 | preempt_disable(); | |
67 | ||
68 | empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); | |
69 | /* The list was empty, raise self-interrupt to start processing. */ | |
70 | if (empty) | |
71 | arch_irq_work_raise(); | |
72 | ||
73 | preempt_enable(); | |
74 | } | |
75 | ||
76 | /* | |
77 | * Enqueue the irq_work @entry, returns true on success, failure when the | |
78 | * @entry was already enqueued by someone else. | |
79 | * | |
80 | * Can be re-enqueued while the callback is still in progress. | |
81 | */ | |
82 | bool irq_work_queue(struct irq_work *work) | |
83 | { | |
84 | if (!irq_work_claim(work)) { | |
85 | /* | |
86 | * Already enqueued, can't do! | |
87 | */ | |
88 | return false; | |
89 | } | |
90 | ||
91 | __irq_work_queue(work); | |
92 | return true; | |
93 | } | |
94 | EXPORT_SYMBOL_GPL(irq_work_queue); | |
95 | ||
96 | /* | |
97 | * Run the irq_work entries on this cpu. Requires to be ran from hardirq | |
98 | * context with local IRQs disabled. | |
99 | */ | |
100 | void irq_work_run(void) | |
101 | { | |
102 | struct irq_work *work; | |
103 | struct llist_head *this_list; | |
104 | struct llist_node *llnode; | |
105 | ||
106 | this_list = &__get_cpu_var(irq_work_list); | |
107 | if (llist_empty(this_list)) | |
108 | return; | |
109 | ||
110 | BUG_ON(!in_irq()); | |
111 | BUG_ON(!irqs_disabled()); | |
112 | ||
113 | llnode = llist_del_all(this_list); | |
114 | while (llnode != NULL) { | |
115 | work = llist_entry(llnode, struct irq_work, llnode); | |
116 | ||
117 | llnode = llist_next(llnode); | |
118 | ||
119 | /* | |
120 | * Clear the PENDING bit, after this point the @work | |
121 | * can be re-used. | |
122 | */ | |
123 | work->flags = IRQ_WORK_BUSY; | |
124 | work->func(work); | |
125 | /* | |
126 | * Clear the BUSY bit and return to the free state if | |
127 | * no-one else claimed it meanwhile. | |
128 | */ | |
129 | (void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); | |
130 | } | |
131 | } | |
132 | EXPORT_SYMBOL_GPL(irq_work_run); | |
133 | ||
134 | /* | |
135 | * Synchronize against the irq_work @entry, ensures the entry is not | |
136 | * currently in use. | |
137 | */ | |
138 | void irq_work_sync(struct irq_work *work) | |
139 | { | |
140 | WARN_ON_ONCE(irqs_disabled()); | |
141 | ||
142 | while (work->flags & IRQ_WORK_BUSY) | |
143 | cpu_relax(); | |
144 | } | |
145 | EXPORT_SYMBOL_GPL(irq_work_sync); |