]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - kernel/irq_work.c
2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
4 * Provides a framework for enqueueing and running callbacks from hardirq
5 * context. The enqueueing is NMI-safe.
8 #include <linux/kernel.h>
9 #include <linux/export.h>
10 #include <linux/irq_work.h>
11 #include <linux/percpu.h>
12 #include <linux/hardirq.h>
13 #include <asm/processor.h>
16 * An entry can be in one of four states:
18 * free NULL, 0 -> {claimed} : free to be used
19 * claimed NULL, 3 -> {pending} : claimed to be enqueued
20 * pending next, 3 -> {busy} : queued, pending callback
21 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
24 #define IRQ_WORK_PENDING 1UL
25 #define IRQ_WORK_BUSY 2UL
26 #define IRQ_WORK_FLAGS 3UL
28 static DEFINE_PER_CPU(struct llist_head
, irq_work_list
);
31 * Claim the entry so that no one else will poke at it.
33 static bool irq_work_claim(struct irq_work
*work
)
35 unsigned long flags
, nflags
;
39 if (flags
& IRQ_WORK_PENDING
)
41 nflags
= flags
| IRQ_WORK_FLAGS
;
42 if (cmpxchg(&work
->flags
, flags
, nflags
) == flags
)
50 void __weak
arch_irq_work_raise(void)
53 * Lame architectures will get the timer tick callback
58 * Queue the entry and raise the IPI if needed.
60 static void __irq_work_queue(struct irq_work
*work
)
66 empty
= llist_add(&work
->llnode
, &__get_cpu_var(irq_work_list
));
67 /* The list was empty, raise self-interrupt to start processing. */
69 arch_irq_work_raise();
75 * Enqueue the irq_work @entry, returns true on success, failure when the
76 * @entry was already enqueued by someone else.
78 * Can be re-enqueued while the callback is still in progress.
80 bool irq_work_queue(struct irq_work
*work
)
82 if (!irq_work_claim(work
)) {
84 * Already enqueued, can't do!
89 __irq_work_queue(work
);
92 EXPORT_SYMBOL_GPL(irq_work_queue
);
95 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
96 * context with local IRQs disabled.
98 void irq_work_run(void)
100 struct irq_work
*work
;
101 struct llist_head
*this_list
;
102 struct llist_node
*llnode
;
104 this_list
= &__get_cpu_var(irq_work_list
);
105 if (llist_empty(this_list
))
109 BUG_ON(!irqs_disabled());
111 llnode
= llist_del_all(this_list
);
112 while (llnode
!= NULL
) {
113 work
= llist_entry(llnode
, struct irq_work
, llnode
);
115 llnode
= llist_next(llnode
);
118 * Clear the PENDING bit, after this point the @work
121 work
->flags
= IRQ_WORK_BUSY
;
124 * Clear the BUSY bit and return to the free state if
125 * no-one else claimed it meanwhile.
127 (void)cmpxchg(&work
->flags
, IRQ_WORK_BUSY
, 0);
130 EXPORT_SYMBOL_GPL(irq_work_run
);
133 * Synchronize against the irq_work @entry, ensures the entry is not
136 void irq_work_sync(struct irq_work
*work
)
138 WARN_ON_ONCE(irqs_disabled());
140 while (work
->flags
& IRQ_WORK_BUSY
)
143 EXPORT_SYMBOL_GPL(irq_work_sync
);