]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - kernel/irq_work.c
2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
4 * Provides a framework for enqueueing and running callbacks from hardirq
5 * context. The enqueueing is NMI-safe.
9 #include <linux/kernel.h>
10 #include <linux/export.h>
11 #include <linux/irq_work.h>
12 #include <linux/percpu.h>
13 #include <linux/hardirq.h>
14 #include <asm/processor.h>
17 * An entry can be in one of four states:
19 * free NULL, 0 -> {claimed} : free to be used
20 * claimed NULL, 3 -> {pending} : claimed to be enqueued
21 * pending next, 3 -> {busy} : queued, pending callback
22 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
25 #define IRQ_WORK_PENDING 1UL
26 #define IRQ_WORK_BUSY 2UL
27 #define IRQ_WORK_FLAGS 3UL
29 static DEFINE_PER_CPU(struct llist_head
, irq_work_list
);
32 * Claim the entry so that no one else will poke at it.
34 static bool irq_work_claim(struct irq_work
*work
)
36 unsigned long flags
, nflags
;
40 if (flags
& IRQ_WORK_PENDING
)
42 nflags
= flags
| IRQ_WORK_FLAGS
;
43 if (cmpxchg(&work
->flags
, flags
, nflags
) == flags
)
51 void __weak
arch_irq_work_raise(void)
54 * Lame architectures will get the timer tick callback
59 * Queue the entry and raise the IPI if needed.
61 static void __irq_work_queue(struct irq_work
*work
)
67 empty
= llist_add(&work
->llnode
, &__get_cpu_var(irq_work_list
));
68 /* The list was empty, raise self-interrupt to start processing. */
70 arch_irq_work_raise();
76 * Enqueue the irq_work @entry, returns true on success, failure when the
77 * @entry was already enqueued by someone else.
79 * Can be re-enqueued while the callback is still in progress.
81 bool irq_work_queue(struct irq_work
*work
)
83 if (!irq_work_claim(work
)) {
85 * Already enqueued, can't do!
90 __irq_work_queue(work
);
93 EXPORT_SYMBOL_GPL(irq_work_queue
);
96 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
97 * context with local IRQs disabled.
99 void irq_work_run(void)
101 struct irq_work
*work
;
102 struct llist_head
*this_list
;
103 struct llist_node
*llnode
;
105 this_list
= &__get_cpu_var(irq_work_list
);
106 if (llist_empty(this_list
))
110 BUG_ON(!irqs_disabled());
112 llnode
= llist_del_all(this_list
);
113 while (llnode
!= NULL
) {
114 work
= llist_entry(llnode
, struct irq_work
, llnode
);
116 llnode
= llist_next(llnode
);
119 * Clear the PENDING bit, after this point the @work
122 work
->flags
= IRQ_WORK_BUSY
;
125 * Clear the BUSY bit and return to the free state if
126 * no-one else claimed it meanwhile.
128 (void)cmpxchg(&work
->flags
, IRQ_WORK_BUSY
, 0);
131 EXPORT_SYMBOL_GPL(irq_work_run
);
134 * Synchronize against the irq_work @entry, ensures the entry is not
137 void irq_work_sync(struct irq_work
*work
)
139 WARN_ON_ONCE(irqs_disabled());
141 while (work
->flags
& IRQ_WORK_BUSY
)
144 EXPORT_SYMBOL_GPL(irq_work_sync
);