]>
Commit | Line | Data |
---|---|---|
62ac665f MP |
1 | #ifndef _LINUX_PERCPU_RWSEM_H |
2 | #define _LINUX_PERCPU_RWSEM_H | |
3 | ||
4 | #include <linux/mutex.h> | |
5 | #include <linux/percpu.h> | |
6 | #include <linux/rcupdate.h> | |
7 | #include <linux/delay.h> | |
8 | ||
9 | struct percpu_rw_semaphore { | |
10 | unsigned __percpu *counters; | |
11 | bool locked; | |
12 | struct mutex mtx; | |
13 | }; | |
14 | ||
5c1eabe6 | 15 | #define light_mb() barrier() |
4b05a1c7 | 16 | #define heavy_mb() synchronize_sched_expedited() |
5c1eabe6 | 17 | |
62ac665f MP |
18 | static inline void percpu_down_read(struct percpu_rw_semaphore *p) |
19 | { | |
1bf11c53 | 20 | rcu_read_lock_sched(); |
62ac665f | 21 | if (unlikely(p->locked)) { |
1bf11c53 | 22 | rcu_read_unlock_sched(); |
62ac665f MP |
23 | mutex_lock(&p->mtx); |
24 | this_cpu_inc(*p->counters); | |
25 | mutex_unlock(&p->mtx); | |
26 | return; | |
27 | } | |
28 | this_cpu_inc(*p->counters); | |
1bf11c53 | 29 | rcu_read_unlock_sched(); |
5c1eabe6 | 30 | light_mb(); /* A, between read of p->locked and read of data, paired with D */ |
62ac665f MP |
31 | } |
32 | ||
33 | static inline void percpu_up_read(struct percpu_rw_semaphore *p) | |
34 | { | |
5c1eabe6 | 35 | light_mb(); /* B, between read of the data and write to p->counter, paired with C */ |
62ac665f MP |
36 | this_cpu_dec(*p->counters); |
37 | } | |
38 | ||
39 | static inline unsigned __percpu_count(unsigned __percpu *counters) | |
40 | { | |
41 | unsigned total = 0; | |
42 | int cpu; | |
43 | ||
44 | for_each_possible_cpu(cpu) | |
45 | total += ACCESS_ONCE(*per_cpu_ptr(counters, cpu)); | |
46 | ||
47 | return total; | |
48 | } | |
49 | ||
50 | static inline void percpu_down_write(struct percpu_rw_semaphore *p) | |
51 | { | |
52 | mutex_lock(&p->mtx); | |
53 | p->locked = true; | |
4b05a1c7 | 54 | synchronize_sched_expedited(); /* make sure that all readers exit the rcu_read_lock_sched region */ |
62ac665f MP |
55 | while (__percpu_count(p->counters)) |
56 | msleep(1); | |
5c1eabe6 | 57 | heavy_mb(); /* C, between read of p->counter and write to data, paired with B */ |
62ac665f MP |
58 | } |
59 | ||
60 | static inline void percpu_up_write(struct percpu_rw_semaphore *p) | |
61 | { | |
5c1eabe6 | 62 | heavy_mb(); /* D, between write to data and write to p->locked, paired with A */ |
62ac665f MP |
63 | p->locked = false; |
64 | mutex_unlock(&p->mtx); | |
65 | } | |
66 | ||
67 | static inline int percpu_init_rwsem(struct percpu_rw_semaphore *p) | |
68 | { | |
69 | p->counters = alloc_percpu(unsigned); | |
70 | if (unlikely(!p->counters)) | |
71 | return -ENOMEM; | |
72 | p->locked = false; | |
73 | mutex_init(&p->mtx); | |
74 | return 0; | |
75 | } | |
76 | ||
77 | static inline void percpu_free_rwsem(struct percpu_rw_semaphore *p) | |
78 | { | |
79 | free_percpu(p->counters); | |
80 | p->counters = NULL; /* catch use after free bugs */ | |
81 | } | |
82 | ||
83 | #endif |