]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __LINUX_SEQLOCK_H |
2 | #define __LINUX_SEQLOCK_H | |
3 | /* | |
4 | * Reader/writer consistent mechanism without starving writers. This type of | |
d08df601 | 5 | * lock for data where the reader wants a consistent set of information |
1da177e4 LT |
6 | * and is willing to retry if the information changes. Readers never |
7 | * block but they may have to retry if a writer is in | |
8 | * progress. Writers do not wait for readers. | |
9 | * | |
10 | * This is not as cache friendly as brlock. Also, this will not work | |
11 | * for data that contains pointers, because any writer could | |
12 | * invalidate a pointer that a reader was following. | |
13 | * | |
14 | * Expected reader usage: | |
15 | * do { | |
16 | * seq = read_seqbegin(&foo); | |
17 | * ... | |
18 | * } while (read_seqretry(&foo, seq)); | |
19 | * | |
20 | * | |
21 | * On non-SMP the spin locks disappear but the writer still needs | |
22 | * to increment the sequence variables because an interrupt routine could | |
23 | * change the state of the data. | |
24 | * | |
25 | * Based on x86_64 vsyscall gettimeofday | |
26 | * by Keith Owens and Andrea Arcangeli | |
27 | */ | |
28 | ||
1da177e4 LT |
29 | #include <linux/spinlock.h> |
30 | #include <linux/preempt.h> | |
56a21052 | 31 | #include <asm/processor.h> |
1da177e4 LT |
32 | |
33 | typedef struct { | |
34 | unsigned sequence; | |
35 | spinlock_t lock; | |
36 | } seqlock_t; | |
37 | ||
38 | /* | |
39 | * These macros triggered gcc-3.x compile-time problems. We think these are | |
40 | * OK now. Be cautious. | |
41 | */ | |
e4d91918 IM |
42 | #define __SEQLOCK_UNLOCKED(lockname) \ |
43 | { 0, __SPIN_LOCK_UNLOCKED(lockname) } | |
1da177e4 | 44 | |
99a3eb38 IM |
45 | #define seqlock_init(x) \ |
46 | do { \ | |
47 | (x)->sequence = 0; \ | |
48 | spin_lock_init(&(x)->lock); \ | |
49 | } while (0) | |
e4d91918 IM |
50 | |
51 | #define DEFINE_SEQLOCK(x) \ | |
52 | seqlock_t x = __SEQLOCK_UNLOCKED(x) | |
1da177e4 LT |
53 | |
54 | /* Lock out other writers and update the count. | |
55 | * Acts like a normal spin_lock/unlock. | |
56 | * Don't need preempt_disable() because that is in the spin_lock already. | |
57 | */ | |
58 | static inline void write_seqlock(seqlock_t *sl) | |
59 | { | |
60 | spin_lock(&sl->lock); | |
61 | ++sl->sequence; | |
20f09390 DW |
62 | smp_wmb(); |
63 | } | |
1da177e4 | 64 | |
20f09390 | 65 | static inline void write_sequnlock(seqlock_t *sl) |
1da177e4 LT |
66 | { |
67 | smp_wmb(); | |
68 | sl->sequence++; | |
69 | spin_unlock(&sl->lock); | |
70 | } | |
71 | ||
1da177e4 | 72 | /* Start of read calculation -- fetch last complete writer token */ |
cde227af | 73 | static __always_inline unsigned read_seqbegin(const seqlock_t *sl) |
1da177e4 | 74 | { |
88a411c0 IM |
75 | unsigned ret; |
76 | ||
77 | repeat: | |
5db1256a | 78 | ret = ACCESS_ONCE(sl->sequence); |
88a411c0 IM |
79 | if (unlikely(ret & 1)) { |
80 | cpu_relax(); | |
81 | goto repeat; | |
82 | } | |
5db1256a | 83 | smp_rmb(); |
88a411c0 | 84 | |
1da177e4 LT |
85 | return ret; |
86 | } | |
87 | ||
88a411c0 IM |
88 | /* |
89 | * Test if reader processed invalid data. | |
90 | * | |
91 | * If sequence value changed then writer changed data while in section. | |
1da177e4 | 92 | */ |
88a411c0 | 93 | static __always_inline int read_seqretry(const seqlock_t *sl, unsigned start) |
1da177e4 LT |
94 | { |
95 | smp_rmb(); | |
88a411c0 | 96 | |
3c22cd57 | 97 | return unlikely(sl->sequence != start); |
1da177e4 LT |
98 | } |
99 | ||
100 | ||
101 | /* | |
102 | * Version using sequence counter only. | |
103 | * This can be used when code has its own mutex protecting the | |
104 | * updating starting before the write_seqcountbeqin() and ending | |
105 | * after the write_seqcount_end(). | |
106 | */ | |
107 | ||
108 | typedef struct seqcount { | |
109 | unsigned sequence; | |
110 | } seqcount_t; | |
111 | ||
112 | #define SEQCNT_ZERO { 0 } | |
113 | #define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0) | |
114 | ||
3c22cd57 NP |
115 | /** |
116 | * __read_seqcount_begin - begin a seq-read critical section (without barrier) | |
117 | * @s: pointer to seqcount_t | |
118 | * Returns: count to be passed to read_seqcount_retry | |
119 | * | |
120 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() | |
121 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
122 | * provided before actually loading any of the variables that are to be | |
123 | * protected in this critical section. | |
124 | * | |
125 | * Use carefully, only in critical code, and comment how the barrier is | |
126 | * provided. | |
127 | */ | |
128 | static inline unsigned __read_seqcount_begin(const seqcount_t *s) | |
1da177e4 | 129 | { |
88a411c0 IM |
130 | unsigned ret; |
131 | ||
132 | repeat: | |
2f624278 | 133 | ret = ACCESS_ONCE(s->sequence); |
88a411c0 IM |
134 | if (unlikely(ret & 1)) { |
135 | cpu_relax(); | |
136 | goto repeat; | |
137 | } | |
1da177e4 LT |
138 | return ret; |
139 | } | |
140 | ||
3c22cd57 NP |
141 | /** |
142 | * read_seqcount_begin - begin a seq-read critical section | |
143 | * @s: pointer to seqcount_t | |
144 | * Returns: count to be passed to read_seqcount_retry | |
145 | * | |
146 | * read_seqcount_begin opens a read critical section of the given seqcount. | |
147 | * Validity of the critical section is tested by checking read_seqcount_retry | |
148 | * function. | |
149 | */ | |
150 | static inline unsigned read_seqcount_begin(const seqcount_t *s) | |
151 | { | |
152 | unsigned ret = __read_seqcount_begin(s); | |
153 | smp_rmb(); | |
154 | return ret; | |
155 | } | |
156 | ||
4f988f15 LT |
157 | /** |
158 | * raw_seqcount_begin - begin a seq-read critical section | |
159 | * @s: pointer to seqcount_t | |
160 | * Returns: count to be passed to read_seqcount_retry | |
161 | * | |
162 | * raw_seqcount_begin opens a read critical section of the given seqcount. | |
163 | * Validity of the critical section is tested by checking read_seqcount_retry | |
164 | * function. | |
165 | * | |
166 | * Unlike read_seqcount_begin(), this function will not wait for the count | |
167 | * to stabilize. If a writer is active when we begin, we will fail the | |
168 | * read_seqcount_retry() instead of stabilizing at the beginning of the | |
169 | * critical section. | |
170 | */ | |
171 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) | |
172 | { | |
173 | unsigned ret = ACCESS_ONCE(s->sequence); | |
174 | smp_rmb(); | |
175 | return ret & ~1; | |
176 | } | |
177 | ||
3c22cd57 NP |
178 | /** |
179 | * __read_seqcount_retry - end a seq-read critical section (without barrier) | |
180 | * @s: pointer to seqcount_t | |
181 | * @start: count, from read_seqcount_begin | |
182 | * Returns: 1 if retry is required, else 0 | |
183 | * | |
184 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() | |
185 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | |
186 | * provided before actually loading any of the variables that are to be | |
187 | * protected in this critical section. | |
188 | * | |
189 | * Use carefully, only in critical code, and comment how the barrier is | |
190 | * provided. | |
191 | */ | |
192 | static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) | |
193 | { | |
194 | return unlikely(s->sequence != start); | |
195 | } | |
196 | ||
197 | /** | |
198 | * read_seqcount_retry - end a seq-read critical section | |
199 | * @s: pointer to seqcount_t | |
200 | * @start: count, from read_seqcount_begin | |
201 | * Returns: 1 if retry is required, else 0 | |
202 | * | |
203 | * read_seqcount_retry closes a read critical section of the given seqcount. | |
204 | * If the critical section was invalid, it must be ignored (and typically | |
205 | * retried). | |
1da177e4 | 206 | */ |
88a411c0 | 207 | static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) |
1da177e4 LT |
208 | { |
209 | smp_rmb(); | |
88a411c0 | 210 | |
3c22cd57 | 211 | return __read_seqcount_retry(s, start); |
1da177e4 LT |
212 | } |
213 | ||
214 | ||
215 | /* | |
216 | * Sequence counter only version assumes that callers are using their | |
217 | * own mutexing. | |
218 | */ | |
219 | static inline void write_seqcount_begin(seqcount_t *s) | |
220 | { | |
221 | s->sequence++; | |
222 | smp_wmb(); | |
223 | } | |
224 | ||
225 | static inline void write_seqcount_end(seqcount_t *s) | |
226 | { | |
227 | smp_wmb(); | |
228 | s->sequence++; | |
229 | } | |
230 | ||
3c22cd57 NP |
231 | /** |
232 | * write_seqcount_barrier - invalidate in-progress read-side seq operations | |
233 | * @s: pointer to seqcount_t | |
234 | * | |
235 | * After write_seqcount_barrier, no read-side seq operations will complete | |
236 | * successfully and see data older than this. | |
237 | */ | |
238 | static inline void write_seqcount_barrier(seqcount_t *s) | |
239 | { | |
240 | smp_wmb(); | |
241 | s->sequence+=2; | |
242 | } | |
243 | ||
1da177e4 LT |
244 | /* |
245 | * Possible sw/hw IRQ protected versions of the interfaces. | |
246 | */ | |
247 | #define write_seqlock_irqsave(lock, flags) \ | |
248 | do { local_irq_save(flags); write_seqlock(lock); } while (0) | |
249 | #define write_seqlock_irq(lock) \ | |
250 | do { local_irq_disable(); write_seqlock(lock); } while (0) | |
251 | #define write_seqlock_bh(lock) \ | |
252 | do { local_bh_disable(); write_seqlock(lock); } while (0) | |
253 | ||
254 | #define write_sequnlock_irqrestore(lock, flags) \ | |
255 | do { write_sequnlock(lock); local_irq_restore(flags); } while(0) | |
256 | #define write_sequnlock_irq(lock) \ | |
257 | do { write_sequnlock(lock); local_irq_enable(); } while(0) | |
258 | #define write_sequnlock_bh(lock) \ | |
259 | do { write_sequnlock(lock); local_bh_enable(); } while(0) | |
260 | ||
1da177e4 | 261 | #endif /* __LINUX_SEQLOCK_H */ |