]> git.proxmox.com Git - mirror_zfs.git/blame - lib/libspl/asm-generic/atomic.c
projects / mirror_zfs.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Add linux libspl support
[mirror_zfs.git] / lib / libspl / asm-generic / atomic.c
CommitLineData
a26baf28
BB		 1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright (c) 2009 by Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <atomic.h>
28#include <assert.h>
29#include <pthread.h>
30
31/*
32 * All operations are implemented by serializing them through a global
33 * pthread mutex. This provides a correct generic implementation.
34 * However all supported architectures are encouraged to provide a
35 * native implementation is assembly for performance reasons.
36 */
37pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
38
39/*
40 * Theses are the void returning variants
41 */
42
43#define ATOMIC_INC(name, type) \
44 void atomic_inc_##name(volatile type *target) \
45 { \
46 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
47 (*target)++; \
48 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
49 }
50
51ATOMIC_INC(long, unsigned long)
52ATOMIC_INC(8, uint8_t)
53ATOMIC_INC(uchar, uchar_t)
54ATOMIC_INC(16, uint16_t)
55ATOMIC_INC(ushort, ushort_t)
56ATOMIC_INC(32, uint32_t)
57ATOMIC_INC(uint, uint_t)
58ATOMIC_INC(ulong, ulong_t)
59ATOMIC_INC(64, uint64_t)
60
61
62#define ATOMIC_DEC(name, type) \
63 void atomic_dec_##name(volatile type *target) \
64 { \
65 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
66 (*target)--; \
67 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
68 }
69
70ATOMIC_DEC(long, unsigned long)
71ATOMIC_DEC(8, uint8_t)
72ATOMIC_DEC(uchar, uchar_t)
73ATOMIC_DEC(16, uint16_t)
74ATOMIC_DEC(ushort, ushort_t)
75ATOMIC_DEC(32, uint32_t)
76ATOMIC_DEC(uint, uint_t)
77ATOMIC_DEC(ulong, ulong_t)
78ATOMIC_DEC(64, uint64_t)
79
80
81#define ATOMIC_ADD(name, type1, type2) \
82 void atomic_add_##name(volatile type1 *target, type2 bits) \
83 { \
84 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
85 *target += bits; \
86 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
87 }
88
89ATOMIC_ADD(8, uint8_t, int8_t)
90ATOMIC_ADD(char, uchar_t, signed char)
91ATOMIC_ADD(16, uint16_t, int16_t)
92ATOMIC_ADD(short, ushort_t, short)
93ATOMIC_ADD(32, uint32_t, int32_t)
94ATOMIC_ADD(int, uint_t, int)
95ATOMIC_ADD(long, ulong_t, long)
96ATOMIC_ADD(64, uint64_t, int64_t)
97
98void atomic_add_ptr(volatile void *target, ssize_t bits)
99{
100 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
101 *(caddr_t *)target += bits;
102 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
103}
104
105
106#define ATOMIC_OR(name, type) \
107 void atomic_or_##name(volatile type *target, type bits) \
108 { \
109 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
110 *target |= bits; \
111 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
112 }
113
114ATOMIC_OR(8, uint8_t)
115ATOMIC_OR(uchar, uchar_t)
116ATOMIC_OR(16, uint16_t)
117ATOMIC_OR(ushort, ushort_t)
118ATOMIC_OR(32, uint32_t)
119ATOMIC_OR(uint, uint_t)
120ATOMIC_OR(ulong, ulong_t)
121ATOMIC_OR(64, uint64_t)
122
123
124#define ATOMIC_AND(name, type) \
125 void atomic_and_##name(volatile type *target, type bits) \
126 { \
127 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
128 *target &= bits; \
129 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
130 }
131
132ATOMIC_AND(8, uint8_t)
133ATOMIC_AND(uchar, uchar_t)
134ATOMIC_AND(16, uint16_t)
135ATOMIC_AND(ushort, ushort_t)
136ATOMIC_AND(32, uint32_t)
137ATOMIC_AND(uint, uint_t)
138ATOMIC_AND(ulong, ulong_t)
139ATOMIC_AND(64, uint64_t)
140
141
142/*
143 * New value returning variants
144 */
145
146#define ATOMIC_INC_NV(name, type) \
147 type atomic_inc_##name##_nv(volatile type *target) \
148 { \
149 type rc; \
150 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
151 rc = (++(*target)); \
152 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
153 return rc; \
154 }
155
156ATOMIC_INC_NV(long, unsigned long)
157ATOMIC_INC_NV(8, uint8_t)
158ATOMIC_INC_NV(uchar, uchar_t)
159ATOMIC_INC_NV(16, uint16_t)
160ATOMIC_INC_NV(ushort, ushort_t)
161ATOMIC_INC_NV(32, uint32_t)
162ATOMIC_INC_NV(uint, uint_t)
163ATOMIC_INC_NV(ulong, ulong_t)
164ATOMIC_INC_NV(64, uint64_t)
165
166
167#define ATOMIC_DEC_NV(name, type) \
168 type atomic_dec_##name##_nv(volatile type *target) \
169 { \
170 type rc; \
171 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
172 rc = (--(*target)); \
173 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
174 return rc; \
175 }
176
177ATOMIC_DEC_NV(long, unsigned long)
178ATOMIC_DEC_NV(8, uint8_t)
179ATOMIC_DEC_NV(uchar, uchar_t)
180ATOMIC_DEC_NV(16, uint16_t)
181ATOMIC_DEC_NV(ushort, ushort_t)
182ATOMIC_DEC_NV(32, uint32_t)
183ATOMIC_DEC_NV(uint, uint_t)
184ATOMIC_DEC_NV(ulong, ulong_t)
185ATOMIC_DEC_NV(64, uint64_t)
186
187
188#define ATOMIC_ADD_NV(name, type1, type2) \
189 type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
190 { \
191 type1 rc; \
192 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
193 rc = (*target += bits); \
194 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
195 return rc; \
196 }
197
198ATOMIC_ADD_NV(8, uint8_t, int8_t)
199ATOMIC_ADD_NV(char, uchar_t, signed char)
200ATOMIC_ADD_NV(16, uint16_t, int16_t)
201ATOMIC_ADD_NV(short, ushort_t, short)
202ATOMIC_ADD_NV(32, uint32_t, int32_t)
203ATOMIC_ADD_NV(int, uint_t, int)
204ATOMIC_ADD_NV(long, ulong_t, long)
205ATOMIC_ADD_NV(64, uint64_t, int64_t)
206
207void *atomic_add_ptr_nv(volatile void *target, ssize_t bits)
208{
209 void *ptr;
210
211 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
212 ptr = (*(caddr_t *)target += bits);
213 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
214
215 return ptr;
216}
217
218
219#define ATOMIC_OR_NV(name, type) \
220 type atomic_or_##name##_nv(volatile type *target, type bits) \
221 { \
222 type rc; \
223 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
224 rc = (*target |= bits); \
225 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
226 return rc; \
227 }
228
229ATOMIC_OR_NV(long, unsigned long)
230ATOMIC_OR_NV(8, uint8_t)
231ATOMIC_OR_NV(uchar, uchar_t)
232ATOMIC_OR_NV(16, uint16_t)
233ATOMIC_OR_NV(ushort, ushort_t)
234ATOMIC_OR_NV(32, uint32_t)
235ATOMIC_OR_NV(uint, uint_t)
236ATOMIC_OR_NV(ulong, ulong_t)
237ATOMIC_OR_NV(64, uint64_t)
238
239
240#define ATOMIC_AND_NV(name, type) \
241 type atomic_and_##name##_nv(volatile type *target, type bits) \
242 { \
243 type rc; \
244 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
245 rc = (*target &= bits); \
246 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
247 return rc; \
248 }
249
250ATOMIC_AND_NV(long, unsigned long)
251ATOMIC_AND_NV(8, uint8_t)
252ATOMIC_AND_NV(uchar, uchar_t)
253ATOMIC_AND_NV(16, uint16_t)
254ATOMIC_AND_NV(ushort, ushort_t)
255ATOMIC_AND_NV(32, uint32_t)
256ATOMIC_AND_NV(uint, uint_t)
257ATOMIC_AND_NV(ulong, ulong_t)
258ATOMIC_AND_NV(64, uint64_t)
259
260
261/*
262 * If *arg1 == arg2, set *arg1 = arg3; return old value
263 */
264
265#define ATOMIC_CAS(name, type) \
266 type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
267 { \
268 type old; \
269 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
270 old = *target; \
271 if (old == arg1) \
272 *target = arg2; \
273 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
274 return old; \
275 }
276
277ATOMIC_CAS(8, uint8_t)
278ATOMIC_CAS(uchar, uchar_t)
279ATOMIC_CAS(16, uint16_t)
280ATOMIC_CAS(ushort, ushort_t)
281ATOMIC_CAS(32, uint32_t)
282ATOMIC_CAS(uint, uint_t)
283ATOMIC_CAS(ulong, ulong_t)
284ATOMIC_CAS(64, uint64_t)
285
286void *atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
287{
288 void *old;
289
290 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
291 old = *(void **)target;
292 if (old == arg1)
293 *(void **)target = arg2;
294 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
295
296 return old;
297}
298
299
300/*
301 * Swap target and return old value
302 */
303
304#define ATOMIC_SWAP(name, type) \
305 type atomic_swap_##name(volatile type *target, type bits) \
306 { \
307 type old; \
308 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
309 old = *target; \
310 *target = bits; \
311 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
312 return old; \
313 }
314
315ATOMIC_SWAP(8, uint8_t)
316ATOMIC_SWAP(uchar, uchar_t)
317ATOMIC_SWAP(16, uint16_t)
318ATOMIC_SWAP(ushort, ushort_t)
319ATOMIC_SWAP(32, uint32_t)
320ATOMIC_SWAP(uint, uint_t)
321ATOMIC_SWAP(ulong, ulong_t)
322ATOMIC_SWAP(64, uint64_t)
323
324void *atomic_swap_ptr(volatile void *target, void *bits)
325{
326 void *old;
327
328 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
329 old = *(void **)target;
330 *(void **)target = bits;
331 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
332
333 return old;
334}
335
336
337int atomic_set_long_excl(volatile ulong_t *target, uint_t value)
338{
339 ulong_t bit;
340
341 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
342 bit = (1UL << value);
343 if ((*target & bit) != 0) {
344 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
345 return -1;
346 }
347 *target |= bit;
348 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
349
350 return 0;
351}
352
353int atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
354{
355 ulong_t bit;
356
357 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
358 bit = (1UL << value);
359 if ((*target & bit) != 0) {
360 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
361 return -1;
362 }
363 *target &= ~bit;
364 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
365
366 return 0;
367}
368
369void membar_enter(void)
370{
371 /* XXX - Implement me */
372}
373
374void membar_exit(void)
375{
376 /* XXX - Implement me */
377}
378
379void membar_producer(void)
380{
381 /* XXX - Implement me */
382}
383
384void membar_consumer(void)
385{
386 /* XXX - Implement me */
387}
388
389/* Legacy kernel interfaces; they will go away (eventually). */
390
391uint8_t cas8(uint8_t *target, uint8_t arg1, uint8_t arg2)
392{
393 return atomic_cas_8(target, arg1, arg2);
394}
395
396uint32_t cas32(uint32_t *target, uint32_t arg1, uint32_t arg2)
397{
398 return atomic_cas_32(target, arg1, arg2);
399}
400
401uint64_t cas64(uint64_t *target, uint64_t arg1, uint64_t arg2)
402{
403 return atomic_cas_64(target, arg1, arg2);
404}
405
406ulong_t caslong(ulong_t *target, ulong_t arg1, ulong_t arg2)
407{
408 return atomic_cas_ulong(target, arg1, arg2);
409}
410
411void *casptr(void *target, void *arg1, void *arg2)
412{
413 return atomic_cas_ptr(target, arg1, arg2);
414}
415
416void atomic_and_long(ulong_t *target, ulong_t bits)
417{
418 return atomic_and_ulong(target, bits);
419}
420
421void atomic_or_long(ulong_t *target, ulong_t bits)
422{
423 return atomic_or_ulong(target, bits);
424}
ZFS On Linux mirror for PVE