]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - tools/testing/selftests/rseq/rseq-ppc.h
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
rseq/selftests: Introduce __rseq_cs_ptr_array, rename __rseq_table to __rseq_cs
[mirror_ubuntu-hirsute-kernel.git] / tools / testing / selftests / rseq / rseq-ppc.h
CommitLineData
2e155fb7
MD		 1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9#define RSEQ_SIG 0x53053053
10
11#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
12#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
13#define rseq_smp_rmb() rseq_smp_lwsync()
14#define rseq_smp_wmb() rseq_smp_lwsync()
15
16#define rseq_smp_load_acquire(p) \
17__extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_smp_lwsync(); \
20 ____p1; \
21})
22
23#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
24
25#define rseq_smp_store_release(p, v) \
26do { \
27 rseq_smp_lwsync(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29} while (0)
30
31#ifdef RSEQ_SKIP_FASTPATH
32#include "rseq-skip.h"
33#else /* !RSEQ_SKIP_FASTPATH */
34
35/*
a3e3131f
MD		 36 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
37 * better handle single-stepping through the restartable critical sections.
2e155fb7
MD		 38 */
39
40#ifdef __PPC64__
41
42#define STORE_WORD "std "
43#define LOAD_WORD "ld "
44#define LOADX_WORD "ldx "
45#define CMP_WORD "cmpd "
46
47#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
48 start_ip, post_commit_offset, abort_ip) \
a3e3131f 49 ".pushsection __rseq_cs, \"aw\"\n\t" \
2e155fb7
MD		 50 ".balign 32\n\t" \
51 __rseq_str(label) ":\n\t" \
52 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
53 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
a3e3131f
MD		 54 ".popsection\n\t" \
55 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
56 ".quad " __rseq_str(label) "b\n\t" \
2e155fb7
MD		 57 ".popsection\n\t"
58
59#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
60 RSEQ_INJECT_ASM(1) \
61 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
62 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
63 "rldicr %%r17, %%r17, 32, 31\n\t" \
64 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
65 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
66 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
67 __rseq_str(label) ":\n\t"
68
4fe2088e
MD		 69/*
70 * Exit points of a rseq critical section consist of all instructions outside
71 * of the critical section where a critical section can either branch to or
72 * reach through the normal course of its execution. The abort IP and the
a3e3131f
MD		 73 * post-commit IP are already part of the __rseq_cs section and should not be
74 * explicitly defined as additional exit points. Knowing all exit points is
4fe2088e
MD		 75 * useful to assist debuggers stepping over the critical section.
76 */
77#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
78 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
79 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
80 ".popsection\n\t"
81
2e155fb7
MD		 82#else /* #ifdef __PPC64__ */
83
84#define STORE_WORD "stw "
85#define LOAD_WORD "lwz "
86#define LOADX_WORD "lwzx "
87#define CMP_WORD "cmpw "
88
89#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
90 start_ip, post_commit_offset, abort_ip) \
a3e3131f 91 ".pushsection __rseq_cs, \"aw\"\n\t" \
2e155fb7
MD		 92 ".balign 32\n\t" \
93 __rseq_str(label) ":\n\t" \
94 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
95 /* 32-bit only supported on BE */ \
96 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
a3e3131f
MD		 97 ".popsection\n\t" \
98 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
99 ".long 0x0, " __rseq_str(label) "b\n\t" \
2e155fb7
MD		 100 ".popsection\n\t"
101
4fe2088e
MD		 102/*
103 * Exit points of a rseq critical section consist of all instructions outside
104 * of the critical section where a critical section can either branch to or
105 * reach through the normal course of its execution. The abort IP and the
a3e3131f
MD		 106 * post-commit IP are already part of the __rseq_cs section and should not be
107 * explicitly defined as additional exit points. Knowing all exit points is
4fe2088e
MD		 108 * useful to assist debuggers stepping over the critical section.
109 */
110#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
111 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
112 /* 32-bit only supported on BE */ \
113 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
114 ".popsection\n\t"
115
2e155fb7
MD		 116#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
117 RSEQ_INJECT_ASM(1) \
118 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
119 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
120 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
121 __rseq_str(label) ":\n\t"
122
123#endif /* #ifdef __PPC64__ */
124
125#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
126 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
127 (post_commit_ip - start_ip), abort_ip)
128
129#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
130 RSEQ_INJECT_ASM(2) \
131 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
132 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
133 "bne- cr7, " __rseq_str(label) "\n\t"
134
135#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
136 ".pushsection __rseq_failure, \"ax\"\n\t" \
137 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
138 __rseq_str(label) ":\n\t" \
139 "b %l[" __rseq_str(abort_label) "]\n\t" \
140 ".popsection\n\t"
141
142/*
143 * RSEQ_ASM_OPs: asm operations for rseq
144 * RSEQ_ASM_OP_R_*: has hard-code registers in it
145 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
146 */
147#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
148 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
149 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
150 "bne- cr7, " __rseq_str(label) "\n\t"
151
152#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
153 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
154 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
155 "beq- cr7, " __rseq_str(label) "\n\t"
156
157#define RSEQ_ASM_OP_STORE(value, var) \
158 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
159
160/* Load @var to r17 */
161#define RSEQ_ASM_OP_R_LOAD(var) \
162 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
163
164/* Store r17 to @var */
165#define RSEQ_ASM_OP_R_STORE(var) \
166 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
167
168/* Add @count to r17 */
169#define RSEQ_ASM_OP_R_ADD(count) \
170 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
171
172/* Load (r17 + voffp) to r17 */
173#define RSEQ_ASM_OP_R_LOADX(voffp) \
174 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
175
176/* TODO: implement a faster memcpy. */
177#define RSEQ_ASM_OP_R_MEMCPY() \
178 "cmpdi %%r19, 0\n\t" \
179 "beq 333f\n\t" \
180 "addi %%r20, %%r20, -1\n\t" \
181 "addi %%r21, %%r21, -1\n\t" \
182 "222:\n\t" \
183 "lbzu %%r18, 1(%%r20)\n\t" \
184 "stbu %%r18, 1(%%r21)\n\t" \
185 "addi %%r19, %%r19, -1\n\t" \
186 "cmpdi %%r19, 0\n\t" \
187 "bne 222b\n\t" \
188 "333:\n\t" \
189
190#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
191 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
192 __rseq_str(post_commit_label) ":\n\t"
193
194#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
195 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
196 __rseq_str(post_commit_label) ":\n\t"
197
198static inline __attribute__((always_inline))
199int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
200{
201 RSEQ_INJECT_C(9)
202
203 __asm__ __volatile__ goto (
204 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 205 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
206#ifdef RSEQ_COMPARE_TWICE
207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
208 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
209#endif
2e155fb7
MD		 210 /* Start rseq by storing table entry pointer into rseq_cs. */
211 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
212 /* cmp cpuid */
213 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
214 RSEQ_INJECT_ASM(3)
215 /* cmp @v equal to @expect */
216 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
217 RSEQ_INJECT_ASM(4)
218#ifdef RSEQ_COMPARE_TWICE
219 /* cmp cpuid */
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
221 /* cmp @v equal to @expect */
222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
223#endif
224 /* final store */
225 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
226 RSEQ_INJECT_ASM(5)
227 RSEQ_ASM_DEFINE_ABORT(4, abort)
228 : /* gcc asm goto does not allow outputs */
229 : [cpu_id] "r" (cpu),
230 [current_cpu_id] "m" (__rseq_abi.cpu_id),
231 [rseq_cs] "m" (__rseq_abi.rseq_cs),
232 [v] "m" (*v),
233 [expect] "r" (expect),
234 [newv] "r" (newv)
235 RSEQ_INJECT_INPUT
236 : "memory", "cc", "r17"
237 RSEQ_INJECT_CLOBBER
238 : abort, cmpfail
239#ifdef RSEQ_COMPARE_TWICE
240 , error1, error2
241#endif
242 );
243 return 0;
244abort:
245 RSEQ_INJECT_FAILED
246 return -1;
247cmpfail:
248 return 1;
249#ifdef RSEQ_COMPARE_TWICE
250error1:
251 rseq_bug("cpu_id comparison failed");
252error2:
253 rseq_bug("expected value comparison failed");
254#endif
255}
256
257static inline __attribute__((always_inline))
258int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
259 off_t voffp, intptr_t *load, int cpu)
260{
261 RSEQ_INJECT_C(9)
262
263 __asm__ __volatile__ goto (
264 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
266#ifdef RSEQ_COMPARE_TWICE
267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
268 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
269#endif
2e155fb7
MD		 270 /* Start rseq by storing table entry pointer into rseq_cs. */
271 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
272 /* cmp cpuid */
273 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
274 RSEQ_INJECT_ASM(3)
275 /* cmp @v not equal to @expectnot */
276 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
277 RSEQ_INJECT_ASM(4)
278#ifdef RSEQ_COMPARE_TWICE
279 /* cmp cpuid */
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
281 /* cmp @v not equal to @expectnot */
282 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
283#endif
284 /* load the value of @v */
285 RSEQ_ASM_OP_R_LOAD(v)
286 /* store it in @load */
287 RSEQ_ASM_OP_R_STORE(load)
288 /* dereference voffp(v) */
289 RSEQ_ASM_OP_R_LOADX(voffp)
290 /* final store the value at voffp(v) */
291 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
292 RSEQ_INJECT_ASM(5)
293 RSEQ_ASM_DEFINE_ABORT(4, abort)
294 : /* gcc asm goto does not allow outputs */
295 : [cpu_id] "r" (cpu),
296 [current_cpu_id] "m" (__rseq_abi.cpu_id),
297 [rseq_cs] "m" (__rseq_abi.rseq_cs),
298 /* final store input */
299 [v] "m" (*v),
300 [expectnot] "r" (expectnot),
301 [voffp] "b" (voffp),
302 [load] "m" (*load)
303 RSEQ_INJECT_INPUT
304 : "memory", "cc", "r17"
305 RSEQ_INJECT_CLOBBER
306 : abort, cmpfail
307#ifdef RSEQ_COMPARE_TWICE
308 , error1, error2
309#endif
310 );
311 return 0;
312abort:
313 RSEQ_INJECT_FAILED
314 return -1;
315cmpfail:
316 return 1;
317#ifdef RSEQ_COMPARE_TWICE
318error1:
319 rseq_bug("cpu_id comparison failed");
320error2:
321 rseq_bug("expected value comparison failed");
322#endif
323}
324
325static inline __attribute__((always_inline))
326int rseq_addv(intptr_t *v, intptr_t count, int cpu)
327{
328 RSEQ_INJECT_C(9)
329
330 __asm__ __volatile__ goto (
331 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 332#ifdef RSEQ_COMPARE_TWICE
333 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
334#endif
2e155fb7
MD		 335 /* Start rseq by storing table entry pointer into rseq_cs. */
336 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
337 /* cmp cpuid */
338 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
339 RSEQ_INJECT_ASM(3)
340#ifdef RSEQ_COMPARE_TWICE
341 /* cmp cpuid */
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
343#endif
344 /* load the value of @v */
345 RSEQ_ASM_OP_R_LOAD(v)
346 /* add @count to it */
347 RSEQ_ASM_OP_R_ADD(count)
348 /* final store */
349 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
350 RSEQ_INJECT_ASM(4)
351 RSEQ_ASM_DEFINE_ABORT(4, abort)
352 : /* gcc asm goto does not allow outputs */
353 : [cpu_id] "r" (cpu),
354 [current_cpu_id] "m" (__rseq_abi.cpu_id),
355 [rseq_cs] "m" (__rseq_abi.rseq_cs),
356 /* final store input */
357 [v] "m" (*v),
358 [count] "r" (count)
359 RSEQ_INJECT_INPUT
360 : "memory", "cc", "r17"
361 RSEQ_INJECT_CLOBBER
362 : abort
363#ifdef RSEQ_COMPARE_TWICE
364 , error1
365#endif
366 );
367 return 0;
368abort:
369 RSEQ_INJECT_FAILED
370 return -1;
371#ifdef RSEQ_COMPARE_TWICE
372error1:
373 rseq_bug("cpu_id comparison failed");
374#endif
375}
376
377static inline __attribute__((always_inline))
378int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
379 intptr_t *v2, intptr_t newv2,
380 intptr_t newv, int cpu)
381{
382 RSEQ_INJECT_C(9)
383
384 __asm__ __volatile__ goto (
385 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 386 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
387#ifdef RSEQ_COMPARE_TWICE
388 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
389 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
390#endif
2e155fb7
MD		 391 /* Start rseq by storing table entry pointer into rseq_cs. */
392 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
393 /* cmp cpuid */
394 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
395 RSEQ_INJECT_ASM(3)
396 /* cmp @v equal to @expect */
397 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
398 RSEQ_INJECT_ASM(4)
399#ifdef RSEQ_COMPARE_TWICE
400 /* cmp cpuid */
401 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
402 /* cmp @v equal to @expect */
403 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
404#endif
405 /* try store */
406 RSEQ_ASM_OP_STORE(newv2, v2)
407 RSEQ_INJECT_ASM(5)
408 /* final store */
409 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
410 RSEQ_INJECT_ASM(6)
411 RSEQ_ASM_DEFINE_ABORT(4, abort)
412 : /* gcc asm goto does not allow outputs */
413 : [cpu_id] "r" (cpu),
414 [current_cpu_id] "m" (__rseq_abi.cpu_id),
415 [rseq_cs] "m" (__rseq_abi.rseq_cs),
416 /* try store input */
417 [v2] "m" (*v2),
418 [newv2] "r" (newv2),
419 /* final store input */
420 [v] "m" (*v),
421 [expect] "r" (expect),
422 [newv] "r" (newv)
423 RSEQ_INJECT_INPUT
424 : "memory", "cc", "r17"
425 RSEQ_INJECT_CLOBBER
426 : abort, cmpfail
427#ifdef RSEQ_COMPARE_TWICE
428 , error1, error2
429#endif
430 );
431 return 0;
432abort:
433 RSEQ_INJECT_FAILED
434 return -1;
435cmpfail:
436 return 1;
437#ifdef RSEQ_COMPARE_TWICE
438error1:
439 rseq_bug("cpu_id comparison failed");
440error2:
441 rseq_bug("expected value comparison failed");
442#endif
443}
444
445static inline __attribute__((always_inline))
446int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
447 intptr_t *v2, intptr_t newv2,
448 intptr_t newv, int cpu)
449{
450 RSEQ_INJECT_C(9)
451
452 __asm__ __volatile__ goto (
453 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 454 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
455#ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
457 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
458#endif
2e155fb7
MD		 459 /* Start rseq by storing table entry pointer into rseq_cs. */
460 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
461 /* cmp cpuid */
462 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
463 RSEQ_INJECT_ASM(3)
464 /* cmp @v equal to @expect */
465 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
466 RSEQ_INJECT_ASM(4)
467#ifdef RSEQ_COMPARE_TWICE
468 /* cmp cpuid */
469 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
470 /* cmp @v equal to @expect */
471 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
472#endif
473 /* try store */
474 RSEQ_ASM_OP_STORE(newv2, v2)
475 RSEQ_INJECT_ASM(5)
476 /* for 'release' */
477 "lwsync\n\t"
478 /* final store */
479 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
480 RSEQ_INJECT_ASM(6)
481 RSEQ_ASM_DEFINE_ABORT(4, abort)
482 : /* gcc asm goto does not allow outputs */
483 : [cpu_id] "r" (cpu),
484 [current_cpu_id] "m" (__rseq_abi.cpu_id),
485 [rseq_cs] "m" (__rseq_abi.rseq_cs),
486 /* try store input */
487 [v2] "m" (*v2),
488 [newv2] "r" (newv2),
489 /* final store input */
490 [v] "m" (*v),
491 [expect] "r" (expect),
492 [newv] "r" (newv)
493 RSEQ_INJECT_INPUT
494 : "memory", "cc", "r17"
495 RSEQ_INJECT_CLOBBER
496 : abort, cmpfail
497#ifdef RSEQ_COMPARE_TWICE
498 , error1, error2
499#endif
500 );
501 return 0;
502abort:
503 RSEQ_INJECT_FAILED
504 return -1;
505cmpfail:
506 return 1;
507#ifdef RSEQ_COMPARE_TWICE
508error1:
509 rseq_bug("cpu_id comparison failed");
510error2:
511 rseq_bug("expected value comparison failed");
512#endif
513}
514
515static inline __attribute__((always_inline))
516int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
517 intptr_t *v2, intptr_t expect2,
518 intptr_t newv, int cpu)
519{
520 RSEQ_INJECT_C(9)
521
522 __asm__ __volatile__ goto (
523 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 524 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
525#ifdef RSEQ_COMPARE_TWICE
526 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
527 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
528 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
529#endif
2e155fb7
MD		 530 /* Start rseq by storing table entry pointer into rseq_cs. */
531 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
532 /* cmp cpuid */
533 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
534 RSEQ_INJECT_ASM(3)
535 /* cmp @v equal to @expect */
536 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
537 RSEQ_INJECT_ASM(4)
538 /* cmp @v2 equal to @expct2 */
539 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
540 RSEQ_INJECT_ASM(5)
541#ifdef RSEQ_COMPARE_TWICE
542 /* cmp cpuid */
543 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
544 /* cmp @v equal to @expect */
545 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
546 /* cmp @v2 equal to @expct2 */
547 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
548#endif
549 /* final store */
550 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
551 RSEQ_INJECT_ASM(6)
552 RSEQ_ASM_DEFINE_ABORT(4, abort)
553 : /* gcc asm goto does not allow outputs */
554 : [cpu_id] "r" (cpu),
555 [current_cpu_id] "m" (__rseq_abi.cpu_id),
556 [rseq_cs] "m" (__rseq_abi.rseq_cs),
557 /* cmp2 input */
558 [v2] "m" (*v2),
559 [expect2] "r" (expect2),
560 /* final store input */
561 [v] "m" (*v),
562 [expect] "r" (expect),
563 [newv] "r" (newv)
564 RSEQ_INJECT_INPUT
565 : "memory", "cc", "r17"
566 RSEQ_INJECT_CLOBBER
567 : abort, cmpfail
568#ifdef RSEQ_COMPARE_TWICE
569 , error1, error2, error3
570#endif
571 );
572 return 0;
573abort:
574 RSEQ_INJECT_FAILED
575 return -1;
576cmpfail:
577 return 1;
578#ifdef RSEQ_COMPARE_TWICE
579error1:
580 rseq_bug("cpu_id comparison failed");
581error2:
582 rseq_bug("1st expected value comparison failed");
583error3:
584 rseq_bug("2nd expected value comparison failed");
585#endif
586}
587
588static inline __attribute__((always_inline))
589int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
590 void *dst, void *src, size_t len,
591 intptr_t newv, int cpu)
592{
593 RSEQ_INJECT_C(9)
594
595 __asm__ __volatile__ goto (
596 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 597 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
598#ifdef RSEQ_COMPARE_TWICE
599 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
601#endif
2e155fb7
MD		 602 /* setup for mempcy */
603 "mr %%r19, %[len]\n\t"
604 "mr %%r20, %[src]\n\t"
605 "mr %%r21, %[dst]\n\t"
606 /* Start rseq by storing table entry pointer into rseq_cs. */
607 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
608 /* cmp cpuid */
609 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
610 RSEQ_INJECT_ASM(3)
611 /* cmp @v equal to @expect */
612 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
613 RSEQ_INJECT_ASM(4)
614#ifdef RSEQ_COMPARE_TWICE
615 /* cmp cpuid */
616 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
617 /* cmp @v equal to @expect */
618 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
619#endif
620 /* try memcpy */
621 RSEQ_ASM_OP_R_MEMCPY()
622 RSEQ_INJECT_ASM(5)
623 /* final store */
624 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
625 RSEQ_INJECT_ASM(6)
626 /* teardown */
627 RSEQ_ASM_DEFINE_ABORT(4, abort)
628 : /* gcc asm goto does not allow outputs */
629 : [cpu_id] "r" (cpu),
630 [current_cpu_id] "m" (__rseq_abi.cpu_id),
631 [rseq_cs] "m" (__rseq_abi.rseq_cs),
632 /* final store input */
633 [v] "m" (*v),
634 [expect] "r" (expect),
635 [newv] "r" (newv),
636 /* try memcpy input */
637 [dst] "r" (dst),
638 [src] "r" (src),
639 [len] "r" (len)
640 RSEQ_INJECT_INPUT
641 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
642 RSEQ_INJECT_CLOBBER
643 : abort, cmpfail
644#ifdef RSEQ_COMPARE_TWICE
645 , error1, error2
646#endif
647 );
648 return 0;
649abort:
650 RSEQ_INJECT_FAILED
651 return -1;
652cmpfail:
653 return 1;
654#ifdef RSEQ_COMPARE_TWICE
655error1:
656 rseq_bug("cpu_id comparison failed");
657error2:
658 rseq_bug("expected value comparison failed");
659#endif
660}
661
662static inline __attribute__((always_inline))
663int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
664 void *dst, void *src, size_t len,
665 intptr_t newv, int cpu)
666{
667 RSEQ_INJECT_C(9)
668
669 __asm__ __volatile__ goto (
670 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
4fe2088e
MD		 671 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
672#ifdef RSEQ_COMPARE_TWICE
673 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
674 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
675#endif
2e155fb7
MD		 676 /* setup for mempcy */
677 "mr %%r19, %[len]\n\t"
678 "mr %%r20, %[src]\n\t"
679 "mr %%r21, %[dst]\n\t"
680 /* Start rseq by storing table entry pointer into rseq_cs. */
681 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
682 /* cmp cpuid */
683 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
684 RSEQ_INJECT_ASM(3)
685 /* cmp @v equal to @expect */
686 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
687 RSEQ_INJECT_ASM(4)
688#ifdef RSEQ_COMPARE_TWICE
689 /* cmp cpuid */
690 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
691 /* cmp @v equal to @expect */
692 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
693#endif
694 /* try memcpy */
695 RSEQ_ASM_OP_R_MEMCPY()
696 RSEQ_INJECT_ASM(5)
697 /* for 'release' */
698 "lwsync\n\t"
699 /* final store */
700 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
701 RSEQ_INJECT_ASM(6)
702 /* teardown */
703 RSEQ_ASM_DEFINE_ABORT(4, abort)
704 : /* gcc asm goto does not allow outputs */
705 : [cpu_id] "r" (cpu),
706 [current_cpu_id] "m" (__rseq_abi.cpu_id),
707 [rseq_cs] "m" (__rseq_abi.rseq_cs),
708 /* final store input */
709 [v] "m" (*v),
710 [expect] "r" (expect),
711 [newv] "r" (newv),
712 /* try memcpy input */
713 [dst] "r" (dst),
714 [src] "r" (src),
715 [len] "r" (len)
716 RSEQ_INJECT_INPUT
717 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
718 RSEQ_INJECT_CLOBBER
719 : abort, cmpfail
720#ifdef RSEQ_COMPARE_TWICE
721 , error1, error2
722#endif
723 );
724 return 0;
725abort:
726 RSEQ_INJECT_FAILED
727 return -1;
728cmpfail:
729 return 1;
730#ifdef RSEQ_COMPARE_TWICE
731error1:
732 rseq_bug("cpu_id comparison failed");
733error2:
734 rseq_bug("expected value comparison failed");
735#endif
736}
737
738#undef STORE_WORD
739#undef LOAD_WORD
740#undef LOADX_WORD
741#undef CMP_WORD
742
743#endif /* !RSEQ_SKIP_FASTPATH */
Ubuntu Hirsute Linux kernel mirror