Gitiles
Code Review Sign In
192.168.1.100 / T108 / c2023d5d03be91dbb983606f0ec5e9e4ea36c8c2 / . / marvell / linux / tools / testing / selftests / rseq / rseq-arm64.h
blob: cbe190a4d0056b2e30537d5f52217c5693877cb9 [file] [log] [blame]
b.liue9582032025-04-17 19:18:16 +0800[diff] [blame]1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm64.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9/*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16#ifdef __AARCH64EB__
17#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18#else
19#define RSEQ_SIG_DATA RSEQ_SIG_CODE
20#endif
21
22#define RSEQ_SIG RSEQ_SIG_DATA
23
24#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28#define rseq_smp_load_acquire(p) \
29__extension__ ({ \
30 __typeof(*p) ____p1; \
31 switch (sizeof(*p)) { \
32 case 1: \
33 asm volatile ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)p) \
35 : "Q" (*p) : "memory"); \
36 break; \
37 case 2: \
38 asm volatile ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)p) \
40 : "Q" (*p) : "memory"); \
41 break; \
42 case 4: \
43 asm volatile ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)p) \
45 : "Q" (*p) : "memory"); \
46 break; \
47 case 8: \
48 asm volatile ("ldar %0, %1" \
49 : "=r" (*(__u64 *)p) \
50 : "Q" (*p) : "memory"); \
51 break; \
52 } \
53 ____p1; \
54})
55
56#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58#define rseq_smp_store_release(p, v) \
59do { \
60 switch (sizeof(*p)) { \
61 case 1: \
62 asm volatile ("stlrb %w1, %0" \
63 : "=Q" (*p) \
64 : "r" ((__u8)v) \
65 : "memory"); \
66 break; \
67 case 2: \
68 asm volatile ("stlrh %w1, %0" \
69 : "=Q" (*p) \
70 : "r" ((__u16)v) \
71 : "memory"); \
72 break; \
73 case 4: \
74 asm volatile ("stlr %w1, %0" \
75 : "=Q" (*p) \
76 : "r" ((__u32)v) \
77 : "memory"); \
78 break; \
79 case 8: \
80 asm volatile ("stlr %1, %0" \
81 : "=Q" (*p) \
82 : "r" ((__u64)v) \
83 : "memory"); \
84 break; \
85 } \
86} while (0)
87
88#ifdef RSEQ_SKIP_FASTPATH
89#include "rseq-skip.h"
90#else /* !RSEQ_SKIP_FASTPATH */
91
92#define RSEQ_ASM_TMP_REG32 "w15"
93#define RSEQ_ASM_TMP_REG "x15"
94#define RSEQ_ASM_TMP_REG_2 "x14"
95
96#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97 post_commit_offset, abort_ip) \
98 " .pushsection __rseq_cs, \"aw\"\n" \
99 " .balign 32\n" \
100 __rseq_str(label) ":\n" \
101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
102 " .quad " __rseq_str(start_ip) ", " \
103 __rseq_str(post_commit_offset) ", " \
104 __rseq_str(abort_ip) "\n" \
105 " .popsection\n\t" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
108 " .popsection\n"
109
110#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
112 (post_commit_ip - start_ip), abort_ip)
113
114/*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
125 " .popsection\n"
126
127#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
128 RSEQ_INJECT_ASM(1) \
129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
131 ", :lo12:" __rseq_str(cs_label) "\n" \
132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
133 __rseq_str(label) ":\n"
134
135#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
136 " b 222f\n" \
137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
138 __rseq_str(label) ":\n" \
139 " b %l[" __rseq_str(abort_label) "]\n" \
140 "222:\n"
141
142#define RSEQ_ASM_OP_STORE(value, var) \
143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
144
145#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147
148#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
149 RSEQ_ASM_OP_STORE(value, var) \
150 __rseq_str(post_commit_label) ":\n"
151
152#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
154 __rseq_str(post_commit_label) ":\n"
155
156#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
159 ", %[" __rseq_str(expect) "]\n" \
160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
161
162#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
165 ", %w[" __rseq_str(expect) "]\n" \
166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
167
168#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
171 ", %[" __rseq_str(expect) "]\n" \
172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
173
174#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
175 RSEQ_INJECT_ASM(2) \
176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
177
178#define RSEQ_ASM_OP_R_LOAD(var) \
179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
180
181#define RSEQ_ASM_OP_R_STORE(var) \
182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183
184#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
186 ", %[" __rseq_str(offset) "]]\n"
187
188#define RSEQ_ASM_OP_R_ADD(count) \
189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
190 ", %[" __rseq_str(count) "]\n"
191
192#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
194 __rseq_str(post_commit_label) ":\n"
195
196#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
197 " cbz %[" __rseq_str(len) "], 333f\n" \
198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
205 "333:\n"
206
207static inline __attribute__((always_inline))
208int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209{
210 RSEQ_INJECT_C(9)
211
212 __asm__ __volatile__ goto (
213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215#ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218#endif
219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
223 RSEQ_INJECT_ASM(4)
224#ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
227#endif
228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
229 RSEQ_INJECT_ASM(5)
230 RSEQ_ASM_DEFINE_ABORT(4, abort)
231 : /* gcc asm goto does not allow outputs */
232 : [cpu_id] "r" (cpu),
233 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
234 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
235 [v] "Qo" (*v),
236 [expect] "r" (expect),
237 [newv] "r" (newv)
238 RSEQ_INJECT_INPUT
239 : "memory", RSEQ_ASM_TMP_REG
240 : abort, cmpfail
241#ifdef RSEQ_COMPARE_TWICE
242 , error1, error2
243#endif
244 );
245 rseq_after_asm_goto();
246 return 0;
247abort:
248 rseq_after_asm_goto();
249 RSEQ_INJECT_FAILED
250 return -1;
251cmpfail:
252 rseq_after_asm_goto();
253 return 1;
254#ifdef RSEQ_COMPARE_TWICE
255error1:
256 rseq_after_asm_goto();
257 rseq_bug("cpu_id comparison failed");
258error2:
259 rseq_after_asm_goto();
260 rseq_bug("expected value comparison failed");
261#endif
262}
263
264static inline __attribute__((always_inline))
265int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
266 long voffp, intptr_t *load, int cpu)
267{
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
272 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
273#ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
275 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
276#endif
277 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
278 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
279 RSEQ_INJECT_ASM(3)
280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
281 RSEQ_INJECT_ASM(4)
282#ifdef RSEQ_COMPARE_TWICE
283 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
285#endif
286 RSEQ_ASM_OP_R_LOAD(v)
287 RSEQ_ASM_OP_R_STORE(load)
288 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
289 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
290 RSEQ_INJECT_ASM(5)
291 RSEQ_ASM_DEFINE_ABORT(4, abort)
292 : /* gcc asm goto does not allow outputs */
293 : [cpu_id] "r" (cpu),
294 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
295 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
296 [v] "Qo" (*v),
297 [expectnot] "r" (expectnot),
298 [load] "Qo" (*load),
299 [voffp] "r" (voffp)
300 RSEQ_INJECT_INPUT
301 : "memory", RSEQ_ASM_TMP_REG
302 : abort, cmpfail
303#ifdef RSEQ_COMPARE_TWICE
304 , error1, error2
305#endif
306 );
307 rseq_after_asm_goto();
308 return 0;
309abort:
310 rseq_after_asm_goto();
311 RSEQ_INJECT_FAILED
312 return -1;
313cmpfail:
314 rseq_after_asm_goto();
315 return 1;
316#ifdef RSEQ_COMPARE_TWICE
317error1:
318 rseq_after_asm_goto();
319 rseq_bug("cpu_id comparison failed");
320error2:
321 rseq_after_asm_goto();
322 rseq_bug("expected value comparison failed");
323#endif
324}
325
326static inline __attribute__((always_inline))
327int rseq_addv(intptr_t *v, intptr_t count, int cpu)
328{
329 RSEQ_INJECT_C(9)
330
331 __asm__ __volatile__ goto (
332 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
333#ifdef RSEQ_COMPARE_TWICE
334 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
335#endif
336 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
337 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
338 RSEQ_INJECT_ASM(3)
339#ifdef RSEQ_COMPARE_TWICE
340 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
341#endif
342 RSEQ_ASM_OP_R_LOAD(v)
343 RSEQ_ASM_OP_R_ADD(count)
344 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
345 RSEQ_INJECT_ASM(4)
346 RSEQ_ASM_DEFINE_ABORT(4, abort)
347 : /* gcc asm goto does not allow outputs */
348 : [cpu_id] "r" (cpu),
349 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
350 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
351 [v] "Qo" (*v),
352 [count] "r" (count)
353 RSEQ_INJECT_INPUT
354 : "memory", RSEQ_ASM_TMP_REG
355 : abort
356#ifdef RSEQ_COMPARE_TWICE
357 , error1
358#endif
359 );
360 rseq_after_asm_goto();
361 return 0;
362abort:
363 rseq_after_asm_goto();
364 RSEQ_INJECT_FAILED
365 return -1;
366#ifdef RSEQ_COMPARE_TWICE
367error1:
368 rseq_after_asm_goto();
369 rseq_bug("cpu_id comparison failed");
370#endif
371}
372
373static inline __attribute__((always_inline))
374int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
375 intptr_t *v2, intptr_t newv2,
376 intptr_t newv, int cpu)
377{
378 RSEQ_INJECT_C(9)
379
380 __asm__ __volatile__ goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
383#ifdef RSEQ_COMPARE_TWICE
384 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
385 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
386#endif
387 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
389 RSEQ_INJECT_ASM(3)
390 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
391 RSEQ_INJECT_ASM(4)
392#ifdef RSEQ_COMPARE_TWICE
393 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
394 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
395#endif
396 RSEQ_ASM_OP_STORE(newv2, v2)
397 RSEQ_INJECT_ASM(5)
398 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
399 RSEQ_INJECT_ASM(6)
400 RSEQ_ASM_DEFINE_ABORT(4, abort)
401 : /* gcc asm goto does not allow outputs */
402 : [cpu_id] "r" (cpu),
403 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
404 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
405 [expect] "r" (expect),
406 [v] "Qo" (*v),
407 [newv] "r" (newv),
408 [v2] "Qo" (*v2),
409 [newv2] "r" (newv2)
410 RSEQ_INJECT_INPUT
411 : "memory", RSEQ_ASM_TMP_REG
412 : abort, cmpfail
413#ifdef RSEQ_COMPARE_TWICE
414 , error1, error2
415#endif
416 );
417 rseq_after_asm_goto();
418 return 0;
419abort:
420 rseq_after_asm_goto();
421 RSEQ_INJECT_FAILED
422 return -1;
423cmpfail:
424 rseq_after_asm_goto();
425 return 1;
426#ifdef RSEQ_COMPARE_TWICE
427error1:
428 rseq_after_asm_goto();
429 rseq_bug("cpu_id comparison failed");
430error2:
431 rseq_after_asm_goto();
432 rseq_bug("expected value comparison failed");
433#endif
434}
435
436static inline __attribute__((always_inline))
437int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
438 intptr_t *v2, intptr_t newv2,
439 intptr_t newv, int cpu)
440{
441 RSEQ_INJECT_C(9)
442
443 __asm__ __volatile__ goto (
444 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
445 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
446#ifdef RSEQ_COMPARE_TWICE
447 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
448 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
449#endif
450 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
451 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
452 RSEQ_INJECT_ASM(3)
453 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
454 RSEQ_INJECT_ASM(4)
455#ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
457 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
458#endif
459 RSEQ_ASM_OP_STORE(newv2, v2)
460 RSEQ_INJECT_ASM(5)
461 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
462 RSEQ_INJECT_ASM(6)
463 RSEQ_ASM_DEFINE_ABORT(4, abort)
464 : /* gcc asm goto does not allow outputs */
465 : [cpu_id] "r" (cpu),
466 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
467 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
468 [expect] "r" (expect),
469 [v] "Qo" (*v),
470 [newv] "r" (newv),
471 [v2] "Qo" (*v2),
472 [newv2] "r" (newv2)
473 RSEQ_INJECT_INPUT
474 : "memory", RSEQ_ASM_TMP_REG
475 : abort, cmpfail
476#ifdef RSEQ_COMPARE_TWICE
477 , error1, error2
478#endif
479 );
480 rseq_after_asm_goto();
481 return 0;
482abort:
483 rseq_after_asm_goto();
484 RSEQ_INJECT_FAILED
485 return -1;
486cmpfail:
487 rseq_after_asm_goto();
488 return 1;
489#ifdef RSEQ_COMPARE_TWICE
490error1:
491 rseq_after_asm_goto();
492 rseq_bug("cpu_id comparison failed");
493error2:
494 rseq_after_asm_goto();
495 rseq_bug("expected value comparison failed");
496#endif
497}
498
499static inline __attribute__((always_inline))
500int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
501 intptr_t *v2, intptr_t expect2,
502 intptr_t newv, int cpu)
503{
504 RSEQ_INJECT_C(9)
505
506 __asm__ __volatile__ goto (
507 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
508 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
509#ifdef RSEQ_COMPARE_TWICE
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
511 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
513#endif
514 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
516 RSEQ_INJECT_ASM(3)
517 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
518 RSEQ_INJECT_ASM(4)
519 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
520 RSEQ_INJECT_ASM(5)
521#ifdef RSEQ_COMPARE_TWICE
522 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
523 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
524 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
525#endif
526 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
527 RSEQ_INJECT_ASM(6)
528 RSEQ_ASM_DEFINE_ABORT(4, abort)
529 : /* gcc asm goto does not allow outputs */
530 : [cpu_id] "r" (cpu),
531 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
532 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
533 [v] "Qo" (*v),
534 [expect] "r" (expect),
535 [v2] "Qo" (*v2),
536 [expect2] "r" (expect2),
537 [newv] "r" (newv)
538 RSEQ_INJECT_INPUT
539 : "memory", RSEQ_ASM_TMP_REG
540 : abort, cmpfail
541#ifdef RSEQ_COMPARE_TWICE
542 , error1, error2, error3
543#endif
544 );
545 rseq_after_asm_goto();
546 return 0;
547abort:
548 rseq_after_asm_goto();
549 RSEQ_INJECT_FAILED
550 return -1;
551cmpfail:
552 rseq_after_asm_goto();
553 return 1;
554#ifdef RSEQ_COMPARE_TWICE
555error1:
556 rseq_after_asm_goto();
557 rseq_bug("cpu_id comparison failed");
558error2:
559 rseq_after_asm_goto();
560 rseq_bug("expected value comparison failed");
561error3:
562 rseq_after_asm_goto();
563 rseq_bug("2nd expected value comparison failed");
564#endif
565}
566
567static inline __attribute__((always_inline))
568int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
569 void *dst, void *src, size_t len,
570 intptr_t newv, int cpu)
571{
572 RSEQ_INJECT_C(9)
573
574 __asm__ __volatile__ goto (
575 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
576 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
577#ifdef RSEQ_COMPARE_TWICE
578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
579 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
580#endif
581 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
582 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
583 RSEQ_INJECT_ASM(3)
584 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
585 RSEQ_INJECT_ASM(4)
586#ifdef RSEQ_COMPARE_TWICE
587 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
588 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
589#endif
590 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
591 RSEQ_INJECT_ASM(5)
592 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
593 RSEQ_INJECT_ASM(6)
594 RSEQ_ASM_DEFINE_ABORT(4, abort)
595 : /* gcc asm goto does not allow outputs */
596 : [cpu_id] "r" (cpu),
597 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
598 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
599 [expect] "r" (expect),
600 [v] "Qo" (*v),
601 [newv] "r" (newv),
602 [dst] "r" (dst),
603 [src] "r" (src),
604 [len] "r" (len)
605 RSEQ_INJECT_INPUT
606 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
607 : abort, cmpfail
608#ifdef RSEQ_COMPARE_TWICE
609 , error1, error2
610#endif
611 );
612 rseq_after_asm_goto();
613 return 0;
614abort:
615 rseq_after_asm_goto();
616 RSEQ_INJECT_FAILED
617 return -1;
618cmpfail:
619 rseq_after_asm_goto();
620 return 1;
621#ifdef RSEQ_COMPARE_TWICE
622error1:
623 rseq_after_asm_goto();
624 rseq_bug("cpu_id comparison failed");
625error2:
626 rseq_after_asm_goto();
627 rseq_bug("expected value comparison failed");
628#endif
629}
630
631static inline __attribute__((always_inline))
632int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
633 void *dst, void *src, size_t len,
634 intptr_t newv, int cpu)
635{
636 RSEQ_INJECT_C(9)
637
638 __asm__ __volatile__ goto (
639 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
640 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
641#ifdef RSEQ_COMPARE_TWICE
642 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
643 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
644#endif
645 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
646 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
647 RSEQ_INJECT_ASM(3)
648 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
649 RSEQ_INJECT_ASM(4)
650#ifdef RSEQ_COMPARE_TWICE
651 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
652 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
653#endif
654 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
655 RSEQ_INJECT_ASM(5)
656 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
657 RSEQ_INJECT_ASM(6)
658 RSEQ_ASM_DEFINE_ABORT(4, abort)
659 : /* gcc asm goto does not allow outputs */
660 : [cpu_id] "r" (cpu),
661 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
662 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
663 [expect] "r" (expect),
664 [v] "Qo" (*v),
665 [newv] "r" (newv),
666 [dst] "r" (dst),
667 [src] "r" (src),
668 [len] "r" (len)
669 RSEQ_INJECT_INPUT
670 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
671 : abort, cmpfail
672#ifdef RSEQ_COMPARE_TWICE
673 , error1, error2
674#endif
675 );
676 rseq_after_asm_goto();
677 return 0;
678abort:
679 rseq_after_asm_goto();
680 RSEQ_INJECT_FAILED
681 return -1;
682cmpfail:
683 rseq_after_asm_goto();
684 return 1;
685#ifdef RSEQ_COMPARE_TWICE
686error1:
687 rseq_after_asm_goto();
688 rseq_bug("cpu_id comparison failed");
689error2:
690 rseq_after_asm_goto();
691 rseq_bug("expected value comparison failed");
692#endif
693}
694
695#endif /* !RSEQ_SKIP_FASTPATH */