Gitiles
Code Review Sign In
192.168.1.100 / T106_DC / 0d1b61e51a97cc1eaa83dbaf31a4345e3b1e1768 / . / ap / libc / glibc / glibc-2.23 / include / atomic.h
blob: 5e8bfffa08dbe9a53287d339001a8a05aa4fa06e [file] [log] [blame]
xf.libdd93d52023-05-12 07:10:14 -0700[diff] [blame]1/* Internal macros for atomic operations for GNU C Library.
2 Copyright (C) 2002-2016 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <http://www.gnu.org/licenses/>. */
19
20#ifndef _ATOMIC_H
21#define _ATOMIC_H 1
22
23/* This header defines three types of macros:
24
25 - atomic arithmetic and logic operation on memory. They all
26 have the prefix "atomic_".
27
28 - conditionally atomic operations of the same kinds. These
29 always behave identical but can be faster when atomicity
30 is not really needed since only one thread has access to
31 the memory location. In that case the code is slower in
32 the multi-thread case. The interfaces have the prefix
33 "catomic_".
34
35 - support functions like barriers. They also have the prefix
36 "atomic_".
37
38 Architectures must provide a few lowlevel macros (the compare
39 and exchange definitions). All others are optional. They
40 should only be provided if the architecture has specific
41 support for the operation.
42
43 As <atomic.h> macros are usually heavily nested and often use local
44 variables to make sure side-effects are evaluated properly, use for
45 macro local variables a per-macro unique prefix. This file uses
46 __atgN_ prefix where N is different in each macro. */
47
48#include <stdlib.h>
49
50#include <atomic-machine.h>
51
52/* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
53 bit width of *MEM. The calling macro puts parens around MEM
54 and following args. */
55#define __atomic_val_bysize(pre, post, mem, ...) \
56 ({ \
57 __typeof (*mem) __atg1_result; \
58 if (sizeof (*mem) == 1) \
59 __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \
60 else if (sizeof (*mem) == 2) \
61 __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \
62 else if (sizeof (*mem) == 4) \
63 __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \
64 else if (sizeof (*mem) == 8) \
65 __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \
66 else \
67 abort (); \
68 __atg1_result; \
69 })
70#define __atomic_bool_bysize(pre, post, mem, ...) \
71 ({ \
72 int __atg2_result; \
73 if (sizeof (*mem) == 1) \
74 __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \
75 else if (sizeof (*mem) == 2) \
76 __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \
77 else if (sizeof (*mem) == 4) \
78 __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \
79 else if (sizeof (*mem) == 8) \
80 __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \
81 else \
82 abort (); \
83 __atg2_result; \
84 })
85
86
87/* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
88 Return the old *MEM value. */
89#if !defined atomic_compare_and_exchange_val_acq \
90 && defined __arch_compare_and_exchange_val_32_acq
91# define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
92 __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \
93 mem, newval, oldval)
94#endif
95
96
97#ifndef catomic_compare_and_exchange_val_acq
98# ifdef __arch_c_compare_and_exchange_val_32_acq
99# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
100 __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \
101 mem, newval, oldval)
102# else
103# define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
104 atomic_compare_and_exchange_val_acq (mem, newval, oldval)
105# endif
106#endif
107
108
109#ifndef catomic_compare_and_exchange_val_rel
110# ifndef atomic_compare_and_exchange_val_rel
111# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
112 catomic_compare_and_exchange_val_acq (mem, newval, oldval)
113# else
114# define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
115 atomic_compare_and_exchange_val_rel (mem, newval, oldval)
116# endif
117#endif
118
119
120#ifndef atomic_compare_and_exchange_val_rel
121# define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
122 atomic_compare_and_exchange_val_acq (mem, newval, oldval)
123#endif
124
125
126/* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
127 Return zero if *MEM was changed or non-zero if no exchange happened. */
128#ifndef atomic_compare_and_exchange_bool_acq
129# ifdef __arch_compare_and_exchange_bool_32_acq
130# define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
131 __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \
132 mem, newval, oldval)
133# else
134# define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
135 ({ /* Cannot use __oldval here, because macros later in this file might \
136 call this macro with __oldval argument. */ \
137 __typeof (oldval) __atg3_old = (oldval); \
138 atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \
139 != __atg3_old; \
140 })
141# endif
142#endif
143
144
145#ifndef catomic_compare_and_exchange_bool_acq
146# ifdef __arch_c_compare_and_exchange_bool_32_acq
147# define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
148 __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \
149 mem, newval, oldval)
150# else
151# define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
152 ({ /* Cannot use __oldval here, because macros later in this file might \
153 call this macro with __oldval argument. */ \
154 __typeof (oldval) __atg4_old = (oldval); \
155 catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \
156 != __atg4_old; \
157 })
158# endif
159#endif
160
161
162#ifndef catomic_compare_and_exchange_bool_rel
163# ifndef atomic_compare_and_exchange_bool_rel
164# define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
165 catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
166# else
167# define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
168 atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
169# endif
170#endif
171
172
173#ifndef atomic_compare_and_exchange_bool_rel
174# define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
175 atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
176#endif
177
178
179/* Store NEWVALUE in *MEM and return the old value. */
180#ifndef atomic_exchange_acq
181# define atomic_exchange_acq(mem, newvalue) \
182 ({ __typeof (*(mem)) __atg5_oldval; \
183 __typeof (mem) __atg5_memp = (mem); \
184 __typeof (*(mem)) __atg5_value = (newvalue); \
185 \
186 do \
187 __atg5_oldval = *__atg5_memp; \
188 while (__builtin_expect \
189 (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
190 __atg5_oldval), 0)); \
191 \
192 __atg5_oldval; })
193#endif
194
195#ifndef atomic_exchange_rel
196# define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
197#endif
198
199
200/* Add VALUE to *MEM and return the old value of *MEM. */
201#ifndef atomic_exchange_and_add_acq
202# ifdef atomic_exchange_and_add
203# define atomic_exchange_and_add_acq(mem, value) \
204 atomic_exchange_and_add (mem, value)
205# else
206# define atomic_exchange_and_add_acq(mem, value) \
207 ({ __typeof (*(mem)) __atg6_oldval; \
208 __typeof (mem) __atg6_memp = (mem); \
209 __typeof (*(mem)) __atg6_value = (value); \
210 \
211 do \
212 __atg6_oldval = *__atg6_memp; \
213 while (__builtin_expect \
214 (atomic_compare_and_exchange_bool_acq (__atg6_memp, \
215 __atg6_oldval \
216 + __atg6_value, \
217 __atg6_oldval), 0)); \
218 \
219 __atg6_oldval; })
220# endif
221#endif
222
223#ifndef atomic_exchange_and_add_rel
224# define atomic_exchange_and_add_rel(mem, value) \
225 atomic_exchange_and_add_acq(mem, value)
226#endif
227
228#ifndef atomic_exchange_and_add
229# define atomic_exchange_and_add(mem, value) \
230 atomic_exchange_and_add_acq(mem, value)
231#endif
232
233#ifndef catomic_exchange_and_add
234# define catomic_exchange_and_add(mem, value) \
235 ({ __typeof (*(mem)) __atg7_oldv; \
236 __typeof (mem) __atg7_memp = (mem); \
237 __typeof (*(mem)) __atg7_value = (value); \
238 \
239 do \
240 __atg7_oldv = *__atg7_memp; \
241 while (__builtin_expect \
242 (catomic_compare_and_exchange_bool_acq (__atg7_memp, \
243 __atg7_oldv \
244 + __atg7_value, \
245 __atg7_oldv), 0)); \
246 \
247 __atg7_oldv; })
248#endif
249
250
251#ifndef atomic_max
252# define atomic_max(mem, value) \
253 do { \
254 __typeof (*(mem)) __atg8_oldval; \
255 __typeof (mem) __atg8_memp = (mem); \
256 __typeof (*(mem)) __atg8_value = (value); \
257 do { \
258 __atg8_oldval = *__atg8_memp; \
259 if (__atg8_oldval >= __atg8_value) \
260 break; \
261 } while (__builtin_expect \
262 (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
263 __atg8_oldval), 0)); \
264 } while (0)
265#endif
266
267
268#ifndef catomic_max
269# define catomic_max(mem, value) \
270 do { \
271 __typeof (*(mem)) __atg9_oldv; \
272 __typeof (mem) __atg9_memp = (mem); \
273 __typeof (*(mem)) __atg9_value = (value); \
274 do { \
275 __atg9_oldv = *__atg9_memp; \
276 if (__atg9_oldv >= __atg9_value) \
277 break; \
278 } while (__builtin_expect \
279 (catomic_compare_and_exchange_bool_acq (__atg9_memp, \
280 __atg9_value, \
281 __atg9_oldv), 0)); \
282 } while (0)
283#endif
284
285
286#ifndef atomic_min
287# define atomic_min(mem, value) \
288 do { \
289 __typeof (*(mem)) __atg10_oldval; \
290 __typeof (mem) __atg10_memp = (mem); \
291 __typeof (*(mem)) __atg10_value = (value); \
292 do { \
293 __atg10_oldval = *__atg10_memp; \
294 if (__atg10_oldval <= __atg10_value) \
295 break; \
296 } while (__builtin_expect \
297 (atomic_compare_and_exchange_bool_acq (__atg10_memp, \
298 __atg10_value, \
299 __atg10_oldval), 0)); \
300 } while (0)
301#endif
302
303
304#ifndef atomic_add
305# define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
306#endif
307
308
309#ifndef catomic_add
310# define catomic_add(mem, value) \
311 (void) catomic_exchange_and_add ((mem), (value))
312#endif
313
314
315#ifndef atomic_increment
316# define atomic_increment(mem) atomic_add ((mem), 1)
317#endif
318
319
320#ifndef catomic_increment
321# define catomic_increment(mem) catomic_add ((mem), 1)
322#endif
323
324
325#ifndef atomic_increment_val
326# define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
327#endif
328
329
330#ifndef catomic_increment_val
331# define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
332#endif
333
334
335/* Add one to *MEM and return true iff it's now zero. */
336#ifndef atomic_increment_and_test
337# define atomic_increment_and_test(mem) \
338 (atomic_exchange_and_add ((mem), 1) + 1 == 0)
339#endif
340
341
342#ifndef atomic_decrement
343# define atomic_decrement(mem) atomic_add ((mem), -1)
344#endif
345
346
347#ifndef catomic_decrement
348# define catomic_decrement(mem) catomic_add ((mem), -1)
349#endif
350
351
352#ifndef atomic_decrement_val
353# define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
354#endif
355
356
357#ifndef catomic_decrement_val
358# define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
359#endif
360
361
362/* Subtract 1 from *MEM and return true iff it's now zero. */
363#ifndef atomic_decrement_and_test
364# define atomic_decrement_and_test(mem) \
365 (atomic_exchange_and_add ((mem), -1) == 1)
366#endif
367
368
369/* Decrement *MEM if it is > 0, and return the old value. */
370#ifndef atomic_decrement_if_positive
371# define atomic_decrement_if_positive(mem) \
372 ({ __typeof (*(mem)) __atg11_oldval; \
373 __typeof (mem) __atg11_memp = (mem); \
374 \
375 do \
376 { \
377 __atg11_oldval = *__atg11_memp; \
378 if (__glibc_unlikely (__atg11_oldval <= 0)) \
379 break; \
380 } \
381 while (__builtin_expect \
382 (atomic_compare_and_exchange_bool_acq (__atg11_memp, \
383 __atg11_oldval - 1, \
384 __atg11_oldval), 0)); \
385 __atg11_oldval; })
386#endif
387
388
389#ifndef atomic_add_negative
390# define atomic_add_negative(mem, value) \
391 ({ __typeof (value) __atg12_value = (value); \
392 atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
393#endif
394
395
396#ifndef atomic_add_zero
397# define atomic_add_zero(mem, value) \
398 ({ __typeof (value) __atg13_value = (value); \
399 atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
400#endif
401
402
403#ifndef atomic_bit_set
404# define atomic_bit_set(mem, bit) \
405 (void) atomic_bit_test_set(mem, bit)
406#endif
407
408
409#ifndef atomic_bit_test_set
410# define atomic_bit_test_set(mem, bit) \
411 ({ __typeof (*(mem)) __atg14_old; \
412 __typeof (mem) __atg14_memp = (mem); \
413 __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \
414 \
415 do \
416 __atg14_old = (*__atg14_memp); \
417 while (__builtin_expect \
418 (atomic_compare_and_exchange_bool_acq (__atg14_memp, \
419 __atg14_old | __atg14_mask,\
420 __atg14_old), 0)); \
421 \
422 __atg14_old & __atg14_mask; })
423#endif
424
425/* Atomically *mem &= mask. */
426#ifndef atomic_and
427# define atomic_and(mem, mask) \
428 do { \
429 __typeof (*(mem)) __atg15_old; \
430 __typeof (mem) __atg15_memp = (mem); \
431 __typeof (*(mem)) __atg15_mask = (mask); \
432 \
433 do \
434 __atg15_old = (*__atg15_memp); \
435 while (__builtin_expect \
436 (atomic_compare_and_exchange_bool_acq (__atg15_memp, \
437 __atg15_old & __atg15_mask, \
438 __atg15_old), 0)); \
439 } while (0)
440#endif
441
442#ifndef catomic_and
443# define catomic_and(mem, mask) \
444 do { \
445 __typeof (*(mem)) __atg20_old; \
446 __typeof (mem) __atg20_memp = (mem); \
447 __typeof (*(mem)) __atg20_mask = (mask); \
448 \
449 do \
450 __atg20_old = (*__atg20_memp); \
451 while (__builtin_expect \
452 (catomic_compare_and_exchange_bool_acq (__atg20_memp, \
453 __atg20_old & __atg20_mask,\
454 __atg20_old), 0)); \
455 } while (0)
456#endif
457
458/* Atomically *mem &= mask and return the old value of *mem. */
459#ifndef atomic_and_val
460# define atomic_and_val(mem, mask) \
461 ({ __typeof (*(mem)) __atg16_old; \
462 __typeof (mem) __atg16_memp = (mem); \
463 __typeof (*(mem)) __atg16_mask = (mask); \
464 \
465 do \
466 __atg16_old = (*__atg16_memp); \
467 while (__builtin_expect \
468 (atomic_compare_and_exchange_bool_acq (__atg16_memp, \
469 __atg16_old & __atg16_mask,\
470 __atg16_old), 0)); \
471 \
472 __atg16_old; })
473#endif
474
475/* Atomically *mem |= mask and return the old value of *mem. */
476#ifndef atomic_or
477# define atomic_or(mem, mask) \
478 do { \
479 __typeof (*(mem)) __atg17_old; \
480 __typeof (mem) __atg17_memp = (mem); \
481 __typeof (*(mem)) __atg17_mask = (mask); \
482 \
483 do \
484 __atg17_old = (*__atg17_memp); \
485 while (__builtin_expect \
486 (atomic_compare_and_exchange_bool_acq (__atg17_memp, \
487 __atg17_old | __atg17_mask, \
488 __atg17_old), 0)); \
489 } while (0)
490#endif
491
492#ifndef catomic_or
493# define catomic_or(mem, mask) \
494 do { \
495 __typeof (*(mem)) __atg18_old; \
496 __typeof (mem) __atg18_memp = (mem); \
497 __typeof (*(mem)) __atg18_mask = (mask); \
498 \
499 do \
500 __atg18_old = (*__atg18_memp); \
501 while (__builtin_expect \
502 (catomic_compare_and_exchange_bool_acq (__atg18_memp, \
503 __atg18_old | __atg18_mask,\
504 __atg18_old), 0)); \
505 } while (0)
506#endif
507
508/* Atomically *mem |= mask and return the old value of *mem. */
509#ifndef atomic_or_val
510# define atomic_or_val(mem, mask) \
511 ({ __typeof (*(mem)) __atg19_old; \
512 __typeof (mem) __atg19_memp = (mem); \
513 __typeof (*(mem)) __atg19_mask = (mask); \
514 \
515 do \
516 __atg19_old = (*__atg19_memp); \
517 while (__builtin_expect \
518 (atomic_compare_and_exchange_bool_acq (__atg19_memp, \
519 __atg19_old | __atg19_mask,\
520 __atg19_old), 0)); \
521 \
522 __atg19_old; })
523#endif
524
525#ifndef atomic_full_barrier
526# define atomic_full_barrier() __asm ("" ::: "memory")
527#endif
528
529
530#ifndef atomic_read_barrier
531# define atomic_read_barrier() atomic_full_barrier ()
532#endif
533
534
535#ifndef atomic_write_barrier
536# define atomic_write_barrier() atomic_full_barrier ()
537#endif
538
539
540#ifndef atomic_forced_read
541# define atomic_forced_read(x) \
542 ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; })
543#endif
544
545/* This is equal to 1 iff the architecture supports 64b atomic operations. */
546#ifndef __HAVE_64B_ATOMICS
547#error Unable to determine if 64-bit atomics are present.
548#endif
549
550/* The following functions are a subset of the atomic operations provided by
551 C11. Usually, a function named atomic_OP_MO(args) is equivalent to C11's
552 atomic_OP_explicit(args, memory_order_MO); exceptions noted below. */
553
554/* Each arch can request to use compiler built-ins for C11 atomics. If it
555 does, all atomics will be based on these. */
556#if USE_ATOMIC_COMPILER_BUILTINS
557
558/* We require 32b atomic operations; some archs also support 64b atomic
559 operations. */
560void __atomic_link_error (void);
561# if __HAVE_64B_ATOMICS == 1
562# define __atomic_check_size(mem) \
563 if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8)) \
564 __atomic_link_error ();
565# else
566# define __atomic_check_size(mem) \
567 if (sizeof (*mem) != 4) \
568 __atomic_link_error ();
569# endif
570
571# define atomic_thread_fence_acquire() \
572 __atomic_thread_fence (__ATOMIC_ACQUIRE)
573# define atomic_thread_fence_release() \
574 __atomic_thread_fence (__ATOMIC_RELEASE)
575# define atomic_thread_fence_seq_cst() \
576 __atomic_thread_fence (__ATOMIC_SEQ_CST)
577
578# define atomic_load_relaxed(mem) \
579 ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_RELAXED); })
580# define atomic_load_acquire(mem) \
581 ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_ACQUIRE); })
582
583# define atomic_store_relaxed(mem, val) \
584 do { \
585 __atomic_check_size((mem)); \
586 __atomic_store_n ((mem), (val), __ATOMIC_RELAXED); \
587 } while (0)
588# define atomic_store_release(mem, val) \
589 do { \
590 __atomic_check_size((mem)); \
591 __atomic_store_n ((mem), (val), __ATOMIC_RELEASE); \
592 } while (0)
593
594/* On failure, this CAS has memory_order_relaxed semantics. */
595# define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \
596 ({ __atomic_check_size((mem)); \
597 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \
598 __ATOMIC_RELAXED, __ATOMIC_RELAXED); })
599# define atomic_compare_exchange_weak_acquire(mem, expected, desired) \
600 ({ __atomic_check_size((mem)); \
601 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \
602 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); })
603# define atomic_compare_exchange_weak_release(mem, expected, desired) \
604 ({ __atomic_check_size((mem)); \
605 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \
606 __ATOMIC_RELEASE, __ATOMIC_RELAXED); })
607
608# define atomic_exchange_acquire(mem, desired) \
609 ({ __atomic_check_size((mem)); \
610 __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); })
611# define atomic_exchange_release(mem, desired) \
612 ({ __atomic_check_size((mem)); \
613 __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); })
614
615# define atomic_fetch_add_relaxed(mem, operand) \
616 ({ __atomic_check_size((mem)); \
617 __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); })
618# define atomic_fetch_add_acquire(mem, operand) \
619 ({ __atomic_check_size((mem)); \
620 __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); })
621# define atomic_fetch_add_release(mem, operand) \
622 ({ __atomic_check_size((mem)); \
623 __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); })
624# define atomic_fetch_add_acq_rel(mem, operand) \
625 ({ __atomic_check_size((mem)); \
626 __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQ_REL); })
627
628# define atomic_fetch_and_acquire(mem, operand) \
629 ({ __atomic_check_size((mem)); \
630 __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); })
631
632# define atomic_fetch_or_relaxed(mem, operand) \
633 ({ __atomic_check_size((mem)); \
634 __atomic_fetch_or ((mem), (operand), __ATOMIC_RELAXED); })
635# define atomic_fetch_or_acquire(mem, operand) \
636 ({ __atomic_check_size((mem)); \
637 __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); })
638
639#else /* !USE_ATOMIC_COMPILER_BUILTINS */
640
641/* By default, we assume that read, write, and full barriers are equivalent
642 to acquire, release, and seq_cst barriers. Archs for which this does not
643 hold have to provide custom definitions of the fences. */
644# ifndef atomic_thread_fence_acquire
645# define atomic_thread_fence_acquire() atomic_read_barrier ()
646# endif
647# ifndef atomic_thread_fence_release
648# define atomic_thread_fence_release() atomic_write_barrier ()
649# endif
650# ifndef atomic_thread_fence_seq_cst
651# define atomic_thread_fence_seq_cst() atomic_full_barrier ()
652# endif
653
654# ifndef atomic_load_relaxed
655# define atomic_load_relaxed(mem) \
656 ({ __typeof (*(mem)) __atg100_val; \
657 __asm ("" : "=r" (__atg100_val) : "0" (*(mem))); \
658 __atg100_val; })
659# endif
660# ifndef atomic_load_acquire
661# define atomic_load_acquire(mem) \
662 ({ __typeof (*(mem)) __atg101_val = atomic_load_relaxed (mem); \
663 atomic_thread_fence_acquire (); \
664 __atg101_val; })
665# endif
666
667# ifndef atomic_store_relaxed
668/* XXX Use inline asm here? */
669# define atomic_store_relaxed(mem, val) do { *(mem) = (val); } while (0)
670# endif
671# ifndef atomic_store_release
672# define atomic_store_release(mem, val) \
673 do { \
674 atomic_thread_fence_release (); \
675 atomic_store_relaxed ((mem), (val)); \
676 } while (0)
677# endif
678
679/* On failure, this CAS has memory_order_relaxed semantics. */
680/* XXX This potentially has one branch more than necessary, but archs
681 currently do not define a CAS that returns both the previous value and
682 the success flag. */
683# ifndef atomic_compare_exchange_weak_acquire
684# define atomic_compare_exchange_weak_acquire(mem, expected, desired) \
685 ({ typeof (*(expected)) __atg102_expected = *(expected); \
686 *(expected) = \
687 atomic_compare_and_exchange_val_acq ((mem), (desired), *(expected)); \
688 *(expected) == __atg102_expected; })
689# endif
690# ifndef atomic_compare_exchange_weak_relaxed
691/* XXX Fall back to CAS with acquire MO because archs do not define a weaker
692 CAS. */
693# define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \
694 atomic_compare_exchange_weak_acquire ((mem), (expected), (desired))
695# endif
696# ifndef atomic_compare_exchange_weak_release
697# define atomic_compare_exchange_weak_release(mem, expected, desired) \
698 ({ typeof (*(expected)) __atg103_expected = *(expected); \
699 *(expected) = \
700 atomic_compare_and_exchange_val_rel ((mem), (desired), *(expected)); \
701 *(expected) == __atg103_expected; })
702# endif
703
704# ifndef atomic_exchange_acquire
705# define atomic_exchange_acquire(mem, val) \
706 atomic_exchange_acq ((mem), (val))
707# endif
708# ifndef atomic_exchange_release
709# define atomic_exchange_release(mem, val) \
710 atomic_exchange_rel ((mem), (val))
711# endif
712
713# ifndef atomic_fetch_add_acquire
714# define atomic_fetch_add_acquire(mem, operand) \
715 atomic_exchange_and_add_acq ((mem), (operand))
716# endif
717# ifndef atomic_fetch_add_relaxed
718/* XXX Fall back to acquire MO because the MO semantics of
719 atomic_exchange_and_add are not documented; the generic version falls back
720 to atomic_exchange_and_add_acq if atomic_exchange_and_add is not defined,
721 and vice versa. */
722# define atomic_fetch_add_relaxed(mem, operand) \
723 atomic_fetch_add_acquire ((mem), (operand))
724# endif
725# ifndef atomic_fetch_add_release
726# define atomic_fetch_add_release(mem, operand) \
727 atomic_exchange_and_add_rel ((mem), (operand))
728# endif
729# ifndef atomic_fetch_add_acq_rel
730# define atomic_fetch_add_acq_rel(mem, operand) \
731 ({ atomic_thread_fence_release (); \
732 atomic_exchange_and_add_acq ((mem), (operand)); })
733# endif
734
735/* XXX The default for atomic_and_val has acquire semantics, but this is not
736 documented. */
737# ifndef atomic_fetch_and_acquire
738# define atomic_fetch_and_acquire(mem, operand) \
739 atomic_and_val ((mem), (operand))
740# endif
741
742/* XXX The default for atomic_or_val has acquire semantics, but this is not
743 documented. */
744# ifndef atomic_fetch_or_acquire
745# define atomic_fetch_or_acquire(mem, operand) \
746 atomic_or_val ((mem), (operand))
747# endif
748/* XXX Fall back to acquire MO because archs do not define a weaker
749 atomic_or_val. */
750# ifndef atomic_fetch_or_relaxed
751# define atomic_fetch_or_relaxed(mem, operand) \
752 atomic_fetch_or_acquire ((mem), (operand))
753# endif
754
755#endif /* !USE_ATOMIC_COMPILER_BUILTINS */
756
757/* This operation does not affect synchronization semantics but can be used
758 in the body of a spin loop to potentially improve its efficiency. */
759#ifndef atomic_spin_nop
760# define atomic_spin_nop() do { /* nothing */ } while (0)
761#endif
762
763#endif /* atomic.h */