Gitiles
Code Review Sign In
192.168.1.100 / T108 / e958203796650e3959a43708d67534bf36f4c83b / . / marvell / linux / arch / riscv / include / asm / cmpxchg.h
blob: 262e5bbb27760840d598e4e940a340fc5d820178 [file] [log] [blame]
b.liue9582032025-04-17 19:18:16 +0800[diff] [blame^]1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2014 Regents of the University of California
4 */
5
6#ifndef _ASM_RISCV_CMPXCHG_H
7#define _ASM_RISCV_CMPXCHG_H
8
9#include <linux/bug.h>
10
11#include <asm/barrier.h>
12#include <asm/fence.h>
13
14#define __xchg_relaxed(ptr, new, size) \
15({ \
16 __typeof__(ptr) __ptr = (ptr); \
17 __typeof__(new) __new = (new); \
18 __typeof__(*(ptr)) __ret; \
19 switch (size) { \
20 case 4: \
21 __asm__ __volatile__ ( \
22 " amoswap.w %0, %2, %1\n" \
23 : "=r" (__ret), "+A" (*__ptr) \
24 : "r" (__new) \
25 : "memory"); \
26 break; \
27 case 8: \
28 __asm__ __volatile__ ( \
29 " amoswap.d %0, %2, %1\n" \
30 : "=r" (__ret), "+A" (*__ptr) \
31 : "r" (__new) \
32 : "memory"); \
33 break; \
34 default: \
35 BUILD_BUG(); \
36 } \
37 __ret; \
38})
39
40#define xchg_relaxed(ptr, x) \
41({ \
42 __typeof__(*(ptr)) _x_ = (x); \
43 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
44 _x_, sizeof(*(ptr))); \
45})
46
47#define __xchg_acquire(ptr, new, size) \
48({ \
49 __typeof__(ptr) __ptr = (ptr); \
50 __typeof__(new) __new = (new); \
51 __typeof__(*(ptr)) __ret; \
52 switch (size) { \
53 case 4: \
54 __asm__ __volatile__ ( \
55 " amoswap.w %0, %2, %1\n" \
56 RISCV_ACQUIRE_BARRIER \
57 : "=r" (__ret), "+A" (*__ptr) \
58 : "r" (__new) \
59 : "memory"); \
60 break; \
61 case 8: \
62 __asm__ __volatile__ ( \
63 " amoswap.d %0, %2, %1\n" \
64 RISCV_ACQUIRE_BARRIER \
65 : "=r" (__ret), "+A" (*__ptr) \
66 : "r" (__new) \
67 : "memory"); \
68 break; \
69 default: \
70 BUILD_BUG(); \
71 } \
72 __ret; \
73})
74
75#define xchg_acquire(ptr, x) \
76({ \
77 __typeof__(*(ptr)) _x_ = (x); \
78 (__typeof__(*(ptr))) __xchg_acquire((ptr), \
79 _x_, sizeof(*(ptr))); \
80})
81
82#define __xchg_release(ptr, new, size) \
83({ \
84 __typeof__(ptr) __ptr = (ptr); \
85 __typeof__(new) __new = (new); \
86 __typeof__(*(ptr)) __ret; \
87 switch (size) { \
88 case 4: \
89 __asm__ __volatile__ ( \
90 RISCV_RELEASE_BARRIER \
91 " amoswap.w %0, %2, %1\n" \
92 : "=r" (__ret), "+A" (*__ptr) \
93 : "r" (__new) \
94 : "memory"); \
95 break; \
96 case 8: \
97 __asm__ __volatile__ ( \
98 RISCV_RELEASE_BARRIER \
99 " amoswap.d %0, %2, %1\n" \
100 : "=r" (__ret), "+A" (*__ptr) \
101 : "r" (__new) \
102 : "memory"); \
103 break; \
104 default: \
105 BUILD_BUG(); \
106 } \
107 __ret; \
108})
109
110#define xchg_release(ptr, x) \
111({ \
112 __typeof__(*(ptr)) _x_ = (x); \
113 (__typeof__(*(ptr))) __xchg_release((ptr), \
114 _x_, sizeof(*(ptr))); \
115})
116
117#define __xchg(ptr, new, size) \
118({ \
119 __typeof__(ptr) __ptr = (ptr); \
120 __typeof__(new) __new = (new); \
121 __typeof__(*(ptr)) __ret; \
122 switch (size) { \
123 case 4: \
124 __asm__ __volatile__ ( \
125 " amoswap.w.aqrl %0, %2, %1\n" \
126 : "=r" (__ret), "+A" (*__ptr) \
127 : "r" (__new) \
128 : "memory"); \
129 break; \
130 case 8: \
131 __asm__ __volatile__ ( \
132 " amoswap.d.aqrl %0, %2, %1\n" \
133 : "=r" (__ret), "+A" (*__ptr) \
134 : "r" (__new) \
135 : "memory"); \
136 break; \
137 default: \
138 BUILD_BUG(); \
139 } \
140 __ret; \
141})
142
143#define xchg(ptr, x) \
144({ \
145 __typeof__(*(ptr)) _x_ = (x); \
146 (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
147})
148
149#define xchg32(ptr, x) \
150({ \
151 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
152 xchg((ptr), (x)); \
153})
154
155#define xchg64(ptr, x) \
156({ \
157 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
158 xchg((ptr), (x)); \
159})
160
161/*
162 * Atomic compare and exchange. Compare OLD with MEM, if identical,
163 * store NEW in MEM. Return the initial value in MEM. Success is
164 * indicated by comparing RETURN with OLD.
165 */
166#define __cmpxchg_relaxed(ptr, old, new, size) \
167({ \
168 __typeof__(ptr) __ptr = (ptr); \
169 __typeof__(*(ptr)) __old = (old); \
170 __typeof__(*(ptr)) __new = (new); \
171 __typeof__(*(ptr)) __ret; \
172 register unsigned int __rc; \
173 switch (size) { \
174 case 4: \
175 __asm__ __volatile__ ( \
176 "0: lr.w %0, %2\n" \
177 " bne %0, %z3, 1f\n" \
178 " sc.w %1, %z4, %2\n" \
179 " bnez %1, 0b\n" \
180 "1:\n" \
181 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
182 : "rJ" ((long)__old), "rJ" (__new) \
183 : "memory"); \
184 break; \
185 case 8: \
186 __asm__ __volatile__ ( \
187 "0: lr.d %0, %2\n" \
188 " bne %0, %z3, 1f\n" \
189 " sc.d %1, %z4, %2\n" \
190 " bnez %1, 0b\n" \
191 "1:\n" \
192 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
193 : "rJ" (__old), "rJ" (__new) \
194 : "memory"); \
195 break; \
196 default: \
197 BUILD_BUG(); \
198 } \
199 __ret; \
200})
201
202#define cmpxchg_relaxed(ptr, o, n) \
203({ \
204 __typeof__(*(ptr)) _o_ = (o); \
205 __typeof__(*(ptr)) _n_ = (n); \
206 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
207 _o_, _n_, sizeof(*(ptr))); \
208})
209
210#define __cmpxchg_acquire(ptr, old, new, size) \
211({ \
212 __typeof__(ptr) __ptr = (ptr); \
213 __typeof__(*(ptr)) __old = (old); \
214 __typeof__(*(ptr)) __new = (new); \
215 __typeof__(*(ptr)) __ret; \
216 register unsigned int __rc; \
217 switch (size) { \
218 case 4: \
219 __asm__ __volatile__ ( \
220 "0: lr.w %0, %2\n" \
221 " bne %0, %z3, 1f\n" \
222 " sc.w %1, %z4, %2\n" \
223 " bnez %1, 0b\n" \
224 RISCV_ACQUIRE_BARRIER \
225 "1:\n" \
226 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
227 : "rJ" ((long)__old), "rJ" (__new) \
228 : "memory"); \
229 break; \
230 case 8: \
231 __asm__ __volatile__ ( \
232 "0: lr.d %0, %2\n" \
233 " bne %0, %z3, 1f\n" \
234 " sc.d %1, %z4, %2\n" \
235 " bnez %1, 0b\n" \
236 RISCV_ACQUIRE_BARRIER \
237 "1:\n" \
238 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
239 : "rJ" (__old), "rJ" (__new) \
240 : "memory"); \
241 break; \
242 default: \
243 BUILD_BUG(); \
244 } \
245 __ret; \
246})
247
248#define cmpxchg_acquire(ptr, o, n) \
249({ \
250 __typeof__(*(ptr)) _o_ = (o); \
251 __typeof__(*(ptr)) _n_ = (n); \
252 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
253 _o_, _n_, sizeof(*(ptr))); \
254})
255
256#define __cmpxchg_release(ptr, old, new, size) \
257({ \
258 __typeof__(ptr) __ptr = (ptr); \
259 __typeof__(*(ptr)) __old = (old); \
260 __typeof__(*(ptr)) __new = (new); \
261 __typeof__(*(ptr)) __ret; \
262 register unsigned int __rc; \
263 switch (size) { \
264 case 4: \
265 __asm__ __volatile__ ( \
266 RISCV_RELEASE_BARRIER \
267 "0: lr.w %0, %2\n" \
268 " bne %0, %z3, 1f\n" \
269 " sc.w %1, %z4, %2\n" \
270 " bnez %1, 0b\n" \
271 "1:\n" \
272 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
273 : "rJ" ((long)__old), "rJ" (__new) \
274 : "memory"); \
275 break; \
276 case 8: \
277 __asm__ __volatile__ ( \
278 RISCV_RELEASE_BARRIER \
279 "0: lr.d %0, %2\n" \
280 " bne %0, %z3, 1f\n" \
281 " sc.d %1, %z4, %2\n" \
282 " bnez %1, 0b\n" \
283 "1:\n" \
284 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
285 : "rJ" (__old), "rJ" (__new) \
286 : "memory"); \
287 break; \
288 default: \
289 BUILD_BUG(); \
290 } \
291 __ret; \
292})
293
294#define cmpxchg_release(ptr, o, n) \
295({ \
296 __typeof__(*(ptr)) _o_ = (o); \
297 __typeof__(*(ptr)) _n_ = (n); \
298 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \
299 _o_, _n_, sizeof(*(ptr))); \
300})
301
302#define __cmpxchg(ptr, old, new, size) \
303({ \
304 __typeof__(ptr) __ptr = (ptr); \
305 __typeof__(*(ptr)) __old = (old); \
306 __typeof__(*(ptr)) __new = (new); \
307 __typeof__(*(ptr)) __ret; \
308 register unsigned int __rc; \
309 switch (size) { \
310 case 4: \
311 __asm__ __volatile__ ( \
312 "0: lr.w %0, %2\n" \
313 " bne %0, %z3, 1f\n" \
314 " sc.w.rl %1, %z4, %2\n" \
315 " bnez %1, 0b\n" \
316 " fence rw, rw\n" \
317 "1:\n" \
318 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
319 : "rJ" ((long)__old), "rJ" (__new) \
320 : "memory"); \
321 break; \
322 case 8: \
323 __asm__ __volatile__ ( \
324 "0: lr.d %0, %2\n" \
325 " bne %0, %z3, 1f\n" \
326 " sc.d.rl %1, %z4, %2\n" \
327 " bnez %1, 0b\n" \
328 " fence rw, rw\n" \
329 "1:\n" \
330 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
331 : "rJ" (__old), "rJ" (__new) \
332 : "memory"); \
333 break; \
334 default: \
335 BUILD_BUG(); \
336 } \
337 __ret; \
338})
339
340#define cmpxchg(ptr, o, n) \
341({ \
342 __typeof__(*(ptr)) _o_ = (o); \
343 __typeof__(*(ptr)) _n_ = (n); \
344 (__typeof__(*(ptr))) __cmpxchg((ptr), \
345 _o_, _n_, sizeof(*(ptr))); \
346})
347
348#define cmpxchg_local(ptr, o, n) \
349 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
350
351#define cmpxchg32(ptr, o, n) \
352({ \
353 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
354 cmpxchg((ptr), (o), (n)); \
355})
356
357#define cmpxchg32_local(ptr, o, n) \
358({ \
359 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
360 cmpxchg_relaxed((ptr), (o), (n)) \
361})
362
363#define cmpxchg64(ptr, o, n) \
364({ \
365 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
366 cmpxchg((ptr), (o), (n)); \
367})
368
369#define cmpxchg64_local(ptr, o, n) \
370({ \
371 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
372 cmpxchg_relaxed((ptr), (o), (n)); \
373})
374
375#endif /* _ASM_RISCV_CMPXCHG_H */