| lh | 9ed821d | 2023-04-07 01:36:19 -0700 | [diff] [blame] | 1 | /* |
| 2 | * include/asm-generic/mutex-xchg.h |
| 3 | * |
| 4 | * Generic implementation of the mutex fastpath, based on xchg(). |
| 5 | * |
| 6 | * NOTE: An xchg based implementation might be less optimal than an atomic |
| 7 | * decrement/increment based implementation. If your architecture |
| 8 | * has a reasonable atomic dec/inc then you should probably use |
| 9 | * asm-generic/mutex-dec.h instead, or you could open-code an |
| 10 | * optimized version in asm/mutex.h. |
| 11 | */ |
| 12 | #ifndef _ASM_GENERIC_MUTEX_XCHG_H |
| 13 | #define _ASM_GENERIC_MUTEX_XCHG_H |
| 14 | |
| 15 | /** |
| 16 | * __mutex_fastpath_lock - try to take the lock by moving the count |
| 17 | * from 1 to a 0 value |
| 18 | * @count: pointer of type atomic_t |
| 19 | * @fail_fn: function to call if the original value was not 1 |
| 20 | * |
| 21 | * Change the count from 1 to a value lower than 1, and call <fail_fn> if it |
| 22 | * wasn't 1 originally. This function MUST leave the value lower than 1 |
| 23 | * even when the "1" assertion wasn't true. |
| 24 | */ |
| 25 | static inline void |
| 26 | __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) |
| 27 | { |
| 28 | if (unlikely(atomic_xchg(count, 0) != 1)) |
| 29 | /* |
| 30 | * We failed to acquire the lock, so mark it contended |
| 31 | * to ensure that any waiting tasks are woken up by the |
| 32 | * unlock slow path. |
| 33 | */ |
| 34 | if (likely(atomic_xchg(count, -1) != 1)) |
| 35 | fail_fn(count); |
| 36 | } |
| 37 | |
| 38 | /** |
| 39 | * __mutex_fastpath_lock_retval - try to take the lock by moving the count |
| 40 | * from 1 to a 0 value |
| 41 | * @count: pointer of type atomic_t |
| 42 | * @fail_fn: function to call if the original value was not 1 |
| 43 | * |
| 44 | * Change the count from 1 to a value lower than 1, and call <fail_fn> if it |
| 45 | * wasn't 1 originally. This function returns 0 if the fastpath succeeds, |
| 46 | * or anything the slow path function returns |
| 47 | */ |
| 48 | static inline int |
| 49 | __mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *)) |
| 50 | { |
| 51 | if (unlikely(atomic_xchg(count, 0) != 1)) |
| 52 | if (likely(atomic_xchg(count, -1) != 1)) |
| 53 | return fail_fn(count); |
| 54 | return 0; |
| 55 | } |
| 56 | |
| 57 | /** |
| 58 | * __mutex_fastpath_unlock - try to promote the mutex from 0 to 1 |
| 59 | * @count: pointer of type atomic_t |
| 60 | * @fail_fn: function to call if the original value was not 0 |
| 61 | * |
| 62 | * try to promote the mutex from 0 to 1. if it wasn't 0, call <function> |
| 63 | * In the failure case, this function is allowed to either set the value to |
| 64 | * 1, or to set it to a value lower than one. |
| 65 | * If the implementation sets it to a value of lower than one, the |
| 66 | * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs |
| 67 | * to return 0 otherwise. |
| 68 | */ |
| 69 | static inline void |
| 70 | __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) |
| 71 | { |
| 72 | if (unlikely(atomic_xchg(count, 1) != 0)) |
| 73 | fail_fn(count); |
| 74 | } |
| 75 | |
| 76 | #define __mutex_slowpath_needs_to_unlock() 0 |
| 77 | |
| 78 | /** |
| 79 | * __mutex_fastpath_trylock - try to acquire the mutex, without waiting |
| 80 | * |
| 81 | * @count: pointer of type atomic_t |
| 82 | * @fail_fn: spinlock based trylock implementation |
| 83 | * |
| 84 | * Change the count from 1 to a value lower than 1, and return 0 (failure) |
| 85 | * if it wasn't 1 originally, or return 1 (success) otherwise. This function |
| 86 | * MUST leave the value lower than 1 even when the "1" assertion wasn't true. |
| 87 | * Additionally, if the value was < 0 originally, this function must not leave |
| 88 | * it to 0 on failure. |
| 89 | * |
| 90 | * If the architecture has no effective trylock variant, it should call the |
| 91 | * <fail_fn> spinlock-based trylock variant unconditionally. |
| 92 | */ |
| 93 | static inline int |
| 94 | __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) |
| 95 | { |
| 96 | int prev = atomic_xchg(count, 0); |
| 97 | |
| 98 | if (unlikely(prev < 0)) { |
| 99 | /* |
| 100 | * The lock was marked contended so we must restore that |
| 101 | * state. If while doing so we get back a prev value of 1 |
| 102 | * then we just own it. |
| 103 | * |
| 104 | * [ In the rare case of the mutex going to 1, to 0, to -1 |
| 105 | * and then back to 0 in this few-instructions window, |
| 106 | * this has the potential to trigger the slowpath for the |
| 107 | * owner's unlock path needlessly, but that's not a problem |
| 108 | * in practice. ] |
| 109 | */ |
| 110 | prev = atomic_xchg(count, prev); |
| 111 | if (prev < 0) |
| 112 | prev = 0; |
| 113 | } |
| 114 | |
| 115 | return prev; |
| 116 | } |
| 117 | |
| 118 | #endif |