| /* Low-level lock implementation. Generic futex-based version. |
| Copyright (C) 2005-2016 Free Software Foundation, Inc. |
| This file is part of the GNU C Library. |
| |
| The GNU C Library is free software; you can redistribute it and/or |
| modify it under the terms of the GNU Lesser General Public |
| License as published by the Free Software Foundation; either |
| version 2.1 of the License, or (at your option) any later version. |
| |
| The GNU C Library is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| Lesser General Public License for more details. |
| |
| You should have received a copy of the GNU Lesser General Public |
| License along with the GNU C Library. If not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| #ifndef _LOWLEVELLOCK_H |
| #define _LOWLEVELLOCK_H 1 |
| |
| #include <atomic.h> |
| #include <lowlevellock-futex.h> |
| |
| /* Low-level locks use a combination of atomic operations (to acquire and |
| release lock ownership) and futex operations (to block until the state |
| of a lock changes). A lock can be in one of three states: |
| 0: not acquired, |
| 1: acquired with no waiters; no other threads are blocked or about to block |
| for changes to the lock state, |
| >1: acquired, possibly with waiters; there may be other threads blocked or |
| about to block for changes to the lock state. |
| |
| We expect that the common case is an uncontended lock, so we just need |
| to transition the lock between states 0 and 1; releasing the lock does |
| not need to wake any other blocked threads. If the lock is contended |
| and a thread decides to block using a futex operation, then this thread |
| needs to first change the state to >1; if this state is observed during |
| lock release, the releasing thread will wake one of the potentially |
| blocked threads. |
| |
| Much of this code takes a 'private' parameter. This may be: |
| LLL_PRIVATE: lock only shared within a process |
| LLL_SHARED: lock may be shared across processes. |
| |
| Condition variables contain an optimization for broadcasts that requeues |
| waiting threads on a lock's futex. Therefore, there is a special |
| variant of the locks (whose name contains "cond") that makes sure to |
| always set the lock state to >1 and not just 1. |
| |
| Robust locks set the lock to the id of the owner. This allows detection |
| of the case where the owner exits without releasing the lock. Flags are |
| OR'd with the owner id to record additional information about lock state. |
| Therefore the states of robust locks are: |
| 0: not acquired |
| id: acquired (by user identified by id & FUTEX_TID_MASK) |
| |
| The following flags may be set in the robust lock value: |
| FUTEX_WAITERS - possibly has waiters |
| FUTEX_OWNER_DIED - owning user has exited without releasing the futex. */ |
| |
| |
| /* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return |
| 0. Otherwise leave lock unchanged and return non-zero to indicate that the |
| lock was not acquired. */ |
| #define lll_trylock(lock) \ |
| atomic_compare_and_exchange_bool_acq (&(lock), 1, 0) |
| |
| /* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and |
| return 0. Otherwise leave lock unchanged and return non-zero to indicate |
| that the lock was not acquired. */ |
| #define lll_cond_trylock(lock) \ |
| atomic_compare_and_exchange_bool_acq (&(lock), 2, 0) |
| |
| extern void __lll_lock_wait_private (int *futex) attribute_hidden; |
| extern void __lll_lock_wait (int *futex, int private) attribute_hidden; |
| extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden; |
| |
| /* This is an expression rather than a statement even though its value is |
| void, so that it can be used in a comma expression or as an expression |
| that's cast to void. */ |
| /* The inner conditional compiles to a call to __lll_lock_wait_private if |
| private is known at compile time to be LLL_PRIVATE, and to a call to |
| __lll_lock_wait otherwise. */ |
| /* If FUTEX is 0 (not acquired), set to 1 (acquired with no waiters) and |
| return. Otherwise, ensure that it is >1 (acquired, possibly with waiters) |
| and then block until we acquire the lock, at which point FUTEX will still be |
| >1. The lock is always acquired on return. */ |
| #define __lll_lock(futex, private) \ |
| ((void) \ |
| ({ \ |
| int *__futex = (futex); \ |
| if (__glibc_unlikely \ |
| (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \ |
| { \ |
| if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ |
| __lll_lock_wait_private (__futex); \ |
| else \ |
| __lll_lock_wait (__futex, private); \ |
| } \ |
| })) |
| #define lll_lock(futex, private) \ |
| __lll_lock (&(futex), private) |
| |
| |
| /* If FUTEX is 0 (not acquired), set to ID (acquired with no waiters) and |
| return 0. Otherwise, ensure that it is set to FUTEX | FUTEX_WAITERS |
| (acquired, possibly with waiters) and block until we acquire the lock. |
| FUTEX will now be ID | FUTEX_WAITERS and we return 0. |
| If the previous owner of the lock dies before we acquire the lock then FUTEX |
| will be the value of id as set by the previous owner, with FUTEX_OWNER_DIED |
| set (FUTEX_WAITERS may or may not be set). We return this value to indicate |
| that the lock is not acquired. */ |
| #define __lll_robust_lock(futex, id, private) \ |
| ({ \ |
| int *__futex = (futex); \ |
| int __val = 0; \ |
| \ |
| if (__glibc_unlikely \ |
| (atomic_compare_and_exchange_bool_acq (__futex, id, 0))) \ |
| __val = __lll_robust_lock_wait (__futex, private); \ |
| __val; \ |
| }) |
| #define lll_robust_lock(futex, id, private) \ |
| __lll_robust_lock (&(futex), id, private) |
| |
| |
| /* This is an expression rather than a statement even though its value is |
| void, so that it can be used in a comma expression or as an expression |
| that's cast to void. */ |
| /* Unconditionally set FUTEX to 2 (acquired, possibly with waiters). If FUTEX |
| was 0 (not acquired) then return. Otherwise, block until the lock is |
| acquired, at which point FUTEX is 2 (acquired, possibly with waiters). The |
| lock is always acquired on return. */ |
| #define __lll_cond_lock(futex, private) \ |
| ((void) \ |
| ({ \ |
| int *__futex = (futex); \ |
| if (__glibc_unlikely (atomic_exchange_acq (__futex, 2) != 0)) \ |
| __lll_lock_wait (__futex, private); \ |
| })) |
| #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private) |
| |
| |
| /* As __lll_robust_lock, but set to ID | FUTEX_WAITERS (acquired, possibly with |
| waiters) if FUTEX is 0. */ |
| #define lll_robust_cond_lock(futex, id, private) \ |
| __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private) |
| |
| |
| extern int __lll_timedlock_wait (int *futex, const struct timespec *, |
| int private) attribute_hidden; |
| extern int __lll_robust_timedlock_wait (int *futex, const struct timespec *, |
| int private) attribute_hidden; |
| |
| |
| /* As __lll_lock, but with a timeout. If the timeout occurs then return |
| ETIMEDOUT. If ABSTIME is invalid, return EINVAL. */ |
| #define __lll_timedlock(futex, abstime, private) \ |
| ({ \ |
| int *__futex = (futex); \ |
| int __val = 0; \ |
| \ |
| if (__glibc_unlikely \ |
| (atomic_compare_and_exchange_bool_acq (__futex, 1, 0))) \ |
| __val = __lll_timedlock_wait (__futex, abstime, private); \ |
| __val; \ |
| }) |
| #define lll_timedlock(futex, abstime, private) \ |
| __lll_timedlock (&(futex), abstime, private) |
| |
| |
| /* As __lll_robust_lock, but with a timeout. If the timeout occurs then return |
| ETIMEDOUT. If ABSTIME is invalid, return EINVAL. */ |
| #define __lll_robust_timedlock(futex, abstime, id, private) \ |
| ({ \ |
| int *__futex = (futex); \ |
| int __val = 0; \ |
| \ |
| if (__glibc_unlikely \ |
| (atomic_compare_and_exchange_bool_acq (__futex, id, 0))) \ |
| __val = __lll_robust_timedlock_wait (__futex, abstime, private); \ |
| __val; \ |
| }) |
| #define lll_robust_timedlock(futex, abstime, id, private) \ |
| __lll_robust_timedlock (&(futex), abstime, id, private) |
| |
| |
| /* This is an expression rather than a statement even though its value is |
| void, so that it can be used in a comma expression or as an expression |
| that's cast to void. */ |
| /* Unconditionally set FUTEX to 0 (not acquired), releasing the lock. If FUTEX |
| was >1 (acquired, possibly with waiters), then wake any waiters. The waiter |
| that acquires the lock will set FUTEX to >1. |
| Evaluate PRIVATE before releasing the lock so that we do not violate the |
| mutex destruction requirements. Specifically, we need to ensure that |
| another thread can destroy the mutex (and reuse its memory) once it |
| acquires the lock and when there will be no further lock acquisitions; |
| thus, we must not access the lock after releasing it, or those accesses |
| could be concurrent with mutex destruction or reuse of the memory. */ |
| #define __lll_unlock(futex, private) \ |
| ((void) \ |
| ({ \ |
| int *__futex = (futex); \ |
| int __private = (private); \ |
| int __oldval = atomic_exchange_rel (__futex, 0); \ |
| if (__glibc_unlikely (__oldval > 1)) \ |
| lll_futex_wake (__futex, 1, __private); \ |
| })) |
| #define lll_unlock(futex, private) \ |
| __lll_unlock (&(futex), private) |
| |
| |
| /* This is an expression rather than a statement even though its value is |
| void, so that it can be used in a comma expression or as an expression |
| that's cast to void. */ |
| /* Unconditionally set FUTEX to 0 (not acquired), releasing the lock. If FUTEX |
| had FUTEX_WAITERS set then wake any waiters. The waiter that acquires the |
| lock will set FUTEX_WAITERS. |
| Evaluate PRIVATE before releasing the lock so that we do not violate the |
| mutex destruction requirements (see __lll_unlock). */ |
| #define __lll_robust_unlock(futex, private) \ |
| ((void) \ |
| ({ \ |
| int *__futex = (futex); \ |
| int __private = (private); \ |
| int __oldval = atomic_exchange_rel (__futex, 0); \ |
| if (__glibc_unlikely (__oldval & FUTEX_WAITERS)) \ |
| lll_futex_wake (__futex, 1, __private); \ |
| })) |
| #define lll_robust_unlock(futex, private) \ |
| __lll_robust_unlock (&(futex), private) |
| |
| |
| #define lll_islocked(futex) \ |
| ((futex) != LLL_LOCK_INITIALIZER) |
| |
| |
| /* Our internal lock implementation is identical to the binary-compatible |
| mutex implementation. */ |
| |
| /* Initializers for lock. */ |
| #define LLL_LOCK_INITIALIZER (0) |
| #define LLL_LOCK_INITIALIZER_LOCKED (1) |
| |
| |
| /* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex |
| wake-up when the clone terminates. The memory location contains the |
| thread ID while the clone is running and is reset to zero by the kernel |
| afterwards. The kernel up to version 3.16.3 does not use the private futex |
| operations for futex wake-up when the clone terminates. */ |
| #define lll_wait_tid(tid) \ |
| do { \ |
| __typeof (tid) __tid; \ |
| while ((__tid = (tid)) != 0) \ |
| lll_futex_wait (&(tid), __tid, LLL_SHARED);\ |
| } while (0) |
| |
| extern int __lll_timedwait_tid (int *, const struct timespec *) |
| attribute_hidden; |
| |
| /* As lll_wait_tid, but with a timeout. If the timeout occurs then return |
| ETIMEDOUT. If ABSTIME is invalid, return EINVAL. */ |
| #define lll_timedwait_tid(tid, abstime) \ |
| ({ \ |
| int __res = 0; \ |
| if ((tid) != 0) \ |
| __res = __lll_timedwait_tid (&(tid), (abstime)); \ |
| __res; \ |
| }) |
| |
| |
| #endif /* lowlevellock.h */ |