1 /* SPDX-License-Identifier: GPL-2.0 */
5 #include <linux/futex.h>
6 #include <linux/sched/wake_q.h>
8 #ifdef CONFIG_PREEMPT_RT
9 #include <linux/rcuwait.h>
12 #include <asm/futex.h>
15 * Futex flags used to encode options to functions and preserve them across
19 # define FLAGS_SHARED 0x01
22 * NOMMU does not have per process address space. Let the compiler optimize
25 # define FLAGS_SHARED 0x00
27 #define FLAGS_CLOCKRT 0x02
28 #define FLAGS_HAS_TIMEOUT 0x04
30 #ifdef CONFIG_FAIL_FUTEX
31 extern bool should_fail_futex(bool fshared);
33 static inline bool should_fail_futex(bool fshared)
40 * Hash buckets are shared by all the futex_keys that hash to the same
41 * location. Each key may have multiple futex_q structures, one for each task
44 struct futex_hash_bucket {
47 struct plist_head chain;
48 } ____cacheline_aligned_in_smp;
51 * Priority Inheritance state:
53 struct futex_pi_state {
55 * list of 'owned' pi_state instances - these have to be
56 * cleaned up in do_exit() if the task exits prematurely:
58 struct list_head list;
63 struct rt_mutex_base pi_mutex;
65 struct task_struct *owner;
72 * struct futex_q - The hashed futex queue entry, one per waiting task
73 * @list: priority-sorted list of tasks waiting on this futex
74 * @task: the task waiting on the futex
75 * @lock_ptr: the hash bucket lock
76 * @key: the key the futex is hashed on
77 * @pi_state: optional priority inheritance state
78 * @rt_waiter: rt_waiter storage for use with requeue_pi
79 * @requeue_pi_key: the requeue_pi target futex key
80 * @bitset: bitset for the optional bitmasked wakeup
81 * @requeue_state: State field for futex_requeue_pi()
82 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
84 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
85 * we can wake only the relevant ones (hashed queues may be shared).
87 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
88 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
89 * The order of wakeup is always to make the first condition true, then
92 * PI futexes are typically woken before they are removed from the hash list via
93 * the rt_mutex code. See futex_unqueue_pi().
96 struct plist_node list;
98 struct task_struct *task;
101 struct futex_pi_state *pi_state;
102 struct rt_mutex_waiter *rt_waiter;
103 union futex_key *requeue_pi_key;
105 atomic_t requeue_state;
106 #ifdef CONFIG_PREEMPT_RT
107 struct rcuwait requeue_wait;
109 } __randomize_layout;
111 extern const struct futex_q futex_q_init;
118 extern int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
119 enum futex_access rw);
121 extern struct hrtimer_sleeper *
122 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
123 int flags, u64 range_ns);
125 extern struct futex_hash_bucket *futex_hash(union futex_key *key);
128 * futex_match - Check whether two futex keys are equal
129 * @key1: Pointer to key1
130 * @key2: Pointer to key2
132 * Return 1 if two futex_keys are equal, 0 otherwise.
134 static inline int futex_match(union futex_key *key1, union futex_key *key2)
137 && key1->both.word == key2->both.word
138 && key1->both.ptr == key2->both.ptr
139 && key1->both.offset == key2->both.offset);
142 extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
143 struct futex_q *q, struct futex_hash_bucket **hb);
144 extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
145 struct hrtimer_sleeper *timeout);
146 extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
148 extern int fault_in_user_writeable(u32 __user *uaddr);
149 extern int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval);
150 extern int futex_get_value_locked(u32 *dest, u32 __user *from);
151 extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
153 extern void __futex_unqueue(struct futex_q *q);
154 extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb);
155 extern int futex_unqueue(struct futex_q *q);
158 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
159 * @q: The futex_q to enqueue
160 * @hb: The destination hash bucket
162 * The hb->lock must be held by the caller, and is released here. A call to
163 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The
164 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
165 * or nothing if the unqueue is done as part of the wake process and the unqueue
166 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
169 static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb)
170 __releases(&hb->lock)
172 __futex_queue(q, hb);
173 spin_unlock(&hb->lock);
176 extern void futex_unqueue_pi(struct futex_q *q);
178 extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
181 * Reflects a new waiter being added to the waitqueue.
183 static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
186 atomic_inc(&hb->waiters);
188 * Full barrier (A), see the ordering comment above.
190 smp_mb__after_atomic();
195 * Reflects a waiter being removed from the waitqueue by wakeup
198 static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
201 atomic_dec(&hb->waiters);
205 static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
209 * Full barrier (B), see the ordering comment above.
212 return atomic_read(&hb->waiters);
218 extern struct futex_hash_bucket *futex_q_lock(struct futex_q *q);
219 extern void futex_q_unlock(struct futex_hash_bucket *hb);
222 extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
223 union futex_key *key,
224 struct futex_pi_state **ps,
225 struct task_struct *task,
226 struct task_struct **exiting,
229 extern int refill_pi_state_cache(void);
230 extern void get_pi_state(struct futex_pi_state *pi_state);
231 extern void put_pi_state(struct futex_pi_state *pi_state);
232 extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
235 * Express the locking dependencies for lockdep:
238 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
243 spin_lock(&hb1->lock);
245 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
249 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
251 spin_unlock(&hb1->lock);
253 spin_unlock(&hb2->lock);
258 extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
259 val, ktime_t *abs_time, u32 bitset, u32 __user
262 extern int futex_requeue(u32 __user *uaddr1, unsigned int flags,
263 u32 __user *uaddr2, int nr_wake, int nr_requeue,
264 u32 *cmpval, int requeue_pi);
266 extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
267 ktime_t *abs_time, u32 bitset);
270 * struct futex_vector - Auxiliary struct for futex_waitv()
271 * @w: Userspace provided data
272 * @q: Kernel side data
274 * Struct used to build an array with all data need for futex_waitv()
276 struct futex_vector {
277 struct futex_waitv w;
281 extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
282 struct hrtimer_sleeper *to);
284 extern int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset);
286 extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
287 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
289 extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags);
291 extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
293 #endif /* _FUTEX_H */