| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
|---|
| 2 | #ifndef _LINUX_SCHED_RCUPDATE_WAIT_H |
|---|
| 3 | #define _LINUX_SCHED_RCUPDATE_WAIT_H |
|---|
| 4 | |
|---|
| 5 | /* |
|---|
| 6 | * RCU synchronization types and methods: |
|---|
| 7 | */ |
|---|
| 8 | |
|---|
| 9 | #include <linux/rcupdate.h> |
|---|
| 10 | #include <linux/completion.h> |
|---|
| 11 | #include <linux/sched.h> |
|---|
| 12 | |
|---|
| 13 | /* |
|---|
| 14 | * Structure allowing asynchronous waiting on RCU. |
|---|
| 15 | */ |
|---|
| 16 | struct rcu_synchronize { |
|---|
| 17 | struct rcu_head head; |
|---|
| 18 | struct completion completion; |
|---|
| 19 | |
|---|
| 20 | /* This is for debugging. */ |
|---|
| 21 | struct rcu_gp_oldstate oldstate; |
|---|
| 22 | }; |
|---|
| 23 | void wakeme_after_rcu(struct rcu_head *head); |
|---|
| 24 | |
|---|
| 25 | void __wait_rcu_gp(bool checktiny, unsigned int state, int n, call_rcu_func_t *crcu_array, |
|---|
| 26 | struct rcu_synchronize *rs_array); |
|---|
| 27 | |
|---|
| 28 | #define _wait_rcu_gp(checktiny, state, ...) \ |
|---|
| 29 | do { \ |
|---|
| 30 | call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ |
|---|
| 31 | struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ |
|---|
| 32 | __wait_rcu_gp(checktiny, state, ARRAY_SIZE(__crcu_array), __crcu_array, __rs_array); \ |
|---|
| 33 | } while (0) |
|---|
| 34 | |
|---|
| 35 | #define wait_rcu_gp(...) _wait_rcu_gp(false, TASK_UNINTERRUPTIBLE, __VA_ARGS__) |
|---|
| 36 | #define wait_rcu_gp_state(state, ...) _wait_rcu_gp(false, state, __VA_ARGS__) |
|---|
| 37 | |
|---|
| 38 | /** |
|---|
| 39 | * synchronize_rcu_mult - Wait concurrently for multiple grace periods |
|---|
| 40 | * @...: List of call_rcu() functions for different grace periods to wait on |
|---|
| 41 | * |
|---|
| 42 | * This macro waits concurrently for multiple types of RCU grace periods. |
|---|
| 43 | * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait |
|---|
| 44 | * on concurrent RCU and RCU-tasks grace periods. Waiting on a given SRCU |
|---|
| 45 | * domain requires you to write a wrapper function for that SRCU domain's |
|---|
| 46 | * call_srcu() function, with this wrapper supplying the pointer to the |
|---|
| 47 | * corresponding srcu_struct. |
|---|
| 48 | * |
|---|
| 49 | * Note that call_rcu_hurry() should be used instead of call_rcu() |
|---|
| 50 | * because in kernels built with CONFIG_RCU_LAZY=y the delay between the |
|---|
| 51 | * invocation of call_rcu() and that of the corresponding RCU callback |
|---|
| 52 | * can be multiple seconds. |
|---|
| 53 | * |
|---|
| 54 | * The first argument tells Tiny RCU's _wait_rcu_gp() not to |
|---|
| 55 | * bother waiting for RCU. The reason for this is because anywhere |
|---|
| 56 | * synchronize_rcu_mult() can be called is automatically already a full |
|---|
| 57 | * grace period. |
|---|
| 58 | */ |
|---|
| 59 | #define synchronize_rcu_mult(...) \ |
|---|
| 60 | _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), TASK_UNINTERRUPTIBLE, __VA_ARGS__) |
|---|
| 61 | |
|---|
| 62 | static inline void cond_resched_rcu(void) |
|---|
| 63 | { |
|---|
| 64 | #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) |
|---|
| 65 | rcu_read_unlock(); |
|---|
| 66 | cond_resched(); |
|---|
| 67 | rcu_read_lock(); |
|---|
| 68 | #endif |
|---|
| 69 | } |
|---|
| 70 | |
|---|
| 71 | // Has the current task blocked within its current RCU read-side |
|---|
| 72 | // critical section? |
|---|
| 73 | static inline bool has_rcu_reader_blocked(void) |
|---|
| 74 | { |
|---|
| 75 | #ifdef CONFIG_PREEMPT_RCU |
|---|
| 76 | return !list_empty(head: ¤t->rcu_node_entry); |
|---|
| 77 | #else |
|---|
| 78 | return false; |
|---|
| 79 | #endif |
|---|
| 80 | } |
|---|
| 81 | |
|---|
| 82 | #endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */ |
|---|
| 83 | |
|---|
