| b.liu | e958203 | 2025-04-17 19:18:16 +0800 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
| 2 | /* |
| 3 | * RCU segmented callback lists, internal-to-rcu header file |
| 4 | * |
| 5 | * Copyright IBM Corporation, 2017 |
| 6 | * |
| 7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
| 8 | */ |
| 9 | |
| 10 | #include <linux/rcu_segcblist.h> |
| 11 | |
| 12 | /* Return number of callbacks in the specified callback list. */ |
| 13 | static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) |
| 14 | { |
| 15 | return READ_ONCE(rclp->len); |
| 16 | } |
| 17 | |
| 18 | /* |
| 19 | * Account for the fact that a previously dequeued callback turned out |
| 20 | * to be marked as lazy. |
| 21 | */ |
| 22 | static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp) |
| 23 | { |
| 24 | rclp->len_lazy--; |
| 25 | } |
| 26 | |
| 27 | void rcu_cblist_init(struct rcu_cblist *rclp); |
| 28 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
| 29 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, |
| 30 | struct rcu_cblist *srclp, |
| 31 | struct rcu_head *rhp); |
| 32 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
| 33 | |
| 34 | /* |
| 35 | * Is the specified rcu_segcblist structure empty? |
| 36 | * |
| 37 | * But careful! The fact that the ->head field is NULL does not |
| 38 | * necessarily imply that there are no callbacks associated with |
| 39 | * this structure. When callbacks are being invoked, they are |
| 40 | * removed as a group. If callback invocation must be preempted, |
| 41 | * the remaining callbacks will be added back to the list. Either |
| 42 | * way, the counts are updated later. |
| 43 | * |
| 44 | * So it is often the case that rcu_segcblist_n_cbs() should be used |
| 45 | * instead. |
| 46 | */ |
| 47 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) |
| 48 | { |
| 49 | return !READ_ONCE(rsclp->head); |
| 50 | } |
| 51 | |
| 52 | /* Return number of callbacks in segmented callback list. */ |
| 53 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) |
| 54 | { |
| 55 | #ifdef CONFIG_RCU_NOCB_CPU |
| 56 | return atomic_long_read(&rsclp->len); |
| 57 | #else |
| 58 | return READ_ONCE(rsclp->len); |
| 59 | #endif |
| 60 | } |
| 61 | |
| 62 | /* Return number of lazy callbacks in segmented callback list. */ |
| 63 | static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp) |
| 64 | { |
| 65 | return rsclp->len_lazy; |
| 66 | } |
| 67 | |
| 68 | /* Return number of lazy callbacks in segmented callback list. */ |
| 69 | static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp) |
| 70 | { |
| 71 | return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy; |
| 72 | } |
| 73 | |
| 74 | /* |
| 75 | * Is the specified rcu_segcblist enabled, for example, not corresponding |
| 76 | * to an offline CPU? |
| 77 | */ |
| 78 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) |
| 79 | { |
| 80 | return rsclp->enabled; |
| 81 | } |
| 82 | |
| 83 | /* Is the specified rcu_segcblist offloaded? */ |
| 84 | static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) |
| 85 | { |
| 86 | return rsclp->offloaded; |
| 87 | } |
| 88 | |
| 89 | /* |
| 90 | * Are all segments following the specified segment of the specified |
| 91 | * rcu_segcblist structure empty of callbacks? (The specified |
| 92 | * segment might well contain callbacks.) |
| 93 | */ |
| 94 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) |
| 95 | { |
| 96 | return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); |
| 97 | } |
| 98 | |
| 99 | void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); |
| 100 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); |
| 101 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); |
| 102 | void rcu_segcblist_offload(struct rcu_segcblist *rsclp); |
| 103 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
| 104 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); |
| 105 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
| 106 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); |
| 107 | bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); |
| 108 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
| 109 | struct rcu_head *rhp, bool lazy); |
| 110 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, |
| 111 | struct rcu_head *rhp, bool lazy); |
| 112 | void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, |
| 113 | struct rcu_cblist *rclp); |
| 114 | void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, |
| 115 | struct rcu_cblist *rclp); |
| 116 | void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, |
| 117 | struct rcu_cblist *rclp); |
| 118 | void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, |
| 119 | struct rcu_cblist *rclp); |
| 120 | void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, |
| 121 | struct rcu_cblist *rclp); |
| 122 | void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, |
| 123 | struct rcu_cblist *rclp); |
| 124 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); |
| 125 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); |
| 126 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
| 127 | struct rcu_segcblist *src_rsclp); |