Gitiles
Code Review Sign In
192.168.1.100 / T103 / 9a80dc99f8ca842c57ed84cccda8a386efda7580 / . / src / kernel / linux / v4.14 / include / linux / srcutree.h
blob: a949f4f9e4d769f37579e8d6e90c8fb636f6029b [file] [log] [blame]
rjw1f884582022-01-06 17:20:42 +0800[diff] [blame]1/*
2 * Sleepable Read-Copy Update mechanism for mutual exclusion,
3 * tree variant.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, you can access it online at
17 * http://www.gnu.org/licenses/gpl-2.0.html.
18 *
19 * Copyright (C) IBM Corporation, 2017
20 *
21 * Author: Paul McKenney <paulmck@us.ibm.com>
22 */
23
24#ifndef _LINUX_SRCU_TREE_H
25#define _LINUX_SRCU_TREE_H
26
27#include <linux/rcu_node_tree.h>
28#include <linux/completion.h>
29
30struct srcu_node;
31struct srcu_struct;
32
33/*
34 * Per-CPU structure feeding into leaf srcu_node, similar in function
35 * to rcu_node.
36 */
37struct srcu_data {
38 /* Read-side state. */
39 unsigned long srcu_lock_count[2]; /* Locks per CPU. */
40 unsigned long srcu_unlock_count[2]; /* Unlocks per CPU. */
41
42 /* Update-side state. */
43 raw_spinlock_t __private lock ____cacheline_internodealigned_in_smp;
44 struct rcu_segcblist srcu_cblist; /* List of callbacks.*/
45 unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */
46 unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
47 bool srcu_cblist_invoking; /* Invoking these CBs? */
48 struct delayed_work work; /* Context for CB invoking. */
49 struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */
50 struct srcu_node *mynode; /* Leaf srcu_node. */
51 unsigned long grpmask; /* Mask for leaf srcu_node */
52 /* ->srcu_data_have_cbs[]. */
53 int cpu;
54 struct srcu_struct *sp;
55};
56
57/*
58 * Node in SRCU combining tree, similar in function to rcu_data.
59 */
60struct srcu_node {
61 raw_spinlock_t __private lock;
62 unsigned long srcu_have_cbs[4]; /* GP seq for children */
63 /* having CBs, but only */
64 /* is > ->srcu_gq_seq. */
65 unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs */
66 /* have CBs for given GP? */
67 unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
68 struct srcu_node *srcu_parent; /* Next up in tree. */
69 int grplo; /* Least CPU for node. */
70 int grphi; /* Biggest CPU for node. */
71};
72
73/*
74 * Per-SRCU-domain structure, similar in function to rcu_state.
75 */
76struct srcu_struct {
77 struct srcu_node node[NUM_RCU_NODES]; /* Combining tree. */
78 struct srcu_node *level[RCU_NUM_LVLS + 1];
79 /* First node at each level. */
80 struct mutex srcu_cb_mutex; /* Serialize CB preparation. */
81 raw_spinlock_t __private lock; /* Protect counters */
82 struct mutex srcu_gp_mutex; /* Serialize GP work. */
83 unsigned int srcu_idx; /* Current rdr array element. */
84 unsigned long srcu_gp_seq; /* Grace-period seq #. */
85 unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */
86 unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
87 unsigned long srcu_last_gp_end; /* Last GP end timestamp (ns) */
88 struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */
89 unsigned long srcu_barrier_seq; /* srcu_barrier seq #. */
90 struct mutex srcu_barrier_mutex; /* Serialize barrier ops. */
91 struct completion srcu_barrier_completion;
92 /* Awaken barrier rq at end. */
93 atomic_t srcu_barrier_cpu_cnt; /* # CPUs not yet posting a */
94 /* callback for the barrier */
95 /* operation. */
96 struct delayed_work work;
97#ifdef CONFIG_DEBUG_LOCK_ALLOC
98 struct lockdep_map dep_map;
99#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
100};
101
102/* Values for state variable (bottom bits of ->srcu_gp_seq). */
103#define SRCU_STATE_IDLE 0
104#define SRCU_STATE_SCAN1 1
105#define SRCU_STATE_SCAN2 2
106
107#define __SRCU_STRUCT_INIT(name) \
108 { \
109 .sda = &name##_srcu_data, \
110 .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \
111 .srcu_gp_seq_needed = 0 - 1, \
112 __SRCU_DEP_MAP_INIT(name) \
113 }
114
115/*
116 * Define and initialize a srcu struct at build time.
117 * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
118 *
119 * Note that although DEFINE_STATIC_SRCU() hides the name from other
120 * files, the per-CPU variable rules nevertheless require that the
121 * chosen name be globally unique. These rules also prohibit use of
122 * DEFINE_STATIC_SRCU() within a function. If these rules are too
123 * restrictive, declare the srcu_struct manually. For example, in
124 * each file:
125 *
126 * static struct srcu_struct my_srcu;
127 *
128 * Then, before the first use of each my_srcu, manually initialize it:
129 *
130 * init_srcu_struct(&my_srcu);
131 *
132 * See include/linux/percpu-defs.h for the rules on per-CPU variables.
133 */
134#define __DEFINE_SRCU(name, is_static) \
135 static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data);\
136 is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
137#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
138#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
139
140void synchronize_srcu_expedited(struct srcu_struct *sp);
141void srcu_barrier(struct srcu_struct *sp);
142void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf);
143
144#endif