]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - kernel/rcu/srcutiny.c
2 * Sleepable Read-Copy Update mechanism for mutual exclusion,
3 * tiny version for non-preemptible single-CPU use.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, you can access it online at
17 * http://www.gnu.org/licenses/gpl-2.0.html.
19 * Copyright (C) IBM Corporation, 2017
21 * Author: Paul McKenney <paulmck@us.ibm.com>
24 #include <linux/export.h>
25 #include <linux/mutex.h>
26 #include <linux/preempt.h>
27 #include <linux/rcupdate_wait.h>
28 #include <linux/sched.h>
29 #include <linux/delay.h>
30 #include <linux/srcu.h>
32 #include <linux/rcu_node_tree.h>
33 #include "rcu_segcblist.h"
36 static int init_srcu_struct_fields(struct srcu_struct
*sp
)
38 sp
->srcu_lock_nesting
[0] = 0;
39 sp
->srcu_lock_nesting
[1] = 0;
40 init_swait_queue_head(&sp
->srcu_wq
);
42 rcu_segcblist_init(&sp
->srcu_cblist
);
43 sp
->srcu_gp_running
= false;
44 sp
->srcu_gp_waiting
= false;
46 INIT_WORK(&sp
->srcu_work
, srcu_drive_gp
);
50 #ifdef CONFIG_DEBUG_LOCK_ALLOC
52 int __init_srcu_struct(struct srcu_struct
*sp
, const char *name
,
53 struct lock_class_key
*key
)
55 /* Don't re-initialize a lock while it is held. */
56 debug_check_no_locks_freed((void *)sp
, sizeof(*sp
));
57 lockdep_init_map(&sp
->dep_map
, name
, key
, 0);
58 return init_srcu_struct_fields(sp
);
60 EXPORT_SYMBOL_GPL(__init_srcu_struct
);
62 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
65 * init_srcu_struct - initialize a sleep-RCU structure
66 * @sp: structure to initialize.
68 * Must invoke this on a given srcu_struct before passing that srcu_struct
69 * to any other function. Each srcu_struct represents a separate domain
72 int init_srcu_struct(struct srcu_struct
*sp
)
74 return init_srcu_struct_fields(sp
);
76 EXPORT_SYMBOL_GPL(init_srcu_struct
);
78 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
81 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
82 * @sp: structure to clean up.
84 * Must invoke this after you are finished using a given srcu_struct that
85 * was initialized via init_srcu_struct(), else you leak memory.
87 void cleanup_srcu_struct(struct srcu_struct
*sp
)
89 WARN_ON(sp
->srcu_lock_nesting
[0] || sp
->srcu_lock_nesting
[1]);
90 flush_work(&sp
->srcu_work
);
91 WARN_ON(rcu_seq_state(sp
->srcu_gp_seq
));
92 WARN_ON(sp
->srcu_gp_running
);
93 WARN_ON(sp
->srcu_gp_waiting
);
94 WARN_ON(!rcu_segcblist_empty(&sp
->srcu_cblist
));
96 EXPORT_SYMBOL_GPL(cleanup_srcu_struct
);
99 * Counts the new reader in the appropriate per-CPU element of the
100 * srcu_struct. Must be called from process context.
101 * Returns an index that must be passed to the matching srcu_read_unlock().
103 int __srcu_read_lock(struct srcu_struct
*sp
)
107 idx
= READ_ONCE(sp
->srcu_idx
);
108 WRITE_ONCE(sp
->srcu_lock_nesting
[idx
], sp
->srcu_lock_nesting
[idx
] + 1);
111 EXPORT_SYMBOL_GPL(__srcu_read_lock
);
114 * Removes the count for the old reader from the appropriate element of
115 * the srcu_struct. Must be called from process context.
117 void __srcu_read_unlock(struct srcu_struct
*sp
, int idx
)
119 int newval
= sp
->srcu_lock_nesting
[idx
] - 1;
121 WRITE_ONCE(sp
->srcu_lock_nesting
[idx
], newval
);
122 if (!newval
&& READ_ONCE(sp
->srcu_gp_waiting
))
123 swake_up(&sp
->srcu_wq
);
125 EXPORT_SYMBOL_GPL(__srcu_read_unlock
);
128 * Workqueue handler to drive one grace period and invoke any callbacks
129 * that become ready as a result. Single-CPU and !PREEMPT operation
130 * means that we get away with murder on synchronization. ;-)
132 void srcu_drive_gp(struct work_struct
*wp
)
135 struct rcu_cblist ready_cbs
;
136 struct srcu_struct
*sp
;
137 struct rcu_head
*rhp
;
139 sp
= container_of(wp
, struct srcu_struct
, srcu_work
);
140 if (sp
->srcu_gp_running
|| rcu_segcblist_empty(&sp
->srcu_cblist
))
141 return; /* Already running or nothing to do. */
143 /* Tag recently arrived callbacks and wait for readers. */
144 WRITE_ONCE(sp
->srcu_gp_running
, true);
145 rcu_segcblist_accelerate(&sp
->srcu_cblist
,
146 rcu_seq_snap(&sp
->srcu_gp_seq
));
147 rcu_seq_start(&sp
->srcu_gp_seq
);
149 WRITE_ONCE(sp
->srcu_idx
, !sp
->srcu_idx
);
150 WRITE_ONCE(sp
->srcu_gp_waiting
, true); /* srcu_read_unlock() wakes! */
151 swait_event(sp
->srcu_wq
, !READ_ONCE(sp
->srcu_lock_nesting
[idx
]));
152 WRITE_ONCE(sp
->srcu_gp_waiting
, false); /* srcu_read_unlock() cheap. */
153 rcu_seq_end(&sp
->srcu_gp_seq
);
155 /* Update callback list based on GP, and invoke ready callbacks. */
156 rcu_segcblist_advance(&sp
->srcu_cblist
,
157 rcu_seq_current(&sp
->srcu_gp_seq
));
158 if (rcu_segcblist_ready_cbs(&sp
->srcu_cblist
)) {
159 rcu_cblist_init(&ready_cbs
);
161 rcu_segcblist_extract_done_cbs(&sp
->srcu_cblist
, &ready_cbs
);
163 rhp
= rcu_cblist_dequeue(&ready_cbs
);
164 for (; rhp
!= NULL
; rhp
= rcu_cblist_dequeue(&ready_cbs
)) {
170 rcu_segcblist_insert_count(&sp
->srcu_cblist
, &ready_cbs
);
173 WRITE_ONCE(sp
->srcu_gp_running
, false);
176 * If more callbacks, reschedule ourselves. This can race with
177 * a call_srcu() at interrupt level, but the ->srcu_gp_running
178 * checks will straighten that out.
180 if (!rcu_segcblist_empty(&sp
->srcu_cblist
))
181 schedule_work(&sp
->srcu_work
);
183 EXPORT_SYMBOL_GPL(srcu_drive_gp
);
186 * Enqueue an SRCU callback on the specified srcu_struct structure,
187 * initiating grace-period processing if it is not already running.
189 void call_srcu(struct srcu_struct
*sp
, struct rcu_head
*head
,
195 local_irq_save(flags
);
196 rcu_segcblist_enqueue(&sp
->srcu_cblist
, head
, false);
197 local_irq_restore(flags
);
198 if (!READ_ONCE(sp
->srcu_gp_running
))
199 schedule_work(&sp
->srcu_work
);
201 EXPORT_SYMBOL_GPL(call_srcu
);
204 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
206 void synchronize_srcu(struct srcu_struct
*sp
)
208 struct rcu_synchronize rs
;
210 init_rcu_head_on_stack(&rs
.head
);
211 init_completion(&rs
.completion
);
212 call_srcu(sp
, &rs
.head
, wakeme_after_rcu
);
213 wait_for_completion(&rs
.completion
);
214 destroy_rcu_head_on_stack(&rs
.head
);
216 EXPORT_SYMBOL_GPL(synchronize_srcu
);