1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Sleepable Read-Copy Update mechanism for mutual exclusion
4 *
5 * Copyright (C) IBM Corporation, 2006
6 * Copyright (C) Fujitsu, 2012
7 *
8 * Author: Paul McKenney <paulmck@linux.ibm.com>
9 * Lai Jiangshan <laijs@cn.fujitsu.com>
10 *
11 * For detailed explanation of Read-Copy Update mechanism see -
12 * Documentation/RCU/ *.txt
13 *
14 */
15
16#ifndef _LINUX_SRCU_H
17#define _LINUX_SRCU_H
18
19#include <linux/mutex.h>
20#include <linux/rcupdate.h>
21#include <linux/workqueue.h>
22#include <linux/rcu_segcblist.h>
23
24struct srcu_struct;
25
26#ifdef CONFIG_DEBUG_LOCK_ALLOC
27
28int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
29 struct lock_class_key *key);
30
31#define init_srcu_struct(ssp) \
32({ \
33 static struct lock_class_key __srcu_key; \
34 \
35 __init_srcu_struct((ssp), #ssp, &__srcu_key); \
36})
37
38#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
39#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
40
41int init_srcu_struct(struct srcu_struct *ssp);
42
43#define __SRCU_DEP_MAP_INIT(srcu_name)
44#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
45
46/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
47#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock().
48#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe().
49#define SRCU_READ_FLAVOR_LITE 0x4 // srcu_read_lock_lite().
50#define SRCU_READ_FLAVOR_FAST 0x8 // srcu_read_lock_fast().
51#define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
52 SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above.
53#define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)
54 // Flavors requiring synchronize_rcu()
55 // instead of smp_mb().
56void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
57
58#ifdef CONFIG_TINY_SRCU
59#include <linux/srcutiny.h>
60#elif defined(CONFIG_TREE_SRCU)
61#include <linux/srcutree.h>
62#else
63#error "Unknown SRCU implementation specified to kernel configuration"
64#endif
65
66void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
67 void (*func)(struct rcu_head *head));
68void cleanup_srcu_struct(struct srcu_struct *ssp);
69void synchronize_srcu(struct srcu_struct *ssp);
70
71#define SRCU_GET_STATE_COMPLETED 0x1
72
73/**
74 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
75 *
76 * Returns a value that poll_state_synchronize_srcu() will always treat
77 * as a cookie whose grace period has already completed.
78 */
79static inline unsigned long get_completed_synchronize_srcu(void)
80{
81 return SRCU_GET_STATE_COMPLETED;
82}
83
84unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
85unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
86bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
87
88// Maximum number of unsigned long values corresponding to
89// not-yet-completed SRCU grace periods.
90#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2
91
92/**
93 * same_state_synchronize_srcu - Are two old-state values identical?
94 * @oldstate1: First old-state value.
95 * @oldstate2: Second old-state value.
96 *
97 * The two old-state values must have been obtained from either
98 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
99 * get_completed_synchronize_srcu(). Returns @true if the two values are
100 * identical and @false otherwise. This allows structures whose lifetimes
101 * are tracked by old-state values to push these values to a list header,
102 * allowing those structures to be slightly smaller.
103 */
104static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
105{
106 return oldstate1 == oldstate2;
107}
108
109#ifdef CONFIG_NEED_SRCU_NMI_SAFE
110int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
111void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
112#else
113static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
114{
115 return __srcu_read_lock(ssp);
116}
117static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
118{
119 __srcu_read_unlock(ssp, idx);
120}
121#endif /* CONFIG_NEED_SRCU_NMI_SAFE */
122
123void srcu_init(void);
124
125#ifdef CONFIG_DEBUG_LOCK_ALLOC
126
127/**
128 * srcu_read_lock_held - might we be in SRCU read-side critical section?
129 * @ssp: The srcu_struct structure to check
130 *
131 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
132 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
133 * this assumes we are in an SRCU read-side critical section unless it can
134 * prove otherwise.
135 *
136 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
137 * and while lockdep is disabled.
138 *
139 * Note that SRCU is based on its own statemachine and it doesn't
140 * relies on normal RCU, it can be called from the CPU which
141 * is in the idle loop from an RCU point of view or offline.
142 */
143static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
144{
145 if (!debug_lockdep_rcu_enabled())
146 return 1;
147 return lock_is_held(lock: &ssp->dep_map);
148}
149
150/*
151 * Annotations provide deadlock detection for SRCU.
152 *
153 * Similar to other lockdep annotations, except there is an additional
154 * srcu_lock_sync(), which is basically an empty *write*-side critical section,
155 * see lock_sync() for more information.
156 */
157
158/* Annotates a srcu_read_lock() */
159static inline void srcu_lock_acquire(struct lockdep_map *map)
160{
161 lock_map_acquire_read(map);
162}
163
164/* Annotates a srcu_read_lock() */
165static inline void srcu_lock_release(struct lockdep_map *map)
166{
167 lock_map_release(map);
168}
169
170/* Annotates a synchronize_srcu() */
171static inline void srcu_lock_sync(struct lockdep_map *map)
172{
173 lock_map_sync(map);
174}
175
176#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
177
178static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
179{
180 return 1;
181}
182
183#define srcu_lock_acquire(m) do { } while (0)
184#define srcu_lock_release(m) do { } while (0)
185#define srcu_lock_sync(m) do { } while (0)
186
187#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
188
189
190/**
191 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
192 * @p: the pointer to fetch and protect for later dereferencing
193 * @ssp: pointer to the srcu_struct, which is used to check that we
194 * really are in an SRCU read-side critical section.
195 * @c: condition to check for update-side use
196 *
197 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
198 * critical section will result in an RCU-lockdep splat, unless @c evaluates
199 * to 1. The @c argument will normally be a logical expression containing
200 * lockdep_is_held() calls.
201 */
202#define srcu_dereference_check(p, ssp, c) \
203 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
204 (c) || srcu_read_lock_held(ssp), __rcu)
205
206/**
207 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
208 * @p: the pointer to fetch and protect for later dereferencing
209 * @ssp: pointer to the srcu_struct, which is used to check that we
210 * really are in an SRCU read-side critical section.
211 *
212 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
213 * is enabled, invoking this outside of an RCU read-side critical
214 * section will result in an RCU-lockdep splat.
215 */
216#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
217
218/**
219 * srcu_dereference_notrace - no tracing and no lockdep calls from here
220 * @p: the pointer to fetch and protect for later dereferencing
221 * @ssp: pointer to the srcu_struct, which is used to check that we
222 * really are in an SRCU read-side critical section.
223 */
224#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
225
226/**
227 * srcu_read_lock - register a new reader for an SRCU-protected structure.
228 * @ssp: srcu_struct in which to register the new reader.
229 *
230 * Enter an SRCU read-side critical section. Note that SRCU read-side
231 * critical sections may be nested. However, it is illegal to
232 * call anything that waits on an SRCU grace period for the same
233 * srcu_struct, whether directly or indirectly. Please note that
234 * one way to indirectly wait on an SRCU grace period is to acquire
235 * a mutex that is held elsewhere while calling synchronize_srcu() or
236 * synchronize_srcu_expedited().
237 *
238 * The return value from srcu_read_lock() is guaranteed to be
239 * non-negative. This value must be passed unaltered to the matching
240 * srcu_read_unlock(). Note that srcu_read_lock() and the matching
241 * srcu_read_unlock() must occur in the same context, for example, it is
242 * illegal to invoke srcu_read_unlock() in an irq handler if the matching
243 * srcu_read_lock() was invoked in process context. Or, for that matter to
244 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
245 * from another.
246 */
247static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
248{
249 int retval;
250
251 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
252 retval = __srcu_read_lock(ssp);
253 srcu_lock_acquire(map: &ssp->dep_map);
254 return retval;
255}
256
257/**
258 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
259 * @ssp: srcu_struct in which to register the new reader.
260 *
261 * Enter an SRCU read-side critical section, but for a light-weight
262 * smp_mb()-free reader. See srcu_read_lock() for more information.
263 *
264 * If srcu_read_lock_fast() is ever used on an srcu_struct structure,
265 * then none of the other flavors may be used, whether before, during,
266 * or after. Note that grace-period auto-expediting is disabled for _fast
267 * srcu_struct structures because auto-expedited grace periods invoke
268 * synchronize_rcu_expedited(), IPIs and all.
269 *
270 * Note that srcu_read_lock_fast() can be invoked only from those contexts
271 * where RCU is watching, that is, from contexts where it would be legal
272 * to invoke rcu_read_lock(). Otherwise, lockdep will complain.
273 */
274static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)
275{
276 struct srcu_ctr __percpu *retval;
277
278 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
279 retval = __srcu_read_lock_fast(ssp);
280 rcu_try_lock_acquire(map: &ssp->dep_map);
281 return retval;
282}
283
284/**
285 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
286 * @ssp: srcu_struct in which to register the new reader.
287 *
288 * Enter a semaphore-like SRCU read-side critical section, but for
289 * a light-weight smp_mb()-free reader. See srcu_read_lock_fast() and
290 * srcu_down_read() for more information.
291 *
292 * The same srcu_struct may be used concurrently by srcu_down_read_fast()
293 * and srcu_read_lock_fast().
294 */
295static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)
296{
297 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
298 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
299 return __srcu_read_lock_fast(ssp);
300}
301
302/**
303 * srcu_read_lock_lite - register a new reader for an SRCU-protected structure.
304 * @ssp: srcu_struct in which to register the new reader.
305 *
306 * Enter an SRCU read-side critical section, but for a light-weight
307 * smp_mb()-free reader. See srcu_read_lock() for more information.
308 *
309 * If srcu_read_lock_lite() is ever used on an srcu_struct structure,
310 * then none of the other flavors may be used, whether before, during,
311 * or after. Note that grace-period auto-expediting is disabled for _lite
312 * srcu_struct structures because auto-expedited grace periods invoke
313 * synchronize_rcu_expedited(), IPIs and all.
314 *
315 * Note that srcu_read_lock_lite() can be invoked only from those contexts
316 * where RCU is watching, that is, from contexts where it would be legal
317 * to invoke rcu_read_lock(). Otherwise, lockdep will complain.
318 */
319static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp)
320{
321 int retval;
322
323 srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_LITE);
324 retval = __srcu_read_lock_lite(ssp);
325 rcu_try_lock_acquire(map: &ssp->dep_map);
326 return retval;
327}
328
329/**
330 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
331 * @ssp: srcu_struct in which to register the new reader.
332 *
333 * Enter an SRCU read-side critical section, but in an NMI-safe manner.
334 * See srcu_read_lock() for more information.
335 *
336 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
337 * then none of the other flavors may be used, whether before, during,
338 * or after.
339 */
340static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
341{
342 int retval;
343
344 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
345 retval = __srcu_read_lock_nmisafe(ssp);
346 rcu_try_lock_acquire(map: &ssp->dep_map);
347 return retval;
348}
349
350/* Used by tracing, cannot be traced and cannot invoke lockdep. */
351static inline notrace int
352srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
353{
354 int retval;
355
356 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
357 retval = __srcu_read_lock(ssp);
358 return retval;
359}
360
361/**
362 * srcu_down_read - register a new reader for an SRCU-protected structure.
363 * @ssp: srcu_struct in which to register the new reader.
364 *
365 * Enter a semaphore-like SRCU read-side critical section. Note that
366 * SRCU read-side critical sections may be nested. However, it is
367 * illegal to call anything that waits on an SRCU grace period for the
368 * same srcu_struct, whether directly or indirectly. Please note that
369 * one way to indirectly wait on an SRCU grace period is to acquire
370 * a mutex that is held elsewhere while calling synchronize_srcu() or
371 * synchronize_srcu_expedited(). But if you want lockdep to help you
372 * keep this stuff straight, you should instead use srcu_read_lock().
373 *
374 * The semaphore-like nature of srcu_down_read() means that the matching
375 * srcu_up_read() can be invoked from some other context, for example,
376 * from some other task or from an irq handler. However, neither
377 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
378 *
379 * Calls to srcu_down_read() may be nested, similar to the manner in
380 * which calls to down_read() may be nested. The same srcu_struct may be
381 * used concurrently by srcu_down_read() and srcu_read_lock().
382 */
383static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
384{
385 WARN_ON_ONCE(in_nmi());
386 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
387 return __srcu_read_lock(ssp);
388}
389
390/**
391 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
392 * @ssp: srcu_struct in which to unregister the old reader.
393 * @idx: return value from corresponding srcu_read_lock().
394 *
395 * Exit an SRCU read-side critical section.
396 */
397static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
398 __releases(ssp)
399{
400 WARN_ON_ONCE(idx & ~0x1);
401 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
402 srcu_lock_release(map: &ssp->dep_map);
403 __srcu_read_unlock(ssp, idx);
404}
405
406/**
407 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
408 * @ssp: srcu_struct in which to unregister the old reader.
409 * @scp: return value from corresponding srcu_read_lock_fast().
410 *
411 * Exit a light-weight SRCU read-side critical section.
412 */
413static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
414 __releases(ssp)
415{
416 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
417 srcu_lock_release(map: &ssp->dep_map);
418 __srcu_read_unlock_fast(ssp, scp);
419}
420
421/**
422 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
423 * @ssp: srcu_struct in which to unregister the old reader.
424 * @scp: return value from corresponding srcu_read_lock_fast().
425 *
426 * Exit an SRCU read-side critical section, but not necessarily from
427 * the same context as the maching srcu_down_read_fast().
428 */
429static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
430 __releases(ssp)
431{
432 WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
433 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
434 __srcu_read_unlock_fast(ssp, scp);
435}
436
437/**
438 * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure.
439 * @ssp: srcu_struct in which to unregister the old reader.
440 * @idx: return value from corresponding srcu_read_lock_lite().
441 *
442 * Exit a light-weight SRCU read-side critical section.
443 */
444static inline void srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
445 __releases(ssp)
446{
447 WARN_ON_ONCE(idx & ~0x1);
448 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);
449 srcu_lock_release(map: &ssp->dep_map);
450 __srcu_read_unlock_lite(ssp, idx);
451}
452
453/**
454 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
455 * @ssp: srcu_struct in which to unregister the old reader.
456 * @idx: return value from corresponding srcu_read_lock_nmisafe().
457 *
458 * Exit an SRCU read-side critical section, but in an NMI-safe manner.
459 */
460static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
461 __releases(ssp)
462{
463 WARN_ON_ONCE(idx & ~0x1);
464 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
465 rcu_lock_release(map: &ssp->dep_map);
466 __srcu_read_unlock_nmisafe(ssp, idx);
467}
468
469/* Used by tracing, cannot be traced and cannot call lockdep. */
470static inline notrace void
471srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
472{
473 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
474 __srcu_read_unlock(ssp, idx);
475}
476
477/**
478 * srcu_up_read - unregister a old reader from an SRCU-protected structure.
479 * @ssp: srcu_struct in which to unregister the old reader.
480 * @idx: return value from corresponding srcu_read_lock().
481 *
482 * Exit an SRCU read-side critical section, but not necessarily from
483 * the same context as the maching srcu_down_read().
484 */
485static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
486 __releases(ssp)
487{
488 WARN_ON_ONCE(idx & ~0x1);
489 WARN_ON_ONCE(in_nmi());
490 srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
491 __srcu_read_unlock(ssp, idx);
492}
493
494/**
495 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
496 *
497 * Converts the preceding srcu_read_unlock into a two-way memory barrier.
498 *
499 * Call this after srcu_read_unlock, to guarantee that all memory operations
500 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
501 * the preceding srcu_read_unlock.
502 */
503static inline void smp_mb__after_srcu_read_unlock(void)
504{
505 /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
506}
507
508/**
509 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
510 *
511 * Converts the preceding srcu_read_lock into a two-way memory barrier.
512 *
513 * Call this after srcu_read_lock, to guarantee that all memory operations
514 * that occur after smp_mb__after_srcu_read_lock will appear to happen after
515 * the preceding srcu_read_lock.
516 */
517static inline void smp_mb__after_srcu_read_lock(void)
518{
519 /* __srcu_read_lock has smp_mb() internally so nothing to do here. */
520}
521
522DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
523 _T->idx = srcu_read_lock(_T->lock),
524 srcu_read_unlock(_T->lock, _T->idx),
525 int idx)
526
527#endif
528

source code of linux/include/linux/srcu.h

Morty Proxy This is a proxified and sanitized view of the page, visit original site.