1 #ifndef __LINUX_SPINLOCK_H
2 #define __LINUX_SPINLOCK_H
3
4 /*
5 * include/linux/spinlock.h - generic spinlock/rwlock declarations
6 *
7 * here's the role of the various spinlock/rwlock related include files:
8 *
9 * on SMP builds:
10 *
11 * asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the
12 * initializers
13 *
14 * linux/spinlock_types.h:
15 * defines the generic type and initializers
16 *
17 * asm/spinlock.h: contains the __raw_spin_*()/etc. lowlevel
18 * implementations, mostly inline assembly code
19 *
20 * (also included on UP-debug builds:)
21 *
22 * linux/spinlock_api_smp.h:
23 * contains the prototypes for the _spin_*() APIs.
24 *
25 * linux/spinlock.h: builds the final spin_*() APIs.
26 *
27 * on UP builds:
28 *
29 * linux/spinlock_type_up.h:
30 * contains the generic, simplified UP spinlock type.
31 * (which is an empty structure on non-debug builds)
32 *
33 * linux/spinlock_types.h:
34 * defines the generic type and initializers
35 *
36 * linux/spinlock_up.h:
37 * contains the __raw_spin_*()/etc. version of UP
38 * builds. (which are NOPs on non-debug, non-preempt
39 * builds)
40 *
41 * (included on UP-non-debug builds:)
42 *
43 * linux/spinlock_api_up.h:
44 * builds the _spin_*() APIs.
45 *
46 * linux/spinlock.h: builds the final spin_*() APIs.
47 *
48 *
49 * Public types and naming conventions:
50 * ------------------------------------
51 * spinlock_t: type: sleep-lock
52 * raw_spinlock_t: type: spin-lock (debug)
53 *
54 * spin_lock([raw_]spinlock_t): API: acquire lock, both types
55 *
56 *
57 * Internal types and naming conventions:
58 * -------------------------------------
59 * __raw_spinlock_t: type: lowlevel spin-lock
60 *
61 * _spin_lock(struct rt_mutex): API: acquire sleep-lock
62 * __spin_lock(raw_spinlock_t): API: acquire spin-lock (highlevel)
63 * _raw_spin_lock(raw_spinlock_t): API: acquire spin-lock (debug)
64 * __raw_spin_lock(__raw_spinlock_t): API: acquire spin-lock (lowlevel)
65 *
66 *
67 * spin_lock(raw_spinlock_t) translates into the following chain of
68 * calls/inlines/macros, if spin-lock debugging is enabled:
69 *
70 * spin_lock() [include/linux/spinlock.h]
71 * -> __spin_lock() [kernel/spinlock.c]
72 * -> _raw_spin_lock() [lib/spinlock_debug.c]
73 * -> __raw_spin_lock() [include/asm/spinlock.h]
74 *
75 * spin_lock(spinlock_t) translates into the following chain of
76 * calls/inlines/macros:
77 *
78 * spin_lock() [include/linux/spinlock.h]
79 * -> _spin_lock() [include/linux/spinlock.h]
80 * -> rt_spin_lock() [kernel/rtmutex.c]
81 * -> rt_spin_lock_fastlock() [kernel/rtmutex.c]
82 * -> rt_spin_lock_slowlock() [kernel/rtmutex.c]
83 */
84
85 #include <linux/preempt.h>
86 #include <linux/linkage.h>
87 #include <linux/compiler.h>
88 #include <linux/thread_info.h>
89 #include <linux/kernel.h>
90 #include <linux/cache.h>
91 #include <linux/stringify.h>
92 #include <linux/bottom_half.h>
93 #include <linux/irqflags.h>
94 #include <linux/pickop.h>
95
96 #include <asm/system.h>
97
98 /*
99 * Pull the raw_spinlock_t and raw_rwlock_t definitions:
100 */
101 #include <linux/spinlock_types.h>
102
103 extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock);
104
105 /*
106 * Pull the __raw*() functions/declarations (UP-nondebug doesnt need them):
107 */
108 #ifdef CONFIG_SMP
109 # include <asm/spinlock.h>
110 #else
111 # include <linux/spinlock_up.h>
112 #endif
113
114 /*
115 * Pull the RT types:
116 */
117 #include <linux/rt_lock.h>
118
119 #ifdef CONFIG_GENERIC_LOCKBREAK
120 #define spin_is_contended(lock) ((lock)->break_lock)
121 #else
122 #define spin_is_contended(lock) __raw_spin_is_contended(&(lock)->raw_lock)
123 #endif
124
125 /*
126 * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
127 */
128 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
129 # include <linux/spinlock_api_smp.h>
130 #else
131 # include <linux/spinlock_api_up.h>
132 #endif
133
134 #ifdef CONFIG_DEBUG_SPINLOCK
135 extern __lockfunc void _raw_spin_lock(raw_spinlock_t *lock);
136 # define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
137 extern __lockfunc int _raw_spin_trylock(raw_spinlock_t *lock);
138 extern __lockfunc void _raw_spin_unlock(raw_spinlock_t *lock);
139 extern __lockfunc void _raw_read_lock(raw_rwlock_t *lock);
140 extern __lockfunc int _raw_read_trylock(raw_rwlock_t *lock);
141 extern __lockfunc void _raw_read_unlock(raw_rwlock_t *lock);
142 extern __lockfunc void _raw_write_lock(raw_rwlock_t *lock);
143 extern __lockfunc int _raw_write_trylock(raw_rwlock_t *lock);
144 extern __lockfunc void _raw_write_unlock(raw_rwlock_t *lock);
145 #else
146 # define _raw_spin_lock(lock) __raw_spin_lock(&(lock)->raw_lock)
147 # define _raw_spin_lock_flags(lock, flags) \
148 __raw_spin_lock_flags(&(lock)->raw_lock, *(flags))
149 # define _raw_spin_trylock(lock) __raw_spin_trylock(&(lock)->raw_lock)
150 # define _raw_spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock)
151 # define _raw_read_lock(rwlock) __raw_read_lock(&(rwlock)->raw_lock)
152 # define _raw_read_trylock(rwlock) __raw_read_trylock(&(rwlock)->raw_lock)
153 # define _raw_read_unlock(rwlock) __raw_read_unlock(&(rwlock)->raw_lock)
154 # define _raw_write_lock(rwlock) __raw_write_lock(&(rwlock)->raw_lock)
155 # define _raw_write_trylock(rwlock) __raw_write_trylock(&(rwlock)->raw_lock)
156 # define _raw_write_unlock(rwlock) __raw_write_unlock(&(rwlock)->raw_lock)
157 #endif
158
159 extern int __bad_spinlock_type(void);
160 extern int __bad_rwlock_type(void);
161
162 extern void
163 __rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key);
164
165 extern void __lockfunc rt_spin_lock(spinlock_t *lock);
166 extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass);
167 extern void __lockfunc rt_spin_unlock(spinlock_t *lock);
168 extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock);
169 extern int __lockfunc
170 rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags);
171 extern int __lockfunc rt_spin_trylock(spinlock_t *lock);
172 extern int _atomic_dec_and_spin_lock(spinlock_t *lock, atomic_t *atomic);
173
174 /*
175 * lockdep-less calls, for derived types like rwlock:
176 * (for trylock they can use rt_mutex_trylock() directly.
177 */
178 extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock);
179 extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock);
180
181 #ifdef CONFIG_PREEMPT_RT
182 # define _spin_lock(l) rt_spin_lock(l)
183 # define _spin_lock_nested(l, s) rt_spin_lock_nested(l, s)
184 # define _spin_lock_bh(l) rt_spin_lock(l)
185 # define _spin_lock_irq(l) rt_spin_lock(l)
186 # define _spin_unlock(l) rt_spin_unlock(l)
187 # define _spin_unlock_no_resched(l) rt_spin_unlock(l)
188 # define _spin_unlock_bh(l) rt_spin_unlock(l)
189 # define _spin_unlock_irq(l) rt_spin_unlock(l)
190 # define _spin_unlock_irqrestore(l, f) rt_spin_unlock(l)
191 static inline unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
192 {
193 rt_spin_lock(lock);
194 return 0;
195 }
196 static inline unsigned long __lockfunc
197 _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
198 {
199 rt_spin_lock_nested(lock, subclass);
200 return 0;
201 }
202 #else
203 static inline unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
204 {
205 return 0;
206 }
207 static inline unsigned long __lockfunc
208 _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
209 {
210 return 0;
211 }
212 # define _spin_lock(l) do { } while (0)
213 # define _spin_lock_nested(l, s) do { } while (0)
214 # define _spin_lock_bh(l) do { } while (0)
215 # define _spin_lock_irq(l) do { } while (0)
216 # define _spin_unlock(l) do { } while (0)
217 # define _spin_unlock_no_resched(l) do { } while (0)
218 # define _spin_unlock_bh(l) do { } while (0)
219 # define _spin_unlock_irq(l) do { } while (0)
220 # define _spin_unlock_irqrestore(l, f) do { } while (0)
221 #endif
222
223 #define _spin_lock_init(sl, n, f, l) \
224 do { \
225 static struct lock_class_key __key; \
226 \
227 __rt_spin_lock_init(sl, n, &__key); \
228 } while (0)
229
230 # ifdef CONFIG_PREEMPT_RT
231 # define _spin_can_lock(l) (!rt_mutex_is_locked(&(l)->lock))
232 # define _spin_is_locked(l) rt_mutex_is_locked(&(l)->lock)
233 # define _spin_unlock_wait(l) rt_spin_unlock_wait(l)
234
235 # define _spin_trylock(l) rt_spin_trylock(l)
236 # define _spin_trylock_bh(l) rt_spin_trylock(l)
237 # define _spin_trylock_irq(l) rt_spin_trylock(l)
238 # define _spin_trylock_irqsave(l,f) rt_spin_trylock_irqsave(l, f)
239 # else
240
241 extern int this_should_never_be_called_on_non_rt(spinlock_t *lock);
242 # define TSNBCONRT(l) this_should_never_be_called_on_non_rt(l)
243 # define _spin_can_lock(l) TSNBCONRT(l)
244 # define _spin_is_locked(l) TSNBCONRT(l)
245 # define _spin_unlock_wait(l) TSNBCONRT(l)
246
247 # define _spin_trylock(l) TSNBCONRT(l)
248 # define _spin_trylock_bh(l) TSNBCONRT(l)
249 # define _spin_trylock_irq(l) TSNBCONRT(l)
250 # define _spin_trylock_irqsave(l,f) TSNBCONRT(l)
251 #endif
252
253 extern void __lockfunc rt_write_lock(rwlock_t *rwlock);
254 extern void __lockfunc rt_read_lock(rwlock_t *rwlock);
255 extern int __lockfunc rt_write_trylock(rwlock_t *rwlock);
256 extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock,
257 unsigned long *flags);
258 extern int __lockfunc rt_read_trylock(rwlock_t *rwlock);
259 extern void __lockfunc rt_write_unlock(rwlock_t *rwlock);
260 extern void __lockfunc rt_read_unlock(rwlock_t *rwlock);
261 extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock);
262 extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock);
263 extern void
264 __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key);
265
266 #define _rwlock_init(rwl, n, f, l) \
267 do { \
268 static struct lock_class_key __key; \
269 \
270 __rt_rwlock_init(rwl, n, &__key); \
271 } while (0)
272
273 #ifdef CONFIG_PREEMPT_RT
274 # define rt_read_can_lock(rwl) (!rt_mutex_is_locked(&(rwl)->lock))
275 # define rt_write_can_lock(rwl) ((rwl)->owners.owner == NULL)
276 #else
277 extern int rt_rwlock_can_lock_never_call_on_non_rt(rwlock_t *rwlock);
278 # define rt_read_can_lock(rwl) rt_rwlock_can_lock_never_call_on_non_rt(rwl)
279 # define rt_write_can_lock(rwl) rt_rwlock_can_lock_never_call_on_non_rt(rwl)
280 #endif
281
282 # define _read_can_lock(rwl) rt_read_can_lock(rwl)
283 # define _write_can_lock(rwl) rt_write_can_lock(rwl)
284
285 # define _read_trylock(rwl) rt_read_trylock(rwl)
286 # define _write_trylock(rwl) rt_write_trylock(rwl)
287 # define _write_trylock_irqsave(rwl, flags) \
288 rt_write_trylock_irqsave(rwl, flags)
289
290 # define _read_lock(rwl) rt_read_lock(rwl)
291 # define _write_lock(rwl) rt_write_lock(rwl)
292 # define _read_unlock(rwl) rt_read_unlock(rwl)
293 # define _write_unlock(rwl) rt_write_unlock(rwl)
294
295 # define _read_lock_bh(rwl) rt_read_lock(rwl)
296 # define _write_lock_bh(rwl) rt_write_lock(rwl)
297 # define _read_unlock_bh(rwl) rt_read_unlock(rwl)
298 # define _write_unlock_bh(rwl) rt_write_unlock(rwl)
299
300 # define _read_lock_irq(rwl) rt_read_lock(rwl)
301 # define _write_lock_irq(rwl) rt_write_lock(rwl)
302 # define _read_unlock_irq(rwl) rt_read_unlock(rwl)
303 # define _write_unlock_irq(rwl) rt_write_unlock(rwl)
304
305 # define _read_lock_irqsave(rwl) rt_read_lock_irqsave(rwl)
306 # define _write_lock_irqsave(rwl) rt_write_lock_irqsave(rwl)
307
308 # define _read_unlock_irqrestore(rwl, f) rt_read_unlock(rwl)
309 # define _write_unlock_irqrestore(rwl, f) rt_write_unlock(rwl)
310
311 #ifdef CONFIG_DEBUG_SPINLOCK
312 extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
313 struct lock_class_key *key);
314 # define _raw_spin_lock_init(lock, name, file, line) \
315 do { \
316 static struct lock_class_key __key; \
317 \
318 __raw_spin_lock_init((lock), #lock, &__key); \
319 } while (0)
320
321 #else
322 #define __raw_spin_lock_init(lock) \
323 do { *(lock) = RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
324 # define _raw_spin_lock_init(lock, name, file, line) __raw_spin_lock_init(lock)
325 #endif
326
327 /*
328 * PICK_SPIN_OP()/PICK_RW_OP() are simple redirectors for PICK_FUNCTION
329 */
330 #define PICK_SPIN_OP(...) \
331 PICK_FUNCTION(raw_spinlock_t *, spinlock_t *, ##__VA_ARGS__)
332 #define PICK_SPIN_OP_RET(...) \
333 PICK_FUNCTION_RET(raw_spinlock_t *, spinlock_t *, ##__VA_ARGS__)
334 #define PICK_RW_OP(...) PICK_FUNCTION(raw_rwlock_t *, rwlock_t *, ##__VA_ARGS__)
335 #define PICK_RW_OP_RET(...) \
336 PICK_FUNCTION_RET(raw_rwlock_t *, rwlock_t *, ##__VA_ARGS__)
337
338 #define spin_lock_init(lock) \
339 PICK_SPIN_OP(_raw_spin_lock_init, _spin_lock_init, lock, #lock, \
340 __FILE__, __LINE__)
341
342 #ifdef CONFIG_DEBUG_SPINLOCK
343 extern void __raw_rwlock_init(raw_rwlock_t *lock, const char *name,
344 struct lock_class_key *key);
345 # define _raw_rwlock_init(lock, name, file, line) \
346 do { \
347 static struct lock_class_key __key; \
348 \
349 __raw_rwlock_init((lock), #lock, &__key); \
350 } while (0)
351 #else
352 #define __raw_rwlock_init(lock) \
353 do { *(lock) = RAW_RW_LOCK_UNLOCKED(lock); } while (0)
354 # define _raw_rwlock_init(lock, name, file, line) __raw_rwlock_init(lock)
355 #endif
356
357 #define rwlock_init(lock) \
358 PICK_RW_OP(_raw_rwlock_init, _rwlock_init, lock, #lock, \
359 __FILE__, __LINE__)
360
361 #define __spin_is_locked(lock) __raw_spin_is_locked(&(lock)->raw_lock)
362
363 #define spin_is_locked(lock) \
364 PICK_SPIN_OP_RET(__spin_is_locked, _spin_is_locked, lock)
365
366 #define __spin_unlock_wait(lock) __raw_spin_unlock_wait(&(lock)->raw_lock)
367
368 #define spin_unlock_wait(lock) \
369 PICK_SPIN_OP(__spin_unlock_wait, _spin_unlock_wait, lock)
370
371 /*
372 * Define the various spin_lock and rw_lock methods. Note we define these
373 * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
374 * methods are defined as nops in the case they are not required.
375 */
376 #define spin_trylock(lock) \
377 __cond_lock(lock, PICK_SPIN_OP_RET(__spin_trylock, _spin_trylock, lock))
378
379 #define read_trylock(lock) \
380 __cond_lock(lock, PICK_RW_OP_RET(__read_trylock, _read_trylock, lock))
381
382 #define write_trylock(lock) \
383 __cond_lock(lock, PICK_RW_OP_RET(__write_trylock, _write_trylock, lock))
384
385 #define write_trylock_irqsave(lock, flags) \
386 __cond_lock(lock, PICK_RW_OP_RET(__write_trylock_irqsave, \
387 _write_trylock_irqsave, lock, &flags))
388
389 #define __spin_can_lock(lock) __raw_spin_can_lock(&(lock)->raw_lock)
390 #define __read_can_lock(lock) __raw_read_can_lock(&(lock)->raw_lock)
391 #define __write_can_lock(lock) __raw_write_can_lock(&(lock)->raw_lock)
392
393 #define spin_can_lock(lock) \
394 __cond_lock(lock, PICK_SPIN_OP_RET(__spin_can_lock, _spin_can_lock,\
395 lock))
396
397 #define read_can_lock(lock) \
398 __cond_lock(lock, PICK_RW_OP_RET(__read_can_lock, _read_can_lock, lock))
399
400 #define write_can_lock(lock) \
401 __cond_lock(lock, PICK_RW_OP_RET(__write_can_lock, _write_can_lock,\
402 lock))
403
404 #define spin_lock(lock) PICK_SPIN_OP(__spin_lock, _spin_lock, lock)
405
406 #ifdef CONFIG_DEBUG_LOCK_ALLOC
407 # define spin_lock_nested(lock, subclass) \
408 PICK_SPIN_OP(__spin_lock_nested, _spin_lock_nested, lock, subclass)
409 #else
410 # define spin_lock_nested(lock, subclass) spin_lock(lock)
411 #endif
412
413 #define write_lock(lock) PICK_RW_OP(__write_lock, _write_lock, lock)
414
415 #define read_lock(lock) PICK_RW_OP(__read_lock, _read_lock, lock)
416
417 # define spin_lock_irqsave(lock, flags) \
418 do { \
419 BUILD_CHECK_IRQ_FLAGS(flags); \
420 flags = PICK_SPIN_OP_RET(__spin_lock_irqsave, _spin_lock_irqsave, \
421 lock); \
422 } while (0)
423
424 #ifdef CONFIG_DEBUG_LOCK_ALLOC
425 # define spin_lock_irqsave_nested(lock, flags, subclass) \
426 do { \
427 BUILD_CHECK_IRQ_FLAGS(flags); \
428 flags = PICK_SPIN_OP_RET(__spin_lock_irqsave_nested, \
429 _spin_lock_irqsave_nested, lock, subclass); \
430 } while (0)
431 #else
432 # define spin_lock_irqsave_nested(lock, flags, subclass) \
433 spin_lock_irqsave(lock, flags)
434 #endif
435
436 # define read_lock_irqsave(lock, flags) \
437 do { \
438 BUILD_CHECK_IRQ_FLAGS(flags); \
439 flags = PICK_RW_OP_RET(__read_lock_irqsave, _read_lock_irqsave, lock);\
440 } while (0)
441
442 # define write_lock_irqsave(lock, flags) \
443 do { \
444 BUILD_CHECK_IRQ_FLAGS(flags); \
445 flags = PICK_RW_OP_RET(__write_lock_irqsave, _write_lock_irqsave,lock);\
446 } while (0)
447
448 #define spin_lock_irq(lock) PICK_SPIN_OP(__spin_lock_irq, _spin_lock_irq, lock)
449
450 #define spin_lock_bh(lock) PICK_SPIN_OP(__spin_lock_bh, _spin_lock_bh, lock)
451
452 #define read_lock_irq(lock) PICK_RW_OP(__read_lock_irq, _read_lock_irq, lock)
453
454 #define read_lock_bh(lock) PICK_RW_OP(__read_lock_bh, _read_lock_bh, lock)
455
456 #define write_lock_irq(lock) PICK_RW_OP(__write_lock_irq, _write_lock_irq, lock)
457
458 #define write_lock_bh(lock) PICK_RW_OP(__write_lock_bh, _write_lock_bh, lock)
459
460 #define spin_unlock(lock) PICK_SPIN_OP(__spin_unlock, _spin_unlock, lock)
461
462 #define read_unlock(lock) PICK_RW_OP(__read_unlock, _read_unlock, lock)
463
464 #define write_unlock(lock) PICK_RW_OP(__write_unlock, _write_unlock, lock)
465
466 #define spin_unlock_no_resched(lock) \
467 PICK_SPIN_OP(__spin_unlock_no_resched, _spin_unlock_no_resched, lock)
468
469 #define spin_unlock_irqrestore(lock, flags) \
470 do { \
471 BUILD_CHECK_IRQ_FLAGS(flags); \
472 PICK_SPIN_OP(__spin_unlock_irqrestore, _spin_unlock_irqrestore, \
473 lock, flags); \
474 } while (0)
475
476 #define spin_unlock_irq(lock) \
477 PICK_SPIN_OP(__spin_unlock_irq, _spin_unlock_irq, lock)
478 #define spin_unlock_bh(lock) \
479 PICK_SPIN_OP(__spin_unlock_bh, _spin_unlock_bh, lock)
480
481 #define read_unlock_irqrestore(lock, flags) \
482 do { \
483 BUILD_CHECK_IRQ_FLAGS(flags); \
484 PICK_RW_OP(__read_unlock_irqrestore, _read_unlock_irqrestore, \
485 lock, flags); \
486 } while (0)
487
488 #define read_unlock_irq(lock) \
489 PICK_RW_OP(__read_unlock_irq, _read_unlock_irq, lock)
490 #define read_unlock_bh(lock) PICK_RW_OP(__read_unlock_bh, _read_unlock_bh, lock)
491
492 #define write_unlock_irqrestore(lock, flags) \
493 do { \
494 BUILD_CHECK_IRQ_FLAGS(flags); \
495 PICK_RW_OP(__write_unlock_irqrestore, _write_unlock_irqrestore, \
496 lock, flags); \
497 } while (0)
498 #define write_unlock_irq(lock) \
499 PICK_RW_OP(__write_unlock_irq, _write_unlock_irq, lock)
500
501 #define write_unlock_bh(lock) \
502 PICK_RW_OP(__write_unlock_bh, _write_unlock_bh, lock)
503
504 #define spin_trylock_bh(lock) \
505 __cond_lock(lock, PICK_SPIN_OP_RET(__spin_trylock_bh, _spin_trylock_bh,\
506 lock))
507
508 #define spin_trylock_irq(lock) \
509 __cond_lock(lock, PICK_SPIN_OP_RET(__spin_trylock_irq, \
510 _spin_trylock_irq, lock))
511
512 #define spin_trylock_irqsave(lock, flags) \
513 __cond_lock(lock, PICK_SPIN_OP_RET(__spin_trylock_irqsave, \
514 _spin_trylock_irqsave, lock, &flags))
515
516 /* "lock on reference count zero" */
517 #ifndef ATOMIC_DEC_AND_LOCK
518 # include <asm/atomic.h>
519 extern int __atomic_dec_and_spin_lock(raw_spinlock_t *lock, atomic_t *atomic);
520 #endif
521
522 #define atomic_dec_and_lock(atomic, lock) \
523 __cond_lock(lock, PICK_SPIN_OP_RET(__atomic_dec_and_spin_lock, \
524 _atomic_dec_and_spin_lock, lock, atomic))
525
526 /*
527 * bit-based spin_lock()
528 *
529 * Don't use this unless you really need to: spin_lock() and spin_unlock()
530 * are significantly faster.
531 */
532 static inline void bit_spin_lock(int bitnum, unsigned long *addr)
533 {
534 /*
535 * Assuming the lock is uncontended, this never enters
536 * the body of the outer loop. If it is contended, then
537 * within the inner loop a non-atomic test is used to
538 * busywait with less bus contention for a good time to
539 * attempt to acquire the lock bit.
540 */
541 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT)
542 while (test_and_set_bit(bitnum, addr))
543 while (test_bit(bitnum, addr))
544 cpu_relax();
545 #endif
546 __acquire(bitlock);
547 }
548
549 /*
550 * Return true if it was acquired
551 */
552 static inline int bit_spin_trylock(int bitnum, unsigned long *addr)
553 {
554 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT)
555 if (test_and_set_bit(bitnum, addr))
556 return 0;
557 #endif
558 __acquire(bitlock);
559 return 1;
560 }
561
562 /*
563 * bit-based spin_unlock()
564 */
565 static inline void bit_spin_unlock(int bitnum, unsigned long *addr)
566 {
567 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT)
568 BUG_ON(!test_bit(bitnum, addr));
569 smp_mb__before_clear_bit();
570 clear_bit(bitnum, addr);
571 #endif
572 __release(bitlock);
573 }
574
575 /*
576 * Return true if the lock is held.
577 */
578 static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
579 {
580 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT)
581 return test_bit(bitnum, addr);
582 #else
583 return 1;
584 #endif
585 }
586
587 /**
588 * __raw_spin_can_lock - would __raw_spin_trylock() succeed?
589 * @lock: the spinlock in question.
590 */
591 #define __raw_spin_can_lock(lock) (!__raw_spin_is_locked(lock))
592
593 /*
594 * Locks two spinlocks l1 and l2.
595 * l1_first indicates if spinlock l1 should be taken first.
596 */
597 static inline void
598 raw_double_spin_lock(raw_spinlock_t *l1, raw_spinlock_t *l2, bool l1_first)
599 __acquires(l1)
600 __acquires(l2)
601 {
602 if (l1_first) {
603 spin_lock(l1);
604 spin_lock(l2);
605 } else {
606 spin_lock(l2);
607 spin_lock(l1);
608 }
609 }
610
611 static inline void
612 double_spin_lock(spinlock_t *l1, spinlock_t *l2, bool l1_first)
613 __acquires(l1)
614 __acquires(l2)
615 {
616 if (l1_first) {
617 spin_lock(l1);
618 spin_lock(l2);
619 } else {
620 spin_lock(l2);
621 spin_lock(l1);
622 }
623 }
624
625
626 /*
627 * Unlocks two spinlocks l1 and l2.
628 * l1_taken_first indicates if spinlock l1 was taken first and therefore
629 * should be released after spinlock l2.
630 */
631 static inline void
632 raw_double_spin_unlock(raw_spinlock_t *l1, raw_spinlock_t *l2,
633 bool l1_taken_first)
634 __releases(l1)
635 __releases(l2)
636 {
637 if (l1_taken_first) {
638 spin_unlock(l2);
639 spin_unlock(l1);
640 } else {
641 spin_unlock(l1);
642 spin_unlock(l2);
643 }
644 }
645
646 static inline void
647 double_spin_unlock(spinlock_t *l1, spinlock_t *l2, bool l1_taken_first)
648 __releases(l1)
649 __releases(l2)
650 {
651 if (l1_taken_first) {
652 spin_unlock(l2);
653 spin_unlock(l1);
654 } else {
655 spin_unlock(l1);
656 spin_unlock(l2);
657 }
658 }
659
660 #endif /* __LINUX_SPINLOCK_H */
661
|
This page was automatically generated by the
LXR engine.
|