1 /*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5 *
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10 *
11 * Note that some architectures have special knowledge about the
12 * stack frames of these functions in their profile_pc. If you
13 * change anything significant here that could change the stack
14 * frame contact the architecture maintainers.
15 */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
24 int __lockfunc __spin_trylock(raw_spinlock_t *lock)
25 {
26 preempt_disable();
27 if (_raw_spin_trylock(lock)) {
28 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
29 return 1;
30 }
31
32 preempt_enable();
33 return 0;
34 }
35 EXPORT_SYMBOL(__spin_trylock);
36
37 int __lockfunc __spin_trylock_irq(raw_spinlock_t *lock)
38 {
39 local_irq_disable();
40 preempt_disable();
41
42 if (_raw_spin_trylock(lock)) {
43 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
44 return 1;
45 }
46
47 __preempt_enable_no_resched();
48 local_irq_enable();
49 preempt_check_resched();
50
51 return 0;
52 }
53 EXPORT_SYMBOL(__spin_trylock_irq);
54
55 int __lockfunc __spin_trylock_irqsave(raw_spinlock_t *lock,
56 unsigned long *flags)
57 {
58 local_irq_save(*flags);
59 preempt_disable();
60
61 if (_raw_spin_trylock(lock)) {
62 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
63 return 1;
64 }
65
66 __preempt_enable_no_resched();
67 local_irq_restore(*flags);
68 preempt_check_resched();
69
70 return 0;
71 }
72 EXPORT_SYMBOL(__spin_trylock_irqsave);
73
74 int __lockfunc __read_trylock(raw_rwlock_t *lock)
75 {
76 preempt_disable();
77 if (_raw_read_trylock(lock)) {
78 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
79 return 1;
80 }
81
82 preempt_enable();
83 return 0;
84 }
85 EXPORT_SYMBOL(__read_trylock);
86
87 int __lockfunc __write_trylock(raw_rwlock_t *lock)
88 {
89 preempt_disable();
90 if (_raw_write_trylock(lock)) {
91 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
92 return 1;
93 }
94
95 preempt_enable();
96 return 0;
97 }
98 EXPORT_SYMBOL(__write_trylock);
99
100 int __lockfunc __write_trylock_irqsave(raw_rwlock_t *lock, unsigned long *flags)
101 {
102 int ret;
103
104 local_irq_save(*flags);
105 ret = __write_trylock(lock);
106 if (ret)
107 return ret;
108
109 local_irq_restore(*flags);
110 return 0;
111 }
112 EXPORT_SYMBOL(__write_trylock_irqsave);
113
114 /*
115 * If lockdep is enabled then we use the non-preemption spin-ops
116 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
117 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
118 */
119 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
120
121 void __lockfunc __read_lock(raw_rwlock_t *lock)
122 {
123 preempt_disable();
124 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
125 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
126 }
127 EXPORT_SYMBOL(__read_lock);
128
129 unsigned long __lockfunc __spin_lock_irqsave(raw_spinlock_t *lock)
130 {
131 unsigned long flags;
132
133 local_irq_save(flags);
134 preempt_disable();
135 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
136 /*
137 * On lockdep we dont want the hand-coded irq-enable of
138 * _raw_spin_lock_flags() code, because lockdep assumes
139 * that interrupts are not re-enabled during lock-acquire:
140 */
141 #ifdef CONFIG_LOCKDEP
142 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
143 #else
144 _raw_spin_lock_flags(lock, &flags);
145 #endif
146 return flags;
147 }
148 EXPORT_SYMBOL(__spin_lock_irqsave);
149
150 void __lockfunc __spin_lock_irq(raw_spinlock_t *lock)
151 {
152 local_irq_disable();
153 preempt_disable();
154 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
155 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
156 }
157 EXPORT_SYMBOL(__spin_lock_irq);
158
159 void __lockfunc __spin_lock_bh(raw_spinlock_t *lock)
160 {
161 local_bh_disable();
162 preempt_disable();
163 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
164 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
165 }
166 EXPORT_SYMBOL(__spin_lock_bh);
167
168 unsigned long __lockfunc __read_lock_irqsave(raw_rwlock_t *lock)
169 {
170 unsigned long flags;
171
172 local_irq_save(flags);
173 preempt_disable();
174 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
175 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
176 return flags;
177 }
178 EXPORT_SYMBOL(__read_lock_irqsave);
179
180 void __lockfunc __read_lock_irq(raw_rwlock_t *lock)
181 {
182 local_irq_disable();
183 preempt_disable();
184 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
185 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
186 }
187 EXPORT_SYMBOL(__read_lock_irq);
188
189 void __lockfunc __read_lock_bh(raw_rwlock_t *lock)
190 {
191 local_bh_disable();
192 preempt_disable();
193 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
194 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
195 }
196 EXPORT_SYMBOL(__read_lock_bh);
197
198 unsigned long __lockfunc __write_lock_irqsave(raw_rwlock_t *lock)
199 {
200 unsigned long flags;
201
202 local_irq_save(flags);
203 preempt_disable();
204 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
205 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
206 return flags;
207 }
208 EXPORT_SYMBOL(__write_lock_irqsave);
209
210 void __lockfunc __write_lock_irq(raw_rwlock_t *lock)
211 {
212 local_irq_disable();
213 preempt_disable();
214 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
215 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
216 }
217 EXPORT_SYMBOL(__write_lock_irq);
218
219 void __lockfunc __write_lock_bh(raw_rwlock_t *lock)
220 {
221 local_bh_disable();
222 preempt_disable();
223 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
224 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
225 }
226 EXPORT_SYMBOL(__write_lock_bh);
227
228 void __lockfunc __spin_lock(raw_spinlock_t *lock)
229 {
230 preempt_disable();
231 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
232 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
233 }
234
235 EXPORT_SYMBOL(__spin_lock);
236
237 void __lockfunc __write_lock(raw_rwlock_t *lock)
238 {
239 preempt_disable();
240 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
241 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
242 }
243
244 EXPORT_SYMBOL(__write_lock);
245
246 #else /* CONFIG_PREEMPT: */
247
248 /*
249 * This could be a long-held lock. We both prepare to spin for a long
250 * time (making _this_ CPU preemptable if possible), and we also signal
251 * towards that other CPU that it should break the lock ASAP.
252 *
253 * (We do this in a function because inlining it would be excessive.)
254 */
255
256 #define BUILD_LOCK_OPS(op, locktype) \
257 void __lockfunc __##op##_lock(locktype##_t *lock) \
258 { \
259 for (;;) { \
260 preempt_disable(); \
261 if (likely(_raw_##op##_trylock(lock))) \
262 break; \
263 preempt_enable(); \
264 \
265 if (!(lock)->break_lock) \
266 (lock)->break_lock = 1; \
267 while (!__raw_##op##_can_lock(&(lock)->raw_lock) && \
268 (lock)->break_lock) \
269 __raw_##op##_relax(&lock->raw_lock); \
270 } \
271 (lock)->break_lock = 0; \
272 } \
273 \
274 EXPORT_SYMBOL(__##op##_lock); \
275 \
276 unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \
277 { \
278 unsigned long flags; \
279 \
280 for (;;) { \
281 preempt_disable(); \
282 local_irq_save(flags); \
283 if (likely(_raw_##op##_trylock(lock))) \
284 break; \
285 local_irq_restore(flags); \
286 preempt_enable(); \
287 \
288 if (!(lock)->break_lock) \
289 (lock)->break_lock = 1; \
290 while (!__raw_##op##_can_lock(&(lock)->raw_lock) && \
291 (lock)->break_lock) \
292 __raw_##op##_relax(&lock->raw_lock); \
293 } \
294 (lock)->break_lock = 0; \
295 return flags; \
296 } \
297 \
298 EXPORT_SYMBOL(__##op##_lock_irqsave); \
299 \
300 void __lockfunc __##op##_lock_irq(locktype##_t *lock) \
301 { \
302 __##op##_lock_irqsave(lock); \
303 } \
304 \
305 EXPORT_SYMBOL(__##op##_lock_irq); \
306 \
307 void __lockfunc __##op##_lock_bh(locktype##_t *lock) \
308 { \
309 unsigned long flags; \
310 \
311 /* */ \
312 /* Careful: we must exclude softirqs too, hence the */ \
313 /* irq-disabling. We use the generic preemption-aware */ \
314 /* function: */ \
315 /**/ \
316 flags = __##op##_lock_irqsave(lock); \
317 local_bh_disable(); \
318 local_irq_restore(flags); \
319 } \
320 \
321 EXPORT_SYMBOL(__##op##_lock_bh)
322
323 /*
324 * Build preemption-friendly versions of the following
325 * lock-spinning functions:
326 *
327 * __[spin|read|write]_lock()
328 * __[spin|read|write]_lock_irq()
329 * __[spin|read|write]_lock_irqsave()
330 * __[spin|read|write]_lock_bh()
331 */
332 BUILD_LOCK_OPS(spin, raw_spinlock);
333 BUILD_LOCK_OPS(read, raw_rwlock);
334 BUILD_LOCK_OPS(write, raw_rwlock);
335
336 #endif /* CONFIG_PREEMPT */
337
338 #ifdef CONFIG_DEBUG_LOCK_ALLOC
339
340 void __lockfunc __spin_lock_nested(raw_spinlock_t *lock, int subclass)
341 {
342 preempt_disable();
343 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
344 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
345 }
346 EXPORT_SYMBOL(__spin_lock_nested);
347
348 unsigned long __lockfunc
349 __spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass)
350 {
351 unsigned long flags;
352
353 local_irq_save(flags);
354 preempt_disable();
355 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
356 /*
357 * On lockdep we dont want the hand-coded irq-enable of
358 * _raw_spin_lock_flags() code, because lockdep assumes
359 * that interrupts are not re-enabled during lock-acquire:
360 */
361 #ifdef CONFIG_LOCKDEP
362 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
363 #else
364 _raw_spin_lock_flags(lock, &flags);
365 #endif
366 return flags;
367 }
368 EXPORT_SYMBOL(__spin_lock_irqsave_nested);
369
370 #endif
371
372 void __lockfunc __spin_unlock(raw_spinlock_t *lock)
373 {
374 spin_release(&lock->dep_map, 1, _RET_IP_);
375 _raw_spin_unlock(lock);
376 preempt_enable();
377 }
378 EXPORT_SYMBOL(__spin_unlock);
379
380 void __lockfunc __spin_unlock_no_resched(raw_spinlock_t *lock)
381 {
382 spin_release(&lock->dep_map, 1, _RET_IP_);
383 _raw_spin_unlock(lock);
384 __preempt_enable_no_resched();
385 }
386 /* not exported */
387
388 void __lockfunc __write_unlock(raw_rwlock_t *lock)
389 {
390 rwlock_release(&lock->dep_map, 1, _RET_IP_);
391 _raw_write_unlock(lock);
392 preempt_enable();
393 }
394 EXPORT_SYMBOL(__write_unlock);
395
396 void __lockfunc __read_unlock(raw_rwlock_t *lock)
397 {
398 rwlock_release(&lock->dep_map, 1, _RET_IP_);
399 _raw_read_unlock(lock);
400 preempt_enable();
401 }
402 EXPORT_SYMBOL(__read_unlock);
403
404 void __lockfunc __spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
405 {
406 spin_release(&lock->dep_map, 1, _RET_IP_);
407 _raw_spin_unlock(lock);
408 __preempt_enable_no_resched();
409 local_irq_restore(flags);
410 preempt_check_resched();
411 }
412 EXPORT_SYMBOL(__spin_unlock_irqrestore);
413
414 void __lockfunc __spin_unlock_irq(raw_spinlock_t *lock)
415 {
416 spin_release(&lock->dep_map, 1, _RET_IP_);
417 _raw_spin_unlock(lock);
418 __preempt_enable_no_resched();
419 local_irq_enable();
420 preempt_check_resched();
421 }
422 EXPORT_SYMBOL(__spin_unlock_irq);
423
424 void __lockfunc __spin_unlock_bh(raw_spinlock_t *lock)
425 {
426 spin_release(&lock->dep_map, 1, _RET_IP_);
427 _raw_spin_unlock(lock);
428 __preempt_enable_no_resched();
429 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
430 }
431 EXPORT_SYMBOL(__spin_unlock_bh);
432
433 void __lockfunc __read_unlock_irqrestore(raw_rwlock_t *lock, unsigned long flags)
434 {
435 rwlock_release(&lock->dep_map, 1, _RET_IP_);
436 _raw_read_unlock(lock);
437 __preempt_enable_no_resched();
438 local_irq_restore(flags);
439 preempt_check_resched();
440 }
441 EXPORT_SYMBOL(__read_unlock_irqrestore);
442
443 void __lockfunc __read_unlock_irq(raw_rwlock_t *lock)
444 {
445 rwlock_release(&lock->dep_map, 1, _RET_IP_);
446 _raw_read_unlock(lock);
447 __preempt_enable_no_resched();
448 local_irq_enable();
449 preempt_check_resched();
450 }
451 EXPORT_SYMBOL(__read_unlock_irq);
452
453 void __lockfunc __read_unlock_bh(raw_rwlock_t *lock)
454 {
455 rwlock_release(&lock->dep_map, 1, _RET_IP_);
456 _raw_read_unlock(lock);
457 __preempt_enable_no_resched();
458 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
459 }
460 EXPORT_SYMBOL(__read_unlock_bh);
461
462 void __lockfunc __write_unlock_irqrestore(raw_rwlock_t *lock, unsigned long flags)
463 {
464 rwlock_release(&lock->dep_map, 1, _RET_IP_);
465 _raw_write_unlock(lock);
466 __preempt_enable_no_resched();
467 local_irq_restore(flags);
468 preempt_check_resched();
469 }
470 EXPORT_SYMBOL(__write_unlock_irqrestore);
471
472 void __lockfunc __write_unlock_irq(raw_rwlock_t *lock)
473 {
474 rwlock_release(&lock->dep_map, 1, _RET_IP_);
475 _raw_write_unlock(lock);
476 __preempt_enable_no_resched();
477 local_irq_enable();
478 preempt_check_resched();
479 }
480 EXPORT_SYMBOL(__write_unlock_irq);
481
482 void __lockfunc __write_unlock_bh(raw_rwlock_t *lock)
483 {
484 rwlock_release(&lock->dep_map, 1, _RET_IP_);
485 _raw_write_unlock(lock);
486 __preempt_enable_no_resched();
487 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
488 }
489 EXPORT_SYMBOL(__write_unlock_bh);
490
491 int __lockfunc __spin_trylock_bh(raw_spinlock_t *lock)
492 {
493 local_bh_disable();
494 preempt_disable();
495 if (_raw_spin_trylock(lock)) {
496 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
497 return 1;
498 }
499
500 __preempt_enable_no_resched();
501 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
502 return 0;
503 }
504 EXPORT_SYMBOL(__spin_trylock_bh);
505
506 int in_lock_functions(unsigned long addr)
507 {
508 /* Linker adds these: start and end of __lockfunc functions */
509 extern char __lock_text_start[], __lock_text_end[];
510
511 return addr >= (unsigned long)__lock_text_start
512 && addr < (unsigned long)__lock_text_end;
513 }
514 EXPORT_SYMBOL(in_lock_functions);
515
516 void notrace __debug_atomic_dec_and_test(atomic_t *v)
517 {
518 static int warn_once = 1;
519
520 if (!atomic_read(v) && warn_once) {
521 warn_once = 0;
522 printk("BUG: atomic counter underflow!\n");
523 WARN_ON(1);
524 }
525 }
526
|
This page was automatically generated by the
LXR engine.
|