1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Softirq-split implemetation by
11 * Copyright (C) 2005 Thomas Gleixner, Ingo Molnar
12 */
13
14 #include <linux/module.h>
15 #include <linux/kallsyms.h>
16 #include <linux/syscalls.h>
17 #include <linux/wait.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/interrupt.h>
20 #include <linux/init.h>
21 #include <linux/delay.h>
22 #include <linux/mm.h>
23 #include <linux/notifier.h>
24 #include <linux/percpu.h>
25 #include <linux/delay.h>
26 #include <linux/cpu.h>
27 #include <linux/freezer.h>
28 #include <linux/kthread.h>
29 #include <linux/rcupdate.h>
30 #include <linux/smp.h>
31 #include <linux/tick.h>
32
33 #include <asm/irq.h>
34 /*
35 - No shared variables, all the data are CPU local.
36 - If a softirq needs serialization, let it serialize itself
37 by its own spinlocks.
38 - Even if softirq is serialized, only local cpu is marked for
39 execution. Hence, we get something sort of weak cpu binding.
40 Though it is still not clear, will it result in better locality
41 or will not.
42
43 Examples:
44 - NET RX softirq. It is multithreaded and does not require
45 any global serialization.
46 - NET TX softirq. It kicks software netdevice queues, hence
47 it is logically serialized per device, but this serialization
48 is invisible to common code.
49 - Tasklets: serialized wrt itself.
50 */
51
52 #ifndef __ARCH_IRQ_STAT
53 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
54 EXPORT_SYMBOL(irq_stat);
55 #endif
56
57 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
58
59 struct softirqdata {
60 int nr;
61 unsigned long cpu;
62 struct task_struct *tsk;
63 #ifdef CONFIG_PREEMPT_SOFTIRQS
64 wait_queue_head_t wait;
65 int running;
66 #endif
67 };
68
69 static DEFINE_PER_CPU(struct softirqdata [MAX_SOFTIRQ], ksoftirqd);
70
71 #ifdef CONFIG_PREEMPT_SOFTIRQS
72 /*
73 * Preempting the softirq causes cases that would not be a
74 * problem when the softirq is not preempted. That is a
75 * process may have code to spin while waiting for a softirq
76 * to finish on another CPU. But if it happens that the
77 * process has preempted the softirq, this could cause a
78 * deadlock.
79 */
80 void wait_for_softirq(int softirq)
81 {
82 struct softirqdata *data = &__get_cpu_var(ksoftirqd)[softirq];
83 if (data->running) {
84 DECLARE_WAITQUEUE(wait, current);
85 set_current_state(TASK_UNINTERRUPTIBLE);
86 add_wait_queue(&data->wait, &wait);
87 if (data->running)
88 schedule();
89 remove_wait_queue(&data->wait, &wait);
90 __set_current_state(TASK_RUNNING);
91 }
92 }
93 #endif
94
95 /*
96 * we cannot loop indefinitely here to avoid userspace starvation,
97 * but we also don't want to introduce a worst case 1/HZ latency
98 * to the pending events, so lets the scheduler to balance
99 * the softirq load for us.
100 */
101 static void wakeup_softirqd(int softirq)
102 {
103 /* Interrupts are disabled: no need to stop preemption */
104 struct task_struct *tsk = __get_cpu_var(ksoftirqd)[softirq].tsk;
105
106 if (unlikely(!tsk))
107 return;
108 /*
109 * Wake up the softirq task:
110 */
111 wake_up_process(tsk);
112 }
113
114 /*
115 * Wake up the softirq threads which have work
116 */
117 static void trigger_softirqs(void)
118 {
119 u32 pending = local_softirq_pending();
120 int curr = 0;
121
122 while (pending) {
123 if (pending & 1)
124 wakeup_softirqd(curr);
125 pending >>= 1;
126 curr++;
127 }
128 }
129
130 #ifndef CONFIG_PREEMPT_HARDIRQS
131
132 /*
133 * This one is for softirq.c-internal use,
134 * where hardirqs are disabled legitimately:
135 */
136 #ifdef CONFIG_TRACE_IRQFLAGS
137 static void __local_bh_disable(unsigned long ip)
138 {
139 unsigned long flags;
140
141 WARN_ON_ONCE(in_irq());
142
143 raw_local_irq_save(flags);
144 add_preempt_count(SOFTIRQ_OFFSET);
145 /*
146 * Were softirqs turned off above:
147 */
148 if (softirq_count() == SOFTIRQ_OFFSET)
149 trace_softirqs_off(ip);
150 raw_local_irq_restore(flags);
151 }
152 #else /* !CONFIG_TRACE_IRQFLAGS */
153 static inline void __local_bh_disable(unsigned long ip)
154 {
155 add_preempt_count(SOFTIRQ_OFFSET);
156 barrier();
157 }
158 #endif /* CONFIG_TRACE_IRQFLAGS */
159
160 void local_bh_disable(void)
161 {
162 __local_bh_disable((unsigned long)__builtin_return_address(0));
163 }
164
165 EXPORT_SYMBOL(local_bh_disable);
166
167 /*
168 * Special-case - softirqs can safely be enabled in
169 * cond_resched_softirq(), or by __do_softirq(),
170 * without processing still-pending softirqs:
171 */
172 void _local_bh_enable(void)
173 {
174 WARN_ON_ONCE(in_irq());
175 WARN_ON_ONCE(!irqs_disabled());
176
177 if (softirq_count() == SOFTIRQ_OFFSET)
178 trace_softirqs_on((unsigned long)__builtin_return_address(0));
179 sub_preempt_count(SOFTIRQ_OFFSET);
180 }
181
182 EXPORT_SYMBOL(_local_bh_enable);
183
184 void local_bh_enable(void)
185 {
186 #ifdef CONFIG_TRACE_IRQFLAGS
187 unsigned long flags;
188
189 WARN_ON_ONCE(in_irq());
190 #endif
191
192 #ifdef CONFIG_TRACE_IRQFLAGS
193 local_irq_save(flags);
194 #endif
195 /*
196 * Are softirqs going to be turned on now:
197 */
198 if (softirq_count() == SOFTIRQ_OFFSET)
199 trace_softirqs_on((unsigned long)__builtin_return_address(0));
200 /*
201 * Keep preemption disabled until we are done with
202 * softirq processing:
203 */
204 sub_preempt_count(SOFTIRQ_OFFSET - 1);
205
206 if (unlikely(!in_interrupt() && local_softirq_pending()))
207 do_softirq();
208
209 dec_preempt_count();
210 #ifdef CONFIG_TRACE_IRQFLAGS
211 local_irq_restore(flags);
212 #endif
213 preempt_check_resched();
214 }
215 EXPORT_SYMBOL(local_bh_enable);
216
217 void local_bh_enable_ip(unsigned long ip)
218 {
219 #ifdef CONFIG_TRACE_IRQFLAGS
220 unsigned long flags;
221
222 WARN_ON_ONCE(in_irq());
223
224 local_irq_save(flags);
225 #endif
226 /*
227 * Are softirqs going to be turned on now:
228 */
229 if (softirq_count() == SOFTIRQ_OFFSET)
230 trace_softirqs_on(ip);
231 /*
232 * Keep preemption disabled until we are done with
233 * softirq processing:
234 */
235 sub_preempt_count(SOFTIRQ_OFFSET - 1);
236
237 if (unlikely(!in_interrupt() && local_softirq_pending()))
238 do_softirq();
239
240 dec_preempt_count();
241 #ifdef CONFIG_TRACE_IRQFLAGS
242 local_irq_restore(flags);
243 #endif
244 preempt_check_resched();
245 }
246 EXPORT_SYMBOL(local_bh_enable_ip);
247
248 #endif
249
250 /*
251 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
252 * and we fall back to softirqd after that.
253 *
254 * This number has been established via experimentation.
255 * The two things to balance is latency against fairness -
256 * we want to handle softirqs as soon as possible, but they
257 * should not be able to lock up the box.
258 */
259 #define MAX_SOFTIRQ_RESTART 20
260
261 static DEFINE_PER_CPU(u32, softirq_running);
262
263 static void ___do_softirq(const int same_prio_only)
264 {
265 int max_restart = MAX_SOFTIRQ_RESTART, max_loops = MAX_SOFTIRQ_RESTART;
266 __u32 pending, available_mask, same_prio_skipped;
267 struct softirq_action *h;
268 struct task_struct *tsk;
269 int cpu, softirq;
270
271 pending = local_softirq_pending();
272 account_system_vtime(current);
273
274 cpu = smp_processor_id();
275 restart:
276 available_mask = -1;
277 softirq = 0;
278 same_prio_skipped = 0;
279 /* Reset the pending bitmask before enabling irqs */
280 set_softirq_pending(0);
281
282 h = softirq_vec;
283
284 do {
285 u32 softirq_mask = 1 << softirq;
286
287 if (pending & 1) {
288 u32 preempt_count = preempt_count();
289
290 #if defined(CONFIG_PREEMPT_SOFTIRQS) && defined(CONFIG_PREEMPT_HARDIRQS)
291 /*
292 * If executed by a same-prio hardirq thread
293 * then skip pending softirqs that belong
294 * to softirq threads with different priority:
295 */
296 if (same_prio_only) {
297 tsk = __get_cpu_var(ksoftirqd)[softirq].tsk;
298 if (tsk && tsk->normal_prio !=
299 current->normal_prio) {
300 same_prio_skipped |= softirq_mask;
301 available_mask &= ~softirq_mask;
302 goto next;
303 }
304 }
305 #endif
306 /*
307 * Is this softirq already being processed?
308 */
309 if (per_cpu(softirq_running, cpu) & softirq_mask) {
310 available_mask &= ~softirq_mask;
311 goto next;
312 }
313 per_cpu(softirq_running, cpu) |= softirq_mask;
314 local_irq_enable();
315
316 h->action(h);
317 if (preempt_count != preempt_count()) {
318 print_symbol("BUG: softirq exited %s with wrong preemption count!\n", (unsigned long) h->action);
319 printk("entered with %08x, exited with %08x.\n", preempt_count, preempt_count());
320 preempt_count() = preempt_count;
321 }
322 rcu_bh_qsctr_inc(cpu);
323 cond_resched_softirq_context();
324 local_irq_disable();
325 per_cpu(softirq_running, cpu) &= ~softirq_mask;
326 }
327 next:
328 h++;
329 softirq++;
330 pending >>= 1;
331 } while (pending);
332
333 or_softirq_pending(same_prio_skipped);
334 pending = local_softirq_pending();
335 if (pending & available_mask) {
336 if (--max_restart)
337 goto restart;
338 /*
339 * With softirq threading there's no reason not to
340 * finish the workload we have:
341 */
342 #ifdef CONFIG_PREEMPT_SOFTIRQS
343 if (--max_loops) {
344 if (printk_ratelimit())
345 printk("INFO: softirq overload: %08x\n", pending);
346 max_restart = MAX_SOFTIRQ_RESTART;
347 goto restart;
348 }
349 if (printk_ratelimit())
350 printk("BUG: softirq loop! %08x\n", pending);
351 #endif
352 }
353
354 if (pending)
355 trigger_softirqs();
356 }
357
358 asmlinkage void __do_softirq(void)
359 {
360 #ifdef CONFIG_PREEMPT_SOFTIRQS
361 /*
362 * 'preempt harder'. Push all softirq processing off to ksoftirqd.
363 */
364 if (softirq_preemption) {
365 if (local_softirq_pending())
366 trigger_softirqs();
367 return;
368 }
369 #endif
370 /*
371 * 'immediate' softirq execution:
372 */
373 __local_bh_disable((unsigned long)__builtin_return_address(0));
374 trace_softirq_enter();
375
376 ___do_softirq(0);
377
378 trace_softirq_exit();
379
380 account_system_vtime(current);
381 _local_bh_enable();
382
383 }
384
385 #ifndef __ARCH_HAS_DO_SOFTIRQ
386
387 asmlinkage void do_softirq(void)
388 {
389 __u32 pending;
390 unsigned long flags;
391
392 if (in_interrupt())
393 return;
394
395 local_irq_save(flags);
396
397 pending = local_softirq_pending();
398
399 if (pending)
400 __do_softirq();
401
402 local_irq_restore(flags);
403 }
404
405 #endif
406
407 /*
408 * Enter an interrupt context.
409 */
410 void irq_enter(void)
411 {
412 #ifdef CONFIG_NO_HZ
413 int cpu = smp_processor_id();
414 if (idle_cpu(cpu) && !in_interrupt())
415 tick_nohz_stop_idle(cpu);
416 #endif
417 __irq_enter();
418 #ifdef CONFIG_NO_HZ
419 if (idle_cpu(cpu))
420 tick_nohz_update_jiffies();
421 #endif
422 }
423
424 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
425 # define invoke_softirq() __do_softirq()
426 #else
427 # define invoke_softirq() do_softirq()
428 #endif
429
430 /*
431 * Exit an interrupt context. Process softirqs if needed and possible:
432 */
433 void irq_exit(void)
434 {
435 account_system_vtime(current);
436 trace_hardirq_exit();
437 sub_preempt_count(IRQ_EXIT_OFFSET);
438 if (!in_interrupt() && local_softirq_pending())
439 invoke_softirq();
440
441 #ifdef CONFIG_NO_HZ
442 /* Make sure that timer wheel updates are propagated */
443 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
444 tick_nohz_stop_sched_tick();
445 rcu_irq_exit();
446 #endif
447 __preempt_enable_no_resched();
448 }
449
450 /*
451 * This function must run with irqs disabled!
452 */
453 inline void raise_softirq_irqoff(unsigned int nr)
454 {
455 __do_raise_softirq_irqoff(nr);
456
457 #ifdef CONFIG_PREEMPT_SOFTIRQS
458 wakeup_softirqd(nr);
459 #endif
460 }
461
462 void raise_softirq(unsigned int nr)
463 {
464 unsigned long flags;
465
466 local_irq_save(flags);
467 raise_softirq_irqoff(nr);
468 local_irq_restore(flags);
469 }
470
471 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
472 {
473 softirq_vec[nr].data = data;
474 softirq_vec[nr].action = action;
475 }
476
477 /* Tasklets */
478 struct tasklet_head
479 {
480 struct tasklet_struct *list;
481 };
482
483 /* Some compilers disobey section attribute on statics when not
484 initialized -- RR */
485 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
486 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
487
488 static void inline
489 __tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr)
490 {
491 if (tasklet_trylock(t)) {
492 again:
493 /* We may have been preempted before tasklet_trylock
494 * and __tasklet_action may have already run.
495 * So double check the sched bit while the takslet
496 * is locked before adding it to the list.
497 */
498 if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
499 WARN_ON(t->next != NULL);
500 t->next = head->list;
501 head->list = t;
502 raise_softirq_irqoff(nr);
503 tasklet_unlock(t);
504 } else {
505 /* This is subtle. If we hit the corner case above
506 * It is possible that we get preempted right here,
507 * and another task has successfully called
508 * tasklet_schedule(), then this function, and
509 * failed on the trylock. Thus we must be sure
510 * before releasing the tasklet lock, that the
511 * SCHED_BIT is clear. Otherwise the tasklet
512 * may get its SCHED_BIT set, but not added to the
513 * list
514 */
515 if (!tasklet_tryunlock(t))
516 goto again;
517 }
518 }
519 }
520
521 void __tasklet_schedule(struct tasklet_struct *t)
522 {
523 unsigned long flags;
524
525 local_irq_save(flags);
526 __tasklet_common_schedule(t, &__get_cpu_var(tasklet_vec), TASKLET_SOFTIRQ);
527 local_irq_restore(flags);
528 }
529
530 EXPORT_SYMBOL(__tasklet_schedule);
531
532 void __tasklet_hi_schedule(struct tasklet_struct *t)
533 {
534 unsigned long flags;
535
536 local_irq_save(flags);
537 __tasklet_common_schedule(t, &__get_cpu_var(tasklet_hi_vec), HI_SOFTIRQ);
538 local_irq_restore(flags);
539 }
540
541 EXPORT_SYMBOL(__tasklet_hi_schedule);
542
543 void tasklet_enable(struct tasklet_struct *t)
544 {
545 if (!atomic_dec_and_test(&t->count))
546 return;
547 if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
548 tasklet_schedule(t);
549 }
550
551 EXPORT_SYMBOL(tasklet_enable);
552
553 void tasklet_hi_enable(struct tasklet_struct *t)
554 {
555 if (!atomic_dec_and_test(&t->count))
556 return;
557 if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
558 tasklet_hi_schedule(t);
559 }
560
561 EXPORT_SYMBOL(tasklet_hi_enable);
562
563 static void
564 __tasklet_action(struct softirq_action *a, struct tasklet_struct *list)
565 {
566 int loops = 1000000;
567
568 while (list) {
569 struct tasklet_struct *t = list;
570
571 list = list->next;
572 /*
573 * Should always succeed - after a tasklist got on the
574 * list (after getting the SCHED bit set from 0 to 1),
575 * nothing but the tasklet softirq it got queued to can
576 * lock it:
577 */
578 if (!tasklet_trylock(t)) {
579 WARN_ON(1);
580 continue;
581 }
582
583 t->next = NULL;
584
585 /*
586 * If we cannot handle the tasklet because it's disabled,
587 * mark it as pending. tasklet_enable() will later
588 * re-schedule the tasklet.
589 */
590 if (unlikely(atomic_read(&t->count))) {
591 out_disabled:
592 /* implicit unlock: */
593 wmb();
594 t->state = TASKLET_STATEF_PENDING;
595 continue;
596 }
597
598 /*
599 * After this point on the tasklet might be rescheduled
600 * on another CPU, but it can only be added to another
601 * CPU's tasklet list if we unlock the tasklet (which we
602 * dont do yet).
603 */
604 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
605 WARN_ON(1);
606
607 again:
608 t->func(t->data);
609
610 /*
611 * Try to unlock the tasklet. We must use cmpxchg, because
612 * another CPU might have scheduled or disabled the tasklet.
613 * We only allow the STATE_RUN -> 0 transition here.
614 */
615 while (!tasklet_tryunlock(t)) {
616 /*
617 * If it got disabled meanwhile, bail out:
618 */
619 if (atomic_read(&t->count))
620 goto out_disabled;
621 /*
622 * If it got scheduled meanwhile, re-execute
623 * the tasklet function:
624 */
625 if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
626 goto again;
627 if (!--loops) {
628 printk("hm, tasklet state: %08lx\n", t->state);
629 WARN_ON(1);
630 tasklet_unlock(t);
631 break;
632 }
633 }
634 }
635 }
636
637 static void tasklet_action(struct softirq_action *a)
638 {
639 struct tasklet_struct *list;
640
641 local_irq_disable();
642 list = __get_cpu_var(tasklet_vec).list;
643 __get_cpu_var(tasklet_vec).list = NULL;
644 local_irq_enable();
645
646 __tasklet_action(a, list);
647 }
648
649 static void tasklet_hi_action(struct softirq_action *a)
650 {
651 struct tasklet_struct *list;
652
653 local_irq_disable();
654 list = __get_cpu_var(tasklet_hi_vec).list;
655 __get_cpu_var(tasklet_hi_vec).list = NULL;
656 local_irq_enable();
657
658 __tasklet_action(a, list);
659 }
660
661 void tasklet_init(struct tasklet_struct *t,
662 void (*func)(unsigned long), unsigned long data)
663 {
664 t->next = NULL;
665 t->state = 0;
666 atomic_set(&t->count, 0);
667 t->func = func;
668 t->data = data;
669 }
670
671 EXPORT_SYMBOL(tasklet_init);
672
673 void tasklet_kill(struct tasklet_struct *t)
674 {
675 if (in_interrupt())
676 printk("Attempt to kill tasklet from interrupt\n");
677
678 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
679 do
680 msleep(1);
681 while (test_bit(TASKLET_STATE_SCHED, &t->state));
682 }
683 tasklet_unlock_wait(t);
684 clear_bit(TASKLET_STATE_SCHED, &t->state);
685 }
686
687 EXPORT_SYMBOL(tasklet_kill);
688
689 void __init softirq_init(void)
690 {
691 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
692 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
693 }
694
695 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
696
697 void tasklet_unlock_wait(struct tasklet_struct *t)
698 {
699 while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
700 /*
701 * Hack for now to avoid this busy-loop:
702 */
703 #ifdef CONFIG_PREEMPT_RT
704 msleep(1);
705 #else
706 barrier();
707 #endif
708 }
709 }
710 EXPORT_SYMBOL(tasklet_unlock_wait);
711
712 #endif
713
714 static int ksoftirqd(void * __data)
715 {
716 struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2 };
717 struct softirqdata *data = __data;
718 u32 softirq_mask = (1 << data->nr);
719 struct softirq_action *h;
720 int cpu = data->cpu;
721
722 #ifdef CONFIG_PREEMPT_SOFTIRQS
723 init_waitqueue_head(&data->wait);
724 #endif
725
726 sys_sched_setscheduler(current->pid, SCHED_FIFO, ¶m);
727 current->flags |= PF_SOFTIRQ;
728 set_current_state(TASK_INTERRUPTIBLE);
729
730 while (!kthread_should_stop()) {
731 preempt_disable();
732 if (!(local_softirq_pending() & softirq_mask)) {
733 sleep_more:
734 __preempt_enable_no_resched();
735 schedule();
736 preempt_disable();
737 }
738
739 __set_current_state(TASK_RUNNING);
740
741 #ifdef CONFIG_PREEMPT_SOFTIRQS
742 data->running = 1;
743 #endif
744
745 while (local_softirq_pending() & softirq_mask) {
746 /* Preempt disable stops cpu going offline.
747 If already offline, we'll be on wrong CPU:
748 don't process */
749 if (cpu_is_offline(cpu))
750 goto wait_to_die;
751
752 local_irq_disable();
753 /*
754 * Is the softirq already being executed by
755 * a hardirq context?
756 */
757 if (per_cpu(softirq_running, cpu) & softirq_mask) {
758 local_irq_enable();
759 set_current_state(TASK_INTERRUPTIBLE);
760 goto sleep_more;
761 }
762 per_cpu(softirq_running, cpu) |= softirq_mask;
763 __preempt_enable_no_resched();
764 set_softirq_pending(local_softirq_pending() & ~softirq_mask);
765 local_bh_disable();
766 local_irq_enable();
767
768 h = &softirq_vec[data->nr];
769 if (h)
770 h->action(h);
771 rcu_bh_qsctr_inc(data->cpu);
772
773 local_irq_disable();
774 per_cpu(softirq_running, cpu) &= ~softirq_mask;
775 _local_bh_enable();
776 local_irq_enable();
777
778 cond_resched();
779 preempt_disable();
780 }
781 preempt_enable();
782 set_current_state(TASK_INTERRUPTIBLE);
783 #ifdef CONFIG_PREEMPT_SOFTIRQS
784 data->running = 0;
785 wake_up(&data->wait);
786 #endif
787 }
788 __set_current_state(TASK_RUNNING);
789 return 0;
790
791 wait_to_die:
792 preempt_enable();
793 /* Wait for kthread_stop */
794 set_current_state(TASK_INTERRUPTIBLE);
795 while (!kthread_should_stop()) {
796 schedule();
797 set_current_state(TASK_INTERRUPTIBLE);
798 }
799 __set_current_state(TASK_RUNNING);
800 return 0;
801 }
802
803 #ifdef CONFIG_HOTPLUG_CPU
804 /*
805 * tasklet_kill_immediate is called to remove a tasklet which can already be
806 * scheduled for execution on @cpu.
807 *
808 * Unlike tasklet_kill, this function removes the tasklet
809 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
810 *
811 * When this function is called, @cpu must be in the CPU_DEAD state.
812 */
813 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
814 {
815 struct tasklet_struct **i;
816
817 BUG_ON(cpu_online(cpu));
818 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
819
820 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
821 return;
822
823 /* CPU is dead, so no lock needed. */
824 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
825 if (*i == t) {
826 *i = t->next;
827 return;
828 }
829 }
830 BUG();
831 }
832
833 void takeover_tasklets(unsigned int cpu)
834 {
835 struct tasklet_struct **i;
836
837 /* CPU is dead, so no lock needed. */
838 local_irq_disable();
839
840 /* Find end, append list for that CPU. */
841 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
842 *i = per_cpu(tasklet_vec, cpu).list;
843 per_cpu(tasklet_vec, cpu).list = NULL;
844 raise_softirq_irqoff(TASKLET_SOFTIRQ);
845
846 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
847 *i = per_cpu(tasklet_hi_vec, cpu).list;
848 per_cpu(tasklet_hi_vec, cpu).list = NULL;
849 raise_softirq_irqoff(HI_SOFTIRQ);
850
851 local_irq_enable();
852 }
853 #endif /* CONFIG_HOTPLUG_CPU */
854
855 static const char *softirq_names [] =
856 {
857 [HI_SOFTIRQ] = "high",
858 [SCHED_SOFTIRQ] = "sched",
859 [TIMER_SOFTIRQ] = "timer",
860 [NET_TX_SOFTIRQ] = "net-tx",
861 [NET_RX_SOFTIRQ] = "net-rx",
862 [BLOCK_SOFTIRQ] = "block",
863 [TASKLET_SOFTIRQ] = "tasklet",
864 #ifdef CONFIG_HIGH_RES_TIMERS
865 [HRTIMER_SOFTIRQ] = "hrtimer",
866 #endif
867 [RCU_SOFTIRQ] = "rcu",
868 };
869
870 static int __cpuinit cpu_callback(struct notifier_block *nfb,
871 unsigned long action,
872 void *hcpu)
873 {
874 int hotcpu = (unsigned long)hcpu, i;
875 struct task_struct *p;
876
877 switch (action) {
878 case CPU_UP_PREPARE:
879 case CPU_UP_PREPARE_FROZEN:
880 for (i = 0; i < MAX_SOFTIRQ; i++) {
881 per_cpu(ksoftirqd, hotcpu)[i].nr = i;
882 per_cpu(ksoftirqd, hotcpu)[i].cpu = hotcpu;
883 per_cpu(ksoftirqd, hotcpu)[i].tsk = NULL;
884 }
885 for (i = 0; i < MAX_SOFTIRQ; i++) {
886 p = kthread_create(ksoftirqd,
887 &per_cpu(ksoftirqd, hotcpu)[i],
888 "sirq-%s/%d", softirq_names[i],
889 hotcpu);
890 if (IS_ERR(p)) {
891 printk("ksoftirqd %d for %i failed\n", i,
892 hotcpu);
893 return NOTIFY_BAD;
894 }
895 kthread_bind(p, hotcpu);
896 per_cpu(ksoftirqd, hotcpu)[i].tsk = p;
897 }
898 break;
899 break;
900 case CPU_ONLINE:
901 case CPU_ONLINE_FROZEN:
902 for (i = 0; i < MAX_SOFTIRQ; i++)
903 wake_up_process(per_cpu(ksoftirqd, hotcpu)[i].tsk);
904 break;
905 #ifdef CONFIG_HOTPLUG_CPU
906 case CPU_UP_CANCELED:
907 case CPU_UP_CANCELED_FROZEN:
908 #if 0
909 for (i = 0; i < MAX_SOFTIRQ; i++) {
910 if (!per_cpu(ksoftirqd, hotcpu)[i].tsk)
911 continue;
912 kthread_bind(per_cpu(ksoftirqd, hotcpu)[i].tsk,
913 any_online_cpu(cpu_online_map));
914 }
915 #endif
916 case CPU_DEAD:
917 case CPU_DEAD_FROZEN: {
918 struct sched_param param;
919
920 for (i = 0; i < MAX_SOFTIRQ; i++) {
921 param.sched_priority = MAX_RT_PRIO-1;
922 p = per_cpu(ksoftirqd, hotcpu)[i].tsk;
923 sched_setscheduler(p, SCHED_FIFO, ¶m);
924 per_cpu(ksoftirqd, hotcpu)[i].tsk = NULL;
925 kthread_stop(p);
926 }
927 takeover_tasklets(hotcpu);
928 break;
929 }
930 #endif /* CONFIG_HOTPLUG_CPU */
931 }
932 return NOTIFY_OK;
933 }
934
935 static struct notifier_block __cpuinitdata cpu_nfb = {
936 .notifier_call = cpu_callback
937 };
938
939 __init int spawn_ksoftirqd(void)
940 {
941 void *cpu = (void *)(long)smp_processor_id();
942 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
943
944 BUG_ON(err == NOTIFY_BAD);
945 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
946 register_cpu_notifier(&cpu_nfb);
947 return 0;
948 }
949
950
951 #ifdef CONFIG_PREEMPT_SOFTIRQS
952
953 int softirq_preemption = 1;
954
955 EXPORT_SYMBOL(softirq_preemption);
956
957 /*
958 * Real-Time Preemption depends on softirq threading:
959 */
960 #ifndef CONFIG_PREEMPT_RT
961
962 static int __init softirq_preempt_setup (char *str)
963 {
964 if (!strncmp(str, "off", 3))
965 softirq_preemption = 0;
966 else
967 get_option(&str, &softirq_preemption);
968 if (!softirq_preemption)
969 printk("turning off softirq preemption!\n");
970
971 return 1;
972 }
973
974 __setup("softirq-preempt=", softirq_preempt_setup);
975 #endif
976 #endif
977
978 #ifdef CONFIG_SMP
979 /*
980 * Call a function on all processors
981 */
982 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
983 {
984 int ret = 0;
985
986 preempt_disable();
987 ret = smp_call_function(func, info, retry, wait);
988 local_irq_disable();
989 func(info);
990 local_irq_enable();
991 preempt_enable();
992 return ret;
993 }
994 EXPORT_SYMBOL(on_each_cpu);
995 #endif
996
|
This page was automatically generated by the
LXR engine.
|