1 /*
2 * linux/arch/arm/kernel/smp.c
3 *
4 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10 #include <linux/module.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/spinlock.h>
14 #include <linux/sched.h>
15 #include <linux/interrupt.h>
16 #include <linux/cache.h>
17 #include <linux/profile.h>
18 #include <linux/errno.h>
19 #include <linux/mm.h>
20 #include <linux/err.h>
21 #include <linux/cpu.h>
22 #include <linux/smp.h>
23 #include <linux/seq_file.h>
24 #include <linux/irq.h>
25
26 #include <asm/atomic.h>
27 #include <asm/cacheflush.h>
28 #include <asm/cpu.h>
29 #include <asm/mmu_context.h>
30 #include <asm/pgtable.h>
31 #include <asm/pgalloc.h>
32 #include <asm/processor.h>
33 #include <asm/tlbflush.h>
34 #include <asm/ptrace.h>
35
36 /*
37 * bitmask of present and online CPUs.
38 * The present bitmask indicates that the CPU is physically present.
39 * The online bitmask indicates that the CPU is up and running.
40 */
41 cpumask_t cpu_possible_map;
42 EXPORT_SYMBOL(cpu_possible_map);
43 cpumask_t cpu_online_map;
44 EXPORT_SYMBOL(cpu_online_map);
45
46 /*
47 * as from 2.5, kernels no longer have an init_tasks structure
48 * so we need some other way of telling a new secondary core
49 * where to place its SVC stack
50 */
51 struct secondary_data secondary_data;
52
53 /*
54 * structures for inter-processor calls
55 * - A collection of single bit ipi messages.
56 */
57 struct ipi_data {
58 spinlock_t lock;
59 unsigned long ipi_count;
60 unsigned long bits;
61 };
62
63 static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
64 .lock = SPIN_LOCK_UNLOCKED,
65 };
66
67 enum ipi_msg_type {
68 IPI_TIMER,
69 IPI_RESCHEDULE,
70 IPI_CALL_FUNC,
71 IPI_CPU_STOP,
72 };
73
74 struct smp_call_struct {
75 void (*func)(void *info);
76 void *info;
77 int wait;
78 cpumask_t pending;
79 cpumask_t unfinished;
80 };
81
82 static struct smp_call_struct * volatile smp_call_function_data;
83 static DEFINE_SPINLOCK(smp_call_function_lock);
84
85 int __cpuinit __cpu_up(unsigned int cpu)
86 {
87 struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu);
88 struct task_struct *idle = ci->idle;
89 pgd_t *pgd;
90 pmd_t *pmd;
91 int ret;
92
93 /*
94 * Spawn a new process manually, if not already done.
95 * Grab a pointer to its task struct so we can mess with it
96 */
97 if (!idle) {
98 idle = fork_idle(cpu);
99 if (IS_ERR(idle)) {
100 printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
101 return PTR_ERR(idle);
102 }
103 ci->idle = idle;
104 }
105
106 /*
107 * Allocate initial page tables to allow the new CPU to
108 * enable the MMU safely. This essentially means a set
109 * of our "standard" page tables, with the addition of
110 * a 1:1 mapping for the physical address of the kernel.
111 */
112 pgd = pgd_alloc(&init_mm);
113 pmd = pmd_offset(pgd, PHYS_OFFSET);
114 *pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) |
115 PMD_TYPE_SECT | PMD_SECT_AP_WRITE);
116
117 /*
118 * We need to tell the secondary core where to find
119 * its stack and the page tables.
120 */
121 secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
122 secondary_data.pgdir = virt_to_phys(pgd);
123 wmb();
124
125 /*
126 * Now bring the CPU into our world.
127 */
128 ret = boot_secondary(cpu, idle);
129 if (ret == 0) {
130 unsigned long timeout;
131
132 /*
133 * CPU was successfully started, wait for it
134 * to come online or time out.
135 */
136 timeout = jiffies + HZ;
137 while (time_before(jiffies, timeout)) {
138 if (cpu_online(cpu))
139 break;
140
141 udelay(10);
142 barrier();
143 }
144
145 if (!cpu_online(cpu))
146 ret = -EIO;
147 }
148
149 secondary_data.stack = NULL;
150 secondary_data.pgdir = 0;
151
152 *pmd_offset(pgd, PHYS_OFFSET) = __pmd(0);
153 pgd_free(&init_mm, pgd);
154
155 if (ret) {
156 printk(KERN_CRIT "CPU%u: processor failed to boot\n", cpu);
157
158 /*
159 * FIXME: We need to clean up the new idle thread. --rmk
160 */
161 }
162
163 return ret;
164 }
165
166 #ifdef CONFIG_HOTPLUG_CPU
167 /*
168 * __cpu_disable runs on the processor to be shutdown.
169 */
170 int __cpuexit __cpu_disable(void)
171 {
172 unsigned int cpu = smp_processor_id();
173 struct task_struct *p;
174 int ret;
175
176 ret = mach_cpu_disable(cpu);
177 if (ret)
178 return ret;
179
180 /*
181 * Take this CPU offline. Once we clear this, we can't return,
182 * and we must not schedule until we're ready to give up the cpu.
183 */
184 cpu_clear(cpu, cpu_online_map);
185
186 /*
187 * OK - migrate IRQs away from this CPU
188 */
189 migrate_irqs();
190
191 /*
192 * Stop the local timer for this CPU.
193 */
194 local_timer_stop(cpu);
195
196 /*
197 * Flush user cache and TLB mappings, and then remove this CPU
198 * from the vm mask set of all processes.
199 */
200 flush_cache_all();
201 local_flush_tlb_all();
202
203 read_lock(&tasklist_lock);
204 for_each_process(p) {
205 if (p->mm)
206 cpu_clear(cpu, p->mm->cpu_vm_mask);
207 }
208 read_unlock(&tasklist_lock);
209
210 return 0;
211 }
212
213 /*
214 * called on the thread which is asking for a CPU to be shutdown -
215 * waits until shutdown has completed, or it is timed out.
216 */
217 void __cpuexit __cpu_die(unsigned int cpu)
218 {
219 if (!platform_cpu_kill(cpu))
220 printk("CPU%u: unable to kill\n", cpu);
221 }
222
223 /*
224 * Called from the idle thread for the CPU which has been shutdown.
225 *
226 * Note that we disable IRQs here, but do not re-enable them
227 * before returning to the caller. This is also the behaviour
228 * of the other hotplug-cpu capable cores, so presumably coming
229 * out of idle fixes this.
230 */
231 void __cpuexit cpu_die(void)
232 {
233 unsigned int cpu = smp_processor_id();
234
235 local_irq_disable();
236 idle_task_exit();
237
238 /*
239 * actual CPU shutdown procedure is at least platform (if not
240 * CPU) specific
241 */
242 platform_cpu_die(cpu);
243
244 /*
245 * Do not return to the idle loop - jump back to the secondary
246 * cpu initialisation. There's some initialisation which needs
247 * to be repeated to undo the effects of taking the CPU offline.
248 */
249 __asm__("mov sp, %0\n"
250 " b secondary_start_kernel"
251 :
252 : "r" (task_stack_page(current) + THREAD_SIZE - 8));
253 }
254 #endif /* CONFIG_HOTPLUG_CPU */
255
256 /*
257 * This is the secondary CPU boot entry. We're using this CPUs
258 * idle thread stack, but a set of temporary page tables.
259 */
260 asmlinkage void __cpuinit secondary_start_kernel(void)
261 {
262 struct mm_struct *mm = &init_mm;
263 unsigned int cpu = smp_processor_id();
264
265 printk("CPU%u: Booted secondary processor\n", cpu);
266
267 /*
268 * All kernel threads share the same mm context; grab a
269 * reference and switch to it.
270 */
271 atomic_inc(&mm->mm_users);
272 atomic_inc(&mm->mm_count);
273 current->active_mm = mm;
274 cpu_set(cpu, mm->cpu_vm_mask);
275 cpu_switch_mm(mm->pgd, mm);
276 enter_lazy_tlb(mm, current);
277 local_flush_tlb_all();
278
279 cpu_init();
280 preempt_disable();
281
282 /*
283 * Give the platform a chance to do its own initialisation.
284 */
285 platform_secondary_init(cpu);
286
287 /*
288 * Enable local interrupts.
289 */
290 local_irq_enable();
291 local_fiq_enable();
292
293 /*
294 * Setup local timer for this CPU.
295 */
296 local_timer_setup(cpu);
297
298 calibrate_delay();
299
300 smp_store_cpu_info(cpu);
301
302 /*
303 * OK, now it's safe to let the boot CPU continue
304 */
305 cpu_set(cpu, cpu_online_map);
306
307 /*
308 * OK, it's off to the idle thread for us
309 */
310 cpu_idle();
311 }
312
313 /*
314 * Called by both boot and secondaries to move global data into
315 * per-processor storage.
316 */
317 void __cpuinit smp_store_cpu_info(unsigned int cpuid)
318 {
319 struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
320
321 cpu_info->loops_per_jiffy = loops_per_jiffy;
322 }
323
324 void __init smp_cpus_done(unsigned int max_cpus)
325 {
326 int cpu;
327 unsigned long bogosum = 0;
328
329 for_each_online_cpu(cpu)
330 bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
331
332 printk(KERN_INFO "SMP: Total of %d processors activated "
333 "(%lu.%02lu BogoMIPS).\n",
334 num_online_cpus(),
335 bogosum / (500000/HZ),
336 (bogosum / (5000/HZ)) % 100);
337 }
338
339 void __init smp_prepare_boot_cpu(void)
340 {
341 unsigned int cpu = smp_processor_id();
342
343 per_cpu(cpu_data, cpu).idle = current;
344 }
345
346 static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg)
347 {
348 unsigned long flags;
349 unsigned int cpu;
350
351 local_irq_save(flags);
352
353 for_each_cpu_mask(cpu, callmap) {
354 struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
355
356 spin_lock(&ipi->lock);
357 ipi->bits |= 1 << msg;
358 spin_unlock(&ipi->lock);
359 }
360
361 /*
362 * Call the platform specific cross-CPU call function.
363 */
364 smp_cross_call(callmap);
365
366 local_irq_restore(flags);
367 }
368
369 /*
370 * You must not call this function with disabled interrupts, from a
371 * hardware interrupt handler, nor from a bottom half handler.
372 */
373 static int smp_call_function_on_cpu(void (*func)(void *info), void *info,
374 int retry, int wait, cpumask_t callmap)
375 {
376 struct smp_call_struct data;
377 unsigned long timeout;
378 int ret = 0;
379
380 data.func = func;
381 data.info = info;
382 data.wait = wait;
383
384 cpu_clear(smp_processor_id(), callmap);
385 if (cpus_empty(callmap))
386 goto out;
387
388 data.pending = callmap;
389 if (wait)
390 data.unfinished = callmap;
391
392 /*
393 * try to get the mutex on smp_call_function_data
394 */
395 spin_lock(&smp_call_function_lock);
396 smp_call_function_data = &data;
397
398 send_ipi_message(callmap, IPI_CALL_FUNC);
399
400 timeout = jiffies + HZ;
401 while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
402 barrier();
403
404 /*
405 * did we time out?
406 */
407 if (!cpus_empty(data.pending)) {
408 /*
409 * this may be causing our panic - report it
410 */
411 printk(KERN_CRIT
412 "CPU%u: smp_call_function timeout for %p(%p)\n"
413 " callmap %lx pending %lx, %swait\n",
414 smp_processor_id(), func, info, *cpus_addr(callmap),
415 *cpus_addr(data.pending), wait ? "" : "no ");
416
417 /*
418 * TRACE
419 */
420 timeout = jiffies + (5 * HZ);
421 while (!cpus_empty(data.pending) && time_before(jiffies, timeout))
422 barrier();
423
424 if (cpus_empty(data.pending))
425 printk(KERN_CRIT " RESOLVED\n");
426 else
427 printk(KERN_CRIT " STILL STUCK\n");
428 }
429
430 /*
431 * whatever happened, we're done with the data, so release it
432 */
433 smp_call_function_data = NULL;
434 spin_unlock(&smp_call_function_lock);
435
436 if (!cpus_empty(data.pending)) {
437 ret = -ETIMEDOUT;
438 goto out;
439 }
440
441 if (wait)
442 while (!cpus_empty(data.unfinished))
443 barrier();
444 out:
445
446 return 0;
447 }
448
449 int smp_call_function(void (*func)(void *info), void *info, int retry,
450 int wait)
451 {
452 return smp_call_function_on_cpu(func, info, retry, wait,
453 cpu_online_map);
454 }
455 EXPORT_SYMBOL_GPL(smp_call_function);
456
457 int smp_call_function_single(int cpu, void (*func)(void *info), void *info,
458 int retry, int wait)
459 {
460 /* prevent preemption and reschedule on another processor */
461 int current_cpu = get_cpu();
462 int ret = 0;
463
464 if (cpu == current_cpu) {
465 local_irq_disable();
466 func(info);
467 local_irq_enable();
468 } else
469 ret = smp_call_function_on_cpu(func, info, retry, wait,
470 cpumask_of_cpu(cpu));
471
472 put_cpu();
473
474 return ret;
475 }
476 EXPORT_SYMBOL_GPL(smp_call_function_single);
477
478 void show_ipi_list(struct seq_file *p)
479 {
480 unsigned int cpu;
481
482 seq_puts(p, "IPI:");
483
484 for_each_present_cpu(cpu)
485 seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);
486
487 seq_putc(p, '\n');
488 }
489
490 void show_local_irqs(struct seq_file *p)
491 {
492 unsigned int cpu;
493
494 seq_printf(p, "LOC: ");
495
496 for_each_present_cpu(cpu)
497 seq_printf(p, "%10u ", irq_stat[cpu].local_timer_irqs);
498
499 seq_putc(p, '\n');
500 }
501
502 static void ipi_timer(void)
503 {
504 irq_enter();
505 local_timer_interrupt();
506 irq_exit();
507 }
508
509 #ifdef CONFIG_LOCAL_TIMERS
510 asmlinkage void __exception do_local_timer(struct pt_regs *regs)
511 {
512 struct pt_regs *old_regs = set_irq_regs(regs);
513 int cpu = smp_processor_id();
514
515 if (local_timer_ack()) {
516 irq_stat[cpu].local_timer_irqs++;
517 ipi_timer();
518 }
519
520 set_irq_regs(old_regs);
521 }
522 #endif
523
524 /*
525 * ipi_call_function - handle IPI from smp_call_function()
526 *
527 * Note that we copy data out of the cross-call structure and then
528 * let the caller know that we're here and have done with their data
529 */
530 static void ipi_call_function(unsigned int cpu)
531 {
532 struct smp_call_struct *data = smp_call_function_data;
533 void (*func)(void *info) = data->func;
534 void *info = data->info;
535 int wait = data->wait;
536
537 cpu_clear(cpu, data->pending);
538
539 func(info);
540
541 if (wait)
542 cpu_clear(cpu, data->unfinished);
543 }
544
545 static DEFINE_SPINLOCK(stop_lock);
546
547 /*
548 * ipi_cpu_stop - handle IPI from smp_send_stop()
549 */
550 static void ipi_cpu_stop(unsigned int cpu)
551 {
552 spin_lock(&stop_lock);
553 printk(KERN_CRIT "CPU%u: stopping\n", cpu);
554 dump_stack();
555 spin_unlock(&stop_lock);
556
557 cpu_clear(cpu, cpu_online_map);
558
559 local_fiq_disable();
560 local_irq_disable();
561
562 while (1)
563 cpu_relax();
564 }
565
566 /*
567 * Main handler for inter-processor interrupts
568 *
569 * For ARM, the ipimask now only identifies a single
570 * category of IPI (Bit 1 IPIs have been replaced by a
571 * different mechanism):
572 *
573 * Bit 0 - Inter-processor function call
574 */
575 asmlinkage void __exception do_IPI(struct pt_regs *regs)
576 {
577 unsigned int cpu = smp_processor_id();
578 struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
579 struct pt_regs *old_regs = set_irq_regs(regs);
580
581 ipi->ipi_count++;
582
583 for (;;) {
584 unsigned long msgs;
585
586 spin_lock(&ipi->lock);
587 msgs = ipi->bits;
588 ipi->bits = 0;
589 spin_unlock(&ipi->lock);
590
591 if (!msgs)
592 break;
593
594 do {
595 unsigned nextmsg;
596
597 nextmsg = msgs & -msgs;
598 msgs &= ~nextmsg;
599 nextmsg = ffz(~nextmsg);
600
601 switch (nextmsg) {
602 case IPI_TIMER:
603 ipi_timer();
604 break;
605
606 case IPI_RESCHEDULE:
607 /*
608 * nothing more to do - eveything is
609 * done on the interrupt return path
610 */
611 break;
612
613 case IPI_CALL_FUNC:
614 ipi_call_function(cpu);
615 break;
616
617 case IPI_CPU_STOP:
618 ipi_cpu_stop(cpu);
619 break;
620
621 default:
622 printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
623 cpu, nextmsg);
624 break;
625 }
626 } while (msgs);
627 }
628
629 set_irq_regs(old_regs);
630 }
631
632 void smp_send_reschedule(int cpu)
633 {
634 send_ipi_message(cpumask_of_cpu(cpu), IPI_RESCHEDULE);
635 }
636
637 void smp_send_timer(void)
638 {
639 cpumask_t mask = cpu_online_map;
640 cpu_clear(smp_processor_id(), mask);
641 send_ipi_message(mask, IPI_TIMER);
642 }
643
644 void smp_timer_broadcast(cpumask_t mask)
645 {
646 send_ipi_message(mask, IPI_TIMER);
647 }
648
649 void smp_send_stop(void)
650 {
651 cpumask_t mask = cpu_online_map;
652 cpu_clear(smp_processor_id(), mask);
653 send_ipi_message(mask, IPI_CPU_STOP);
654 }
655
656 /*
657 * not supported here
658 */
659 int setup_profiling_timer(unsigned int multiplier)
660 {
661 return -EINVAL;
662 }
663
664 static int
665 on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait,
666 cpumask_t mask)
667 {
668 int ret = 0;
669
670 preempt_disable();
671
672 ret = smp_call_function_on_cpu(func, info, retry, wait, mask);
673 if (cpu_isset(smp_processor_id(), mask))
674 func(info);
675
676 preempt_enable();
677
678 return ret;
679 }
680
681 /**********************************************************************/
682
683 /*
684 * TLB operations
685 */
686 struct tlb_args {
687 struct vm_area_struct *ta_vma;
688 unsigned long ta_start;
689 unsigned long ta_end;
690 };
691
692 static inline void ipi_flush_tlb_all(void *ignored)
693 {
694 local_flush_tlb_all();
695 }
696
697 static inline void ipi_flush_tlb_mm(void *arg)
698 {
699 struct mm_struct *mm = (struct mm_struct *)arg;
700
701 local_flush_tlb_mm(mm);
702 }
703
704 static inline void ipi_flush_tlb_page(void *arg)
705 {
706 struct tlb_args *ta = (struct tlb_args *)arg;
707
708 local_flush_tlb_page(ta->ta_vma, ta->ta_start);
709 }
710
711 static inline void ipi_flush_tlb_kernel_page(void *arg)
712 {
713 struct tlb_args *ta = (struct tlb_args *)arg;
714
715 local_flush_tlb_kernel_page(ta->ta_start);
716 }
717
718 static inline void ipi_flush_tlb_range(void *arg)
719 {
720 struct tlb_args *ta = (struct tlb_args *)arg;
721
722 local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
723 }
724
725 static inline void ipi_flush_tlb_kernel_range(void *arg)
726 {
727 struct tlb_args *ta = (struct tlb_args *)arg;
728
729 local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
730 }
731
732 void flush_tlb_all(void)
733 {
734 on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1);
735 }
736
737 void flush_tlb_mm(struct mm_struct *mm)
738 {
739 cpumask_t mask = mm->cpu_vm_mask;
740
741 on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask);
742 }
743
744 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
745 {
746 cpumask_t mask = vma->vm_mm->cpu_vm_mask;
747 struct tlb_args ta;
748
749 ta.ta_vma = vma;
750 ta.ta_start = uaddr;
751
752 on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask);
753 }
754
755 void flush_tlb_kernel_page(unsigned long kaddr)
756 {
757 struct tlb_args ta;
758
759 ta.ta_start = kaddr;
760
761 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1, 1);
762 }
763
764 void flush_tlb_range(struct vm_area_struct *vma,
765 unsigned long start, unsigned long end)
766 {
767 cpumask_t mask = vma->vm_mm->cpu_vm_mask;
768 struct tlb_args ta;
769
770 ta.ta_vma = vma;
771 ta.ta_start = start;
772 ta.ta_end = end;
773
774 on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask);
775 }
776
777 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
778 {
779 struct tlb_args ta;
780
781 ta.ta_start = start;
782 ta.ta_end = end;
783
784 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1, 1);
785 }
786
|
This page was automatically generated by the
LXR engine.
|