1 /*
2 * x86 SMP booting functions
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
6 * Copyright 2001 Andi Kleen, SuSE Labs.
7 *
8 * Much of the core SMP work is based on previous work by Thomas Radke, to
9 * whom a great many thanks are extended.
10 *
11 * Thanks to Intel for making available several different Pentium,
12 * Pentium Pro and Pentium-II/Xeon MP machines.
13 * Original development of Linux SMP code supported by Caldera.
14 *
15 * This code is released under the GNU General Public License version 2
16 *
17 * Fixes
18 * Felix Koop : NR_CPUS used properly
19 * Jose Renau : Handle single CPU case.
20 * Alan Cox : By repeated request 8) - Total BogoMIP report.
21 * Greg Wright : Fix for kernel stacks panic.
22 * Erich Boleyn : MP v1.4 and additional changes.
23 * Matthias Sattler : Changes for 2.1 kernel map.
24 * Michel Lespinasse : Changes for 2.1 kernel map.
25 * Michael Chastain : Change trampoline.S to gnu as.
26 * Alan Cox : Dumb bug: 'B' step PPro's are fine
27 * Ingo Molnar : Added APIC timers, based on code
28 * from Jose Renau
29 * Ingo Molnar : various cleanups and rewrites
30 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
31 * Maciej W. Rozycki : Bits for genuine 82489DX APICs
32 * Andi Kleen : Changed for SMP boot into long mode.
33 * Rusty Russell : Hacked into shape for new "hotplug" boot process.
34 * Andi Kleen : Converted to new state machine.
35 * Various cleanups.
36 * Probably mostly hotplug CPU ready now.
37 * Ashok Raj : CPU hotplug support
38 */
39
40
41 #include <linux/init.h>
42
43 #include <linux/mm.h>
44 #include <linux/kernel_stat.h>
45 #include <linux/bootmem.h>
46 #include <linux/thread_info.h>
47 #include <linux/module.h>
48 #include <linux/delay.h>
49 #include <linux/mc146818rtc.h>
50 #include <linux/smp.h>
51 #include <linux/kdebug.h>
52
53 #include <asm/mtrr.h>
54 #include <asm/pgalloc.h>
55 #include <asm/desc.h>
56 #include <asm/tlbflush.h>
57 #include <asm/proto.h>
58 #include <asm/nmi.h>
59 #include <asm/irq.h>
60 #include <asm/hw_irq.h>
61 #include <asm/numa.h>
62
63 /* Number of siblings per CPU package */
64 int smp_num_siblings = 1;
65 EXPORT_SYMBOL(smp_num_siblings);
66
67 /* Last level cache ID of each logical CPU */
68 DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
69
70 /* Bitmask of currently online CPUs */
71 cpumask_t cpu_online_map __read_mostly;
72
73 EXPORT_SYMBOL(cpu_online_map);
74
75 /*
76 * Private maps to synchronize booting between AP and BP.
77 * Probably not needed anymore, but it makes for easier debugging. -AK
78 */
79 cpumask_t cpu_callin_map;
80 cpumask_t cpu_callout_map;
81 cpumask_t cpu_possible_map;
82 EXPORT_SYMBOL(cpu_possible_map);
83
84 /* Per CPU bogomips and other parameters */
85 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
86 EXPORT_PER_CPU_SYMBOL(cpu_info);
87
88 /* Set when the idlers are all forked */
89 int smp_threads_ready;
90
91 /* representing HT siblings of each logical CPU */
92 DEFINE_PER_CPU(cpumask_t, cpu_sibling_map);
93 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
94
95 /* representing HT and core siblings of each logical CPU */
96 DEFINE_PER_CPU(cpumask_t, cpu_core_map);
97 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
98
99 /*
100 * Trampoline 80x86 program as an array.
101 */
102
103 extern const unsigned char trampoline_data[];
104 extern const unsigned char trampoline_end[];
105
106 /* State of each CPU */
107 DEFINE_PER_CPU(int, cpu_state) = { 0 };
108
109 /*
110 * Store all idle threads, this can be reused instead of creating
111 * a new thread. Also avoids complicated thread destroy functionality
112 * for idle threads.
113 */
114 #ifdef CONFIG_HOTPLUG_CPU
115 /*
116 * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
117 * removed after init for !CONFIG_HOTPLUG_CPU.
118 */
119 static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
120 #define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
121 #define set_idle_for_cpu(x,p) (per_cpu(idle_thread_array, x) = (p))
122 #else
123 struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
124 #define get_idle_for_cpu(x) (idle_thread_array[(x)])
125 #define set_idle_for_cpu(x,p) (idle_thread_array[(x)] = (p))
126 #endif
127
128
129 /*
130 * Currently trivial. Write the real->protected mode
131 * bootstrap into the page concerned. The caller
132 * has made sure it's suitably aligned.
133 */
134
135 static unsigned long __cpuinit setup_trampoline(void)
136 {
137 void *tramp = __va(SMP_TRAMPOLINE_BASE);
138 memcpy(tramp, trampoline_data, trampoline_end - trampoline_data);
139 return virt_to_phys(tramp);
140 }
141
142 /*
143 * The bootstrap kernel entry code has set these up. Save them for
144 * a given CPU
145 */
146
147 static void __cpuinit smp_store_cpu_info(int id)
148 {
149 struct cpuinfo_x86 *c = &cpu_data(id);
150
151 *c = boot_cpu_data;
152 c->cpu_index = id;
153 identify_cpu(c);
154 print_cpu_info(c);
155 }
156
157 static atomic_t init_deasserted __cpuinitdata;
158
159 /*
160 * Report back to the Boot Processor.
161 * Running on AP.
162 */
163 void __cpuinit smp_callin(void)
164 {
165 int cpuid, phys_id;
166 unsigned long timeout;
167
168 /*
169 * If waken up by an INIT in an 82489DX configuration
170 * we may get here before an INIT-deassert IPI reaches
171 * our local APIC. We have to wait for the IPI or we'll
172 * lock up on an APIC access.
173 */
174 while (!atomic_read(&init_deasserted))
175 cpu_relax();
176
177 /*
178 * (This works even if the APIC is not enabled.)
179 */
180 phys_id = GET_APIC_ID(apic_read(APIC_ID));
181 cpuid = smp_processor_id();
182 if (cpu_isset(cpuid, cpu_callin_map)) {
183 panic("smp_callin: phys CPU#%d, CPU#%d already present??\n",
184 phys_id, cpuid);
185 }
186 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
187
188 /*
189 * STARTUP IPIs are fragile beasts as they might sometimes
190 * trigger some glue motherboard logic. Complete APIC bus
191 * silence for 1 second, this overestimates the time the
192 * boot CPU is spending to send the up to 2 STARTUP IPIs
193 * by a factor of two. This should be enough.
194 */
195
196 /*
197 * Waiting 2s total for startup (udelay is not yet working)
198 */
199 timeout = jiffies + 2*HZ;
200 while (time_before(jiffies, timeout)) {
201 /*
202 * Has the boot CPU finished it's STARTUP sequence?
203 */
204 if (cpu_isset(cpuid, cpu_callout_map))
205 break;
206 cpu_relax();
207 }
208
209 if (!time_before(jiffies, timeout)) {
210 panic("smp_callin: CPU%d started up but did not get a callout!\n",
211 cpuid);
212 }
213
214 /*
215 * the boot CPU has finished the init stage and is spinning
216 * on callin_map until we finish. We are free to set up this
217 * CPU, first the APIC. (this is probably redundant on most
218 * boards)
219 */
220
221 Dprintk("CALLIN, before setup_local_APIC().\n");
222 setup_local_APIC();
223 end_local_APIC_setup();
224
225 /*
226 * Get our bogomips.
227 *
228 * Need to enable IRQs because it can take longer and then
229 * the NMI watchdog might kill us.
230 */
231 local_irq_enable();
232 calibrate_delay();
233 local_irq_disable();
234 Dprintk("Stack at about %p\n",&cpuid);
235
236 /*
237 * Save our processor parameters
238 */
239 smp_store_cpu_info(cpuid);
240
241 /*
242 * Allow the master to continue.
243 */
244 cpu_set(cpuid, cpu_callin_map);
245 }
246
247 /* maps the cpu to the sched domain representing multi-core */
248 cpumask_t cpu_coregroup_map(int cpu)
249 {
250 struct cpuinfo_x86 *c = &cpu_data(cpu);
251 /*
252 * For perf, we return last level cache shared map.
253 * And for power savings, we return cpu_core_map
254 */
255 if (sched_mc_power_savings || sched_smt_power_savings)
256 return per_cpu(cpu_core_map, cpu);
257 else
258 return c->llc_shared_map;
259 }
260
261 /* representing cpus for which sibling maps can be computed */
262 static cpumask_t cpu_sibling_setup_map;
263
264 static inline void set_cpu_sibling_map(int cpu)
265 {
266 int i;
267 struct cpuinfo_x86 *c = &cpu_data(cpu);
268
269 cpu_set(cpu, cpu_sibling_setup_map);
270
271 if (smp_num_siblings > 1) {
272 for_each_cpu_mask(i, cpu_sibling_setup_map) {
273 if (c->phys_proc_id == cpu_data(i).phys_proc_id &&
274 c->cpu_core_id == cpu_data(i).cpu_core_id) {
275 cpu_set(i, per_cpu(cpu_sibling_map, cpu));
276 cpu_set(cpu, per_cpu(cpu_sibling_map, i));
277 cpu_set(i, per_cpu(cpu_core_map, cpu));
278 cpu_set(cpu, per_cpu(cpu_core_map, i));
279 cpu_set(i, c->llc_shared_map);
280 cpu_set(cpu, cpu_data(i).llc_shared_map);
281 }
282 }
283 } else {
284 cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
285 }
286
287 cpu_set(cpu, c->llc_shared_map);
288
289 if (current_cpu_data.x86_max_cores == 1) {
290 per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu);
291 c->booted_cores = 1;
292 return;
293 }
294
295 for_each_cpu_mask(i, cpu_sibling_setup_map) {
296 if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
297 per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
298 cpu_set(i, c->llc_shared_map);
299 cpu_set(cpu, cpu_data(i).llc_shared_map);
300 }
301 if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
302 cpu_set(i, per_cpu(cpu_core_map, cpu));
303 cpu_set(cpu, per_cpu(cpu_core_map, i));
304 /*
305 * Does this new cpu bringup a new core?
306 */
307 if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) {
308 /*
309 * for each core in package, increment
310 * the booted_cores for this new cpu
311 */
312 if (first_cpu(per_cpu(cpu_sibling_map, i)) == i)
313 c->booted_cores++;
314 /*
315 * increment the core count for all
316 * the other cpus in this package
317 */
318 if (i != cpu)
319 cpu_data(i).booted_cores++;
320 } else if (i != cpu && !c->booted_cores)
321 c->booted_cores = cpu_data(i).booted_cores;
322 }
323 }
324 }
325
326 /*
327 * Setup code on secondary processor (after comming out of the trampoline)
328 */
329 void __cpuinit start_secondary(void)
330 {
331 /*
332 * Dont put anything before smp_callin(), SMP
333 * booting is too fragile that we want to limit the
334 * things done here to the most necessary things.
335 */
336 cpu_init();
337 preempt_disable();
338 smp_callin();
339
340 /* otherwise gcc will move up the smp_processor_id before the cpu_init */
341 barrier();
342
343 /*
344 * Check TSC sync first:
345 */
346 check_tsc_sync_target();
347
348 if (nmi_watchdog == NMI_IO_APIC) {
349 disable_8259A_irq(0);
350 enable_NMI_through_LVT0();
351 enable_8259A_irq(0);
352 }
353
354 /*
355 * The sibling maps must be set before turing the online map on for
356 * this cpu
357 */
358 set_cpu_sibling_map(smp_processor_id());
359
360 /*
361 * We need to hold call_lock, so there is no inconsistency
362 * between the time smp_call_function() determines number of
363 * IPI recipients, and the time when the determination is made
364 * for which cpus receive the IPI in genapic_flat.c. Holding this
365 * lock helps us to not include this cpu in a currently in progress
366 * smp_call_function().
367 */
368 lock_ipi_call_lock();
369 spin_lock(&vector_lock);
370
371 /* Setup the per cpu irq handling data structures */
372 __setup_vector_irq(smp_processor_id());
373 /*
374 * Allow the master to continue.
375 */
376 cpu_set(smp_processor_id(), cpu_online_map);
377 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
378 spin_unlock(&vector_lock);
379
380 unlock_ipi_call_lock();
381
382 setup_secondary_clock();
383
384 cpu_idle();
385 }
386
387 extern volatile unsigned long init_rsp;
388 extern void (*initial_code)(void);
389
390 #ifdef APIC_DEBUG
391 static void inquire_remote_apic(int apicid)
392 {
393 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
394 char *names[] = { "ID", "VERSION", "SPIV" };
395 int timeout;
396 u32 status;
397
398 printk(KERN_INFO "Inquiring remote APIC #%d...\n", apicid);
399
400 for (i = 0; i < ARRAY_SIZE(regs); i++) {
401 printk(KERN_INFO "... APIC #%d %s: ", apicid, names[i]);
402
403 /*
404 * Wait for idle.
405 */
406 status = safe_apic_wait_icr_idle();
407 if (status)
408 printk(KERN_CONT
409 "a previous APIC delivery may have failed\n");
410
411 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
412 apic_write(APIC_ICR, APIC_DM_REMRD | regs[i]);
413
414 timeout = 0;
415 do {
416 udelay(100);
417 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
418 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
419
420 switch (status) {
421 case APIC_ICR_RR_VALID:
422 status = apic_read(APIC_RRR);
423 printk(KERN_CONT "%08x\n", status);
424 break;
425 default:
426 printk(KERN_CONT "failed\n");
427 }
428 }
429 }
430 #endif
431
432 /*
433 * Kick the secondary to wake up.
434 */
435 static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int start_rip)
436 {
437 unsigned long send_status, accept_status = 0;
438 int maxlvt, num_starts, j;
439
440 Dprintk("Asserting INIT.\n");
441
442 /*
443 * Turn INIT on target chip
444 */
445 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
446
447 /*
448 * Send IPI
449 */
450 apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
451 | APIC_DM_INIT);
452
453 Dprintk("Waiting for send to finish...\n");
454 send_status = safe_apic_wait_icr_idle();
455
456 mdelay(10);
457
458 Dprintk("Deasserting INIT.\n");
459
460 /* Target chip */
461 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
462
463 /* Send IPI */
464 apic_write(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
465
466 Dprintk("Waiting for send to finish...\n");
467 send_status = safe_apic_wait_icr_idle();
468
469 mb();
470 atomic_set(&init_deasserted, 1);
471
472 num_starts = 2;
473
474 /*
475 * Run STARTUP IPI loop.
476 */
477 Dprintk("#startup loops: %d.\n", num_starts);
478
479 maxlvt = lapic_get_maxlvt();
480
481 for (j = 1; j <= num_starts; j++) {
482 Dprintk("Sending STARTUP #%d.\n",j);
483 apic_write(APIC_ESR, 0);
484 apic_read(APIC_ESR);
485 Dprintk("After apic_write.\n");
486
487 /*
488 * STARTUP IPI
489 */
490
491 /* Target chip */
492 apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
493
494 /* Boot on the stack */
495 /* Kick the second */
496 apic_write(APIC_ICR, APIC_DM_STARTUP | (start_rip >> 12));
497
498 /*
499 * Give the other CPU some time to accept the IPI.
500 */
501 udelay(300);
502
503 Dprintk("Startup point 1.\n");
504
505 Dprintk("Waiting for send to finish...\n");
506 send_status = safe_apic_wait_icr_idle();
507
508 /*
509 * Give the other CPU some time to accept the IPI.
510 */
511 udelay(200);
512 /*
513 * Due to the Pentium erratum 3AP.
514 */
515 if (maxlvt > 3) {
516 apic_write(APIC_ESR, 0);
517 }
518 accept_status = (apic_read(APIC_ESR) & 0xEF);
519 if (send_status || accept_status)
520 break;
521 }
522 Dprintk("After Startup.\n");
523
524 if (send_status)
525 printk(KERN_ERR "APIC never delivered???\n");
526 if (accept_status)
527 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
528
529 return (send_status | accept_status);
530 }
531
532 struct create_idle {
533 struct work_struct work;
534 struct task_struct *idle;
535 struct completion done;
536 int cpu;
537 };
538
539 static void __cpuinit do_fork_idle(struct work_struct *work)
540 {
541 struct create_idle *c_idle =
542 container_of(work, struct create_idle, work);
543
544 c_idle->idle = fork_idle(c_idle->cpu);
545 complete(&c_idle->done);
546 }
547
548 /*
549 * Boot one CPU.
550 */
551 static int __cpuinit do_boot_cpu(int cpu, int apicid)
552 {
553 unsigned long boot_error;
554 int timeout;
555 unsigned long start_rip;
556 struct create_idle c_idle = {
557 .cpu = cpu,
558 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
559 };
560 INIT_WORK(&c_idle.work, do_fork_idle);
561
562 /* allocate memory for gdts of secondary cpus. Hotplug is considered */
563 if (!cpu_gdt_descr[cpu].address &&
564 !(cpu_gdt_descr[cpu].address = get_zeroed_page(GFP_KERNEL))) {
565 printk(KERN_ERR "Failed to allocate GDT for CPU %d\n", cpu);
566 return -1;
567 }
568
569 /* Allocate node local memory for AP pdas */
570 if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) {
571 struct x8664_pda *newpda, *pda;
572 int node = cpu_to_node(cpu);
573 pda = cpu_pda(cpu);
574 newpda = kmalloc_node(sizeof (struct x8664_pda), GFP_ATOMIC,
575 node);
576 if (newpda) {
577 memcpy(newpda, pda, sizeof (struct x8664_pda));
578 cpu_pda(cpu) = newpda;
579 } else
580 printk(KERN_ERR
581 "Could not allocate node local PDA for CPU %d on node %d\n",
582 cpu, node);
583 }
584
585 alternatives_smp_switch(1);
586
587 c_idle.idle = get_idle_for_cpu(cpu);
588
589 if (c_idle.idle) {
590 c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
591 (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
592 init_idle(c_idle.idle, cpu);
593 goto do_rest;
594 }
595
596 /*
597 * During cold boot process, keventd thread is not spun up yet.
598 * When we do cpu hot-add, we create idle threads on the fly, we should
599 * not acquire any attributes from the calling context. Hence the clean
600 * way to create kernel_threads() is to do that from keventd().
601 * We do the current_is_keventd() due to the fact that ACPI notifier
602 * was also queuing to keventd() and when the caller is already running
603 * in context of keventd(), we would end up with locking up the keventd
604 * thread.
605 */
606 if (!keventd_up() || current_is_keventd())
607 c_idle.work.func(&c_idle.work);
608 else {
609 schedule_work(&c_idle.work);
610 wait_for_completion(&c_idle.done);
611 }
612
613 if (IS_ERR(c_idle.idle)) {
614 printk("failed fork for CPU %d\n", cpu);
615 return PTR_ERR(c_idle.idle);
616 }
617
618 set_idle_for_cpu(cpu, c_idle.idle);
619
620 do_rest:
621
622 cpu_pda(cpu)->pcurrent = c_idle.idle;
623
624 start_rip = setup_trampoline();
625
626 init_rsp = c_idle.idle->thread.sp;
627 load_sp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
628 initial_code = start_secondary;
629 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
630
631 printk(KERN_INFO "Booting processor %d/%d APIC 0x%x\n", cpu,
632 cpus_weight(cpu_present_map),
633 apicid);
634
635 /*
636 * This grunge runs the startup process for
637 * the targeted processor.
638 */
639
640 atomic_set(&init_deasserted, 0);
641
642 Dprintk("Setting warm reset code and vector.\n");
643
644 CMOS_WRITE(0xa, 0xf);
645 local_flush_tlb();
646 Dprintk("1.\n");
647 *((volatile unsigned short *) phys_to_virt(0x469)) = start_rip >> 4;
648 Dprintk("2.\n");
649 *((volatile unsigned short *) phys_to_virt(0x467)) = start_rip & 0xf;
650 Dprintk("3.\n");
651
652 /*
653 * Be paranoid about clearing APIC errors.
654 */
655 apic_write(APIC_ESR, 0);
656 apic_read(APIC_ESR);
657
658 /*
659 * Status is now clean
660 */
661 boot_error = 0;
662
663 /*
664 * Starting actual IPI sequence...
665 */
666 boot_error = wakeup_secondary_via_INIT(apicid, start_rip);
667
668 if (!boot_error) {
669 /*
670 * allow APs to start initializing.
671 */
672 Dprintk("Before Callout %d.\n", cpu);
673 cpu_set(cpu, cpu_callout_map);
674 Dprintk("After Callout %d.\n", cpu);
675
676 /*
677 * Wait 5s total for a response
678 */
679 for (timeout = 0; timeout < 50000; timeout++) {
680 if (cpu_isset(cpu, cpu_callin_map))
681 break; /* It has booted */
682 udelay(100);
683 }
684
685 if (cpu_isset(cpu, cpu_callin_map)) {
686 /* number CPUs logically, starting from 1 (BSP is 0) */
687 Dprintk("CPU has booted.\n");
688 } else {
689 boot_error = 1;
690 if (*((volatile unsigned char *)phys_to_virt(SMP_TRAMPOLINE_BASE))
691 == 0xA5)
692 /* trampoline started but...? */
693 printk("Stuck ??\n");
694 else
695 /* trampoline code not run */
696 printk("Not responding.\n");
697 #ifdef APIC_DEBUG
698 inquire_remote_apic(apicid);
699 #endif
700 }
701 }
702 if (boot_error) {
703 cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
704 clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
705 clear_node_cpumask(cpu); /* was set by numa_add_cpu */
706 cpu_clear(cpu, cpu_present_map);
707 cpu_clear(cpu, cpu_possible_map);
708 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
709 return -EIO;
710 }
711
712 return 0;
713 }
714
715 cycles_t cacheflush_time;
716 unsigned long cache_decay_ticks;
717
718 /*
719 * Cleanup possible dangling ends...
720 */
721 static __cpuinit void smp_cleanup_boot(void)
722 {
723 /*
724 * Paranoid: Set warm reset code and vector here back
725 * to default values.
726 */
727 CMOS_WRITE(0, 0xf);
728
729 /*
730 * Reset trampoline flag
731 */
732 *((volatile int *) phys_to_virt(0x467)) = 0;
733 }
734
735 /*
736 * Fall back to non SMP mode after errors.
737 *
738 * RED-PEN audit/test this more. I bet there is more state messed up here.
739 */
740 static __init void disable_smp(void)
741 {
742 cpu_present_map = cpumask_of_cpu(0);
743 cpu_possible_map = cpumask_of_cpu(0);
744 if (smp_found_config)
745 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id);
746 else
747 phys_cpu_present_map = physid_mask_of_physid(0);
748 cpu_set(0, per_cpu(cpu_sibling_map, 0));
749 cpu_set(0, per_cpu(cpu_core_map, 0));
750 }
751
752 #ifdef CONFIG_HOTPLUG_CPU
753
754 int additional_cpus __initdata = -1;
755
756 /*
757 * cpu_possible_map should be static, it cannot change as cpu's
758 * are onlined, or offlined. The reason is per-cpu data-structures
759 * are allocated by some modules at init time, and dont expect to
760 * do this dynamically on cpu arrival/departure.
761 * cpu_present_map on the other hand can change dynamically.
762 * In case when cpu_hotplug is not compiled, then we resort to current
763 * behaviour, which is cpu_possible == cpu_present.
764 * - Ashok Raj
765 *
766 * Three ways to find out the number of additional hotplug CPUs:
767 * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
768 * - The user can overwrite it with additional_cpus=NUM
769 * - Otherwise don't reserve additional CPUs.
770 * We do this because additional CPUs waste a lot of memory.
771 * -AK
772 */
773 __init void prefill_possible_map(void)
774 {
775 int i;
776 int possible;
777
778 if (additional_cpus == -1) {
779 if (disabled_cpus > 0)
780 additional_cpus = disabled_cpus;
781 else
782 additional_cpus = 0;
783 }
784 possible = num_processors + additional_cpus;
785 if (possible > NR_CPUS)
786 possible = NR_CPUS;
787
788 printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
789 possible,
790 max_t(int, possible - num_processors, 0));
791
792 for (i = 0; i < possible; i++)
793 cpu_set(i, cpu_possible_map);
794 }
795 #endif
796
797 /*
798 * Various sanity checks.
799 */
800 static int __init smp_sanity_check(unsigned max_cpus)
801 {
802 if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
803 printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
804 hard_smp_processor_id());
805 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
806 }
807
808 /*
809 * If we couldn't find an SMP configuration at boot time,
810 * get out of here now!
811 */
812 if (!smp_found_config) {
813 printk(KERN_NOTICE "SMP motherboard not detected.\n");
814 disable_smp();
815 if (APIC_init_uniprocessor())
816 printk(KERN_NOTICE "Local APIC not detected."
817 " Using dummy APIC emulation.\n");
818 return -1;
819 }
820
821 /*
822 * Should not be necessary because the MP table should list the boot
823 * CPU too, but we do it for the sake of robustness anyway.
824 */
825 if (!physid_isset(boot_cpu_id, phys_cpu_present_map)) {
826 printk(KERN_NOTICE "weird, boot CPU (#%d) not listed by the BIOS.\n",
827 boot_cpu_id);
828 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
829 }
830
831 /*
832 * If we couldn't find a local APIC, then get out of here now!
833 */
834 if (!cpu_has_apic) {
835 printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
836 boot_cpu_id);
837 printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
838 nr_ioapics = 0;
839 return -1;
840 }
841
842 /*
843 * If SMP should be disabled, then really disable it!
844 */
845 if (!max_cpus) {
846 printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
847 nr_ioapics = 0;
848 return -1;
849 }
850
851 return 0;
852 }
853
854 static void __init smp_cpu_index_default(void)
855 {
856 int i;
857 struct cpuinfo_x86 *c;
858
859 for_each_cpu_mask(i, cpu_possible_map) {
860 c = &cpu_data(i);
861 /* mark all to hotplug */
862 c->cpu_index = NR_CPUS;
863 }
864 }
865
866 /*
867 * Prepare for SMP bootup. The MP table or ACPI has been read
868 * earlier. Just do some sanity checking here and enable APIC mode.
869 */
870 void __init smp_prepare_cpus(unsigned int max_cpus)
871 {
872 smp_cpu_index_default();
873 current_cpu_data = boot_cpu_data;
874 current_thread_info()->cpu = 0; /* needed? */
875 set_cpu_sibling_map(0);
876
877 if (smp_sanity_check(max_cpus) < 0) {
878 printk(KERN_INFO "SMP disabled\n");
879 disable_smp();
880 return;
881 }
882
883
884 /*
885 * Switch from PIC to APIC mode.
886 */
887 setup_local_APIC();
888
889 /*
890 * Enable IO APIC before setting up error vector
891 */
892 if (!skip_ioapic_setup && nr_ioapics)
893 enable_IO_APIC();
894 end_local_APIC_setup();
895
896 if (GET_APIC_ID(apic_read(APIC_ID)) != boot_cpu_id) {
897 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
898 GET_APIC_ID(apic_read(APIC_ID)), boot_cpu_id);
899 /* Or can we switch back to PIC here? */
900 }
901
902 /*
903 * Now start the IO-APICs
904 */
905 if (!skip_ioapic_setup && nr_ioapics)
906 setup_IO_APIC();
907 else
908 nr_ioapics = 0;
909
910 /*
911 * Set up local APIC timer on boot CPU.
912 */
913
914 setup_boot_clock();
915 }
916
917 /*
918 * Early setup to make printk work.
919 */
920 void __init smp_prepare_boot_cpu(void)
921 {
922 int me = smp_processor_id();
923 /* already set me in cpu_online_map in boot_cpu_init() */
924 cpu_set(me, cpu_callout_map);
925 per_cpu(cpu_state, me) = CPU_ONLINE;
926 }
927
928 /*
929 * Entry point to boot a CPU.
930 */
931 int __cpuinit __cpu_up(unsigned int cpu)
932 {
933 int apicid = cpu_present_to_apicid(cpu);
934 unsigned long flags;
935 int err;
936
937 WARN_ON(irqs_disabled());
938
939 Dprintk("++++++++++++++++++++=_---CPU UP %u\n", cpu);
940
941 if (apicid == BAD_APICID || apicid == boot_cpu_id ||
942 !physid_isset(apicid, phys_cpu_present_map)) {
943 printk("__cpu_up: bad cpu %d\n", cpu);
944 return -EINVAL;
945 }
946
947 /*
948 * Already booted CPU?
949 */
950 if (cpu_isset(cpu, cpu_callin_map)) {
951 Dprintk("do_boot_cpu %d Already started\n", cpu);
952 return -ENOSYS;
953 }
954
955 /*
956 * Save current MTRR state in case it was changed since early boot
957 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
958 */
959 mtrr_save_state();
960
961 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
962 /* Boot it! */
963 err = do_boot_cpu(cpu, apicid);
964 if (err < 0) {
965 Dprintk("do_boot_cpu failed %d\n", err);
966 return err;
967 }
968
969 /* Unleash the CPU! */
970 Dprintk("waiting for cpu %d\n", cpu);
971
972 /*
973 * Make sure and check TSC sync:
974 */
975 local_irq_save(flags);
976 check_tsc_sync_source(cpu);
977 local_irq_restore(flags);
978
979 while (!cpu_isset(cpu, cpu_online_map))
980 cpu_relax();
981 err = 0;
982
983 return err;
984 }
985
986 /*
987 * Finish the SMP boot.
988 */
989 void __init smp_cpus_done(unsigned int max_cpus)
990 {
991 smp_cleanup_boot();
992 setup_ioapic_dest();
993 check_nmi_watchdog();
994 }
995
996 #ifdef CONFIG_HOTPLUG_CPU
997
998 static void remove_siblinginfo(int cpu)
999 {
1000 int sibling;
1001 struct cpuinfo_x86 *c = &cpu_data(cpu);
1002
1003 for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) {
1004 cpu_clear(cpu, per_cpu(cpu_core_map, sibling));
1005 /*
1006 * last thread sibling in this cpu core going down
1007 */
1008 if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1)
1009 cpu_data(sibling).booted_cores--;
1010 }
1011
1012 for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu))
1013 cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling));
1014 cpus_clear(per_cpu(cpu_sibling_map, cpu));
1015 cpus_clear(per_cpu(cpu_core_map, cpu));
1016 c->phys_proc_id = 0;
1017 c->cpu_core_id = 0;
1018 cpu_clear(cpu, cpu_sibling_setup_map);
1019 }
1020
1021 static void __ref remove_cpu_from_maps(void)
1022 {
1023 int cpu = smp_processor_id();
1024
1025 cpu_clear(cpu, cpu_callout_map);
1026 cpu_clear(cpu, cpu_callin_map);
1027 clear_bit(cpu, (unsigned long *)&cpu_initialized); /* was set by cpu_init() */
1028 clear_node_cpumask(cpu);
1029 }
1030
1031 int __cpu_disable(void)
1032 {
1033 int cpu = smp_processor_id();
1034
1035 /*
1036 * Perhaps use cpufreq to drop frequency, but that could go
1037 * into generic code.
1038 *
1039 * We won't take down the boot processor on i386 due to some
1040 * interrupts only being able to be serviced by the BSP.
1041 * Especially so if we're not using an IOAPIC -zwane
1042 */
1043 if (cpu == 0)
1044 return -EBUSY;
1045
1046 if (nmi_watchdog == NMI_LOCAL_APIC)
1047 stop_apic_nmi_watchdog(NULL);
1048 clear_local_APIC();
1049
1050 /*
1051 * HACK:
1052 * Allow any queued timer interrupts to get serviced
1053 * This is only a temporary solution until we cleanup
1054 * fixup_irqs as we do for IA64.
1055 */
1056 local_irq_enable();
1057 mdelay(1);
1058
1059 local_irq_disable();
1060 remove_siblinginfo(cpu);
1061
1062 spin_lock(&vector_lock);
1063 /* It's now safe to remove this processor from the online map */
1064 cpu_clear(cpu, cpu_online_map);
1065 spin_unlock(&vector_lock);
1066 remove_cpu_from_maps();
1067 fixup_irqs(cpu_online_map);
1068 return 0;
1069 }
1070
1071 void __cpu_die(unsigned int cpu)
1072 {
1073 /* We don't do anything here: idle task is faking death itself. */
1074 unsigned int i;
1075
1076 for (i = 0; i < 10; i++) {
1077 /* They ack this in play_dead by setting CPU_DEAD */
1078 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1079 printk ("CPU %d is now offline\n", cpu);
1080 if (1 == num_online_cpus())
1081 alternatives_smp_switch(0);
1082 return;
1083 }
1084 msleep(100);
1085 }
1086 printk(KERN_ERR "CPU %u didn't die...\n", cpu);
1087 }
1088
1089 static __init int setup_additional_cpus(char *s)
1090 {
1091 return s && get_option(&s, &additional_cpus) ? 0 : -EINVAL;
1092 }
1093 early_param("additional_cpus", setup_additional_cpus);
1094
1095 #else /* ... !CONFIG_HOTPLUG_CPU */
1096
1097 int __cpu_disable(void)
1098 {
1099 return -ENOSYS;
1100 }
1101
1102 void __cpu_die(unsigned int cpu)
1103 {
1104 /* We said "no" in __cpu_disable */
1105 BUG();
1106 }
1107 #endif /* CONFIG_HOTPLUG_CPU */
1108
|
This page was automatically generated by the
LXR engine.
|