| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249 |
- /* SMP bring-up: ACPI CPU list, AP trampoline, INIT-SIPI and per-CPU idle.
- *
- * No MP-table fallback and no IOAPIC: this targets ACPI/MADT machines (QEMU
- * and the Atom board). If ACPI is absent or has one CPU, smp_init() leaves
- * the system exactly as it was.
- */
- #include "smp.h"
- #include "acpi.h"
- #include "cpu.h"
- #include "idle.h"
- #include "lapic.h"
- #include "paging.h"
- #include "process.h"
- #include "sched.h"
- #include "serial.h"
- #include "timer.h"
- #include "tss.h"
- #define AP_TRAMP_ADDR 0x6000u
- #define AP_INFO_ADDR 0x6F00u
- extern unsigned char _binary_ap_tramp_bin_start[];
- extern unsigned char _binary_ap_tramp_bin_end[];
- extern void idt_load_ap(void);
- static volatile unsigned int ap_started[MAX_CPUS];
- static unsigned int online_aps;
- unsigned int smp_sched_enabled;
- static void delay_ms(unsigned int ms)
- {
- unsigned int t0 = get_ticks_ms();
- while ((unsigned int)(get_ticks_ms() - t0) < ms)
- cpu_idle();
- }
- static int copy_trampoline(void)
- {
- unsigned int n = (unsigned int)(_binary_ap_tramp_bin_end -
- _binary_ap_tramp_bin_start);
- unsigned char *dst = (unsigned char *)AP_TRAMP_ADDR;
- unsigned int i;
- if (n == 0 || n > 0xC00)
- return 0;
- for (i = 0; i < n; i++)
- dst[i] = _binary_ap_tramp_bin_start[i];
- __asm__ volatile ("wbinvd" ::: "memory");
- return 1;
- }
- static void start_one_ap(unsigned int cpu, unsigned int top)
- {
- volatile unsigned int *info = (volatile unsigned int *)AP_INFO_ADDR;
- unsigned int apic = acpi_cpu_apic_id(cpu);
- unsigned int t0;
- info[0] = paging_kernel_cr3();
- info[1] = top;
- info[2] = (unsigned int)smp_ap_main;
- info[3] = cpu;
- __sync_synchronize();
- ap_started[cpu] = 0;
- lapic_send_init(apic);
- delay_ms(10);
- lapic_send_sipi(apic, AP_TRAMP_ADDR >> 12);
- /* The spec recommends a second SIPI if the first one was missed; some
- * firmware/boards drop it while the AP is still finishing INIT. */
- t0 = get_ticks_ms();
- while ((unsigned int)(get_ticks_ms() - t0) < 10) {
- if (ap_started[cpu])
- return;
- cpu_idle();
- }
- if (!ap_started[cpu])
- lapic_send_sipi(apic, AP_TRAMP_ADDR >> 12);
- t0 = get_ticks_ms();
- while ((unsigned int)(get_ticks_ms() - t0) < 500) {
- if (ap_started[cpu])
- return;
- cpu_idle();
- }
- /* It did not check in; don't let it start later with stale info. */
- lapic_send_init(apic);
- }
- void smp_init(void)
- {
- unsigned int n = acpi_cpu_count();
- unsigned int i;
- online_aps = 0;
- if (n <= 1) {
- serial_puts(SERIAL_COM1, "smp: unavailable, 1 cpu\n");
- return;
- }
- if (!acpi_lapic_base())
- return;
- lapic_set_base(acpi_lapic_base());
- if (!lapic_init_bsp()) {
- serial_puts(SERIAL_COM1, "smp: BSP LAPIC init failed\n");
- return;
- }
- cpus[0].lapic_id = lapic_id();
- if (!copy_trampoline()) {
- serial_puts(SERIAL_COM1, "smp: bad AP trampoline, staying UP\n");
- return;
- }
- for (i = 1; i < n; i++) {
- unsigned int top = process_init_ap(i);
- if (!top) {
- serial_puts(SERIAL_COM1, "smp: no stack for AP\n");
- break;
- }
- start_one_ap(i, top);
- if (!ap_started[i]) {
- serial_puts(SERIAL_COM1, "smp: AP ");
- serial_putdec(SERIAL_COM1, i);
- serial_puts(SERIAL_COM1, " did not start\n");
- break;
- }
- online_aps++;
- cpu_count = online_aps + 1;
- }
- serial_puts(SERIAL_COM1, "smp: ");
- serial_putdec(SERIAL_COM1, cpu_count);
- serial_puts(SERIAL_COM1, " cpu online\n");
- }
- void smp_ap_main(unsigned int cpu)
- {
- struct process *idle;
- if (cpu == 0 || cpu >= MAX_CPUS) {
- for (;;)
- __asm__ volatile ("cli; hlt");
- }
- gdt_tss_init(cpu);
- idt_load_ap();
- /* An AP starts with the firmware's CR4; enable the SSE state before it
- * can reach the drawing fast paths (the BSP does this lazily). */
- cpu_enable_sse();
- cpus[cpu].id = cpu;
- cpus[cpu].lapic_id = lapic_id();
- idle = cpus[cpu].idle;
- if (!idle)
- for (;;)
- __asm__ volatile ("cli; hlt");
- cpus[cpu].current = idle;
- idle->state = PROC_RUNNING;
- idle->on_cpu = cpu;
- idle->last_cpu = cpu;
- cpus[cpu].tss->esp0 = idle->kstack_top;
- lapic_init_ap();
- if (!lapic_timer_start(LAPIC_TIMER_VECTOR))
- for (;;)
- __asm__ volatile ("cli; hlt");
- ap_started[cpu] = 1;
- __sync_synchronize();
- __asm__ volatile ("sti");
- for (;;)
- cpu_idle();
- }
- /* Called from the reschedule IPI stub (and from smp_resched_cpu when the
- * target is the current CPU): mark the CPU and shorten its idle timer so the
- * next tick schedules promptly even when the PIT/LAPIC period is 10 ms. */
- void smp_resched_wake(void)
- {
- struct cpu *c = this_cpu();
- c->need_resched = 1;
- if (c->id == 0) {
- timer_set_hz(1000);
- } else {
- c->tick_ms = 1;
- lapic_timer_set_ms(1);
- }
- }
- void smp_resched_cpu(unsigned int cpu)
- {
- if (cpu >= cpu_count || cpu >= MAX_CPUS)
- return;
- if (cpu == this_cpu()->id) {
- smp_resched_wake();
- return;
- }
- if (cpus[cpu].lapic_id == 0xFFFFFFFFu)
- return;
- lapic_send_ipi(cpus[cpu].lapic_id, LAPIC_RESCHED_VECTOR);
- }
- void smp_enable_sched(void)
- {
- unsigned int i;
- if (cpu_count <= 1)
- return;
- smp_sched_enabled = 1;
- for (i = 1; i < cpu_count; i++)
- smp_resched_cpu(i);
- }
- void smp_stop_aps(void)
- {
- unsigned int i, n;
- /* The kexec trampoline is about to overwrite the kernel. Stop taking
- * interrupts first (also on UP, so the final copy cannot be
- * preempted), park every AP, and wait long enough for INIT to take
- * effect. An AP may have been inside a scheduler critical section, so
- * the lock is reinitialized afterwards; the old run queue is dead
- * anyway and the new kernel re-creates everything. */
- __asm__ volatile ("cli");
- if (cpu_count <= 1)
- return;
- for (i = 1; i < cpu_count; i++) {
- if (cpus[i].lapic_id != 0xFFFFFFFFu)
- lapic_send_init(cpus[i].lapic_id);
- }
- /* A few million PAUSE iterations is well over the 10 ms the INIT
- * protocol requires on the Atom/P5 range, and costs nothing on the
- * few occasions a kexec runs. */
- for (n = 0; n < 4000000u; n++)
- __asm__ volatile ("pause");
- sched_lock_reset();
- }
- void smp_dump(void)
- {
- serial_puts(SERIAL_COM1, "smp: online ");
- serial_putdec(SERIAL_COM1, cpu_count);
- serial_puts(SERIAL_COM1, "\n");
- }
|