smp.c 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. /* SMP bring-up: ACPI CPU list, AP trampoline, INIT-SIPI and per-CPU idle.
  2. *
  3. * No MP-table fallback and no IOAPIC: this targets ACPI/MADT machines (QEMU
  4. * and the Atom board). If ACPI is absent or has one CPU, smp_init() leaves
  5. * the system exactly as it was.
  6. */
  7. #include "smp.h"
  8. #include "acpi.h"
  9. #include "bootcon.h"
  10. #include "cpu.h"
  11. #include "idle.h"
  12. #include "lapic.h"
  13. #include "paging.h"
  14. #include "process.h"
  15. #include "sched.h"
  16. #include "serial.h"
  17. #include "timer.h"
  18. #include "tss.h"
  19. #define AP_TRAMP_ADDR 0x6000u
  20. #define AP_INFO_ADDR 0x6F00u
  21. extern unsigned char _binary_ap_tramp_bin_start[];
  22. extern unsigned char _binary_ap_tramp_bin_end[];
  23. extern void idt_load_ap(void);
  24. static volatile unsigned int ap_started[MAX_CPUS];
  25. static unsigned int online_aps;
  26. unsigned int smp_sched_enabled;
  27. static void delay_ms(unsigned int ms)
  28. {
  29. unsigned int t0 = get_ticks_ms();
  30. while ((unsigned int)(get_ticks_ms() - t0) < ms)
  31. cpu_idle();
  32. }
  33. static int copy_trampoline(void)
  34. {
  35. unsigned int n = (unsigned int)(_binary_ap_tramp_bin_end -
  36. _binary_ap_tramp_bin_start);
  37. unsigned char *dst = (unsigned char *)AP_TRAMP_ADDR;
  38. unsigned int i;
  39. if (n == 0 || n > 0xC00)
  40. return 0;
  41. for (i = 0; i < n; i++)
  42. dst[i] = _binary_ap_tramp_bin_start[i];
  43. __asm__ volatile ("wbinvd" ::: "memory");
  44. return 1;
  45. }
  46. static void start_one_ap(unsigned int cpu, unsigned int top)
  47. {
  48. volatile unsigned int *info = (volatile unsigned int *)AP_INFO_ADDR;
  49. unsigned int apic = acpi_cpu_apic_id(cpu);
  50. unsigned int t0;
  51. info[0] = paging_kernel_cr3();
  52. info[1] = top;
  53. info[2] = (unsigned int)smp_ap_main;
  54. info[3] = cpu;
  55. info[4] = 0;
  56. __sync_synchronize();
  57. ap_started[cpu] = 0;
  58. lapic_send_init(apic);
  59. delay_ms(10);
  60. lapic_send_sipi(apic, AP_TRAMP_ADDR >> 12);
  61. /* The spec recommends a second SIPI if the first one was missed; some
  62. * firmware/boards drop it while the AP is still finishing INIT. */
  63. t0 = get_ticks_ms();
  64. while ((unsigned int)(get_ticks_ms() - t0) < 10) {
  65. if (ap_started[cpu])
  66. return;
  67. cpu_idle();
  68. }
  69. if (!ap_started[cpu])
  70. lapic_send_sipi(apic, AP_TRAMP_ADDR >> 12);
  71. t0 = get_ticks_ms();
  72. while ((unsigned int)(get_ticks_ms() - t0) < 500) {
  73. if (ap_started[cpu])
  74. return;
  75. cpu_idle();
  76. }
  77. /* It did not check in; don't let it start later with stale info. */
  78. lapic_send_init(apic);
  79. }
  80. void smp_init(void)
  81. {
  82. unsigned int n = acpi_cpu_count();
  83. unsigned int i;
  84. online_aps = 0;
  85. if (n <= 1) {
  86. serial_puts(SERIAL_COM1, "smp: unavailable, 1 cpu\n");
  87. return;
  88. }
  89. if (!acpi_lapic_base())
  90. return;
  91. lapic_set_base(acpi_lapic_base());
  92. if (!lapic_init_bsp()) {
  93. serial_puts(SERIAL_COM1, "smp: BSP LAPIC init failed\n");
  94. return;
  95. }
  96. cpus[0].lapic_id = lapic_id();
  97. if (!copy_trampoline()) {
  98. serial_puts(SERIAL_COM1, "smp: bad AP trampoline, staying UP\n");
  99. return;
  100. }
  101. for (i = 1; i < n; i++) {
  102. unsigned int top = process_init_ap(i);
  103. if (!top) {
  104. serial_puts(SERIAL_COM1, "smp: no stack for AP\n");
  105. break;
  106. }
  107. bootcon_puts("smp: ap ");
  108. bootcon_dec(i);
  109. bootcon_puts("\n");
  110. start_one_ap(i, top);
  111. if (!ap_started[i]) {
  112. serial_puts(SERIAL_COM1, "smp: AP ");
  113. serial_putdec(SERIAL_COM1, i);
  114. serial_puts(SERIAL_COM1, " did not start\n");
  115. break;
  116. }
  117. online_aps++;
  118. cpu_count = online_aps + 1;
  119. bootcon_puts("smp: ap ok\n");
  120. /* CR0 as the AP woke up (its CD/NW are cleared by the
  121. * trampoline): 0x60000011 means it would have run uncached. */
  122. serial_puts(SERIAL_COM1, "smp: AP ");
  123. serial_putdec(SERIAL_COM1, i);
  124. serial_puts(SERIAL_COM1, " start cr0 ");
  125. serial_puthex(SERIAL_COM1,
  126. ((volatile unsigned int *)AP_INFO_ADDR)[4]);
  127. serial_puts(SERIAL_COM1, "\n");
  128. }
  129. serial_puts(SERIAL_COM1, "smp: ");
  130. serial_putdec(SERIAL_COM1, cpu_count);
  131. serial_puts(SERIAL_COM1, " cpu online\n");
  132. }
  133. void smp_ap_main(unsigned int cpu)
  134. {
  135. struct process *idle;
  136. if (cpu == 0 || cpu >= MAX_CPUS) {
  137. for (;;)
  138. __asm__ volatile ("cli; hlt");
  139. }
  140. gdt_tss_init(cpu);
  141. idt_load_ap();
  142. /* An AP starts with the firmware's CR4; enable the SSE state before it
  143. * can reach the drawing fast paths (the BSP does this lazily). */
  144. cpu_enable_sse();
  145. cpu_init_fpu();
  146. cpus[cpu].id = cpu;
  147. cpus[cpu].lapic_id = lapic_id();
  148. idle = cpus[cpu].idle;
  149. if (!idle)
  150. for (;;)
  151. __asm__ volatile ("cli; hlt");
  152. cpus[cpu].current = idle;
  153. idle->state = PROC_RUNNING;
  154. idle->on_cpu = cpu;
  155. idle->last_cpu = cpu;
  156. cpus[cpu].tss->esp0 = idle->kstack_top;
  157. lapic_init_ap();
  158. if (!lapic_timer_start(LAPIC_TIMER_VECTOR))
  159. for (;;)
  160. __asm__ volatile ("cli; hlt");
  161. ap_started[cpu] = 1;
  162. __sync_synchronize();
  163. __asm__ volatile ("sti");
  164. for (;;)
  165. cpu_idle();
  166. }
  167. /* Called from the reschedule IPI stub (and from smp_resched_cpu when the
  168. * target is the current CPU): mark the CPU and shorten its idle timer so the
  169. * next tick schedules promptly even when the PIT/LAPIC period is 10 ms. */
  170. void smp_resched_wake(void)
  171. {
  172. struct cpu *c = this_cpu();
  173. c->need_resched = 1;
  174. if (c->id == 0) {
  175. timer_set_hz(1000);
  176. } else {
  177. c->tick_ms = 1;
  178. lapic_timer_set_ms(1);
  179. }
  180. }
  181. void smp_resched_cpu(unsigned int cpu)
  182. {
  183. if (cpu >= cpu_count || cpu >= MAX_CPUS)
  184. return;
  185. if (cpu == this_cpu()->id) {
  186. smp_resched_wake();
  187. return;
  188. }
  189. if (cpus[cpu].lapic_id == 0xFFFFFFFFu)
  190. return;
  191. lapic_send_ipi(cpus[cpu].lapic_id, LAPIC_RESCHED_VECTOR);
  192. }
  193. void smp_enable_sched(void)
  194. {
  195. unsigned int i;
  196. if (cpu_count <= 1)
  197. return;
  198. smp_sched_enabled = 1;
  199. for (i = 1; i < cpu_count; i++)
  200. smp_resched_cpu(i);
  201. }
  202. void smp_stop_aps(void)
  203. {
  204. unsigned int i, n;
  205. /* The kexec trampoline is about to overwrite the kernel. Stop taking
  206. * interrupts first (also on UP, so the final copy cannot be
  207. * preempted), park every AP, and wait long enough for INIT to take
  208. * effect. An AP may have been inside a scheduler critical section, so
  209. * the lock is reinitialized afterwards; the old run queue is dead
  210. * anyway and the new kernel re-creates everything. */
  211. __asm__ volatile ("cli");
  212. if (cpu_count <= 1)
  213. return;
  214. for (i = 1; i < cpu_count; i++) {
  215. if (cpus[i].lapic_id != 0xFFFFFFFFu)
  216. lapic_send_init(cpus[i].lapic_id);
  217. }
  218. /* A few million PAUSE iterations is well over the 10 ms the INIT
  219. * protocol requires on the Atom/P5 range, and costs nothing on the
  220. * few occasions a kexec runs. */
  221. for (n = 0; n < 4000000u; n++)
  222. __asm__ volatile ("pause");
  223. sched_lock_reset();
  224. }
  225. void smp_dump(void)
  226. {
  227. serial_puts(SERIAL_COM1, "smp: online ");
  228. serial_putdec(SERIAL_COM1, cpu_count);
  229. serial_puts(SERIAL_COM1, "\n");
  230. }