kernel.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572
  1. /* Kernel: entry point at 0x7E00, timer (PIT 1 kHz), process init, terminal. */
  2. #include "acpi.h"
  3. #include "asix.h"
  4. #include "bootcon.h"
  5. #include "bootinfo.h"
  6. #include "cpu.h"
  7. #include "idle.h"
  8. #include "io.h"
  9. #include "lapic.h"
  10. #include "panic.h"
  11. #include "kmalloc.h"
  12. #include "mm.h"
  13. #include "mouse.h"
  14. #include "netdev.h"
  15. #include "net.h"
  16. #include "dhcp.h"
  17. #include "netlog.h"
  18. #include "paging.h"
  19. #include "power.h"
  20. #include "process.h"
  21. #include "ramfs.h"
  22. #include "sched.h"
  23. #include "sys_gfx.h"
  24. #include "thermal.h"
  25. #include "timer.h"
  26. #include "tpec.h"
  27. #include "tss.h"
  28. #include "usbhid.h"
  29. #include "wm.h"
  30. #include "serial.h"
  31. #include "smp.h"
  32. #include "version.h"
  33. #include "vga.h"
  34. #include "vfs.h"
  35. static inline void outb(unsigned short port, unsigned char v)
  36. {
  37. __asm__ volatile ("outb %0, %1" : : "a"(v), "Nd"(port));
  38. }
  39. volatile unsigned int ticks;
  40. static unsigned int timer_hz;
  41. static unsigned int tick_ms = 1;
  42. struct idt_entry {
  43. unsigned short offset_lo;
  44. unsigned short selector;
  45. unsigned char zero;
  46. unsigned char type;
  47. unsigned short offset_hi;
  48. } __attribute__((packed));
  49. struct idtr_desc {
  50. unsigned short limit;
  51. unsigned int base;
  52. } __attribute__((packed));
  53. static struct idt_entry idt[256];
  54. static struct idtr_desc idtr;
  55. extern void pit_irq_stub(void);
  56. extern void kbd_irq_stub(void);
  57. extern void mouse_irq_stub(void);
  58. extern void fdc_irq_stub(void);
  59. extern void default_irq_stub(void);
  60. extern void resched_stub(void);
  61. extern void lapic_timer_stub(void);
  62. extern void lapic_resched_stub(void);
  63. extern void lapic_spurious_stub(void);
  64. extern void syscall_stub(void);
  65. extern unsigned int exc_stub_table[];
  66. unsigned int get_ticks_ms(void)
  67. {
  68. return ticks;
  69. }
  70. unsigned int timer_tick_ms(void)
  71. {
  72. return tick_ms;
  73. }
  74. void timer_tick_advance(void)
  75. {
  76. ticks += tick_ms;
  77. }
  78. void timer_set_hz(unsigned int hz)
  79. {
  80. unsigned int flags, divisor;
  81. if (hz < 100 || hz > 1000)
  82. hz = 1000;
  83. if (hz == timer_hz)
  84. return;
  85. __asm__ volatile ("pushfl; popl %0" : "=r"(flags));
  86. __asm__ volatile ("cli" ::: "memory");
  87. timer_hz = hz;
  88. tick_ms = 1000 / hz;
  89. divisor = 1193182u / hz;
  90. outb(0x43, 0x36);
  91. outb(0x40, (unsigned char)(divisor & 0xFF));
  92. outb(0x40, (unsigned char)((divisor >> 8) & 0xFF));
  93. if (flags & (1u << 9))
  94. __asm__ volatile ("sti" ::: "memory");
  95. }
  96. void exc_handler(struct exc_state *st);
  97. void exc_handler(struct exc_state *st)
  98. {
  99. unsigned int vector = st->vector;
  100. unsigned int cr2 = 0;
  101. static const char * const names[] = {
  102. "#DE Divide Error", "#DB Debug", "NMI", "#BP Breakpoint",
  103. "#OF Overflow", "#BR Bound Range", "#UD Invalid Opcode",
  104. "#NM No FPU", "#DF Double Fault", "Coprocessor Overrun",
  105. "#TS Invalid TSS", "#NP Seg Not Present", "#SS Stack Fault",
  106. "#GP General Protection", "#PF Page Fault", "Reserved",
  107. "#MF FPU Error", "#AC Alignment Check", "#MC Machine Check",
  108. "#XM SIMD Error",
  109. };
  110. __asm__ volatile ("mov %%cr2, %0" : "=r"(cr2));
  111. /* A fault from ring 3 is a bug in a user command (or its ELF): kill that
  112. * process and keep the kernel running. Kernel faults still halt. */
  113. if ((st->cs & 3) == 3) {
  114. serial_puts(SERIAL_COM1, "\n*** user fault vec ");
  115. serial_puthex(SERIAL_COM1, vector);
  116. serial_puts(SERIAL_COM1, " eip=");
  117. serial_puthex(SERIAL_COM1, st->eip);
  118. serial_puts(SERIAL_COM1, " cr2=");
  119. serial_puthex(SERIAL_COM1, cr2);
  120. serial_puts(SERIAL_COM1, " -- process killed\n");
  121. io_putstr("\n*** fault: ");
  122. io_putstr(vector < 20 ? names[vector] : "unknown vector");
  123. io_putstr(" -- command killed\n");
  124. process_exit(); /* marks us dead and waits to be switched out */
  125. }
  126. /* Fatal kernel fault: the banner goes straight into the LFB (no serial
  127. * console needed) and the full state is mirrored to COM1. */
  128. panic_report(st, vector < 20 ? names[vector] : "unknown vector");
  129. for (;;)
  130. __asm__ volatile ("cli; hlt");
  131. }
  132. static void set_idt_gate(int vec, unsigned int handler)
  133. {
  134. idt[vec].offset_lo = (unsigned short)(handler & 0xFFFF);
  135. idt[vec].selector = 0x08;
  136. idt[vec].zero = 0;
  137. idt[vec].type = 0x8E;
  138. idt[vec].offset_hi = (unsigned short)((handler >> 16) & 0xFFFF);
  139. }
  140. /* Trap gate (DPL=3) so ring-3 can `int` into it; a trap gate does NOT clear
  141. * IF, so a blocking syscall (e.g. sleep) can still be preempted by the timer. */
  142. static void set_idt_gate_user(int vec, unsigned int handler)
  143. {
  144. idt[vec].offset_lo = (unsigned short)(handler & 0xFFFF);
  145. idt[vec].selector = 0x08;
  146. idt[vec].zero = 0;
  147. idt[vec].type = 0xEF;
  148. idt[vec].offset_hi = (unsigned short)((handler >> 16) & 0xFFFF);
  149. }
  150. static void timer_init(void)
  151. {
  152. int i;
  153. /* Remap PICs: master at 0x20, slave at 0x28. */
  154. outb(0x20, 0x11);
  155. outb(0xA0, 0x11);
  156. outb(0x21, 0x20);
  157. outb(0xA1, 0x28);
  158. outb(0x21, 0x04);
  159. outb(0xA1, 0x02);
  160. outb(0x21, 0x01);
  161. outb(0xA1, 0x01);
  162. outb(0x21, 0xFC);
  163. outb(0xA1, 0xFF);
  164. /* Default handler for all 256 vectors: absorbs any unexpected
  165. interrupt (including spurious IRQ 7) with an EOI + iret. */
  166. for (i = 0; i < 256; i++)
  167. set_idt_gate(i, (unsigned int)default_irq_stub);
  168. /* Exception handlers 0-31: ring-3 faults kill the process, kernel
  169. faults print the panic banner and halt (see exc_handler). */
  170. for (i = 0; i < 32; i++)
  171. set_idt_gate(i, exc_stub_table[i]);
  172. /* Hardware IRQ handlers. */
  173. set_idt_gate(0x20, (unsigned int)pit_irq_stub);
  174. set_idt_gate(0x21, (unsigned int)kbd_irq_stub);
  175. set_idt_gate(0x2c, (unsigned int)mouse_irq_stub);
  176. set_idt_gate(0x26, (unsigned int)fdc_irq_stub);
  177. /* Local-APIC vectors: the AP timer and reschedule IPI, plus the
  178. * spurious vector the LAPIC can deliver at any time. */
  179. set_idt_gate(LAPIC_TIMER_VECTOR, (unsigned int)lapic_timer_stub);
  180. set_idt_gate(LAPIC_RESCHED_VECTOR, (unsigned int)lapic_resched_stub);
  181. set_idt_gate(LAPIC_SPURIOUS_VECTOR, (unsigned int)lapic_spurious_stub);
  182. /* Syscall gate (ring-3 can int 0x80). */
  183. set_idt_gate_user(0x80, (unsigned int)syscall_stub);
  184. /* Kernel-only voluntary reschedule gate (int 0x81). */
  185. set_idt_gate(0x81, (unsigned int)resched_stub);
  186. idtr.limit = (unsigned short)(sizeof(idt) - 1);
  187. idtr.base = (unsigned int)idt;
  188. __asm__ volatile ("lidt %0" : : "m"(idtr));
  189. /* Start the PIT at 1 kHz; schedule() drops it to 100 Hz while the BSP
  190. * is idle (see timer_set_hz). */
  191. timer_set_hz(1000);
  192. __asm__ volatile ("sti");
  193. }
  194. /* The BSP built the IDT once; APs only need to load the same pointer. */
  195. void idt_load_ap(void)
  196. {
  197. __asm__ volatile ("lidt %0" : : "m"(idtr));
  198. }
  199. extern unsigned char __bss_start[], __bss_end[];
  200. static int kworker_pid = -1;
  201. /* Parts of the system that must keep running independently of the WM:
  202. * input routing, cursor blink, kexec recovery, netlog flushing, gfx-session
  203. * cleanup and the shadow-buffer flush. */
  204. static void system_housekeeping(void)
  205. {
  206. process_reap_dead();
  207. mouse_poll_update();
  208. #ifndef NOWM
  209. wm_pump_keyboard();
  210. wm_blink_cursor();
  211. #endif
  212. netlog_flush();
  213. #ifndef NOWM
  214. wm_handle_kexec();
  215. wm_server_supervise();
  216. if (wm_server_can_reap())
  217. sys_gfx_collect();
  218. if (!wm_server_frame_active())
  219. vga_fb_flush_budget(VGA_FLUSH_BUDGET_BYTES);
  220. #else
  221. /* Diagnostic build: keep the direct-LFB boot console visible and do
  222. * not let the WM shadow flush overwrite it. */
  223. #endif
  224. }
  225. static void kworker_main(void)
  226. {
  227. for (;;) {
  228. system_housekeeping();
  229. process_sleep_ms(2);
  230. }
  231. }
  232. /* Idle is not a normal runnable task: schedule() selects it only when no
  233. * READY process exists. The kworker does the housekeeping; if it dies (or
  234. * could not be created), idle takes over as a degraded fallback. */
  235. static void kernel_idle_loop(void)
  236. {
  237. for (;;) {
  238. if (kworker_pid <= 0 ||
  239. !process_is_alive((unsigned int)kworker_pid)) {
  240. /* Idle must never sleep on the BKL: if a user process on
  241. * another CPU owns it, just keep idling and retry. */
  242. if (bkl_trylock()) {
  243. system_housekeeping();
  244. bkl_unlock();
  245. }
  246. }
  247. cpu_idle();
  248. }
  249. }
  250. /* Boot-stage tracing: COM1 (netlog mirrors it) plus the direct-LFB boot
  251. * console, so a machine without serial still shows where the boot stops. */
  252. static unsigned int boot_t0;
  253. static void boot_trace(const char *what)
  254. {
  255. unsigned int ms = get_ticks_ms() - boot_t0;
  256. bootcon_puts("[");
  257. bootcon_puts(what);
  258. bootcon_puts("] +");
  259. bootcon_dec(ms);
  260. bootcon_puts(" ms\n");
  261. serial_puts(SERIAL_COM1, "[boot] ");
  262. serial_puts(SERIAL_COM1, what);
  263. serial_puts(SERIAL_COM1, " +");
  264. serial_putdec(SERIAL_COM1, ms);
  265. serial_puts(SERIAL_COM1, " ms\n");
  266. }
  267. static void boot_val(const char *name, unsigned int v)
  268. {
  269. bootcon_puts(name);
  270. bootcon_puts(": ");
  271. bootcon_dec(v);
  272. bootcon_puts("\n");
  273. serial_puts(SERIAL_COM1, name);
  274. serial_puts(SERIAL_COM1, ": ");
  275. serial_putdec(SERIAL_COM1, v);
  276. serial_puts(SERIAL_COM1, "\n");
  277. }
  278. /* Bounded PIT self-test. If the 8259 does not deliver IRQ0, every later
  279. * delay/timeout would spin on a frozen tick counter; say so and stop. */
  280. static int pit_ticking(void)
  281. {
  282. unsigned int t0 = get_ticks_ms();
  283. unsigned int n;
  284. for (n = 0; n < 5000000u; n++) {
  285. if (get_ticks_ms() != t0)
  286. return 1;
  287. __asm__ volatile ("pause");
  288. }
  289. return 0;
  290. }
  291. /* A kexec/net-dev reload starts a fresh kernel with zeroed BSS, so the
  292. * leased address and resolver would otherwise fall back to the static
  293. * default until someone types dhcp. The previous kernel stashed them in
  294. * the boot-info reserved words; apply them before the boot DHCP attempt. */
  295. static int net_restore_boot_config(void)
  296. {
  297. const volatile unsigned int *p;
  298. unsigned char ip[4], mask[4], gw[4], dns[4];
  299. unsigned int i;
  300. if (!bootinfo_valid() || !(bootinfo_flags() & BOOTINFO_FLAG_NET))
  301. return 0;
  302. p = bootinfo_words();
  303. for (i = 0; i < 4; i++) {
  304. ip[i] = (unsigned char)(p[7] >> (8 * i));
  305. mask[i] = (unsigned char)(p[8] >> (8 * i));
  306. gw[i] = (unsigned char)(p[9] >> (8 * i));
  307. dns[i] = (unsigned char)(p[10] >> (8 * i));
  308. }
  309. if (!(ip[0] || ip[1] || ip[2] || ip[3]))
  310. return 0;
  311. net_configure(ip, mask, gw);
  312. if (dns[0] || dns[1] || dns[2] || dns[3])
  313. net_set_dns(dns);
  314. boot_trace("net retained");
  315. /* A resolver is what net-dev users notice missing; without one, let
  316. * the boot DHCP attempt learn it. */
  317. return dns[0] || dns[1] || dns[2] || dns[3];
  318. }
  319. void kernel_main(void);
  320. void kernel_main(void)
  321. {
  322. unsigned char *p = __bss_start;
  323. int net_kept;
  324. while (p < __bss_end)
  325. *p++ = 0;
  326. /* Earliest possible output: straight into the boot framebuffer, before
  327. * paging, drivers, the heap or the serial port. Everything below logs
  328. * here as well as to COM1, so a machine without serial (the X201) can
  329. * show where the boot stops. */
  330. bootcon_init();
  331. bootcon_puts(OS_NAME " v" OS_VERSION "\n");
  332. serial_init(SERIAL_COM1);
  333. serial_ring_init();
  334. /* Network console mirror from the very first line on; the ring holds the
  335. * early boot output until the WM loop starts flushing it. */
  336. netlog_init();
  337. serial_puts(SERIAL_COM1, "\n" OS_NAME " v" OS_VERSION " booted\n");
  338. boot_trace("start");
  339. /* GDT/TSS and the per-CPU GS base must exist before anything that can
  340. * use current_proc()/this_cpu(). The BSS is already cleared and the
  341. * GDT/TSS live at identity-mapped addresses. */
  342. gdt_tss_init(0);
  343. boot_trace("gdt");
  344. mm_init();
  345. mm_dump();
  346. boot_trace("mm");
  347. boot_val("mm free KB", mm_free_pages() * 4);
  348. kmalloc_init();
  349. /* Report the VBE geometry before paging, so a boot that dies in the
  350. * translation switch still says which mode the BIOS picked. */
  351. bootcon_fb_info();
  352. paging_init();
  353. boot_trace("paging");
  354. /* Keep a RAM copy of the boot console text; the WM hides the
  355. * framebuffer console later and this is what `bootlog` prints. */
  356. bootcon_log_init();
  357. /* paging_init only ORs in PSE; make sure the BSP's SSE state is on
  358. * before any drawing path can select an SSE2 helper. */
  359. cpu_enable_sse();
  360. /* ACPI needs paging for table access; a missing or unusable MADT simply
  361. * leaves the machine at one CPU. */
  362. acpi_init();
  363. acpi_dump();
  364. boot_trace("acpi");
  365. boot_val("acpi cpus", acpi_cpu_count());
  366. io_init();
  367. boot_trace("io");
  368. /* P-states/deep C-states first: idle_init() picks its default from
  369. * the power module (deep MWAIT on Nehalem, HLT elsewhere). */
  370. power_init();
  371. idle_init();
  372. thermal_init();
  373. /* Board sensors: the ThinkPad EC probe is gated on SMBIOS identifying a
  374. * Lenovo/IBM machine, so it is a no-op everywhere else. */
  375. tpec_init();
  376. boot_trace("idle");
  377. /* process_init must run before timer_init: once the PIT is started (sti
  378. * in timer_init) the context-switch stub runs on every tick and reads
  379. * the current process, its CR3 and the TSS, so those must be valid. */
  380. process_init();
  381. boot_trace("process");
  382. timer_init();
  383. boot_t0 = get_ticks_ms();
  384. boot_trace("timer");
  385. if (!pit_ticking()) {
  386. bootcon_puts("*** PIT NOT TICKING ***\n");
  387. serial_puts(SERIAL_COM1, "*** PIT NOT TICKING ***\n");
  388. for (;;)
  389. __asm__ volatile ("cli; hlt");
  390. }
  391. boot_val("timer ticks", get_ticks_ms());
  392. /* The PIT (and its sti) is live, so the AP bring-up can use it as a
  393. * timeout clock. APs idle until smp_enable_sched() after boot. */
  394. smp_init();
  395. boot_trace("smp");
  396. boot_val("smp cpus", cpu_count);
  397. mouse_init();
  398. boot_trace("ps2");
  399. /* USB HID backend (mouse + keyboard): controller bring-up + first
  400. * scan; devices plugged in later are picked up by the hotplug
  401. * watcher in the WM loop. */
  402. asix_register();
  403. usbhid_init();
  404. boot_trace("usb");
  405. vga_shadow_init();
  406. vga_fb_flush();
  407. boot_trace("vga");
  408. /* The command pack: read-only, mounted at /ramfs (see ramfs.c). The
  409. * boot path copied it into RAM (the legacy stage's low gap, the UEFI
  410. * loader or kexec); ramfs_init takes its own copy so the boot region
  411. * can be released. The mount is created even without a pack, so a running
  412. * board can be recovered with `make cmds-dev`/`make net-cmds`. */
  413. {
  414. unsigned int pack_addr = bootinfo_pack_addr();
  415. unsigned int pack_len = bootinfo_pack_size();
  416. int ok = 0;
  417. if (!pack_addr || !pack_len)
  418. serial_puts(SERIAL_COM1, "ramfs: no boot pack\n");
  419. else {
  420. serial_puts(SERIAL_COM1, "ramfs: boot pack at ");
  421. serial_puthex(SERIAL_COM1, pack_addr);
  422. serial_puts(SERIAL_COM1, ", ");
  423. serial_putdec(SERIAL_COM1, pack_len);
  424. serial_puts(SERIAL_COM1, " bytes\n");
  425. ok = ramfs_init((const void *)pack_addr, pack_len) == 0;
  426. if (!ok)
  427. serial_puts(SERIAL_COM1, "ramfs: bad boot pack\n");
  428. else {
  429. serial_puts(SERIAL_COM1, "ramfs: pack ok, ");
  430. serial_putdec(SERIAL_COM1, ramfs_count());
  431. serial_puts(SERIAL_COM1, " files, ");
  432. serial_putdec(SERIAL_COM1, ramfs_pack_size());
  433. serial_puts(SERIAL_COM1, " bytes\n");
  434. }
  435. mm_free_range(pack_addr, pack_len);
  436. }
  437. (void)ok;
  438. vfs_mount_ramfs();
  439. }
  440. boot_trace("ramfs");
  441. boot_val("ramfs files", ramfs_count());
  442. #ifndef NOWM
  443. /* Start the WM before touching disks: the first frame can be painted
  444. * while storage and the network are still being probed. */
  445. wm_server_init();
  446. boot_trace("wm init");
  447. /* The WM's first frame covers this console; stop drawing before it can
  448. * run, or a later stage line would appear on top of the desktop. */
  449. bootcon_set_enabled(0);
  450. wm_server_start();
  451. boot_trace("wm");
  452. #else
  453. /* Diagnostic build: boot to the console, no WM, no terminal. */
  454. bootcon_puts("WM disabled (diagnostic build)\n");
  455. serial_puts(SERIAL_COM1, "nowm: window manager disabled\n");
  456. boot_trace("wm off");
  457. #endif
  458. /* Probe the NIC (link negotiation is asynchronous now) before the
  459. * kworker starts, so its netlog flushes have a device to use. */
  460. netdev_active();
  461. net_kept = net_restore_boot_config();
  462. boot_trace("net");
  463. /* The idle process is the kernel_main context; housekeeping runs in
  464. * its own process so it cannot be starved by a busy system. */
  465. kworker_pid = process_create("kworker", kworker_main);
  466. if (kworker_pid < 0) {
  467. serial_puts(SERIAL_COM1, "kworker: cannot create, using idle fallback\n");
  468. kworker_pid = -1;
  469. } else {
  470. process_set_weight((unsigned int)kworker_pid, 4);
  471. process_set_affinity((unsigned int)kworker_pid, 1);
  472. }
  473. boot_trace("kworker");
  474. /* Slow storage bring-up happens behind the usable desktop now. */
  475. boot_trace("storage begin");
  476. vfs_mount_all();
  477. boot_trace("storage");
  478. /* The first terminal is spawned only now: while the boot scan was
  479. * rebuilding the mount table, no user command could touch the disks.
  480. * The desktop itself was already painted from /ramfs. */
  481. #ifndef NOWM
  482. wm_spawn_terminal();
  483. boot_trace("terminal");
  484. #else
  485. boot_trace("terminal off");
  486. #endif
  487. /* With a NIC present, try DHCP; on failure the static default stays.
  488. * Deliberately after the WM: a missing cable or DHCP server must not
  489. * delay the first frame. */
  490. if (netdev_active() && !net_kept)
  491. dhcp_run(4000);
  492. boot_trace(net_kept ? "dhcp skipped" : "dhcp");
  493. /* Dump the active NIC's counters after the boot DHCP attempt. With
  494. * the ASIX as the management link this appears in netmon even when
  495. * the active e1000e cannot pass traffic. */
  496. {
  497. const struct netdev *nd = netdev_active();
  498. if (nd && (nd->dump_brief || nd->dump_regs)) {
  499. io_putstr("\n[boot netregs: ");
  500. io_putstr(nd->name);
  501. io_putstr("]\n");
  502. if (nd->dump_brief)
  503. nd->dump_brief();
  504. else
  505. nd->dump_regs();
  506. }
  507. }
  508. /* Boot-time device/FS setup is complete; APs may now take user work.
  509. * Kernel threads stay pinned to the BSP and serialize on the BKL. */
  510. smp_enable_sched();
  511. boot_trace("sched");
  512. kernel_idle_loop();
  513. }