/* Kernel: entry point at 0x7E00, timer (PIT 1 kHz), process init, terminal. */ #include "acpi.h" #include "asix.h" #include "bootcon.h" #include "bootinfo.h" #include "cpu.h" #include "cros_ec.h" #include "idle.h" #include "io.h" #include "ioapic.h" #include "lapic.h" #include "panic.h" #include "kmalloc.h" #include "mm.h" #include "mouse.h" #include "netdev.h" #include "net.h" #include "dhcp.h" #include "netlog.h" #include "paging.h" #include "power.h" #include "process.h" #include "ramfs.h" #include "cursor.h" #include "rndis.h" #include "rtl8821au.h" #include "sched.h" #include "sys_wm.h" #include "thermal.h" #include "timer.h" #include "tpec.h" #include "tss.h" #include "usbhid.h" #include "usbmsc.h" #include "wm.h" #include "serial.h" #include "smp.h" #include "version.h" #include "vga.h" #include "vfs.h" #include "wifi_cfg.h" static inline void outb(unsigned short port, unsigned char v) { __asm__ volatile ("outb %0, %1" : : "a"(v), "Nd"(port)); } volatile unsigned int ticks; static unsigned int timer_hz; static unsigned int tick_ms = 1; struct idt_entry { unsigned short offset_lo; unsigned short selector; unsigned char zero; unsigned char type; unsigned short offset_hi; } __attribute__((packed)); struct idtr_desc { unsigned short limit; unsigned int base; } __attribute__((packed)); static struct idt_entry idt[256]; static struct idtr_desc idtr; extern void pit_irq_stub(void); extern void kbd_irq_stub(void); extern void mouse_irq_stub(void); extern void fdc_irq_stub(void); extern void default_irq_stub(void); extern void resched_stub(void); extern void lapic_timer_stub(void); extern void lapic_resched_stub(void); extern void lapic_spurious_stub(void); extern void syscall_stub(void); extern unsigned int exc_stub_table[]; unsigned int get_ticks_ms(void) { return ticks; } unsigned int timer_tick_ms(void) { return tick_ms; } void timer_tick_advance(void) { ticks += tick_ms; } void timer_set_hz(unsigned int hz) { unsigned int flags, divisor; if (hz < 100 || hz > 1000) hz = 1000; if (hz == timer_hz) return; __asm__ volatile ("pushfl; popl %0" : "=r"(flags)); __asm__ volatile ("cli" ::: "memory"); timer_hz = hz; tick_ms = 1000 / hz; divisor = 1193182u / hz; outb(0x43, 0x36); outb(0x40, (unsigned char)(divisor & 0xFF)); outb(0x40, (unsigned char)((divisor >> 8) & 0xFF)); if (flags & (1u << 9)) __asm__ volatile ("sti" ::: "memory"); } void exc_handler(struct exc_state *st); void exc_handler(struct exc_state *st) { unsigned int vector = st->vector; unsigned int cr2 = 0; static const char * const names[] = { "#DE Divide Error", "#DB Debug", "NMI", "#BP Breakpoint", "#OF Overflow", "#BR Bound Range", "#UD Invalid Opcode", "#NM No FPU", "#DF Double Fault", "Coprocessor Overrun", "#TS Invalid TSS", "#NP Seg Not Present", "#SS Stack Fault", "#GP General Protection", "#PF Page Fault", "Reserved", "#MF FPU Error", "#AC Alignment Check", "#MC Machine Check", "#XM SIMD Error", }; __asm__ volatile ("mov %%cr2, %0" : "=r"(cr2)); /* A fault from ring 3 is a bug in a user command (or its ELF): kill that * process and keep the kernel running. Kernel faults still halt. */ if ((st->cs & 3) == 3) { struct process *p = current_proc(); serial_puts(SERIAL_COM1, "\n*** user fault vec "); serial_puthex(SERIAL_COM1, vector); serial_puts(SERIAL_COM1, " eip="); serial_puthex(SERIAL_COM1, st->eip); serial_puts(SERIAL_COM1, " cr2="); serial_puthex(SERIAL_COM1, cr2); if (p) { serial_puts(SERIAL_COM1, " pid="); serial_putdec(SERIAL_COM1, p->pid); serial_puts(SERIAL_COM1, " "); serial_puts(SERIAL_COM1, p->name); } serial_puts(SERIAL_COM1, " -- process killed\n"); io_putstr("\n*** fault: "); io_putstr(vector < 20 ? names[vector] : "unknown vector"); io_putstr(" -- command killed\n"); process_exit(); /* marks us dead and waits to be switched out */ } /* Fatal kernel fault: the banner goes straight into the LFB (no serial * console needed) and the full state is mirrored to COM1. */ panic_report(st, vector < 20 ? names[vector] : "unknown vector"); for (;;) __asm__ volatile ("cli; hlt"); } static void set_idt_gate(int vec, unsigned int handler) { idt[vec].offset_lo = (unsigned short)(handler & 0xFFFF); idt[vec].selector = 0x08; idt[vec].zero = 0; idt[vec].type = 0x8E; idt[vec].offset_hi = (unsigned short)((handler >> 16) & 0xFFFF); } /* Trap gate (DPL=3) so ring-3 can `int` into it; a trap gate does NOT clear * IF, so a blocking syscall (e.g. sleep) can still be preempted by the timer. */ static void set_idt_gate_user(int vec, unsigned int handler) { idt[vec].offset_lo = (unsigned short)(handler & 0xFFFF); idt[vec].selector = 0x08; idt[vec].zero = 0; idt[vec].type = 0xEF; idt[vec].offset_hi = (unsigned short)((handler >> 16) & 0xFFFF); } static void timer_init(void) { int i; /* Remap PICs: master at 0x20, slave at 0x28. */ outb(0x20, 0x11); outb(0xA0, 0x11); outb(0x21, 0x20); outb(0xA1, 0x28); outb(0x21, 0x04); outb(0xA1, 0x02); outb(0x21, 0x01); outb(0xA1, 0x01); outb(0x21, 0xFC); outb(0xA1, 0xFF); /* Default handler for all 256 vectors: absorbs any unexpected interrupt (including spurious IRQ 7) with an EOI + iret. */ for (i = 0; i < 256; i++) set_idt_gate(i, (unsigned int)default_irq_stub); /* Exception handlers 0-31: ring-3 faults kill the process, kernel faults print the panic banner and halt (see exc_handler). */ for (i = 0; i < 32; i++) set_idt_gate(i, exc_stub_table[i]); /* Hardware IRQ handlers. */ set_idt_gate(0x20, (unsigned int)pit_irq_stub); set_idt_gate(0x21, (unsigned int)kbd_irq_stub); set_idt_gate(0x2c, (unsigned int)mouse_irq_stub); set_idt_gate(0x26, (unsigned int)fdc_irq_stub); /* Local-APIC vectors: the AP timer and reschedule IPI, plus the * spurious vector the LAPIC can deliver at any time. */ set_idt_gate(LAPIC_TIMER_VECTOR, (unsigned int)lapic_timer_stub); set_idt_gate(LAPIC_RESCHED_VECTOR, (unsigned int)lapic_resched_stub); set_idt_gate(LAPIC_SPURIOUS_VECTOR, (unsigned int)lapic_spurious_stub); /* Syscall gate (ring-3 can int 0x80). */ set_idt_gate_user(0x80, (unsigned int)syscall_stub); /* Kernel-only voluntary reschedule gate (int 0x81). */ set_idt_gate(0x81, (unsigned int)resched_stub); idtr.limit = (unsigned short)(sizeof(idt) - 1); idtr.base = (unsigned int)idt; __asm__ volatile ("lidt %0" : : "m"(idtr)); /* Start the PIT at 1 kHz; schedule() drops it to 100 Hz while the BSP * is idle (see timer_set_hz). */ timer_set_hz(1000); __asm__ volatile ("sti"); } /* The BSP built the IDT once; APs only need to load the same pointer. */ void idt_load_ap(void) { __asm__ volatile ("lidt %0" : : "m"(idtr)); } extern unsigned char __bss_start[], __bss_end[]; static int kworker_pid = -1; /* Parts of the system that must keep running independently of the WM: * input routing, thermal polling, cursor blink, kexec recovery, netlog * flushing, gfx-session cleanup and the shadow-buffer flush. */ static void system_housekeeping(void) { process_reap_dead(); thermal_housekeeping(); mouse_poll_update(); #ifndef NOWM wm_pump_keyboard(); #endif netlog_flush(); #ifndef NOWM wm_handle_kexec(); wm_server_supervise(); if (!wm_server_frame_active()) vga_fb_flush_budget(VGA_FLUSH_BUDGET_BYTES); #else /* Diagnostic build: keep the direct-LFB boot console visible and do * not let the WM shadow flush overwrite it. */ #endif } static void kworker_main(void) { for (;;) { system_housekeeping(); process_sleep_ms(2); } } /* Idle is not a normal runnable task: schedule() selects it only when no * READY process exists. The kworker does the housekeeping; if it dies (or * could not be created), idle takes over as a degraded fallback. */ static void kernel_idle_loop(void) { for (;;) { if (kworker_pid <= 0 || !process_is_alive((unsigned int)kworker_pid)) { /* Idle must never sleep on the BKL: if a user process on * another CPU owns it, just keep idling and retry. */ if (bkl_trylock()) { system_housekeeping(); bkl_unlock(); } } cpu_idle(); } } /* Boot-stage tracing: COM1 (netlog mirrors it) plus the direct-LFB boot * console, so a machine without serial still shows where the boot stops. */ static unsigned int boot_t0; static void boot_trace(const char *what) { unsigned int ms = get_ticks_ms() - boot_t0; bootcon_puts("["); bootcon_puts(what); bootcon_puts("] +"); bootcon_dec(ms); bootcon_puts(" ms\n"); serial_puts(SERIAL_COM1, "[boot] "); serial_puts(SERIAL_COM1, what); serial_puts(SERIAL_COM1, " +"); serial_putdec(SERIAL_COM1, ms); serial_puts(SERIAL_COM1, " ms\n"); } static void boot_val(const char *name, unsigned int v) { bootcon_puts(name); bootcon_puts(": "); bootcon_dec(v); bootcon_puts("\n"); serial_puts(SERIAL_COM1, name); serial_puts(SERIAL_COM1, ": "); serial_putdec(SERIAL_COM1, v); serial_puts(SERIAL_COM1, "\n"); } /* Bounded PIT self-test. If the 8259 does not deliver IRQ0, every later * delay/timeout would spin on a frozen tick counter; say so and stop. */ static int pit_ticking(void) { unsigned int t0 = get_ticks_ms(); unsigned int n; for (n = 0; n < 5000000u; n++) { if (get_ticks_ms() != t0) return 1; __asm__ volatile ("pause"); } return 0; } /* A kexec/net-dev reload starts a fresh kernel with zeroed BSS, so the * leased address and resolver would otherwise fall back to the static * default until someone types dhcp. The previous kernel stashed them in * the boot-info reserved words; apply them before the boot DHCP attempt. */ static int net_restore_boot_config(void) { const volatile unsigned int *p; unsigned char ip[4], mask[4], gw[4], dns[4]; unsigned int i; if (!bootinfo_valid() || !(bootinfo_flags() & BOOTINFO_FLAG_NET)) return 0; p = bootinfo_words(); for (i = 0; i < 4; i++) { ip[i] = (unsigned char)(p[7] >> (8 * i)); mask[i] = (unsigned char)(p[8] >> (8 * i)); gw[i] = (unsigned char)(p[9] >> (8 * i)); dns[i] = (unsigned char)(p[10] >> (8 * i)); } if (!(ip[0] || ip[1] || ip[2] || ip[3])) return 0; net_configure(ip, mask, gw); if (dns[0] || dns[1] || dns[2] || dns[3]) net_set_dns(dns); boot_trace("net retained"); /* A resolver is what net-dev users notice missing; without one, let * the boot DHCP attempt learn it. */ return dns[0] || dns[1] || dns[2] || dns[3]; } /* Saved WiFi networks on the writable /data volume: let the normal * supplicant scan and join any known network in the background. It leaves a * wired link that is physically up alone, and a missing /data or wifi helper * is harmless. */ static void boot_wifi_autoconnect(void) { unsigned int sz = 0; int is_dir = 0; if (vfs_stat(WIFI_CRED_PATH, &sz, &is_dir) == 0 && !is_dir && sz > 0) { struct vfs_file f; struct vfs_mount *m = 0; if (vfs_load_file("/ramfs/bin/wifi", &f, &m) == 0) { if (process_create_user("wifi", f.data, f.size, "autoconnect", "/") > 0) serial_puts(SERIAL_COM1, "wifi: autoconnect started\n"); vfs_release_file(m, &f); } } } /* Boot-time storage scan in its own kernel thread. A process (unlike the * idle context kernel_main runs in) may sleep, so the probes' long waits * (floppy motor, ATA/AHCI timeouts) go through vfs_scan_relax() and the WM * and the first terminal run meanwhile. Disk file operations wait for * vfs_scan_busy; /ramfs and /share do not. The WiFi autoconnect needs * /data, so it runs here after the mount. */ static volatile int boot_dhcp_done; static void boot_storage_main(void) { vfs_mount_all(); boot_trace("storage"); /* As before, WiFi only starts after the boot DHCP attempt on the * wired NIC, so two DHCP clients never overlap. */ while (!boot_dhcp_done) process_sleep_ms(20); boot_wifi_autoconnect(); boot_trace("wifi auto"); } void kernel_main(void); void kernel_main(void) { unsigned char *p = __bss_start; int net_kept, storage_inline; while (p < __bss_end) *p++ = 0; /* Earliest possible output: straight into the boot framebuffer, before * paging, drivers, the heap or the serial port. Everything below logs * here as well as to COM1, so a machine without serial (the X201) can * show where the boot stops. */ bootcon_init(); bootcon_puts(OS_NAME " v" OS_VERSION "\n"); serial_init(SERIAL_COM1); serial_ring_init(); /* Network console mirror from the very first line on; the ring holds the * early boot output until the WM loop starts flushing it. */ netlog_init(); serial_puts(SERIAL_COM1, "\n" OS_NAME " v" OS_VERSION " booted\n"); boot_trace("start"); /* GDT/TSS and the per-CPU GS base must exist before anything that can * use current_proc()/this_cpu(). The BSS is already cleared and the * GDT/TSS live at identity-mapped addresses. */ gdt_tss_init(0); boot_trace("gdt"); mm_init(); mm_dump(); boot_trace("mm"); boot_val("mm free KB", mm_free_pages() * 4); kmalloc_init(); /* Report the VBE geometry before paging, so a boot that dies in the * translation switch still says which mode the BIOS picked. */ bootcon_fb_info(); paging_init(); boot_trace("paging"); /* Keep a RAM copy of the boot console text; the WM hides the * framebuffer console later and this is what `bootlog` prints. */ bootcon_log_init(); /* paging_init only ORs in PSE; make sure the BSP's SSE state is on * before any drawing path can select an SSE2 helper. */ cpu_enable_sse(); /* ACPI needs paging for table access; a missing or unusable MADT simply * leaves the machine at one CPU. */ acpi_init(); acpi_dump(); /* The kernel keeps the 8259 on LAPIC LINT0; firmware-left IOAPIC * entries would otherwise deliver into the PIC/LAPIC vector space. */ ioapic_mask_all(); boot_trace("acpi"); boot_val("acpi cpus", acpi_cpu_count()); io_init(); boot_trace("io"); /* P-states/deep C-states first: idle_init() picks its default from * the power module (deep MWAIT on Nehalem, HLT elsewhere). */ power_init(); idle_init(); boot_trace("power"); thermal_init(); boot_trace("thermal"); /* Board sensors: the ThinkPad EC probe is gated on SMBIOS identifying a * Lenovo/IBM machine, so it is a no-op everywhere else. The Chrome EC * probe is skipped when the ThinkPad EC was found. */ tpec_init(); cros_ec_init(); boot_trace("ec"); /* process_init must run before timer_init: once the PIT is started (sti * in timer_init) the context-switch stub runs on every tick and reads * the current process, its CR3 and the TSS, so those must be valid. */ process_init(); boot_trace("process"); timer_init(); boot_t0 = get_ticks_ms(); boot_trace("timer"); if (!pit_ticking()) { bootcon_puts("*** PIT NOT TICKING ***\n"); serial_puts(SERIAL_COM1, "*** PIT NOT TICKING ***\n"); for (;;) __asm__ volatile ("cli; hlt"); } boot_val("timer ticks", get_ticks_ms()); /* The PIT (and its sti) is live, so the AP bring-up can use it as a * timeout clock. APs idle until smp_enable_sched() after boot. */ smp_init(); boot_trace("smp"); boot_val("smp cpus", cpu_count); mouse_init(); boot_trace("ps2"); vga_shadow_init(); vga_fb_flush(); boot_trace("vga"); /* The command pack: read-only, mounted at /ramfs (see ramfs.c). The * boot path copied it into RAM (the legacy stage's low gap, the UEFI * loader or kexec); ramfs_init takes its own copy so the boot region * can be released. The mount is created even without a pack, so a running * board can be recovered with `make cmds-dev`/`make net-cmds`. */ { unsigned int pack_addr = bootinfo_pack_addr(); unsigned int pack_len = bootinfo_pack_size(); int ok = 0, gz = 0; if (!pack_addr || !pack_len) serial_puts(SERIAL_COM1, "ramfs: no boot pack\n"); else { const unsigned char *p = (const unsigned char *)pack_addr; gz = pack_len >= 2 && p[0] == 0x1F && p[1] == 0x8B; serial_puts(SERIAL_COM1, "ramfs: boot pack at "); serial_puthex(SERIAL_COM1, pack_addr); serial_puts(SERIAL_COM1, ", "); serial_putdec(SERIAL_COM1, pack_len); serial_puts(SERIAL_COM1, gz ? " bytes (gzip)\n" : " bytes\n"); if (gz) ramfs_set_boot_image((const void *)pack_addr, pack_len, 1); ok = ramfs_init((const void *)pack_addr, pack_len) == 0; if (!ok) { serial_puts(SERIAL_COM1, "ramfs: bad boot pack\n"); ramfs_set_boot_image(0, 0, 0); mm_free_range(pack_addr, pack_len); } else { serial_puts(SERIAL_COM1, "ramfs: pack ok, "); serial_putdec(SERIAL_COM1, ramfs_count()); serial_puts(SERIAL_COM1, " files, "); serial_putdec(SERIAL_COM1, ramfs_pack_size()); serial_puts(SERIAL_COM1, " bytes\n"); cursor_load(); if (!gz) { /* ramfs has its own copy now; keep that * as the self-update image. */ ramfs_set_boot_image(ramfs_pack_base(), ramfs_pack_size(), 0); mm_free_range(pack_addr, pack_len); } /* A gzip boot image stays reserved so * `install self` can write it to P2. */ } } (void)ok; vfs_mount_ramfs(); /* /share too: the WM and the first terminal may use it while the * bootscan thread is still probing the disks, and a path that * does not resolve yet would wait for the whole scan. */ vfs_mount_share(); } boot_trace("ramfs"); boot_val("ramfs files", ramfs_count()); #ifndef NOWM /* Start the WM before touching disks: the first frame can be painted * while storage and the network are still being probed. */ wm_server_init(); boot_trace("wm init"); wm_server_start(); boot_trace("wm"); #else /* Diagnostic build: boot to the console, no WM, no terminal. */ bootcon_puts("WM disabled (diagnostic build)\n"); serial_puts(SERIAL_COM1, "nowm: window manager disabled\n"); boot_trace("wm off"); #endif /* Publish the storage scan before anything can run a command: disk * paths wait for vfs_scan_busy (owned by the idle context until the * bootscan thread's vfs_mount_all() takes it over), /ramfs and /share * do not. Then let the WM spawn the first terminal at once; neither * it nor the WM needs the NIC probe or the disks below. */ vfs_scan_owner_pid = 0; vfs_scan_busy = 1; #ifndef NOWM wm_server_boot_ready(); boot_trace("terminal ready"); #else boot_trace("terminal off"); #endif #ifndef NOWM /* Let the WM put the first terminal on screen before the USB bring-up * below: kernel_main is the idle context, so halting here hands the * CPU to the WM and the new terminal. Bounded, in case no terminal * ever presents (missing /ramfs/bin/term). */ { unsigned int t0 = get_ticks_ms(); while (!wm_first_window_presented && get_ticks_ms() - t0 < 1000) __asm__ volatile ("sti; hlt"); } boot_trace("terminal shown"); #endif /* USB HID backend (mouse + keyboard) and the USB device drivers: * controller bring-up + first scan; devices plugged in later are * picked up by the hotplug watcher. Non-preemptible, exactly like * the bring-up before the WM existed: port resets and the SOF a * freshly enabled low-speed device needs within 3 ms must not wait * for a WM frame. Before the NIC probe: ASIX/RNDIS/RTL8821AU bind * during this scan, and USB disks before the bootscan thread. */ asix_register(); usbmsc_register(); rtl8821au_register(); rndis_register(); preempt_disable(); usbhid_init(); preempt_enable(); boot_trace("usb"); /* Probe the NIC (link negotiation is asynchronous now) before the * kworker starts, so its netlog flushes have a device to use. */ netdev_active(); net_kept = net_restore_boot_config(); boot_trace("net"); /* The idle process is the kernel_main context; housekeeping runs in * its own process so it cannot be starved by a busy system. */ kworker_pid = process_create("kworker", kworker_main); if (kworker_pid < 0) { serial_puts(SERIAL_COM1, "kworker: cannot create, using idle fallback\n"); kworker_pid = -1; } else { process_set_weight((unsigned int)kworker_pid, 4); process_set_affinity((unsigned int)kworker_pid, 1); } boot_trace("kworker"); /* Slow storage bring-up happens behind the usable desktop, in a * thread that may sleep through its probe timeouts. The scan was * published above; its vfs_mount_all() takes over the ownership. * Without a thread slot, scan inline as before. */ boot_trace("storage begin"); storage_inline = 0; if (process_create("bootscan", boot_storage_main) <= 0) { vfs_mount_all(); boot_trace("storage"); storage_inline = 1; } /* With a linked NIC present, try DHCP; on failure the static default * stays. A wired NIC without a cable is skipped so the WiFi fallback * below does not have to wait out the 4 s timeout first. Deliberately * after the WM: a missing cable or DHCP server must not delay the first * frame. */ { const struct netdev *nd = netdev_active(); int linked = nd && netdev_link(nd); if (linked && !net_kept) dhcp_run(4000); boot_trace(net_kept ? "dhcp skipped" : (linked ? "dhcp" : "dhcp no link")); } boot_dhcp_done = 1; if (storage_inline) { boot_wifi_autoconnect(); boot_trace("wifi auto"); } /* Dump the active NIC's counters after the boot DHCP attempt. With * the ASIX as the management link this appears in netmon even when * the active e1000e cannot pass traffic. */ { const struct netdev *nd = netdev_active(); if (nd && (nd->dump_brief || nd->dump_regs)) { io_putstr("\n[boot netregs: "); io_putstr(nd->name); io_putstr("]\n"); if (nd->dump_brief) nd->dump_brief(); else nd->dump_regs(); } } /* Boot-time device/FS setup is complete; APs may now take user work. * Kernel threads stay pinned to the BSP and serialize on the BKL. */ smp_enable_sched(); boot_trace("sched"); kernel_idle_loop(); }