| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548 |
- /* Paging: identity-map RAM + the VESA LFB, enable CR0.PG (4 MB pages).
- *
- * This is the "paging on but flat" stage: every physical address is mapped
- * 1:1, so all existing code (kernel, stacks, DMA, UHCI, VGA shadow) keeps
- * working unchanged. Per-process page directories come later (Phase 3b/4).
- *
- * Uses 4 MB pages via CR4.PSE (supported since the original Pentium), so the
- * whole map is a single 4 KB page directory.
- */
- #include "paging.h"
- #include "bootcon.h"
- #include "cpu.h"
- #include "mm.h"
- #include "mtrr.h"
- #include "process.h"
- #include "sched.h"
- #include "serial.h"
- #include "vga.h"
- #include "wm_abi.h"
- /* The RAM identity map is made of supervisor 4 MB PDEs; the ring-3 windows
- * (program/stack, canvases, WM slots, heap) are user page tables above it.
- * They must never share a PDE. */
- _Static_assert(MM_MAX_PHYS <= USER_BASE, "identity map overlaps user space");
- _Static_assert(MM_MAX_PHYS <= USER_WIN_BASE &&
- MM_MAX_PHYS <= WM_CONTENT_VA_BASE &&
- MM_MAX_PHYS <= USER_HEAP_BASE,
- "identity map overlaps a ring-3 window");
- _Static_assert(MM_KMAP_VA >= USER_STACK_TOP &&
- MM_KMAP_VA + 0x400000u <= USER_WIN_BASE &&
- (MM_KMAP_VA & 0x3FFFFFu) == 0,
- "kmap window must be a free 4 MB slot below USER_WIN_BASE");
- /* kmap: KMAP_SLOTS nested mappings per CPU in the shared window page table. */
- #define KMAP_PDE (MM_KMAP_VA >> 22)
- #define KMAP_SLOTS 4
- static unsigned int *kmap_pt;
- static unsigned int kmap_depth[MAX_CPUS];
- /* Every PDE writer below leaves the kmap slot alone: a BIOS-placed ACPI
- * table, MMIO BAR or LFB in that 4 MB range cannot be identity-mapped. */
- static int kmap_pde_guard(unsigned int i)
- {
- if (i != KMAP_PDE)
- return 0;
- serial_puts(SERIAL_COM1, "paging: refusing to map over the kmap window\n");
- return 1;
- }
- /* CR4 bits that must not survive a BIOS protected-mode call. A BIOS that
- * returns with CR4.PAE or EFER.LME set makes the first CR0.PG enable either
- * walk the 32-bit page directory as a PAE PDPT or enter long mode, which
- * triple-faults before the kernel can print anything. The X201's old
- * INT 15h AH=87h rootfs copy was one source of this state; clear the bits
- * defensively on every boot path before enabling paging. */
- #define CR4_PSE (1u << 4)
- #define CR4_PAE (1u << 5)
- #define CR4_PGE (1u << 7)
- #define CR4_LA57 (1u << 12)
- #define CR4_PCIDE (1u << 17)
- #define CR4_SMEP (1u << 20)
- #define CR4_SMAP (1u << 21)
- #define MSR_EFER 0xC0000080u
- #define EFER_LME (1u << 8)
- /* Page-directory entry flags (4 MB page). */
- #define PDE_PRESENT 0x001
- #define PDE_WRITABLE 0x002
- #define PDE_USER 0x004
- #define PDE_PWT 0x008
- #define PDE_PCD 0x010
- #define PDE_PS 0x080 /* 4 MB page */
- /* How much of the LFB to map. 16 MB covers any mode up to 1920x1080x32
- * (~8.3 MB) with margin, and is independent of the BIOS's VRAM report. */
- #define LFB_MAP_SIZE 0x1000000
- static unsigned int *page_dir;
- static void uc_page_cr3(unsigned int cr3, unsigned int va);
- static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
- {
- __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
- }
- static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
- {
- __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
- }
- static int cpu_has_long_mode(void)
- {
- unsigned int eax, ebx, ecx, edx;
- if (!cpu_has_cpuid())
- return 0;
- eax = 0x80000000u;
- __asm__ volatile ("cpuid"
- : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
- if (eax < 0x80000001u)
- return 0;
- eax = 0x80000001u;
- __asm__ volatile ("cpuid"
- : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
- return (edx & (1u << 29)) != 0; /* LM */
- }
- /* Put the CPU into the state this 32-bit kernel expects before it installs
- * its own page directory. A BIOS protected-mode call (notably the old
- * high-pack INT 15h AH=87h path) may leave CR4.PAE/PGE or EFER.LME set;
- * CR0.PG is off here, so clearing them is safe on every CPU (reserved bits
- * are written as zero). */
- void paging_prepare_cpu(void)
- {
- unsigned int cr4, lo, hi;
- __asm__ volatile ("movl %%cr4, %0" : "=r"(cr4));
- cr4 &= ~(CR4_PAE | CR4_PGE | CR4_LA57 | CR4_PCIDE |
- CR4_SMEP | CR4_SMAP);
- cr4 |= CR4_PSE;
- __asm__ volatile ("movl %0, %%cr4" : : "r"(cr4) : "memory");
- if (cpu_has_long_mode()) {
- rdmsr(MSR_EFER, &lo, &hi);
- if (lo & EFER_LME) {
- lo &= ~EFER_LME;
- wrmsr(MSR_EFER, lo, hi);
- serial_puts(SERIAL_COM1, "paging: cleared EFER.LME\n");
- }
- }
- }
- /* LFB page-type decision, kept so it can be re-applied after other MMIO
- * mappings (the GPU BARs cover the framebuffer address) have set PCD. */
- static unsigned int lfb_wc_base, lfb_wc_size;
- static int lfb_wc_ok;
- /* Write the LFB PDEs: WC-covered pages cacheable (effective type WC), the
- * rest uncacheable. */
- static void lfb_apply_pages(unsigned int *pd)
- {
- const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
- unsigned int base, p0, p1, i;
- __asm__ volatile ("" : "+r"(fb));
- if (!fb->valid || !fb->lfb_base)
- return;
- base = fb->lfb_base;
- p0 = base >> 22;
- /* Compute the end page from the page index, not from base+size: for a
- * base above 0xFF000000 the 32-bit addition wraps to a small number
- * and the loop would map nothing, faulting on the first LFB write
- * after CR0.PG comes on. */
- p1 = p0 + (LFB_MAP_SIZE >> 22);
- if (p1 > 1024)
- p1 = 1024;
- for (i = p0; i < p1; i++) {
- unsigned int page = i << 22;
- unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
- if (kmap_pde_guard(i))
- continue;
- if (!lfb_wc_ok || page < lfb_wc_base ||
- page + 0x400000 > lfb_wc_base + lfb_wc_size)
- flags |= PDE_PCD;
- pd[i] = page | flags;
- }
- }
- /* Re-apply the framebuffer mapping; call after any MMIO mapping that may have
- * covered the same addresses (the NVIDIA BAR3 aperture includes the VBE LFB). */
- void paging_lfb_refresh(void)
- {
- if (!lfb_wc_ok || !page_dir)
- return;
- lfb_apply_pages(page_dir);
- serial_puts(SERIAL_COM1, "paging: LFB write-combining re-applied\n");
- }
- /* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
- int paging_lfb_wc(void)
- {
- return lfb_wc_ok;
- }
- void paging_map_ram(unsigned int base, unsigned int size)
- {
- unsigned int p0 = base >> 22;
- unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
- unsigned int i;
- if (!page_dir)
- return;
- for (i = p0; i < p1 && i < 1024; i++)
- if (!kmap_pde_guard(i))
- page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE |
- PDE_PRESENT;
- }
- void paging_init(void)
- {
- unsigned int i;
- const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
- bootcon_puts("paging: prepare\n");
- paging_prepare_cpu();
- /* Opaque asm hides the fixed 0x600 pointer from -Warray-bounds. */
- __asm__ volatile ("" : "+r"(fb));
- bootcon_puts("paging: alloc\n");
- page_dir = (unsigned int *)mm_alloc_page();
- if (!page_dir) {
- serial_puts(SERIAL_COM1, "paging: out of memory\n");
- for (;;)
- __asm__ volatile ("cli; hlt");
- }
- /* Zero the whole page directory. mm_alloc_page does NOT clear memory, so
- * on real hardware the high PDEs (indices 32..1023, incl. the user region
- * at 0x100) would otherwise be garbage. QEMU zeroes guest RAM, which is
- * why this only bites on real silicon. */
- for (i = 0; i < 1024; i++)
- page_dir[i] = 0;
- /* Identity-map RAM [0, MM_MAX_PHYS) with 4 MB pages (write-back). */
- for (i = 0; i < MM_MAX_PHYS / 0x400000; i++)
- page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
- /* The kmap window: a supervisor page table every process directory
- * shares (paging_new_directory copies the PDE). */
- kmap_pt = (unsigned int *)mm_alloc_page();
- if (!kmap_pt) {
- serial_puts(SERIAL_COM1, "paging: out of memory (kmap)\n");
- for (;;)
- __asm__ volatile ("cli; hlt");
- }
- for (i = 0; i < 1024; i++)
- kmap_pt[i] = 0;
- page_dir[KMAP_PDE] = (unsigned int)kmap_pt | PDE_WRITABLE | PDE_PRESENT;
- /* The OHCI ED/TD/interrupt-buffer page is DMA'd by the SB950's legacy
- * USB controller, which does not snoop dirty CPU cache lines. Split
- * the containing 4 MB page now, before paging is enabled, so every
- * process directory clones the uncacheable mapping. */
- {
- extern unsigned char ohci_dma_page[];
- unsigned int va = (unsigned int)ohci_dma_page;
- if ((va >> 22) == 0)
- uc_page_cr3((unsigned int)page_dir, va);
- }
- /* Identity-map the framebuffer and make VRAM writes batch. Use the
- * actual framebuffer size (not the whole mapping) so the WC range
- * cannot swallow unrelated MMIO, and install the MTRR *before* choosing
- * the page type: a PCD=1 page is uncacheable no matter what the MTRRs
- * say, which made every VRAM dword a separate PCI write (~30 MB/s).
- * Only the pages fully covered by the WC entry become cacheable (their
- * effective type is then WC); everything else stays UC. */
- if (fb->valid && fb->lfb_base) {
- unsigned int want = (unsigned int)fb->pitch * fb->height;
- if (want == 0 || want > LFB_MAP_SIZE)
- want = LFB_MAP_SIZE;
- bootcon_puts("paging: mtrr\n");
- lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
- &lfb_wc_base, &lfb_wc_size);
- bootcon_puts("paging: map\n");
- lfb_apply_pages(page_dir);
- /* Tell the truth about VRAM writes: with cacheable pages the WC
- * MTRR applies (fast bursts); otherwise every dword is a separate
- * uncached write. */
- serial_puts(SERIAL_COM1, lfb_wc_ok
- ? "paging: LFB write-combining\n"
- : "paging: LFB uncached (no WC MTRR)\n");
- }
- /* Load CR3, enable PSE, then PG + WP. The jump flushes the prefetch
- * queue after the paging flip. */
- bootcon_puts("paging: enable\n");
- __asm__ volatile (
- "movl %0, %%cr3\n\t"
- "movl %%cr4, %%eax\n\t"
- "orl $0x10, %%eax\n\t" /* CR4.PSE */
- "movl %%eax, %%cr4\n\t"
- "movl %%cr0, %%eax\n\t"
- "orl $0x80010000, %%eax\n\t" /* CR0.PG | CR0.WP */
- "movl %%eax, %%cr0\n\t"
- "jmp 1f\n\t"
- "1:\n\t"
- :
- : "r"((unsigned int)page_dir)
- : "eax", "memory");
- bootcon_puts("paging: on\n");
- serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
- }
- unsigned int paging_current_cr3(void)
- {
- unsigned int cr3;
- __asm__ volatile ("movl %%cr3, %0" : "=r"(cr3));
- return cr3 & 0xFFFFF000u;
- }
- unsigned int paging_kernel_cr3(void)
- {
- return (unsigned int)page_dir;
- }
- unsigned int paging_new_directory(void)
- {
- unsigned int *pd = (unsigned int *)mm_alloc_page();
- unsigned int i;
- if (!pd)
- return 0;
- for (i = 0; i < 1024; i++)
- pd[i] = page_dir[i];
- return (unsigned int)pd;
- }
- int paging_map_page(unsigned int cr3, unsigned int va, unsigned int phys,
- unsigned int flags)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int pde = va >> 22;
- unsigned int pte = (va >> 12) & 0x3FF;
- unsigned int *pt;
- if (!(pd[pde] & PDE_PRESENT)) {
- unsigned int pt_phys = mm_alloc_page();
- unsigned int i;
- if (!pt_phys)
- return 0;
- pt = (unsigned int *)pt_phys;
- for (i = 0; i < 1024; i++)
- pt[i] = 0;
- pd[pde] = pt_phys | PDE_USER | PDE_WRITABLE | PDE_PRESENT;
- } else {
- pt = (unsigned int *)(pd[pde] & 0xFFFFF000);
- }
- pt[pte] = (phys & 0xFFFFF000) | flags;
- return 1;
- }
- unsigned int paging_page_phys(unsigned int cr3, unsigned int va)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int pde = pd[va >> 22];
- unsigned int pte = (va >> 12) & 0x3FF;
- if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
- return 0;
- {
- unsigned int e = ((unsigned int *)(pde & 0xFFFFF000))[pte];
- return (e & PTE_PRESENT) ? (e & 0xFFFFF000) : 0;
- }
- }
- void paging_unmap_page(unsigned int cr3, unsigned int va)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int pde = va >> 22;
- unsigned int pte = (va >> 12) & 0x3FF;
- if (!(pd[pde] & PDE_PRESENT))
- return;
- ((unsigned int *)(pd[pde] & 0xFFFFF000))[pte] = 0;
- }
- int paging_user_range_ok(unsigned int cr3, unsigned int va,
- unsigned int size, int write)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int end = va + size;
- unsigned int p;
- if (!pd || size == 0 || end < va)
- return 0;
- for (p = va & 0xFFFFF000u; p < end; p += PAGE_SIZE) {
- unsigned int pde = pd[p >> 22];
- unsigned int *pt;
- unsigned int pte;
- if (!(pde & PDE_PRESENT) || (pde & PDE_PS) || !(pde & PDE_USER))
- return 0;
- pt = (unsigned int *)(pde & 0xFFFFF000);
- pte = pt[(p >> 12) & 0x3FF];
- if (!(pte & PTE_PRESENT) || !(pte & PTE_USER))
- return 0;
- if (write && !(pte & PTE_WRITABLE))
- return 0;
- }
- return 1;
- }
- void paging_release_user_range(unsigned int cr3, unsigned int va,
- unsigned int size, int free_phys)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int end = va + size;
- unsigned int first, last, i, j;
- if (!pd || size == 0 || end < va)
- return;
- first = va >> 22;
- last = (end - 1) >> 22;
- for (i = first; i <= last && i < 1024; i++) {
- unsigned int pde = pd[i];
- unsigned int *pt;
- int empty = 1;
- /* Kernel identity mappings are 4 MB pages and must never be
- * touched here. */
- if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
- continue;
- pt = (unsigned int *)(pde & 0xFFFFF000);
- for (j = 0; j < 1024; j++) {
- unsigned int pva = (i << 22) | (j << 12);
- if (pva < va || pva >= end)
- continue;
- if (pt[j] & PTE_PRESENT) {
- /* Only user pages are owned by the process; a
- * supervisor mapping in this range is left for
- * its owner. */
- if (free_phys && (pt[j] & PTE_USER))
- mm_free_page(pt[j] & 0xFFFFF000);
- pt[j] = 0;
- }
- }
- for (j = 0; j < 1024; j++)
- if (pt[j] & PTE_PRESENT) {
- empty = 0;
- break;
- }
- if (empty) {
- mm_free_page(pde & 0xFFFFF000);
- pd[i] = 0;
- }
- }
- }
- /* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
- * the range is rounded up to a whole number of them). */
- void paging_map_mmio_cr3(unsigned int cr3, unsigned int phys_base, unsigned int size)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int p0 = phys_base >> 22;
- unsigned int p1 = (phys_base + size + 0x3FFFFFu) >> 22;
- unsigned int i;
- for (i = p0; i < p1 && i < 1024; i++)
- if (!kmap_pde_guard(i))
- pd[i] = (i << 22) | PDE_PS | PDE_PCD |
- PDE_WRITABLE | PDE_PRESENT;
- }
- void paging_map_mmio(unsigned int phys_base, unsigned int size)
- {
- paging_map_mmio_cr3((unsigned int)page_dir, phys_base, size);
- }
- /* Split the 4 MB page containing `va` in one directory and set PCD on that
- * 4 KB page. The kernel identity map is WB; legacy OHCI DMA on the SB950
- * does not reliably snoop dirty descriptor lines, so the ED/TD page must
- * bypass the cache. */
- static void uc_page_cr3(unsigned int cr3, unsigned int va)
- {
- unsigned int *pd = (unsigned int *)cr3;
- unsigned int pde = va >> 22;
- unsigned int pte = (va >> 12) & 0x3FFu;
- unsigned int entry = pd[pde];
- unsigned int *pt;
- unsigned int i;
- if (!(entry & PDE_PRESENT))
- return;
- if (entry & PDE_PS) {
- unsigned int base = entry & 0xFFC00000u;
- unsigned int flags = (entry & 0xFFFu) & ~PDE_PS;
- unsigned int table = mm_alloc_page();
- if (!table)
- return;
- pt = (unsigned int *)table;
- for (i = 0; i < 1024; i++)
- pt[i] = (base + i * 0x1000u) | flags;
- pd[pde] = table | flags;
- } else {
- pt = (unsigned int *)(entry & 0xFFFFF000u);
- }
- pt[pte] |= PDE_PCD;
- }
- /* Temporary kernel mapping of one frame. Lowmem is returned through the
- * identity map; a highmem frame gets this CPU's next window slot with
- * preemption disabled, so the task cannot migrate or be switched out while
- * the slot is live. Nothing may sleep or yield before kunmap(). */
- void *kmap(unsigned int phys)
- {
- struct cpu *c;
- unsigned int slot, va;
- phys &= 0xFFFFF000u;
- if (phys < MM_MAX_PHYS)
- return (void *)phys;
- preempt_disable();
- c = this_cpu();
- if (c->id >= MAX_CPUS || kmap_depth[c->id] >= KMAP_SLOTS) {
- serial_puts(SERIAL_COM1, "paging: kmap slots exhausted\n");
- for (;;)
- __asm__ volatile ("cli; hlt");
- }
- slot = c->id * KMAP_SLOTS + kmap_depth[c->id]++;
- va = MM_KMAP_VA + slot * PAGE_SIZE;
- kmap_pt[slot] = phys | PTE_WRITABLE | PTE_PRESENT;
- __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
- return (void *)va;
- }
- /* Undo the most recent kmap() on this CPU (mappings nest LIFO). */
- void kunmap(void *addr)
- {
- unsigned int va = (unsigned int)addr & 0xFFFFF000u;
- struct cpu *c;
- unsigned int slot;
- if (va < MM_MAX_PHYS)
- return;
- c = this_cpu();
- slot = (va - MM_KMAP_VA) / PAGE_SIZE;
- kmap_pt[slot] = 0;
- __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
- if (kmap_depth[c->id])
- kmap_depth[c->id]--;
- preempt_enable();
- }
|