|
|
@@ -38,15 +38,26 @@ _Static_assert(MM_KMAP_VA >= USER_STACK_TOP &&
|
|
|
static unsigned int *kmap_pt;
|
|
|
static unsigned int kmap_depth[MAX_CPUS];
|
|
|
|
|
|
-/* Every PDE writer below leaves the kmap slot alone: a BIOS-placed ACPI
|
|
|
- * table, MMIO BAR or LFB in that 4 MB range cannot be identity-mapped. */
|
|
|
-static int kmap_pde_guard(unsigned int i)
|
|
|
-{
|
|
|
- if (i != KMAP_PDE)
|
|
|
- return 0;
|
|
|
- serial_puts(SERIAL_COM1, "paging: refusing to map over the kmap window\n");
|
|
|
- return 1;
|
|
|
-}
|
|
|
+/* ioremap window: [IOMAP_BASE, 4 GB), kernel-only, 4 KB pages. Its page
|
|
|
+ * tables are allocated on first use and their PDEs copied into every live
|
|
|
+ * process directory (process_sync_kernel_pde), so nothing is mapped at its
|
|
|
+ * physical address above lowmem and ring-3 space is free up to IOMAP_BASE. */
|
|
|
+_Static_assert(IOMAP_BASE >= USER_HEAP_BASE + USER_HEAP_MAX &&
|
|
|
+ IOMAP_BASE >= WM_CONTENT_VA_BASE +
|
|
|
+ WM_CONTENT_SLOT * WM_MAX_WINDOWS &&
|
|
|
+ (IOMAP_BASE & 0x3FFFFFu) == 0,
|
|
|
+ "ioremap window overlaps ring-3 space");
|
|
|
+#define IOMAP_MAX 96
|
|
|
+struct iomap {
|
|
|
+ unsigned int phys; /* page-aligned */
|
|
|
+ unsigned int size; /* page multiple */
|
|
|
+ unsigned int va;
|
|
|
+ int type;
|
|
|
+};
|
|
|
+static struct iomap iomaps[IOMAP_MAX];
|
|
|
+static unsigned int n_iomaps;
|
|
|
+static unsigned int iomap_next = IOMAP_BASE; /* bump pointer; 0 = full */
|
|
|
+static void *lfb_va;
|
|
|
|
|
|
/* CR4 bits that must not survive a BIOS protected-mode call. A BIOS that
|
|
|
* returns with CR4.PAE or EFER.LME set makes the first CR0.PG enable either
|
|
|
@@ -138,8 +149,9 @@ void paging_prepare_cpu(void)
|
|
|
static unsigned int lfb_wc_base, lfb_wc_size;
|
|
|
static int lfb_wc_ok;
|
|
|
|
|
|
-/* Write the LFB PDEs: WC-covered pages cacheable (effective type WC), the
|
|
|
- * rest uncacheable. */
|
|
|
+/* Write the LFB PDEs for a framebuffer in lowmem (stolen system RAM below
|
|
|
+ * 1 GB): WC-covered pages cacheable (effective type WC), the rest
|
|
|
+ * uncacheable. An LFB above lowmem goes through ioremap() instead. */
|
|
|
static void lfb_apply_pages(unsigned int *pd)
|
|
|
{
|
|
|
const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
|
|
|
@@ -161,8 +173,6 @@ static void lfb_apply_pages(unsigned int *pd)
|
|
|
unsigned int page = i << 22;
|
|
|
unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
|
|
|
|
|
|
- if (kmap_pde_guard(i))
|
|
|
- continue;
|
|
|
if (!lfb_wc_ok || page < lfb_wc_base ||
|
|
|
page + 0x400000 > lfb_wc_base + lfb_wc_size)
|
|
|
flags |= PDE_PCD;
|
|
|
@@ -170,34 +180,15 @@ static void lfb_apply_pages(unsigned int *pd)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-/* Re-apply the framebuffer mapping; call after any MMIO mapping that may have
|
|
|
- * covered the same addresses (the NVIDIA BAR3 aperture includes the VBE LFB). */
|
|
|
-void paging_lfb_refresh(void)
|
|
|
-{
|
|
|
- if (!lfb_wc_ok || !page_dir)
|
|
|
- return;
|
|
|
- lfb_apply_pages(page_dir);
|
|
|
- serial_puts(SERIAL_COM1, "paging: LFB write-combining re-applied\n");
|
|
|
-}
|
|
|
-
|
|
|
/* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
|
|
|
int paging_lfb_wc(void)
|
|
|
{
|
|
|
return lfb_wc_ok;
|
|
|
}
|
|
|
|
|
|
-void paging_map_ram(unsigned int base, unsigned int size)
|
|
|
+void *paging_lfb_va(void)
|
|
|
{
|
|
|
- unsigned int p0 = base >> 22;
|
|
|
- unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
|
|
|
- unsigned int i;
|
|
|
-
|
|
|
- if (!page_dir)
|
|
|
- return;
|
|
|
- for (i = p0; i < p1 && i < 1024; i++)
|
|
|
- if (!kmap_pde_guard(i))
|
|
|
- page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE |
|
|
|
- PDE_PRESENT;
|
|
|
+ return lfb_va;
|
|
|
}
|
|
|
|
|
|
void paging_init(void)
|
|
|
@@ -269,7 +260,18 @@ void paging_init(void)
|
|
|
lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
|
|
|
&lfb_wc_base, &lfb_wc_size);
|
|
|
bootcon_puts("paging: map\n");
|
|
|
- lfb_apply_pages(page_dir);
|
|
|
+ if ((unsigned long long)fb->lfb_base + LFB_MAP_SIZE <= MM_MAX_PHYS) {
|
|
|
+ lfb_apply_pages(page_dir);
|
|
|
+ lfb_va = (void *)fb->lfb_base;
|
|
|
+ } else {
|
|
|
+ unsigned long long end = (unsigned long long)fb->lfb_base +
|
|
|
+ LFB_MAP_SIZE;
|
|
|
+ unsigned int sz = end > 0x100000000ull
|
|
|
+ ? (unsigned int)(0x100000000ull - fb->lfb_base)
|
|
|
+ : LFB_MAP_SIZE;
|
|
|
+
|
|
|
+ lfb_va = ioremap(fb->lfb_base, sz, IOREMAP_MMIO);
|
|
|
+ }
|
|
|
/* Tell the truth about VRAM writes: with cacheable pages the WC
|
|
|
* MTRR applies (fast bursts); otherwise every dword is a separate
|
|
|
* uncached write. */
|
|
|
@@ -295,6 +297,8 @@ void paging_init(void)
|
|
|
: "r"((unsigned int)page_dir)
|
|
|
: "eax", "memory");
|
|
|
|
|
|
+ /* The boot console drew through the physical LFB until now. */
|
|
|
+ bootcon_set_lfb(lfb_va);
|
|
|
bootcon_puts("paging: on\n");
|
|
|
serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
|
|
|
}
|
|
|
@@ -453,22 +457,159 @@ void paging_release_user_range(unsigned int cr3, unsigned int va,
|
|
|
|
|
|
/* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
|
|
|
* the range is rounded up to a whole number of them). */
|
|
|
-void paging_map_mmio_cr3(unsigned int cr3, unsigned int phys_base, unsigned int size)
|
|
|
+/* MMIO inside lowmem (a BAR the firmware put below 1 GB): make the 4 MB
|
|
|
+ * identity pages uncacheable, as the identity mapper always did, in the
|
|
|
+ * kernel directory and every process copy. */
|
|
|
+static void lowmem_uncached(unsigned int phys, unsigned int size)
|
|
|
{
|
|
|
- unsigned int *pd = (unsigned int *)cr3;
|
|
|
- unsigned int p0 = phys_base >> 22;
|
|
|
- unsigned int p1 = (phys_base + size + 0x3FFFFFu) >> 22;
|
|
|
+ unsigned int p0 = phys >> 22;
|
|
|
+ unsigned int p1 = (unsigned int)(((unsigned long long)phys + size +
|
|
|
+ 0x3FFFFFu) >> 22);
|
|
|
unsigned int i;
|
|
|
|
|
|
- for (i = p0; i < p1 && i < 1024; i++)
|
|
|
- if (!kmap_pde_guard(i))
|
|
|
- pd[i] = (i << 22) | PDE_PS | PDE_PCD |
|
|
|
- PDE_WRITABLE | PDE_PRESENT;
|
|
|
+ for (i = p0; i < p1 && i < MM_MAX_PHYS / 0x400000u; i++) {
|
|
|
+ if (!(page_dir[i] & PDE_PS))
|
|
|
+ continue; /* split PDE 0 keeps its own page types */
|
|
|
+ page_dir[i] = (i << 22) | PDE_PS | PDE_PCD | PDE_WRITABLE |
|
|
|
+ PDE_PRESENT;
|
|
|
+ process_sync_kernel_pde(i);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
-void paging_map_mmio(unsigned int phys_base, unsigned int size)
|
|
|
+/* Page type for one 4 KB window page: RAM is write-back; MMIO is uncached,
|
|
|
+ * except pages the LFB's WC MTRR covers, which are cacheable (effective
|
|
|
+ * type WC) in every alias, so a GPU BAR that contains the framebuffer and
|
|
|
+ * the framebuffer mapping itself never disagree. */
|
|
|
+static unsigned int iomap_pte_flags(unsigned int phys, int type)
|
|
|
{
|
|
|
- paging_map_mmio_cr3((unsigned int)page_dir, phys_base, size);
|
|
|
+ unsigned int f = PTE_PRESENT | PTE_WRITABLE;
|
|
|
+
|
|
|
+ if (type == IOREMAP_RAM)
|
|
|
+ return f;
|
|
|
+ if (lfb_wc_ok && phys >= lfb_wc_base &&
|
|
|
+ (unsigned long long)phys + PAGE_SIZE <=
|
|
|
+ (unsigned long long)lfb_wc_base + lfb_wc_size)
|
|
|
+ return f;
|
|
|
+ return f | PDE_PCD;
|
|
|
+}
|
|
|
+
|
|
|
+static int iomap_set(unsigned int va, unsigned int phys, unsigned int flags)
|
|
|
+{
|
|
|
+ unsigned int pde = va >> 22;
|
|
|
+ unsigned int *pt;
|
|
|
+
|
|
|
+ if (!(page_dir[pde] & PDE_PRESENT)) {
|
|
|
+ unsigned int t = mm_alloc_page();
|
|
|
+ unsigned int i;
|
|
|
+
|
|
|
+ if (!t)
|
|
|
+ return 0;
|
|
|
+ pt = (unsigned int *)t;
|
|
|
+ for (i = 0; i < 1024; i++)
|
|
|
+ pt[i] = 0;
|
|
|
+ page_dir[pde] = t | PDE_WRITABLE | PDE_PRESENT;
|
|
|
+ process_sync_kernel_pde(pde);
|
|
|
+ } else {
|
|
|
+ pt = (unsigned int *)(page_dir[pde] & 0xFFFFF000u);
|
|
|
+ }
|
|
|
+ pt[(va >> 12) & 0x3FFu] = (phys & 0xFFFFF000u) | flags;
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+void *ioremap(unsigned int phys, unsigned int size, int type)
|
|
|
+{
|
|
|
+ unsigned long long end;
|
|
|
+ unsigned int p0, bytes, va, i;
|
|
|
+
|
|
|
+ if (size == 0)
|
|
|
+ size = 1;
|
|
|
+ end = (unsigned long long)phys + size;
|
|
|
+ if (end > 0x100000000ull || !page_dir)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ /* Lowmem is identity-mapped already. */
|
|
|
+ if (end <= MM_MAX_PHYS) {
|
|
|
+ if (type == IOREMAP_MMIO)
|
|
|
+ lowmem_uncached(phys, size);
|
|
|
+ return (void *)phys;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Reuse a mapping of the same type that already covers the range. */
|
|
|
+ for (i = 0; i < n_iomaps; i++) {
|
|
|
+ const struct iomap *m = &iomaps[i];
|
|
|
+
|
|
|
+ if (m->type == type && phys >= m->phys &&
|
|
|
+ end <= (unsigned long long)m->phys + m->size)
|
|
|
+ return (void *)(m->va + (phys - m->phys));
|
|
|
+ }
|
|
|
+
|
|
|
+ /* Grow the newest mapping in place when the request starts inside it
|
|
|
+ * and only runs past its end (a table header mapped first, then the
|
|
|
+ * whole table): its pages are the last ones in the window. */
|
|
|
+ if (n_iomaps && iomap_next) {
|
|
|
+ struct iomap *m = &iomaps[n_iomaps - 1];
|
|
|
+ unsigned long long mend = (unsigned long long)m->phys + m->size;
|
|
|
+ unsigned long long need = (end + PAGE_SIZE - 1) &
|
|
|
+ ~(unsigned long long)(PAGE_SIZE - 1);
|
|
|
+
|
|
|
+ if (m->type == type && phys >= m->phys && phys < mend &&
|
|
|
+ m->va + m->size == iomap_next &&
|
|
|
+ (unsigned long long)iomap_next + (need - mend) <=
|
|
|
+ 0x100000000ull) {
|
|
|
+ unsigned int extra = (unsigned int)(need - mend);
|
|
|
+
|
|
|
+ for (i = 0; i < extra; i += PAGE_SIZE)
|
|
|
+ if (!iomap_set(iomap_next + i,
|
|
|
+ (unsigned int)mend + i,
|
|
|
+ iomap_pte_flags((unsigned int)mend + i,
|
|
|
+ type)))
|
|
|
+ return 0;
|
|
|
+ m->size += extra;
|
|
|
+ iomap_next = ((unsigned long long)iomap_next + extra >=
|
|
|
+ 0x100000000ull) ? 0 : iomap_next + extra;
|
|
|
+ return (void *)(m->va + (phys - m->phys));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ p0 = phys & 0xFFFFF000u;
|
|
|
+ bytes = (unsigned int)(((end + PAGE_SIZE - 1) &
|
|
|
+ ~(unsigned long long)(PAGE_SIZE - 1)) - p0);
|
|
|
+ if (n_iomaps >= IOMAP_MAX || !iomap_next ||
|
|
|
+ (unsigned long long)iomap_next + bytes > 0x100000000ull) {
|
|
|
+ serial_puts(SERIAL_COM1, "paging: ioremap window full for ");
|
|
|
+ serial_puthex(SERIAL_COM1, phys);
|
|
|
+ serial_puts(SERIAL_COM1, "\n");
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ va = iomap_next;
|
|
|
+ for (i = 0; i < bytes; i += PAGE_SIZE)
|
|
|
+ if (!iomap_set(va + i, p0 + i, iomap_pte_flags(p0 + i, type))) {
|
|
|
+ serial_puts(SERIAL_COM1, "paging: ioremap out of memory\n");
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+ iomap_next = ((unsigned long long)va + bytes >= 0x100000000ull)
|
|
|
+ ? 0 : va + bytes;
|
|
|
+ iomaps[n_iomaps].phys = p0;
|
|
|
+ iomaps[n_iomaps].size = bytes;
|
|
|
+ iomaps[n_iomaps].va = va;
|
|
|
+ iomaps[n_iomaps].type = type;
|
|
|
+ n_iomaps++;
|
|
|
+ return (void *)(va + (phys - p0));
|
|
|
+}
|
|
|
+
|
|
|
+/* The window address of `phys` if some ioremap() covers it, else 0. */
|
|
|
+void *ioremap_lookup(unsigned int phys)
|
|
|
+{
|
|
|
+ unsigned int i;
|
|
|
+
|
|
|
+ for (i = 0; i < n_iomaps; i++) {
|
|
|
+ const struct iomap *m = &iomaps[i];
|
|
|
+
|
|
|
+ if (phys >= m->phys &&
|
|
|
+ (unsigned long long)phys < (unsigned long long)m->phys + m->size)
|
|
|
+ return (void *)(m->va + (phys - m->phys));
|
|
|
+ }
|
|
|
+ return 0;
|
|
|
}
|
|
|
|
|
|
/* Split the 4 MB page containing `va` in one directory and set PCD on that
|
|
|
@@ -546,3 +687,35 @@ void kunmap(void *addr)
|
|
|
kmap_depth[c->id]--;
|
|
|
preempt_enable();
|
|
|
}
|
|
|
+
|
|
|
+static int pa_is_ram(unsigned int pa)
|
|
|
+{
|
|
|
+ unsigned int i, n = mm_e820_count();
|
|
|
+
|
|
|
+ for (i = 0; i < n; i++) {
|
|
|
+ const struct e820_entry *e = mm_e820_entry(i);
|
|
|
+
|
|
|
+ if (e->type == E820_RAM && pa >= e->base &&
|
|
|
+ (unsigned long long)pa < e->base + e->length)
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+volatile unsigned char *paging_debug_map(unsigned int pa, void **km)
|
|
|
+{
|
|
|
+ unsigned char *v;
|
|
|
+
|
|
|
+ *km = 0;
|
|
|
+ if (pa < MM_MAX_PHYS)
|
|
|
+ return (volatile unsigned char *)pa;
|
|
|
+ if (pa_is_ram(pa)) {
|
|
|
+ *km = kmap(pa & 0xFFFFF000u);
|
|
|
+ return (volatile unsigned char *)*km + (pa & 0xFFFu);
|
|
|
+ }
|
|
|
+ v = ioremap_lookup(pa); /* includes the offset */
|
|
|
+ if (v)
|
|
|
+ return v;
|
|
|
+ v = ioremap(pa & 0xFFFFF000u, 0x1000, IOREMAP_MMIO);
|
|
|
+ return v ? v + (pa & 0xFFFu) : 0;
|
|
|
+}
|