paging.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721
  1. /* Paging: identity-map RAM + the VESA LFB, enable CR0.PG (4 MB pages).
  2. *
  3. * This is the "paging on but flat" stage: every physical address is mapped
  4. * 1:1, so all existing code (kernel, stacks, DMA, UHCI, VGA shadow) keeps
  5. * working unchanged. Per-process page directories come later (Phase 3b/4).
  6. *
  7. * Uses 4 MB pages via CR4.PSE (supported since the original Pentium), so the
  8. * whole map is a single 4 KB page directory.
  9. */
  10. #include "paging.h"
  11. #include "bootcon.h"
  12. #include "cpu.h"
  13. #include "mm.h"
  14. #include "mtrr.h"
  15. #include "process.h"
  16. #include "sched.h"
  17. #include "serial.h"
  18. #include "vga.h"
  19. #include "wm_abi.h"
  20. /* The RAM identity map is made of supervisor 4 MB PDEs; the ring-3 windows
  21. * (program/stack, canvases, WM slots, heap) are user page tables above it.
  22. * They must never share a PDE. */
  23. _Static_assert(MM_MAX_PHYS <= USER_BASE, "identity map overlaps user space");
  24. _Static_assert(MM_MAX_PHYS <= USER_WIN_BASE &&
  25. MM_MAX_PHYS <= WM_CONTENT_VA_BASE &&
  26. MM_MAX_PHYS <= USER_HEAP_BASE,
  27. "identity map overlaps a ring-3 window");
  28. _Static_assert(MM_KMAP_VA >= USER_STACK_TOP &&
  29. MM_KMAP_VA + 0x400000u <= USER_WIN_BASE &&
  30. (MM_KMAP_VA & 0x3FFFFFu) == 0,
  31. "kmap window must be a free 4 MB slot below USER_WIN_BASE");
  32. /* kmap: KMAP_SLOTS nested mappings per CPU in the shared window page table. */
  33. #define KMAP_PDE (MM_KMAP_VA >> 22)
  34. #define KMAP_SLOTS 4
  35. static unsigned int *kmap_pt;
  36. static unsigned int kmap_depth[MAX_CPUS];
  37. /* ioremap window: [IOMAP_BASE, 4 GB), kernel-only, 4 KB pages. Its page
  38. * tables are allocated on first use and their PDEs copied into every live
  39. * process directory (process_sync_kernel_pde), so nothing is mapped at its
  40. * physical address above lowmem and ring-3 space is free up to IOMAP_BASE. */
  41. _Static_assert(IOMAP_BASE >= USER_HEAP_BASE + USER_HEAP_MAX &&
  42. IOMAP_BASE >= WM_CONTENT_VA_BASE +
  43. WM_CONTENT_SLOT * WM_MAX_WINDOWS &&
  44. (IOMAP_BASE & 0x3FFFFFu) == 0,
  45. "ioremap window overlaps ring-3 space");
  46. #define IOMAP_MAX 96
  47. struct iomap {
  48. unsigned int phys; /* page-aligned */
  49. unsigned int size; /* page multiple */
  50. unsigned int va;
  51. int type;
  52. };
  53. static struct iomap iomaps[IOMAP_MAX];
  54. static unsigned int n_iomaps;
  55. static unsigned int iomap_next = IOMAP_BASE; /* bump pointer; 0 = full */
  56. static void *lfb_va;
  57. /* CR4 bits that must not survive a BIOS protected-mode call. A BIOS that
  58. * returns with CR4.PAE or EFER.LME set makes the first CR0.PG enable either
  59. * walk the 32-bit page directory as a PAE PDPT or enter long mode, which
  60. * triple-faults before the kernel can print anything. The X201's old
  61. * INT 15h AH=87h rootfs copy was one source of this state; clear the bits
  62. * defensively on every boot path before enabling paging. */
  63. #define CR4_PSE (1u << 4)
  64. #define CR4_PAE (1u << 5)
  65. #define CR4_PGE (1u << 7)
  66. #define CR4_LA57 (1u << 12)
  67. #define CR4_PCIDE (1u << 17)
  68. #define CR4_SMEP (1u << 20)
  69. #define CR4_SMAP (1u << 21)
  70. #define MSR_EFER 0xC0000080u
  71. #define EFER_LME (1u << 8)
  72. /* Page-directory entry flags (4 MB page). */
  73. #define PDE_PRESENT 0x001
  74. #define PDE_WRITABLE 0x002
  75. #define PDE_USER 0x004
  76. #define PDE_PWT 0x008
  77. #define PDE_PCD 0x010
  78. #define PDE_PS 0x080 /* 4 MB page */
  79. /* How much of the LFB to map. 16 MB covers any mode up to 1920x1080x32
  80. * (~8.3 MB) with margin, and is independent of the BIOS's VRAM report. */
  81. #define LFB_MAP_SIZE 0x1000000
  82. static unsigned int *page_dir;
  83. static void uc_page_cr3(unsigned int cr3, unsigned int va);
  84. static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
  85. {
  86. __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
  87. }
  88. static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
  89. {
  90. __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
  91. }
  92. static int cpu_has_long_mode(void)
  93. {
  94. unsigned int eax, ebx, ecx, edx;
  95. if (!cpu_has_cpuid())
  96. return 0;
  97. eax = 0x80000000u;
  98. __asm__ volatile ("cpuid"
  99. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  100. if (eax < 0x80000001u)
  101. return 0;
  102. eax = 0x80000001u;
  103. __asm__ volatile ("cpuid"
  104. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  105. return (edx & (1u << 29)) != 0; /* LM */
  106. }
  107. /* Put the CPU into the state this 32-bit kernel expects before it installs
  108. * its own page directory. A BIOS protected-mode call (notably the old
  109. * high-pack INT 15h AH=87h path) may leave CR4.PAE/PGE or EFER.LME set;
  110. * CR0.PG is off here, so clearing them is safe on every CPU (reserved bits
  111. * are written as zero). */
  112. void paging_prepare_cpu(void)
  113. {
  114. unsigned int cr4, lo, hi;
  115. __asm__ volatile ("movl %%cr4, %0" : "=r"(cr4));
  116. cr4 &= ~(CR4_PAE | CR4_PGE | CR4_LA57 | CR4_PCIDE |
  117. CR4_SMEP | CR4_SMAP);
  118. cr4 |= CR4_PSE;
  119. __asm__ volatile ("movl %0, %%cr4" : : "r"(cr4) : "memory");
  120. if (cpu_has_long_mode()) {
  121. rdmsr(MSR_EFER, &lo, &hi);
  122. if (lo & EFER_LME) {
  123. lo &= ~EFER_LME;
  124. wrmsr(MSR_EFER, lo, hi);
  125. serial_puts(SERIAL_COM1, "paging: cleared EFER.LME\n");
  126. }
  127. }
  128. }
  129. /* LFB page-type decision, kept so it can be re-applied after other MMIO
  130. * mappings (the GPU BARs cover the framebuffer address) have set PCD. */
  131. static unsigned int lfb_wc_base, lfb_wc_size;
  132. static int lfb_wc_ok;
  133. /* Write the LFB PDEs for a framebuffer in lowmem (stolen system RAM below
  134. * 1 GB): WC-covered pages cacheable (effective type WC), the rest
  135. * uncacheable. An LFB above lowmem goes through ioremap() instead. */
  136. static void lfb_apply_pages(unsigned int *pd)
  137. {
  138. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  139. unsigned int base, p0, p1, i;
  140. __asm__ volatile ("" : "+r"(fb));
  141. if (!fb->valid || !fb->lfb_base)
  142. return;
  143. base = fb->lfb_base;
  144. p0 = base >> 22;
  145. /* Compute the end page from the page index, not from base+size: for a
  146. * base above 0xFF000000 the 32-bit addition wraps to a small number
  147. * and the loop would map nothing, faulting on the first LFB write
  148. * after CR0.PG comes on. */
  149. p1 = p0 + (LFB_MAP_SIZE >> 22);
  150. if (p1 > 1024)
  151. p1 = 1024;
  152. for (i = p0; i < p1; i++) {
  153. unsigned int page = i << 22;
  154. unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  155. if (!lfb_wc_ok || page < lfb_wc_base ||
  156. page + 0x400000 > lfb_wc_base + lfb_wc_size)
  157. flags |= PDE_PCD;
  158. pd[i] = page | flags;
  159. }
  160. }
  161. /* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
  162. int paging_lfb_wc(void)
  163. {
  164. return lfb_wc_ok;
  165. }
  166. void *paging_lfb_va(void)
  167. {
  168. return lfb_va;
  169. }
  170. void paging_init(void)
  171. {
  172. unsigned int i;
  173. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  174. bootcon_puts("paging: prepare\n");
  175. paging_prepare_cpu();
  176. /* Opaque asm hides the fixed 0x600 pointer from -Warray-bounds. */
  177. __asm__ volatile ("" : "+r"(fb));
  178. bootcon_puts("paging: alloc\n");
  179. page_dir = (unsigned int *)mm_alloc_page();
  180. if (!page_dir) {
  181. serial_puts(SERIAL_COM1, "paging: out of memory\n");
  182. for (;;)
  183. __asm__ volatile ("cli; hlt");
  184. }
  185. /* Zero the whole page directory. mm_alloc_page does NOT clear memory, so
  186. * on real hardware the high PDEs (indices 32..1023, incl. the user region
  187. * at 0x100) would otherwise be garbage. QEMU zeroes guest RAM, which is
  188. * why this only bites on real silicon. */
  189. for (i = 0; i < 1024; i++)
  190. page_dir[i] = 0;
  191. /* Identity-map RAM [0, MM_MAX_PHYS) with 4 MB pages (write-back). */
  192. for (i = 0; i < MM_MAX_PHYS / 0x400000; i++)
  193. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  194. /* The kmap window: a supervisor page table every process directory
  195. * shares (paging_new_directory copies the PDE). */
  196. kmap_pt = (unsigned int *)mm_alloc_page();
  197. if (!kmap_pt) {
  198. serial_puts(SERIAL_COM1, "paging: out of memory (kmap)\n");
  199. for (;;)
  200. __asm__ volatile ("cli; hlt");
  201. }
  202. for (i = 0; i < 1024; i++)
  203. kmap_pt[i] = 0;
  204. page_dir[KMAP_PDE] = (unsigned int)kmap_pt | PDE_WRITABLE | PDE_PRESENT;
  205. /* The OHCI ED/TD/interrupt-buffer page is DMA'd by the SB950's legacy
  206. * USB controller, which does not snoop dirty CPU cache lines. Split
  207. * the containing 4 MB page now, before paging is enabled, so every
  208. * process directory clones the uncacheable mapping. */
  209. {
  210. extern unsigned char ohci_dma_page[];
  211. unsigned int va = (unsigned int)ohci_dma_page;
  212. if ((va >> 22) == 0)
  213. uc_page_cr3((unsigned int)page_dir, va);
  214. }
  215. /* Identity-map the framebuffer and make VRAM writes batch. Use the
  216. * actual framebuffer size (not the whole mapping) so the WC range
  217. * cannot swallow unrelated MMIO, and install the MTRR *before* choosing
  218. * the page type: a PCD=1 page is uncacheable no matter what the MTRRs
  219. * say, which made every VRAM dword a separate PCI write (~30 MB/s).
  220. * Only the pages fully covered by the WC entry become cacheable (their
  221. * effective type is then WC); everything else stays UC. */
  222. if (fb->valid && fb->lfb_base) {
  223. unsigned int want = (unsigned int)fb->pitch * fb->height;
  224. if (want == 0 || want > LFB_MAP_SIZE)
  225. want = LFB_MAP_SIZE;
  226. bootcon_puts("paging: mtrr\n");
  227. lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
  228. &lfb_wc_base, &lfb_wc_size);
  229. bootcon_puts("paging: map\n");
  230. if ((unsigned long long)fb->lfb_base + LFB_MAP_SIZE <= MM_MAX_PHYS) {
  231. lfb_apply_pages(page_dir);
  232. lfb_va = (void *)fb->lfb_base;
  233. } else {
  234. unsigned long long end = (unsigned long long)fb->lfb_base +
  235. LFB_MAP_SIZE;
  236. unsigned int sz = end > 0x100000000ull
  237. ? (unsigned int)(0x100000000ull - fb->lfb_base)
  238. : LFB_MAP_SIZE;
  239. lfb_va = ioremap(fb->lfb_base, sz, IOREMAP_MMIO);
  240. }
  241. /* Tell the truth about VRAM writes: with cacheable pages the WC
  242. * MTRR applies (fast bursts); otherwise every dword is a separate
  243. * uncached write. */
  244. serial_puts(SERIAL_COM1, lfb_wc_ok
  245. ? "paging: LFB write-combining\n"
  246. : "paging: LFB uncached (no WC MTRR)\n");
  247. }
  248. /* Load CR3, enable PSE, then PG + WP. The jump flushes the prefetch
  249. * queue after the paging flip. */
  250. bootcon_puts("paging: enable\n");
  251. __asm__ volatile (
  252. "movl %0, %%cr3\n\t"
  253. "movl %%cr4, %%eax\n\t"
  254. "orl $0x10, %%eax\n\t" /* CR4.PSE */
  255. "movl %%eax, %%cr4\n\t"
  256. "movl %%cr0, %%eax\n\t"
  257. "orl $0x80010000, %%eax\n\t" /* CR0.PG | CR0.WP */
  258. "movl %%eax, %%cr0\n\t"
  259. "jmp 1f\n\t"
  260. "1:\n\t"
  261. :
  262. : "r"((unsigned int)page_dir)
  263. : "eax", "memory");
  264. /* The boot console drew through the physical LFB until now. */
  265. bootcon_set_lfb(lfb_va);
  266. bootcon_puts("paging: on\n");
  267. serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
  268. }
  269. unsigned int paging_current_cr3(void)
  270. {
  271. unsigned int cr3;
  272. __asm__ volatile ("movl %%cr3, %0" : "=r"(cr3));
  273. return cr3 & 0xFFFFF000u;
  274. }
  275. unsigned int paging_kernel_cr3(void)
  276. {
  277. return (unsigned int)page_dir;
  278. }
  279. unsigned int paging_new_directory(void)
  280. {
  281. unsigned int *pd = (unsigned int *)mm_alloc_page();
  282. unsigned int i;
  283. if (!pd)
  284. return 0;
  285. for (i = 0; i < 1024; i++)
  286. pd[i] = page_dir[i];
  287. return (unsigned int)pd;
  288. }
  289. int paging_map_page(unsigned int cr3, unsigned int va, unsigned int phys,
  290. unsigned int flags)
  291. {
  292. unsigned int *pd = (unsigned int *)cr3;
  293. unsigned int pde = va >> 22;
  294. unsigned int pte = (va >> 12) & 0x3FF;
  295. unsigned int *pt;
  296. if (!(pd[pde] & PDE_PRESENT)) {
  297. unsigned int pt_phys = mm_alloc_page();
  298. unsigned int i;
  299. if (!pt_phys)
  300. return 0;
  301. pt = (unsigned int *)pt_phys;
  302. for (i = 0; i < 1024; i++)
  303. pt[i] = 0;
  304. pd[pde] = pt_phys | PDE_USER | PDE_WRITABLE | PDE_PRESENT;
  305. } else {
  306. pt = (unsigned int *)(pd[pde] & 0xFFFFF000);
  307. }
  308. pt[pte] = (phys & 0xFFFFF000) | flags;
  309. return 1;
  310. }
  311. unsigned int paging_page_phys(unsigned int cr3, unsigned int va)
  312. {
  313. unsigned int *pd = (unsigned int *)cr3;
  314. unsigned int pde = pd[va >> 22];
  315. unsigned int pte = (va >> 12) & 0x3FF;
  316. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  317. return 0;
  318. {
  319. unsigned int e = ((unsigned int *)(pde & 0xFFFFF000))[pte];
  320. return (e & PTE_PRESENT) ? (e & 0xFFFFF000) : 0;
  321. }
  322. }
  323. void paging_unmap_page(unsigned int cr3, unsigned int va)
  324. {
  325. unsigned int *pd = (unsigned int *)cr3;
  326. unsigned int pde = va >> 22;
  327. unsigned int pte = (va >> 12) & 0x3FF;
  328. if (!(pd[pde] & PDE_PRESENT))
  329. return;
  330. ((unsigned int *)(pd[pde] & 0xFFFFF000))[pte] = 0;
  331. }
  332. int paging_user_range_ok(unsigned int cr3, unsigned int va,
  333. unsigned int size, int write)
  334. {
  335. unsigned int *pd = (unsigned int *)cr3;
  336. unsigned int end = va + size;
  337. unsigned int p;
  338. if (!pd || size == 0 || end < va)
  339. return 0;
  340. for (p = va & 0xFFFFF000u; p < end; p += PAGE_SIZE) {
  341. unsigned int pde = pd[p >> 22];
  342. unsigned int *pt;
  343. unsigned int pte;
  344. if (!(pde & PDE_PRESENT) || (pde & PDE_PS) || !(pde & PDE_USER))
  345. return 0;
  346. pt = (unsigned int *)(pde & 0xFFFFF000);
  347. pte = pt[(p >> 12) & 0x3FF];
  348. if (!(pte & PTE_PRESENT) || !(pte & PTE_USER))
  349. return 0;
  350. if (write && !(pte & PTE_WRITABLE))
  351. return 0;
  352. }
  353. return 1;
  354. }
  355. void paging_release_user_range(unsigned int cr3, unsigned int va,
  356. unsigned int size, int free_phys)
  357. {
  358. unsigned int *pd = (unsigned int *)cr3;
  359. unsigned int end = va + size;
  360. unsigned int first, last, i, j;
  361. if (!pd || size == 0 || end < va)
  362. return;
  363. first = va >> 22;
  364. last = (end - 1) >> 22;
  365. for (i = first; i <= last && i < 1024; i++) {
  366. unsigned int pde = pd[i];
  367. unsigned int *pt;
  368. int empty = 1;
  369. /* Kernel identity mappings are 4 MB pages and must never be
  370. * touched here. */
  371. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  372. continue;
  373. pt = (unsigned int *)(pde & 0xFFFFF000);
  374. for (j = 0; j < 1024; j++) {
  375. unsigned int pva = (i << 22) | (j << 12);
  376. if (pva < va || pva >= end)
  377. continue;
  378. if (pt[j] & PTE_PRESENT) {
  379. /* Only user pages are owned by the process; a
  380. * supervisor mapping in this range is left for
  381. * its owner. */
  382. if (free_phys && (pt[j] & PTE_USER))
  383. mm_free_page(pt[j] & 0xFFFFF000);
  384. pt[j] = 0;
  385. }
  386. }
  387. for (j = 0; j < 1024; j++)
  388. if (pt[j] & PTE_PRESENT) {
  389. empty = 0;
  390. break;
  391. }
  392. if (empty) {
  393. mm_free_page(pde & 0xFFFFF000);
  394. pd[i] = 0;
  395. }
  396. }
  397. }
  398. /* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
  399. * the range is rounded up to a whole number of them). */
  400. /* MMIO inside lowmem (a BAR the firmware put below 1 GB): make the 4 MB
  401. * identity pages uncacheable, as the identity mapper always did, in the
  402. * kernel directory and every process copy. */
  403. static void lowmem_uncached(unsigned int phys, unsigned int size)
  404. {
  405. unsigned int p0 = phys >> 22;
  406. unsigned int p1 = (unsigned int)(((unsigned long long)phys + size +
  407. 0x3FFFFFu) >> 22);
  408. unsigned int i;
  409. for (i = p0; i < p1 && i < MM_MAX_PHYS / 0x400000u; i++) {
  410. if (!(page_dir[i] & PDE_PS))
  411. continue; /* split PDE 0 keeps its own page types */
  412. page_dir[i] = (i << 22) | PDE_PS | PDE_PCD | PDE_WRITABLE |
  413. PDE_PRESENT;
  414. process_sync_kernel_pde(i);
  415. }
  416. }
  417. /* Page type for one 4 KB window page: RAM is write-back; MMIO is uncached,
  418. * except pages the LFB's WC MTRR covers, which are cacheable (effective
  419. * type WC) in every alias, so a GPU BAR that contains the framebuffer and
  420. * the framebuffer mapping itself never disagree. */
  421. static unsigned int iomap_pte_flags(unsigned int phys, int type)
  422. {
  423. unsigned int f = PTE_PRESENT | PTE_WRITABLE;
  424. if (type == IOREMAP_RAM)
  425. return f;
  426. if (lfb_wc_ok && phys >= lfb_wc_base &&
  427. (unsigned long long)phys + PAGE_SIZE <=
  428. (unsigned long long)lfb_wc_base + lfb_wc_size)
  429. return f;
  430. return f | PDE_PCD;
  431. }
  432. static int iomap_set(unsigned int va, unsigned int phys, unsigned int flags)
  433. {
  434. unsigned int pde = va >> 22;
  435. unsigned int *pt;
  436. if (!(page_dir[pde] & PDE_PRESENT)) {
  437. unsigned int t = mm_alloc_page();
  438. unsigned int i;
  439. if (!t)
  440. return 0;
  441. pt = (unsigned int *)t;
  442. for (i = 0; i < 1024; i++)
  443. pt[i] = 0;
  444. page_dir[pde] = t | PDE_WRITABLE | PDE_PRESENT;
  445. process_sync_kernel_pde(pde);
  446. } else {
  447. pt = (unsigned int *)(page_dir[pde] & 0xFFFFF000u);
  448. }
  449. pt[(va >> 12) & 0x3FFu] = (phys & 0xFFFFF000u) | flags;
  450. return 1;
  451. }
  452. void *ioremap(unsigned int phys, unsigned int size, int type)
  453. {
  454. unsigned long long end;
  455. unsigned int p0, bytes, va, i;
  456. if (size == 0)
  457. size = 1;
  458. end = (unsigned long long)phys + size;
  459. if (end > 0x100000000ull || !page_dir)
  460. return 0;
  461. /* Lowmem is identity-mapped already. */
  462. if (end <= MM_MAX_PHYS) {
  463. if (type == IOREMAP_MMIO)
  464. lowmem_uncached(phys, size);
  465. return (void *)phys;
  466. }
  467. /* Reuse a mapping of the same type that already covers the range. */
  468. for (i = 0; i < n_iomaps; i++) {
  469. const struct iomap *m = &iomaps[i];
  470. if (m->type == type && phys >= m->phys &&
  471. end <= (unsigned long long)m->phys + m->size)
  472. return (void *)(m->va + (phys - m->phys));
  473. }
  474. /* Grow the newest mapping in place when the request starts inside it
  475. * and only runs past its end (a table header mapped first, then the
  476. * whole table): its pages are the last ones in the window. */
  477. if (n_iomaps && iomap_next) {
  478. struct iomap *m = &iomaps[n_iomaps - 1];
  479. unsigned long long mend = (unsigned long long)m->phys + m->size;
  480. unsigned long long need = (end + PAGE_SIZE - 1) &
  481. ~(unsigned long long)(PAGE_SIZE - 1);
  482. if (m->type == type && phys >= m->phys && phys < mend &&
  483. m->va + m->size == iomap_next &&
  484. (unsigned long long)iomap_next + (need - mend) <=
  485. 0x100000000ull) {
  486. unsigned int extra = (unsigned int)(need - mend);
  487. for (i = 0; i < extra; i += PAGE_SIZE)
  488. if (!iomap_set(iomap_next + i,
  489. (unsigned int)mend + i,
  490. iomap_pte_flags((unsigned int)mend + i,
  491. type)))
  492. return 0;
  493. m->size += extra;
  494. iomap_next = ((unsigned long long)iomap_next + extra >=
  495. 0x100000000ull) ? 0 : iomap_next + extra;
  496. return (void *)(m->va + (phys - m->phys));
  497. }
  498. }
  499. p0 = phys & 0xFFFFF000u;
  500. bytes = (unsigned int)(((end + PAGE_SIZE - 1) &
  501. ~(unsigned long long)(PAGE_SIZE - 1)) - p0);
  502. if (n_iomaps >= IOMAP_MAX || !iomap_next ||
  503. (unsigned long long)iomap_next + bytes > 0x100000000ull) {
  504. serial_puts(SERIAL_COM1, "paging: ioremap window full for ");
  505. serial_puthex(SERIAL_COM1, phys);
  506. serial_puts(SERIAL_COM1, "\n");
  507. return 0;
  508. }
  509. va = iomap_next;
  510. for (i = 0; i < bytes; i += PAGE_SIZE)
  511. if (!iomap_set(va + i, p0 + i, iomap_pte_flags(p0 + i, type))) {
  512. serial_puts(SERIAL_COM1, "paging: ioremap out of memory\n");
  513. return 0;
  514. }
  515. iomap_next = ((unsigned long long)va + bytes >= 0x100000000ull)
  516. ? 0 : va + bytes;
  517. iomaps[n_iomaps].phys = p0;
  518. iomaps[n_iomaps].size = bytes;
  519. iomaps[n_iomaps].va = va;
  520. iomaps[n_iomaps].type = type;
  521. n_iomaps++;
  522. return (void *)(va + (phys - p0));
  523. }
  524. /* The window address of `phys` if some ioremap() covers it, else 0. */
  525. void *ioremap_lookup(unsigned int phys)
  526. {
  527. unsigned int i;
  528. for (i = 0; i < n_iomaps; i++) {
  529. const struct iomap *m = &iomaps[i];
  530. if (phys >= m->phys &&
  531. (unsigned long long)phys < (unsigned long long)m->phys + m->size)
  532. return (void *)(m->va + (phys - m->phys));
  533. }
  534. return 0;
  535. }
  536. /* Split the 4 MB page containing `va` in one directory and set PCD on that
  537. * 4 KB page. The kernel identity map is WB; legacy OHCI DMA on the SB950
  538. * does not reliably snoop dirty descriptor lines, so the ED/TD page must
  539. * bypass the cache. */
  540. static void uc_page_cr3(unsigned int cr3, unsigned int va)
  541. {
  542. unsigned int *pd = (unsigned int *)cr3;
  543. unsigned int pde = va >> 22;
  544. unsigned int pte = (va >> 12) & 0x3FFu;
  545. unsigned int entry = pd[pde];
  546. unsigned int *pt;
  547. unsigned int i;
  548. if (!(entry & PDE_PRESENT))
  549. return;
  550. if (entry & PDE_PS) {
  551. unsigned int base = entry & 0xFFC00000u;
  552. unsigned int flags = (entry & 0xFFFu) & ~PDE_PS;
  553. unsigned int table = mm_alloc_page();
  554. if (!table)
  555. return;
  556. pt = (unsigned int *)table;
  557. for (i = 0; i < 1024; i++)
  558. pt[i] = (base + i * 0x1000u) | flags;
  559. pd[pde] = table | flags;
  560. } else {
  561. pt = (unsigned int *)(entry & 0xFFFFF000u);
  562. }
  563. pt[pte] |= PDE_PCD;
  564. }
  565. /* Temporary kernel mapping of one frame. Lowmem is returned through the
  566. * identity map; a highmem frame gets this CPU's next window slot with
  567. * preemption disabled, so the task cannot migrate or be switched out while
  568. * the slot is live. Nothing may sleep or yield before kunmap(). */
  569. void *kmap(unsigned int phys)
  570. {
  571. struct cpu *c;
  572. unsigned int slot, va;
  573. phys &= 0xFFFFF000u;
  574. if (phys < MM_MAX_PHYS)
  575. return (void *)phys;
  576. preempt_disable();
  577. c = this_cpu();
  578. if (c->id >= MAX_CPUS || kmap_depth[c->id] >= KMAP_SLOTS) {
  579. serial_puts(SERIAL_COM1, "paging: kmap slots exhausted\n");
  580. for (;;)
  581. __asm__ volatile ("cli; hlt");
  582. }
  583. slot = c->id * KMAP_SLOTS + kmap_depth[c->id]++;
  584. va = MM_KMAP_VA + slot * PAGE_SIZE;
  585. kmap_pt[slot] = phys | PTE_WRITABLE | PTE_PRESENT;
  586. __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
  587. return (void *)va;
  588. }
  589. /* Undo the most recent kmap() on this CPU (mappings nest LIFO). */
  590. void kunmap(void *addr)
  591. {
  592. unsigned int va = (unsigned int)addr & 0xFFFFF000u;
  593. struct cpu *c;
  594. unsigned int slot;
  595. if (va < MM_MAX_PHYS)
  596. return;
  597. c = this_cpu();
  598. slot = (va - MM_KMAP_VA) / PAGE_SIZE;
  599. kmap_pt[slot] = 0;
  600. __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
  601. if (kmap_depth[c->id])
  602. kmap_depth[c->id]--;
  603. preempt_enable();
  604. }
  605. static int pa_is_ram(unsigned int pa)
  606. {
  607. unsigned int i, n = mm_e820_count();
  608. for (i = 0; i < n; i++) {
  609. const struct e820_entry *e = mm_e820_entry(i);
  610. if (e->type == E820_RAM && pa >= e->base &&
  611. (unsigned long long)pa < e->base + e->length)
  612. return 1;
  613. }
  614. return 0;
  615. }
  616. volatile unsigned char *paging_debug_map(unsigned int pa, void **km)
  617. {
  618. unsigned char *v;
  619. *km = 0;
  620. if (pa < MM_MAX_PHYS)
  621. return (volatile unsigned char *)pa;
  622. if (pa_is_ram(pa)) {
  623. *km = kmap(pa & 0xFFFFF000u);
  624. return (volatile unsigned char *)*km + (pa & 0xFFFu);
  625. }
  626. v = ioremap_lookup(pa); /* includes the offset */
  627. if (v)
  628. return v;
  629. v = ioremap(pa & 0xFFFFF000u, 0x1000, IOREMAP_MMIO);
  630. return v ? v + (pa & 0xFFFu) : 0;
  631. }