paging.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455
  1. /* Paging: identity-map RAM + the VESA LFB, enable CR0.PG (4 MB pages).
  2. *
  3. * This is the "paging on but flat" stage: every physical address is mapped
  4. * 1:1, so all existing code (kernel, stacks, DMA, UHCI, VGA shadow) keeps
  5. * working unchanged. Per-process page directories come later (Phase 3b/4).
  6. *
  7. * Uses 4 MB pages via CR4.PSE (supported since the original Pentium), so the
  8. * whole map is a single 4 KB page directory.
  9. */
  10. #include "paging.h"
  11. #include "bootcon.h"
  12. #include "cpu.h"
  13. #include "mm.h"
  14. #include "mtrr.h"
  15. #include "serial.h"
  16. #include "vga.h"
  17. /* CR4 bits that must not survive a BIOS protected-mode call. A BIOS that
  18. * returns with CR4.PAE or EFER.LME set makes the first CR0.PG enable either
  19. * walk the 32-bit page directory as a PAE PDPT or enter long mode, which
  20. * triple-faults before the kernel can print anything. The X201's old
  21. * INT 15h AH=87h rootfs copy was one source of this state; clear the bits
  22. * defensively on every boot path before enabling paging. */
  23. #define CR4_PSE (1u << 4)
  24. #define CR4_PAE (1u << 5)
  25. #define CR4_PGE (1u << 7)
  26. #define CR4_LA57 (1u << 12)
  27. #define CR4_PCIDE (1u << 17)
  28. #define CR4_SMEP (1u << 20)
  29. #define CR4_SMAP (1u << 21)
  30. #define MSR_EFER 0xC0000080u
  31. #define EFER_LME (1u << 8)
  32. /* Page-directory entry flags (4 MB page). */
  33. #define PDE_PRESENT 0x001
  34. #define PDE_WRITABLE 0x002
  35. #define PDE_USER 0x004
  36. #define PDE_PWT 0x008
  37. #define PDE_PCD 0x010
  38. #define PDE_PS 0x080 /* 4 MB page */
  39. /* How much of the LFB to map. 16 MB covers any mode up to 1920x1080x32
  40. * (~8.3 MB) with margin, and is independent of the BIOS's VRAM report. */
  41. #define LFB_MAP_SIZE 0x1000000
  42. static unsigned int *page_dir;
  43. static void uc_page_cr3(unsigned int cr3, unsigned int va);
  44. static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
  45. {
  46. __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
  47. }
  48. static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
  49. {
  50. __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
  51. }
  52. static int cpu_has_long_mode(void)
  53. {
  54. unsigned int eax, ebx, ecx, edx;
  55. if (!cpu_has_cpuid())
  56. return 0;
  57. eax = 0x80000000u;
  58. __asm__ volatile ("cpuid"
  59. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  60. if (eax < 0x80000001u)
  61. return 0;
  62. eax = 0x80000001u;
  63. __asm__ volatile ("cpuid"
  64. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  65. return (edx & (1u << 29)) != 0; /* LM */
  66. }
  67. /* Put the CPU into the state this 32-bit kernel expects before it installs
  68. * its own page directory. A BIOS protected-mode call (notably the old
  69. * high-pack INT 15h AH=87h path) may leave CR4.PAE/PGE or EFER.LME set;
  70. * CR0.PG is off here, so clearing them is safe on every CPU (reserved bits
  71. * are written as zero). */
  72. void paging_prepare_cpu(void)
  73. {
  74. unsigned int cr4, lo, hi;
  75. __asm__ volatile ("movl %%cr4, %0" : "=r"(cr4));
  76. cr4 &= ~(CR4_PAE | CR4_PGE | CR4_LA57 | CR4_PCIDE |
  77. CR4_SMEP | CR4_SMAP);
  78. cr4 |= CR4_PSE;
  79. __asm__ volatile ("movl %0, %%cr4" : : "r"(cr4) : "memory");
  80. if (cpu_has_long_mode()) {
  81. rdmsr(MSR_EFER, &lo, &hi);
  82. if (lo & EFER_LME) {
  83. lo &= ~EFER_LME;
  84. wrmsr(MSR_EFER, lo, hi);
  85. serial_puts(SERIAL_COM1, "paging: cleared EFER.LME\n");
  86. }
  87. }
  88. }
  89. /* LFB page-type decision, kept so it can be re-applied after other MMIO
  90. * mappings (the GPU BARs cover the framebuffer address) have set PCD. */
  91. static unsigned int lfb_wc_base, lfb_wc_size;
  92. static int lfb_wc_ok;
  93. /* Write the LFB PDEs: WC-covered pages cacheable (effective type WC), the
  94. * rest uncacheable. */
  95. static void lfb_apply_pages(unsigned int *pd)
  96. {
  97. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  98. unsigned int base, p0, p1, i;
  99. __asm__ volatile ("" : "+r"(fb));
  100. if (!fb->valid || !fb->lfb_base)
  101. return;
  102. base = fb->lfb_base;
  103. p0 = base >> 22;
  104. /* Compute the end page from the page index, not from base+size: for a
  105. * base above 0xFF000000 the 32-bit addition wraps to a small number
  106. * and the loop would map nothing, faulting on the first LFB write
  107. * after CR0.PG comes on. */
  108. p1 = p0 + (LFB_MAP_SIZE >> 22);
  109. if (p1 > 1024)
  110. p1 = 1024;
  111. for (i = p0; i < p1; i++) {
  112. unsigned int page = i << 22;
  113. unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  114. if (!lfb_wc_ok || page < lfb_wc_base ||
  115. page + 0x400000 > lfb_wc_base + lfb_wc_size)
  116. flags |= PDE_PCD;
  117. pd[i] = page | flags;
  118. }
  119. }
  120. /* Re-apply the framebuffer mapping; call after any MMIO mapping that may have
  121. * covered the same addresses (the NVIDIA BAR3 aperture includes the VBE LFB). */
  122. void paging_lfb_refresh(void)
  123. {
  124. if (!lfb_wc_ok || !page_dir)
  125. return;
  126. lfb_apply_pages(page_dir);
  127. serial_puts(SERIAL_COM1, "paging: LFB write-combining re-applied\n");
  128. }
  129. /* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
  130. int paging_lfb_wc(void)
  131. {
  132. return lfb_wc_ok;
  133. }
  134. void paging_map_ram(unsigned int base, unsigned int size)
  135. {
  136. unsigned int p0 = base >> 22;
  137. unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
  138. unsigned int i;
  139. if (!page_dir)
  140. return;
  141. for (i = p0; i < p1 && i < 1024; i++)
  142. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  143. }
  144. void paging_init(void)
  145. {
  146. unsigned int i;
  147. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  148. bootcon_puts("paging: prepare\n");
  149. paging_prepare_cpu();
  150. /* Opaque asm hides the fixed 0x600 pointer from -Warray-bounds. */
  151. __asm__ volatile ("" : "+r"(fb));
  152. bootcon_puts("paging: alloc\n");
  153. page_dir = (unsigned int *)mm_alloc_page();
  154. if (!page_dir) {
  155. serial_puts(SERIAL_COM1, "paging: out of memory\n");
  156. for (;;)
  157. __asm__ volatile ("cli; hlt");
  158. }
  159. /* Zero the whole page directory. mm_alloc_page does NOT clear memory, so
  160. * on real hardware the high PDEs (indices 32..1023, incl. the user region
  161. * at 0x100) would otherwise be garbage. QEMU zeroes guest RAM, which is
  162. * why this only bites on real silicon. */
  163. for (i = 0; i < 1024; i++)
  164. page_dir[i] = 0;
  165. /* Identity-map RAM [0, MM_MAX_PHYS) with 4 MB pages (write-back). */
  166. for (i = 0; i < MM_MAX_PHYS / 0x400000; i++)
  167. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  168. /* The OHCI ED/TD/interrupt-buffer page is DMA'd by the SB950's legacy
  169. * USB controller, which does not snoop dirty CPU cache lines. Split
  170. * the containing 4 MB page now, before paging is enabled, so every
  171. * process directory clones the uncacheable mapping. */
  172. {
  173. extern unsigned char ohci_dma_page[];
  174. unsigned int va = (unsigned int)ohci_dma_page;
  175. if ((va >> 22) == 0)
  176. uc_page_cr3((unsigned int)page_dir, va);
  177. }
  178. /* Identity-map the framebuffer and make VRAM writes batch. Use the
  179. * actual framebuffer size (not the whole mapping) so the WC range
  180. * cannot swallow unrelated MMIO, and install the MTRR *before* choosing
  181. * the page type: a PCD=1 page is uncacheable no matter what the MTRRs
  182. * say, which made every VRAM dword a separate PCI write (~30 MB/s).
  183. * Only the pages fully covered by the WC entry become cacheable (their
  184. * effective type is then WC); everything else stays UC. */
  185. if (fb->valid && fb->lfb_base) {
  186. unsigned int want = (unsigned int)fb->pitch * fb->height;
  187. if (want == 0 || want > LFB_MAP_SIZE)
  188. want = LFB_MAP_SIZE;
  189. bootcon_puts("paging: mtrr\n");
  190. lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
  191. &lfb_wc_base, &lfb_wc_size);
  192. bootcon_puts("paging: map\n");
  193. lfb_apply_pages(page_dir);
  194. /* Tell the truth about VRAM writes: with cacheable pages the WC
  195. * MTRR applies (fast bursts); otherwise every dword is a separate
  196. * uncached write. */
  197. serial_puts(SERIAL_COM1, lfb_wc_ok
  198. ? "paging: LFB write-combining\n"
  199. : "paging: LFB uncached (no WC MTRR)\n");
  200. }
  201. /* Load CR3, enable PSE, then PG + WP. The jump flushes the prefetch
  202. * queue after the paging flip. */
  203. bootcon_puts("paging: enable\n");
  204. __asm__ volatile (
  205. "movl %0, %%cr3\n\t"
  206. "movl %%cr4, %%eax\n\t"
  207. "orl $0x10, %%eax\n\t" /* CR4.PSE */
  208. "movl %%eax, %%cr4\n\t"
  209. "movl %%cr0, %%eax\n\t"
  210. "orl $0x80010000, %%eax\n\t" /* CR0.PG | CR0.WP */
  211. "movl %%eax, %%cr0\n\t"
  212. "jmp 1f\n\t"
  213. "1:\n\t"
  214. :
  215. : "r"((unsigned int)page_dir)
  216. : "eax", "memory");
  217. bootcon_puts("paging: on\n");
  218. serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
  219. }
  220. unsigned int paging_current_cr3(void)
  221. {
  222. unsigned int cr3;
  223. __asm__ volatile ("movl %%cr3, %0" : "=r"(cr3));
  224. return cr3 & 0xFFFFF000u;
  225. }
  226. unsigned int paging_kernel_cr3(void)
  227. {
  228. return (unsigned int)page_dir;
  229. }
  230. unsigned int paging_new_directory(void)
  231. {
  232. unsigned int *pd = (unsigned int *)mm_alloc_page();
  233. unsigned int i;
  234. if (!pd)
  235. return 0;
  236. for (i = 0; i < 1024; i++)
  237. pd[i] = page_dir[i];
  238. return (unsigned int)pd;
  239. }
  240. int paging_map_page(unsigned int cr3, unsigned int va, unsigned int phys,
  241. unsigned int flags)
  242. {
  243. unsigned int *pd = (unsigned int *)cr3;
  244. unsigned int pde = va >> 22;
  245. unsigned int pte = (va >> 12) & 0x3FF;
  246. unsigned int *pt;
  247. if (!(pd[pde] & PDE_PRESENT)) {
  248. unsigned int pt_phys = mm_alloc_page();
  249. unsigned int i;
  250. if (!pt_phys)
  251. return 0;
  252. pt = (unsigned int *)pt_phys;
  253. for (i = 0; i < 1024; i++)
  254. pt[i] = 0;
  255. pd[pde] = pt_phys | PDE_USER | PDE_WRITABLE | PDE_PRESENT;
  256. } else {
  257. pt = (unsigned int *)(pd[pde] & 0xFFFFF000);
  258. }
  259. pt[pte] = (phys & 0xFFFFF000) | flags;
  260. return 1;
  261. }
  262. unsigned int paging_page_phys(unsigned int cr3, unsigned int va)
  263. {
  264. unsigned int *pd = (unsigned int *)cr3;
  265. unsigned int pde = pd[va >> 22];
  266. unsigned int pte = (va >> 12) & 0x3FF;
  267. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  268. return 0;
  269. {
  270. unsigned int e = ((unsigned int *)(pde & 0xFFFFF000))[pte];
  271. return (e & PTE_PRESENT) ? (e & 0xFFFFF000) : 0;
  272. }
  273. }
  274. void paging_unmap_page(unsigned int cr3, unsigned int va)
  275. {
  276. unsigned int *pd = (unsigned int *)cr3;
  277. unsigned int pde = va >> 22;
  278. unsigned int pte = (va >> 12) & 0x3FF;
  279. if (!(pd[pde] & PDE_PRESENT))
  280. return;
  281. ((unsigned int *)(pd[pde] & 0xFFFFF000))[pte] = 0;
  282. }
  283. int paging_user_range_ok(unsigned int cr3, unsigned int va,
  284. unsigned int size, int write)
  285. {
  286. unsigned int *pd = (unsigned int *)cr3;
  287. unsigned int end = va + size;
  288. unsigned int p;
  289. if (!pd || size == 0 || end < va)
  290. return 0;
  291. for (p = va & 0xFFFFF000u; p < end; p += PAGE_SIZE) {
  292. unsigned int pde = pd[p >> 22];
  293. unsigned int *pt;
  294. unsigned int pte;
  295. if (!(pde & PDE_PRESENT) || (pde & PDE_PS) || !(pde & PDE_USER))
  296. return 0;
  297. pt = (unsigned int *)(pde & 0xFFFFF000);
  298. pte = pt[(p >> 12) & 0x3FF];
  299. if (!(pte & PTE_PRESENT) || !(pte & PTE_USER))
  300. return 0;
  301. if (write && !(pte & PTE_WRITABLE))
  302. return 0;
  303. }
  304. return 1;
  305. }
  306. void paging_release_user_range(unsigned int cr3, unsigned int va,
  307. unsigned int size, int free_phys)
  308. {
  309. unsigned int *pd = (unsigned int *)cr3;
  310. unsigned int end = va + size;
  311. unsigned int first, last, i, j;
  312. if (!pd || size == 0 || end < va)
  313. return;
  314. first = va >> 22;
  315. last = (end - 1) >> 22;
  316. for (i = first; i <= last && i < 1024; i++) {
  317. unsigned int pde = pd[i];
  318. unsigned int *pt;
  319. int empty = 1;
  320. /* Kernel identity mappings are 4 MB pages and must never be
  321. * touched here. */
  322. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  323. continue;
  324. pt = (unsigned int *)(pde & 0xFFFFF000);
  325. for (j = 0; j < 1024; j++) {
  326. unsigned int pva = (i << 22) | (j << 12);
  327. if (pva < va || pva >= end)
  328. continue;
  329. if (pt[j] & PTE_PRESENT) {
  330. /* Only user pages are owned by the process; a
  331. * supervisor mapping in this range is left for
  332. * its owner. */
  333. if (free_phys && (pt[j] & PTE_USER))
  334. mm_free_page(pt[j] & 0xFFFFF000);
  335. pt[j] = 0;
  336. }
  337. }
  338. for (j = 0; j < 1024; j++)
  339. if (pt[j] & PTE_PRESENT) {
  340. empty = 0;
  341. break;
  342. }
  343. if (empty) {
  344. mm_free_page(pde & 0xFFFFF000);
  345. pd[i] = 0;
  346. }
  347. }
  348. }
  349. /* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
  350. * the range is rounded up to a whole number of them). */
  351. void paging_map_mmio_cr3(unsigned int cr3, unsigned int phys_base, unsigned int size)
  352. {
  353. unsigned int *pd = (unsigned int *)cr3;
  354. unsigned int p0 = phys_base >> 22;
  355. unsigned int p1 = (phys_base + size + 0x3FFFFFu) >> 22;
  356. unsigned int i;
  357. for (i = p0; i < p1 && i < 1024; i++)
  358. pd[i] = (i << 22) | PDE_PS | PDE_PCD |
  359. PDE_WRITABLE | PDE_PRESENT;
  360. }
  361. void paging_map_mmio(unsigned int phys_base, unsigned int size)
  362. {
  363. paging_map_mmio_cr3((unsigned int)page_dir, phys_base, size);
  364. }
  365. /* Split the 4 MB page containing `va` in one directory and set PCD on that
  366. * 4 KB page. The kernel identity map is WB; legacy OHCI DMA on the SB950
  367. * does not reliably snoop dirty descriptor lines, so the ED/TD page must
  368. * bypass the cache. */
  369. static void uc_page_cr3(unsigned int cr3, unsigned int va)
  370. {
  371. unsigned int *pd = (unsigned int *)cr3;
  372. unsigned int pde = va >> 22;
  373. unsigned int pte = (va >> 12) & 0x3FFu;
  374. unsigned int entry = pd[pde];
  375. unsigned int *pt;
  376. unsigned int i;
  377. if (!(entry & PDE_PRESENT))
  378. return;
  379. if (entry & PDE_PS) {
  380. unsigned int base = entry & 0xFFC00000u;
  381. unsigned int flags = (entry & 0xFFFu) & ~PDE_PS;
  382. unsigned int table = mm_alloc_page();
  383. if (!table)
  384. return;
  385. pt = (unsigned int *)table;
  386. for (i = 0; i < 1024; i++)
  387. pt[i] = (base + i * 0x1000u) | flags;
  388. pd[pde] = table | flags;
  389. } else {
  390. pt = (unsigned int *)(entry & 0xFFFFF000u);
  391. }
  392. pt[pte] |= PDE_PCD;
  393. }