paging.c 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323
  1. /* Paging: identity-map RAM + the VESA LFB, enable CR0.PG (4 MB pages).
  2. *
  3. * This is the "paging on but flat" stage: every physical address is mapped
  4. * 1:1, so all existing code (kernel, stacks, DMA, UHCI, VGA shadow) keeps
  5. * working unchanged. Per-process page directories come later (Phase 3b/4).
  6. *
  7. * Uses 4 MB pages via CR4.PSE (supported since the original Pentium), so the
  8. * whole map is a single 4 KB page directory.
  9. */
  10. #include "paging.h"
  11. #include "mm.h"
  12. #include "mtrr.h"
  13. #include "serial.h"
  14. #include "vga.h"
  15. /* Page-directory entry flags (4 MB page). */
  16. #define PDE_PRESENT 0x001
  17. #define PDE_WRITABLE 0x002
  18. #define PDE_USER 0x004
  19. #define PDE_PWT 0x008
  20. #define PDE_PCD 0x010
  21. #define PDE_PS 0x080 /* 4 MB page */
  22. /* How much of the LFB to map. 16 MB covers any mode up to 1280x1024x32
  23. * (~5 MB) with margin, and is independent of the BIOS's VRAM report. */
  24. #define LFB_MAP_SIZE 0x1000000
  25. static unsigned int *page_dir;
  26. /* LFB page-type decision, kept so it can be re-applied after other MMIO
  27. * mappings (the GPU BARs cover the framebuffer address) have set PCD. */
  28. static unsigned int lfb_wc_base, lfb_wc_size;
  29. static int lfb_wc_ok;
  30. /* Write the LFB PDEs: WC-covered pages cacheable (effective type WC), the
  31. * rest uncacheable. */
  32. static void lfb_apply_pages(unsigned int *pd)
  33. {
  34. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  35. unsigned int base, p0, p1, i;
  36. __asm__ volatile ("" : "+r"(fb));
  37. if (!fb->valid || !fb->lfb_base)
  38. return;
  39. base = fb->lfb_base;
  40. p0 = base >> 22;
  41. p1 = (base + LFB_MAP_SIZE) >> 22;
  42. for (i = p0; i < p1 && i < 1024; i++) {
  43. unsigned int page = i << 22;
  44. unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  45. if (!lfb_wc_ok || page < lfb_wc_base ||
  46. page + 0x400000 > lfb_wc_base + lfb_wc_size)
  47. flags |= PDE_PCD;
  48. pd[i] = page | flags;
  49. }
  50. }
  51. /* Re-apply the framebuffer mapping; call after any MMIO mapping that may have
  52. * covered the same addresses (the NVIDIA BAR3 aperture includes the VBE LFB). */
  53. void paging_lfb_refresh(void)
  54. {
  55. if (!lfb_wc_ok || !page_dir)
  56. return;
  57. lfb_apply_pages(page_dir);
  58. serial_puts(SERIAL_COM1, "paging: LFB write-combining re-applied\n");
  59. }
  60. /* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
  61. int paging_lfb_wc(void)
  62. {
  63. return lfb_wc_ok;
  64. }
  65. void paging_map_ram(unsigned int base, unsigned int size)
  66. {
  67. unsigned int p0 = base >> 22;
  68. unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
  69. unsigned int i;
  70. if (!page_dir)
  71. return;
  72. for (i = p0; i < p1 && i < 1024; i++)
  73. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  74. }
  75. void paging_init(void)
  76. {
  77. unsigned int i;
  78. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  79. /* Opaque asm hides the fixed 0x600 pointer from -Warray-bounds. */
  80. __asm__ volatile ("" : "+r"(fb));
  81. page_dir = (unsigned int *)mm_alloc_page();
  82. if (!page_dir) {
  83. serial_puts(SERIAL_COM1, "paging: out of memory\n");
  84. for (;;)
  85. __asm__ volatile ("cli; hlt");
  86. }
  87. /* Zero the whole page directory. mm_alloc_page does NOT clear memory, so
  88. * on real hardware the high PDEs (indices 32..1023, incl. the user region
  89. * at 0x100) would otherwise be garbage. QEMU zeroes guest RAM, which is
  90. * why this only bites on real silicon. */
  91. for (i = 0; i < 1024; i++)
  92. page_dir[i] = 0;
  93. /* Identity-map RAM [0, MM_MAX_PHYS) with 4 MB pages (write-back). */
  94. for (i = 0; i < MM_MAX_PHYS / 0x400000; i++)
  95. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  96. /* Identity-map the framebuffer and make VRAM writes batch. Use the
  97. * actual framebuffer size (not the whole mapping) so the WC range
  98. * cannot swallow unrelated MMIO, and install the MTRR *before* choosing
  99. * the page type: a PCD=1 page is uncacheable no matter what the MTRRs
  100. * say, which made every VRAM dword a separate PCI write (~30 MB/s).
  101. * Only the pages fully covered by the WC entry become cacheable (their
  102. * effective type is then WC); everything else stays UC. */
  103. if (fb->valid && fb->lfb_base) {
  104. unsigned int want = (unsigned int)fb->pitch * fb->height;
  105. if (want == 0 || want > LFB_MAP_SIZE)
  106. want = LFB_MAP_SIZE;
  107. lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
  108. &lfb_wc_base, &lfb_wc_size);
  109. lfb_apply_pages(page_dir);
  110. /* Tell the truth about VRAM writes: with cacheable pages the WC
  111. * MTRR applies (fast bursts); otherwise every dword is a separate
  112. * uncached write. */
  113. serial_puts(SERIAL_COM1, lfb_wc_ok
  114. ? "paging: LFB write-combining\n"
  115. : "paging: LFB uncached (no WC MTRR)\n");
  116. }
  117. /* Load CR3, enable PSE, then PG + WP. The jump flushes the prefetch
  118. * queue after the paging flip. */
  119. __asm__ volatile (
  120. "movl %0, %%cr3\n\t"
  121. "movl %%cr4, %%eax\n\t"
  122. "orl $0x10, %%eax\n\t" /* CR4.PSE */
  123. "movl %%eax, %%cr4\n\t"
  124. "movl %%cr0, %%eax\n\t"
  125. "orl $0x80010000, %%eax\n\t" /* CR0.PG | CR0.WP */
  126. "movl %%eax, %%cr0\n\t"
  127. "jmp 1f\n\t"
  128. "1:\n\t"
  129. :
  130. : "r"((unsigned int)page_dir)
  131. : "eax", "memory");
  132. serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
  133. }
  134. unsigned int paging_current_cr3(void)
  135. {
  136. unsigned int cr3;
  137. __asm__ volatile ("movl %%cr3, %0" : "=r"(cr3));
  138. return cr3 & 0xFFFFF000u;
  139. }
  140. unsigned int paging_kernel_cr3(void)
  141. {
  142. return (unsigned int)page_dir;
  143. }
  144. unsigned int paging_new_directory(void)
  145. {
  146. unsigned int *pd = (unsigned int *)mm_alloc_page();
  147. unsigned int i;
  148. if (!pd)
  149. return 0;
  150. for (i = 0; i < 1024; i++)
  151. pd[i] = page_dir[i];
  152. return (unsigned int)pd;
  153. }
  154. int paging_map_page(unsigned int cr3, unsigned int va, unsigned int phys,
  155. unsigned int flags)
  156. {
  157. unsigned int *pd = (unsigned int *)cr3;
  158. unsigned int pde = va >> 22;
  159. unsigned int pte = (va >> 12) & 0x3FF;
  160. unsigned int *pt;
  161. if (!(pd[pde] & PDE_PRESENT)) {
  162. unsigned int pt_phys = mm_alloc_page();
  163. unsigned int i;
  164. if (!pt_phys)
  165. return 0;
  166. pt = (unsigned int *)pt_phys;
  167. for (i = 0; i < 1024; i++)
  168. pt[i] = 0;
  169. pd[pde] = pt_phys | PDE_USER | PDE_WRITABLE | PDE_PRESENT;
  170. } else {
  171. pt = (unsigned int *)(pd[pde] & 0xFFFFF000);
  172. }
  173. pt[pte] = (phys & 0xFFFFF000) | flags;
  174. return 1;
  175. }
  176. unsigned int paging_page_phys(unsigned int cr3, unsigned int va)
  177. {
  178. unsigned int *pd = (unsigned int *)cr3;
  179. unsigned int pde = pd[va >> 22];
  180. unsigned int pte = (va >> 12) & 0x3FF;
  181. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  182. return 0;
  183. {
  184. unsigned int e = ((unsigned int *)(pde & 0xFFFFF000))[pte];
  185. return (e & PTE_PRESENT) ? (e & 0xFFFFF000) : 0;
  186. }
  187. }
  188. void paging_unmap_page(unsigned int cr3, unsigned int va)
  189. {
  190. unsigned int *pd = (unsigned int *)cr3;
  191. unsigned int pde = va >> 22;
  192. unsigned int pte = (va >> 12) & 0x3FF;
  193. if (!(pd[pde] & PDE_PRESENT))
  194. return;
  195. ((unsigned int *)(pd[pde] & 0xFFFFF000))[pte] = 0;
  196. }
  197. int paging_user_range_ok(unsigned int cr3, unsigned int va,
  198. unsigned int size, int write)
  199. {
  200. unsigned int *pd = (unsigned int *)cr3;
  201. unsigned int end = va + size;
  202. unsigned int p;
  203. if (!pd || size == 0 || end < va)
  204. return 0;
  205. for (p = va & 0xFFFFF000u; p < end; p += PAGE_SIZE) {
  206. unsigned int pde = pd[p >> 22];
  207. unsigned int *pt;
  208. unsigned int pte;
  209. if (!(pde & PDE_PRESENT) || (pde & PDE_PS) || !(pde & PDE_USER))
  210. return 0;
  211. pt = (unsigned int *)(pde & 0xFFFFF000);
  212. pte = pt[(p >> 12) & 0x3FF];
  213. if (!(pte & PTE_PRESENT) || !(pte & PTE_USER))
  214. return 0;
  215. if (write && !(pte & PTE_WRITABLE))
  216. return 0;
  217. }
  218. return 1;
  219. }
  220. void paging_release_user_range(unsigned int cr3, unsigned int va,
  221. unsigned int size, int free_phys)
  222. {
  223. unsigned int *pd = (unsigned int *)cr3;
  224. unsigned int end = va + size;
  225. unsigned int first, last, i, j;
  226. if (!pd || size == 0 || end < va)
  227. return;
  228. first = va >> 22;
  229. last = (end - 1) >> 22;
  230. for (i = first; i <= last && i < 1024; i++) {
  231. unsigned int pde = pd[i];
  232. unsigned int *pt;
  233. int empty = 1;
  234. /* Kernel identity mappings are 4 MB pages and must never be
  235. * touched here. */
  236. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  237. continue;
  238. pt = (unsigned int *)(pde & 0xFFFFF000);
  239. for (j = 0; j < 1024; j++) {
  240. unsigned int pva = (i << 22) | (j << 12);
  241. if (pva < va || pva >= end)
  242. continue;
  243. if (pt[j] & PTE_PRESENT) {
  244. /* Only user pages are owned by the process; a
  245. * supervisor mapping in this range is left for
  246. * its owner. */
  247. if (free_phys && (pt[j] & PTE_USER))
  248. mm_free_page(pt[j] & 0xFFFFF000);
  249. pt[j] = 0;
  250. }
  251. }
  252. for (j = 0; j < 1024; j++)
  253. if (pt[j] & PTE_PRESENT) {
  254. empty = 0;
  255. break;
  256. }
  257. if (empty) {
  258. mm_free_page(pde & 0xFFFFF000);
  259. pd[i] = 0;
  260. }
  261. }
  262. }
  263. /* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
  264. * the range is rounded up to a whole number of them). */
  265. void paging_map_mmio_cr3(unsigned int cr3, unsigned int phys_base, unsigned int size)
  266. {
  267. unsigned int *pd = (unsigned int *)cr3;
  268. unsigned int p0 = phys_base >> 22;
  269. unsigned int p1 = (phys_base + size + 0x3FFFFFu) >> 22;
  270. unsigned int i;
  271. for (i = p0; i < p1 && i < 1024; i++)
  272. pd[i] = (i << 22) | PDE_PS | PDE_PCD |
  273. PDE_WRITABLE | PDE_PRESENT;
  274. }
  275. void paging_map_mmio(unsigned int phys_base, unsigned int size)
  276. {
  277. paging_map_mmio_cr3((unsigned int)page_dir, phys_base, size);
  278. }