paging.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. /* Paging: identity-map RAM + the VESA LFB, enable CR0.PG (4 MB pages).
  2. *
  3. * This is the "paging on but flat" stage: every physical address is mapped
  4. * 1:1, so all existing code (kernel, stacks, DMA, UHCI, VGA shadow) keeps
  5. * working unchanged. Per-process page directories come later (Phase 3b/4).
  6. *
  7. * Uses 4 MB pages via CR4.PSE (supported since the original Pentium), so the
  8. * whole map is a single 4 KB page directory.
  9. */
  10. #include "paging.h"
  11. #include "bootcon.h"
  12. #include "cpu.h"
  13. #include "mm.h"
  14. #include "mtrr.h"
  15. #include "process.h"
  16. #include "sched.h"
  17. #include "serial.h"
  18. #include "vga.h"
  19. #include "wm_abi.h"
  20. /* The RAM identity map is made of supervisor 4 MB PDEs; the ring-3 windows
  21. * (program/stack, canvases, WM slots, heap) are user page tables above it.
  22. * They must never share a PDE. */
  23. _Static_assert(MM_MAX_PHYS <= USER_BASE, "identity map overlaps user space");
  24. _Static_assert(MM_MAX_PHYS <= USER_WIN_BASE &&
  25. MM_MAX_PHYS <= WM_CONTENT_VA_BASE &&
  26. MM_MAX_PHYS <= USER_HEAP_BASE,
  27. "identity map overlaps a ring-3 window");
  28. _Static_assert(MM_KMAP_VA >= USER_STACK_TOP &&
  29. MM_KMAP_VA + 0x400000u <= USER_WIN_BASE &&
  30. (MM_KMAP_VA & 0x3FFFFFu) == 0,
  31. "kmap window must be a free 4 MB slot below USER_WIN_BASE");
  32. /* kmap: KMAP_SLOTS nested mappings per CPU in the shared window page table. */
  33. #define KMAP_PDE (MM_KMAP_VA >> 22)
  34. #define KMAP_SLOTS 4
  35. static unsigned int *kmap_pt;
  36. static unsigned int kmap_depth[MAX_CPUS];
  37. /* Every PDE writer below leaves the kmap slot alone: a BIOS-placed ACPI
  38. * table, MMIO BAR or LFB in that 4 MB range cannot be identity-mapped. */
  39. static int kmap_pde_guard(unsigned int i)
  40. {
  41. if (i != KMAP_PDE)
  42. return 0;
  43. serial_puts(SERIAL_COM1, "paging: refusing to map over the kmap window\n");
  44. return 1;
  45. }
  46. /* CR4 bits that must not survive a BIOS protected-mode call. A BIOS that
  47. * returns with CR4.PAE or EFER.LME set makes the first CR0.PG enable either
  48. * walk the 32-bit page directory as a PAE PDPT or enter long mode, which
  49. * triple-faults before the kernel can print anything. The X201's old
  50. * INT 15h AH=87h rootfs copy was one source of this state; clear the bits
  51. * defensively on every boot path before enabling paging. */
  52. #define CR4_PSE (1u << 4)
  53. #define CR4_PAE (1u << 5)
  54. #define CR4_PGE (1u << 7)
  55. #define CR4_LA57 (1u << 12)
  56. #define CR4_PCIDE (1u << 17)
  57. #define CR4_SMEP (1u << 20)
  58. #define CR4_SMAP (1u << 21)
  59. #define MSR_EFER 0xC0000080u
  60. #define EFER_LME (1u << 8)
  61. /* Page-directory entry flags (4 MB page). */
  62. #define PDE_PRESENT 0x001
  63. #define PDE_WRITABLE 0x002
  64. #define PDE_USER 0x004
  65. #define PDE_PWT 0x008
  66. #define PDE_PCD 0x010
  67. #define PDE_PS 0x080 /* 4 MB page */
  68. /* How much of the LFB to map. 16 MB covers any mode up to 1920x1080x32
  69. * (~8.3 MB) with margin, and is independent of the BIOS's VRAM report. */
  70. #define LFB_MAP_SIZE 0x1000000
  71. static unsigned int *page_dir;
  72. static void uc_page_cr3(unsigned int cr3, unsigned int va);
  73. static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
  74. {
  75. __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
  76. }
  77. static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
  78. {
  79. __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
  80. }
  81. static int cpu_has_long_mode(void)
  82. {
  83. unsigned int eax, ebx, ecx, edx;
  84. if (!cpu_has_cpuid())
  85. return 0;
  86. eax = 0x80000000u;
  87. __asm__ volatile ("cpuid"
  88. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  89. if (eax < 0x80000001u)
  90. return 0;
  91. eax = 0x80000001u;
  92. __asm__ volatile ("cpuid"
  93. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  94. return (edx & (1u << 29)) != 0; /* LM */
  95. }
  96. /* Put the CPU into the state this 32-bit kernel expects before it installs
  97. * its own page directory. A BIOS protected-mode call (notably the old
  98. * high-pack INT 15h AH=87h path) may leave CR4.PAE/PGE or EFER.LME set;
  99. * CR0.PG is off here, so clearing them is safe on every CPU (reserved bits
  100. * are written as zero). */
  101. void paging_prepare_cpu(void)
  102. {
  103. unsigned int cr4, lo, hi;
  104. __asm__ volatile ("movl %%cr4, %0" : "=r"(cr4));
  105. cr4 &= ~(CR4_PAE | CR4_PGE | CR4_LA57 | CR4_PCIDE |
  106. CR4_SMEP | CR4_SMAP);
  107. cr4 |= CR4_PSE;
  108. __asm__ volatile ("movl %0, %%cr4" : : "r"(cr4) : "memory");
  109. if (cpu_has_long_mode()) {
  110. rdmsr(MSR_EFER, &lo, &hi);
  111. if (lo & EFER_LME) {
  112. lo &= ~EFER_LME;
  113. wrmsr(MSR_EFER, lo, hi);
  114. serial_puts(SERIAL_COM1, "paging: cleared EFER.LME\n");
  115. }
  116. }
  117. }
  118. /* LFB page-type decision, kept so it can be re-applied after other MMIO
  119. * mappings (the GPU BARs cover the framebuffer address) have set PCD. */
  120. static unsigned int lfb_wc_base, lfb_wc_size;
  121. static int lfb_wc_ok;
  122. /* Write the LFB PDEs: WC-covered pages cacheable (effective type WC), the
  123. * rest uncacheable. */
  124. static void lfb_apply_pages(unsigned int *pd)
  125. {
  126. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  127. unsigned int base, p0, p1, i;
  128. __asm__ volatile ("" : "+r"(fb));
  129. if (!fb->valid || !fb->lfb_base)
  130. return;
  131. base = fb->lfb_base;
  132. p0 = base >> 22;
  133. /* Compute the end page from the page index, not from base+size: for a
  134. * base above 0xFF000000 the 32-bit addition wraps to a small number
  135. * and the loop would map nothing, faulting on the first LFB write
  136. * after CR0.PG comes on. */
  137. p1 = p0 + (LFB_MAP_SIZE >> 22);
  138. if (p1 > 1024)
  139. p1 = 1024;
  140. for (i = p0; i < p1; i++) {
  141. unsigned int page = i << 22;
  142. unsigned int flags = PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  143. if (kmap_pde_guard(i))
  144. continue;
  145. if (!lfb_wc_ok || page < lfb_wc_base ||
  146. page + 0x400000 > lfb_wc_base + lfb_wc_size)
  147. flags |= PDE_PCD;
  148. pd[i] = page | flags;
  149. }
  150. }
  151. /* Re-apply the framebuffer mapping; call after any MMIO mapping that may have
  152. * covered the same addresses (the NVIDIA BAR3 aperture includes the VBE LFB). */
  153. void paging_lfb_refresh(void)
  154. {
  155. if (!lfb_wc_ok || !page_dir)
  156. return;
  157. lfb_apply_pages(page_dir);
  158. serial_puts(SERIAL_COM1, "paging: LFB write-combining re-applied\n");
  159. }
  160. /* 1 when the LFB is mapped for write-combining (WC MTRR + cacheable pages). */
  161. int paging_lfb_wc(void)
  162. {
  163. return lfb_wc_ok;
  164. }
  165. void paging_map_ram(unsigned int base, unsigned int size)
  166. {
  167. unsigned int p0 = base >> 22;
  168. unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
  169. unsigned int i;
  170. if (!page_dir)
  171. return;
  172. for (i = p0; i < p1 && i < 1024; i++)
  173. if (!kmap_pde_guard(i))
  174. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE |
  175. PDE_PRESENT;
  176. }
  177. void paging_init(void)
  178. {
  179. unsigned int i;
  180. const volatile struct vga_boot_fb_info *fb = VGA_BOOT_FB;
  181. bootcon_puts("paging: prepare\n");
  182. paging_prepare_cpu();
  183. /* Opaque asm hides the fixed 0x600 pointer from -Warray-bounds. */
  184. __asm__ volatile ("" : "+r"(fb));
  185. bootcon_puts("paging: alloc\n");
  186. page_dir = (unsigned int *)mm_alloc_page();
  187. if (!page_dir) {
  188. serial_puts(SERIAL_COM1, "paging: out of memory\n");
  189. for (;;)
  190. __asm__ volatile ("cli; hlt");
  191. }
  192. /* Zero the whole page directory. mm_alloc_page does NOT clear memory, so
  193. * on real hardware the high PDEs (indices 32..1023, incl. the user region
  194. * at 0x100) would otherwise be garbage. QEMU zeroes guest RAM, which is
  195. * why this only bites on real silicon. */
  196. for (i = 0; i < 1024; i++)
  197. page_dir[i] = 0;
  198. /* Identity-map RAM [0, MM_MAX_PHYS) with 4 MB pages (write-back). */
  199. for (i = 0; i < MM_MAX_PHYS / 0x400000; i++)
  200. page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
  201. /* The kmap window: a supervisor page table every process directory
  202. * shares (paging_new_directory copies the PDE). */
  203. kmap_pt = (unsigned int *)mm_alloc_page();
  204. if (!kmap_pt) {
  205. serial_puts(SERIAL_COM1, "paging: out of memory (kmap)\n");
  206. for (;;)
  207. __asm__ volatile ("cli; hlt");
  208. }
  209. for (i = 0; i < 1024; i++)
  210. kmap_pt[i] = 0;
  211. page_dir[KMAP_PDE] = (unsigned int)kmap_pt | PDE_WRITABLE | PDE_PRESENT;
  212. /* The OHCI ED/TD/interrupt-buffer page is DMA'd by the SB950's legacy
  213. * USB controller, which does not snoop dirty CPU cache lines. Split
  214. * the containing 4 MB page now, before paging is enabled, so every
  215. * process directory clones the uncacheable mapping. */
  216. {
  217. extern unsigned char ohci_dma_page[];
  218. unsigned int va = (unsigned int)ohci_dma_page;
  219. if ((va >> 22) == 0)
  220. uc_page_cr3((unsigned int)page_dir, va);
  221. }
  222. /* Identity-map the framebuffer and make VRAM writes batch. Use the
  223. * actual framebuffer size (not the whole mapping) so the WC range
  224. * cannot swallow unrelated MMIO, and install the MTRR *before* choosing
  225. * the page type: a PCD=1 page is uncacheable no matter what the MTRRs
  226. * say, which made every VRAM dword a separate PCI write (~30 MB/s).
  227. * Only the pages fully covered by the WC entry become cacheable (their
  228. * effective type is then WC); everything else stays UC. */
  229. if (fb->valid && fb->lfb_base) {
  230. unsigned int want = (unsigned int)fb->pitch * fb->height;
  231. if (want == 0 || want > LFB_MAP_SIZE)
  232. want = LFB_MAP_SIZE;
  233. bootcon_puts("paging: mtrr\n");
  234. lfb_wc_ok = mtrr_set_wc(fb->lfb_base, want,
  235. &lfb_wc_base, &lfb_wc_size);
  236. bootcon_puts("paging: map\n");
  237. lfb_apply_pages(page_dir);
  238. /* Tell the truth about VRAM writes: with cacheable pages the WC
  239. * MTRR applies (fast bursts); otherwise every dword is a separate
  240. * uncached write. */
  241. serial_puts(SERIAL_COM1, lfb_wc_ok
  242. ? "paging: LFB write-combining\n"
  243. : "paging: LFB uncached (no WC MTRR)\n");
  244. }
  245. /* Load CR3, enable PSE, then PG + WP. The jump flushes the prefetch
  246. * queue after the paging flip. */
  247. bootcon_puts("paging: enable\n");
  248. __asm__ volatile (
  249. "movl %0, %%cr3\n\t"
  250. "movl %%cr4, %%eax\n\t"
  251. "orl $0x10, %%eax\n\t" /* CR4.PSE */
  252. "movl %%eax, %%cr4\n\t"
  253. "movl %%cr0, %%eax\n\t"
  254. "orl $0x80010000, %%eax\n\t" /* CR0.PG | CR0.WP */
  255. "movl %%eax, %%cr0\n\t"
  256. "jmp 1f\n\t"
  257. "1:\n\t"
  258. :
  259. : "r"((unsigned int)page_dir)
  260. : "eax", "memory");
  261. bootcon_puts("paging: on\n");
  262. serial_puts(SERIAL_COM1, "paging: enabled (identity map)\n");
  263. }
  264. unsigned int paging_current_cr3(void)
  265. {
  266. unsigned int cr3;
  267. __asm__ volatile ("movl %%cr3, %0" : "=r"(cr3));
  268. return cr3 & 0xFFFFF000u;
  269. }
  270. unsigned int paging_kernel_cr3(void)
  271. {
  272. return (unsigned int)page_dir;
  273. }
  274. unsigned int paging_new_directory(void)
  275. {
  276. unsigned int *pd = (unsigned int *)mm_alloc_page();
  277. unsigned int i;
  278. if (!pd)
  279. return 0;
  280. for (i = 0; i < 1024; i++)
  281. pd[i] = page_dir[i];
  282. return (unsigned int)pd;
  283. }
  284. int paging_map_page(unsigned int cr3, unsigned int va, unsigned int phys,
  285. unsigned int flags)
  286. {
  287. unsigned int *pd = (unsigned int *)cr3;
  288. unsigned int pde = va >> 22;
  289. unsigned int pte = (va >> 12) & 0x3FF;
  290. unsigned int *pt;
  291. if (!(pd[pde] & PDE_PRESENT)) {
  292. unsigned int pt_phys = mm_alloc_page();
  293. unsigned int i;
  294. if (!pt_phys)
  295. return 0;
  296. pt = (unsigned int *)pt_phys;
  297. for (i = 0; i < 1024; i++)
  298. pt[i] = 0;
  299. pd[pde] = pt_phys | PDE_USER | PDE_WRITABLE | PDE_PRESENT;
  300. } else {
  301. pt = (unsigned int *)(pd[pde] & 0xFFFFF000);
  302. }
  303. pt[pte] = (phys & 0xFFFFF000) | flags;
  304. return 1;
  305. }
  306. unsigned int paging_page_phys(unsigned int cr3, unsigned int va)
  307. {
  308. unsigned int *pd = (unsigned int *)cr3;
  309. unsigned int pde = pd[va >> 22];
  310. unsigned int pte = (va >> 12) & 0x3FF;
  311. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  312. return 0;
  313. {
  314. unsigned int e = ((unsigned int *)(pde & 0xFFFFF000))[pte];
  315. return (e & PTE_PRESENT) ? (e & 0xFFFFF000) : 0;
  316. }
  317. }
  318. void paging_unmap_page(unsigned int cr3, unsigned int va)
  319. {
  320. unsigned int *pd = (unsigned int *)cr3;
  321. unsigned int pde = va >> 22;
  322. unsigned int pte = (va >> 12) & 0x3FF;
  323. if (!(pd[pde] & PDE_PRESENT))
  324. return;
  325. ((unsigned int *)(pd[pde] & 0xFFFFF000))[pte] = 0;
  326. }
  327. int paging_user_range_ok(unsigned int cr3, unsigned int va,
  328. unsigned int size, int write)
  329. {
  330. unsigned int *pd = (unsigned int *)cr3;
  331. unsigned int end = va + size;
  332. unsigned int p;
  333. if (!pd || size == 0 || end < va)
  334. return 0;
  335. for (p = va & 0xFFFFF000u; p < end; p += PAGE_SIZE) {
  336. unsigned int pde = pd[p >> 22];
  337. unsigned int *pt;
  338. unsigned int pte;
  339. if (!(pde & PDE_PRESENT) || (pde & PDE_PS) || !(pde & PDE_USER))
  340. return 0;
  341. pt = (unsigned int *)(pde & 0xFFFFF000);
  342. pte = pt[(p >> 12) & 0x3FF];
  343. if (!(pte & PTE_PRESENT) || !(pte & PTE_USER))
  344. return 0;
  345. if (write && !(pte & PTE_WRITABLE))
  346. return 0;
  347. }
  348. return 1;
  349. }
  350. void paging_release_user_range(unsigned int cr3, unsigned int va,
  351. unsigned int size, int free_phys)
  352. {
  353. unsigned int *pd = (unsigned int *)cr3;
  354. unsigned int end = va + size;
  355. unsigned int first, last, i, j;
  356. if (!pd || size == 0 || end < va)
  357. return;
  358. first = va >> 22;
  359. last = (end - 1) >> 22;
  360. for (i = first; i <= last && i < 1024; i++) {
  361. unsigned int pde = pd[i];
  362. unsigned int *pt;
  363. int empty = 1;
  364. /* Kernel identity mappings are 4 MB pages and must never be
  365. * touched here. */
  366. if (!(pde & PDE_PRESENT) || (pde & PDE_PS))
  367. continue;
  368. pt = (unsigned int *)(pde & 0xFFFFF000);
  369. for (j = 0; j < 1024; j++) {
  370. unsigned int pva = (i << 22) | (j << 12);
  371. if (pva < va || pva >= end)
  372. continue;
  373. if (pt[j] & PTE_PRESENT) {
  374. /* Only user pages are owned by the process; a
  375. * supervisor mapping in this range is left for
  376. * its owner. */
  377. if (free_phys && (pt[j] & PTE_USER))
  378. mm_free_page(pt[j] & 0xFFFFF000);
  379. pt[j] = 0;
  380. }
  381. }
  382. for (j = 0; j < 1024; j++)
  383. if (pt[j] & PTE_PRESENT) {
  384. empty = 0;
  385. break;
  386. }
  387. if (empty) {
  388. mm_free_page(pde & 0xFFFFF000);
  389. pd[i] = 0;
  390. }
  391. }
  392. }
  393. /* Identity-map an MMIO range uncacheable into one page directory (4 MB pages;
  394. * the range is rounded up to a whole number of them). */
  395. void paging_map_mmio_cr3(unsigned int cr3, unsigned int phys_base, unsigned int size)
  396. {
  397. unsigned int *pd = (unsigned int *)cr3;
  398. unsigned int p0 = phys_base >> 22;
  399. unsigned int p1 = (phys_base + size + 0x3FFFFFu) >> 22;
  400. unsigned int i;
  401. for (i = p0; i < p1 && i < 1024; i++)
  402. if (!kmap_pde_guard(i))
  403. pd[i] = (i << 22) | PDE_PS | PDE_PCD |
  404. PDE_WRITABLE | PDE_PRESENT;
  405. }
  406. void paging_map_mmio(unsigned int phys_base, unsigned int size)
  407. {
  408. paging_map_mmio_cr3((unsigned int)page_dir, phys_base, size);
  409. }
  410. /* Split the 4 MB page containing `va` in one directory and set PCD on that
  411. * 4 KB page. The kernel identity map is WB; legacy OHCI DMA on the SB950
  412. * does not reliably snoop dirty descriptor lines, so the ED/TD page must
  413. * bypass the cache. */
  414. static void uc_page_cr3(unsigned int cr3, unsigned int va)
  415. {
  416. unsigned int *pd = (unsigned int *)cr3;
  417. unsigned int pde = va >> 22;
  418. unsigned int pte = (va >> 12) & 0x3FFu;
  419. unsigned int entry = pd[pde];
  420. unsigned int *pt;
  421. unsigned int i;
  422. if (!(entry & PDE_PRESENT))
  423. return;
  424. if (entry & PDE_PS) {
  425. unsigned int base = entry & 0xFFC00000u;
  426. unsigned int flags = (entry & 0xFFFu) & ~PDE_PS;
  427. unsigned int table = mm_alloc_page();
  428. if (!table)
  429. return;
  430. pt = (unsigned int *)table;
  431. for (i = 0; i < 1024; i++)
  432. pt[i] = (base + i * 0x1000u) | flags;
  433. pd[pde] = table | flags;
  434. } else {
  435. pt = (unsigned int *)(entry & 0xFFFFF000u);
  436. }
  437. pt[pte] |= PDE_PCD;
  438. }
  439. /* Temporary kernel mapping of one frame. Lowmem is returned through the
  440. * identity map; a highmem frame gets this CPU's next window slot with
  441. * preemption disabled, so the task cannot migrate or be switched out while
  442. * the slot is live. Nothing may sleep or yield before kunmap(). */
  443. void *kmap(unsigned int phys)
  444. {
  445. struct cpu *c;
  446. unsigned int slot, va;
  447. phys &= 0xFFFFF000u;
  448. if (phys < MM_MAX_PHYS)
  449. return (void *)phys;
  450. preempt_disable();
  451. c = this_cpu();
  452. if (c->id >= MAX_CPUS || kmap_depth[c->id] >= KMAP_SLOTS) {
  453. serial_puts(SERIAL_COM1, "paging: kmap slots exhausted\n");
  454. for (;;)
  455. __asm__ volatile ("cli; hlt");
  456. }
  457. slot = c->id * KMAP_SLOTS + kmap_depth[c->id]++;
  458. va = MM_KMAP_VA + slot * PAGE_SIZE;
  459. kmap_pt[slot] = phys | PTE_WRITABLE | PTE_PRESENT;
  460. __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
  461. return (void *)va;
  462. }
  463. /* Undo the most recent kmap() on this CPU (mappings nest LIFO). */
  464. void kunmap(void *addr)
  465. {
  466. unsigned int va = (unsigned int)addr & 0xFFFFF000u;
  467. struct cpu *c;
  468. unsigned int slot;
  469. if (va < MM_MAX_PHYS)
  470. return;
  471. c = this_cpu();
  472. slot = (va - MM_KMAP_VA) / PAGE_SIZE;
  473. kmap_pt[slot] = 0;
  474. __asm__ volatile ("invlpg (%0)" : : "r"(va) : "memory");
  475. if (kmap_depth[c->id])
  476. kmap_depth[c->id]--;
  477. preempt_enable();
  478. }