mtrr.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. /*
  2. * MTRR write-combining.
  3. *
  4. * VRAM writes (the shadow flush) used to be separate uncached PCI
  5. * transactions: ~33 MB/s on the AT3IONT-I, so a full screen took 150 ms.
  6. * Marking the LFB range write-combining (WC) via an MTRR lets the CPU batch
  7. * those writes into bursts -- but only if the *pages* allow caching too:
  8. * paging.c used to map the LFB with PCD=1, which is UC regardless of the
  9. * MTRR, so the WC entry alone did nothing. paging.c now installs the MTRR
  10. * first (through mtrr_set_wc(), which reports whether it covered the range)
  11. * and maps only the covered pages cacheable, leaving the rest UC.
  12. *
  13. * Two hard rules (the first key press froze an ASUS AT3IONT-I; these were
  14. * the only places where the kernel touched cacheability behind the BIOS's
  15. * back, and overwriting the BIOS's MMIO-cacheability entries is the prime
  16. * suspect, so both are fixed to be safe regardless):
  17. *
  18. * 1. NEVER overwrite an enabled BIOS MTRR entry. Only ever add the WC
  19. * range to a free variable entry; if none is free, skip WC entirely.
  20. * 2. Get the physmask direction right: mask bits are "compare" bits and
  21. * are therefore the bitwise COMPLEMENT of (size - 1). (The previous
  22. * version had them inverted, so e.g. a 16 MB request marked the first
  23. * 4 KB of every 16 MB block as WC instead of the framebuffer range.)
  24. *
  25. * The MTRR layout found at boot and the result are dumped to serial, so this
  26. * can be inspected on machines without a VGA console.
  27. */
  28. #include "mtrr.h"
  29. #include "serial.h"
  30. #define MSR_MTRRCAP 0x0FE
  31. #define MSR_MTRR_DEF_TYPE 0x2FF
  32. #define MSR_MTRR_PHYSBASE0 0x200
  33. #define MSR_MTRR_PHYSMASK0 0x201
  34. #define MTRR_TYPE_WC 0x01
  35. #define MTRR_VALID 0x800
  36. static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
  37. {
  38. __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
  39. }
  40. static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
  41. {
  42. __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
  43. }
  44. static int cpu_has_mtrr(void)
  45. {
  46. unsigned int eax, ebx, ecx, edx;
  47. __asm__ volatile ("cpuid"
  48. : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
  49. : "a"(1), "c"(0));
  50. return (edx & (1u << 12)) != 0;
  51. }
  52. static int cpu_is_amd(void)
  53. {
  54. unsigned int eax = 0, ebx, ecx, edx;
  55. __asm__ volatile ("cpuid"
  56. : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
  57. /* "AuthenticAMD" */
  58. return ebx == 0x68747541u && edx == 0x69746E65u &&
  59. ecx == 0x444D4163u;
  60. }
  61. /* Mask bits for physical address bits 32..35+. Those are reserved (writing
  62. * 1 to an unimplemented bit faults) so ask CPUID for the real width. All
  63. * ones for the implemented high bits means "must match PHYSBASE", which is
  64. * zero, so addresses >= 4 GB never match the WC entry. */
  65. static unsigned int mtrr_hi_mask(void)
  66. {
  67. unsigned int eax, ebx, ecx, edx;
  68. unsigned int bits, n;
  69. __asm__ volatile ("cpuid"
  70. : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
  71. : "a"(0x80000000u), "c"(0));
  72. if (eax < 0x80000008u)
  73. return 0;
  74. __asm__ volatile ("cpuid"
  75. : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
  76. : "a"(0x80000008u), "c"(0));
  77. bits = eax & 0xFF;
  78. if (bits <= 32)
  79. return 0;
  80. n = bits - 32;
  81. if (n >= 32)
  82. return 0xFFFFFFFFu;
  83. return (1u << n) - 1;
  84. }
  85. static char *app(char *p, const char *s)
  86. {
  87. while (*s)
  88. *p++ = *s++;
  89. return p;
  90. }
  91. static char *hex8(char *p, unsigned int v)
  92. {
  93. static const char h[] = "0123456789ABCDEF";
  94. int i;
  95. for (i = 7; i >= 0; i--)
  96. *p++ = h[(v >> (i * 4)) & 0xF];
  97. return p;
  98. }
  99. static const char *type_name(unsigned int t)
  100. {
  101. switch (t & 7) {
  102. case 0: return "UC";
  103. case 1: return "WC";
  104. case 4: return "WT";
  105. case 5: return "WP";
  106. case 6: return "WB";
  107. default: return "??";
  108. }
  109. }
  110. /* Dump MTRRdefType and the variable entries to serial. */
  111. static void dump_mtrrs(unsigned int vcnt)
  112. {
  113. char buf[56];
  114. char *p;
  115. unsigned int i, lo, hi, mlo, mhi;
  116. rdmsr(MSR_MTRR_DEF_TYPE, &lo, &hi);
  117. p = buf;
  118. p = app(p, "MTRRdef ");
  119. p = hex8(p, lo);
  120. p = app(p, " vcnt ");
  121. *p++ = (char)('0' + (vcnt > 9 ? 9 : vcnt));
  122. *p = '\0';
  123. serial_puts(SERIAL_COM1, buf);
  124. serial_puts(SERIAL_COM1, "\n");
  125. for (i = 0; i < vcnt && i < 8; i++) {
  126. rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
  127. rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
  128. p = buf;
  129. *p++ = 'M';
  130. *p++ = (char)('0' + i);
  131. *p++ = ' ';
  132. if (!(mlo & MTRR_VALID)) {
  133. p = app(p, "-");
  134. } else {
  135. p = app(p, "b=");
  136. p = hex8(p, lo & 0xFFFFF000u);
  137. p = app(p, " m=");
  138. p = hex8(p, mlo & 0xFFFFF000u);
  139. *p++ = ' ';
  140. p = app(p, type_name(lo));
  141. }
  142. *p = '\0';
  143. serial_puts(SERIAL_COM1, buf);
  144. serial_puts(SERIAL_COM1, "\n");
  145. }
  146. }
  147. /* AMD family 15h WC path. The Intel path's read-modify-write of
  148. * MSR_MTRRdefType is what reset the 970A-DS3P during the first paging init.
  149. * AMD only has eight variable MTRRs, and the LFB is below 4 GB, so this path
  150. * never touches MTRRdefType and writes a zero high mask (all 32..63 address
  151. * bits are don't-care); that avoids every reserved-bit write while still
  152. * covering the framebuffer range. Only a free entry is used. */
  153. static int mtrr_set_wc_amd(unsigned int base, unsigned int size,
  154. unsigned int *wc_base, unsigned int *wc_size)
  155. {
  156. unsigned int lo, hi, mlo, mhi;
  157. unsigned int vcnt, i, use = ~0u;
  158. unsigned int region_size, align, physmask;
  159. rdmsr(MSR_MTRRCAP, &lo, &hi);
  160. vcnt = lo & 0xFF;
  161. if (vcnt > 8)
  162. vcnt = 8;
  163. if (vcnt == 0)
  164. return 0;
  165. rdmsr(MSR_MTRR_DEF_TYPE, &lo, &hi);
  166. if (!(lo & MTRR_VALID)) {
  167. serial_puts(SERIAL_COM1, "mtrr: AMD MTRR disabled by BIOS\n");
  168. return 0;
  169. }
  170. region_size = 0x1000;
  171. while (region_size < size) {
  172. region_size <<= 1;
  173. if (region_size == 0)
  174. return 0;
  175. }
  176. align = base & (~base + 1u);
  177. if (align == 0 || align > 0x80000000u)
  178. align = 0x80000000u;
  179. if (region_size > align)
  180. region_size = align;
  181. if (region_size < 0x1000)
  182. return 0;
  183. base &= ~(region_size - 1);
  184. for (i = 0; i < vcnt; i++) {
  185. unsigned int esize;
  186. rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
  187. rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
  188. if (!(mlo & MTRR_VALID) || (lo & 0xFF) != MTRR_TYPE_WC ||
  189. (lo & 0xFFFFF000u) != base)
  190. continue;
  191. esize = ((~mlo) & 0xFFFFF000u) + 0x1000;
  192. if (esize < region_size)
  193. continue;
  194. serial_puts(SERIAL_COM1,
  195. "mtrr: AMD LFB already write-combining\n");
  196. if (wc_base)
  197. *wc_base = base;
  198. if (wc_size)
  199. *wc_size = esize;
  200. return 1;
  201. }
  202. for (i = 0; i < vcnt; i++) {
  203. rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
  204. if (!(mlo & MTRR_VALID)) {
  205. use = i;
  206. break;
  207. }
  208. }
  209. if (use == ~0u) {
  210. serial_puts(SERIAL_COM1,
  211. "mtrr: AMD no free variable entry; skipping WC\n");
  212. return 0;
  213. }
  214. physmask = (~(region_size - 1) & 0xFFFFF000u) | MTRR_VALID;
  215. wrmsr(MSR_MTRR_PHYSBASE0 + 2 * use, base | MTRR_TYPE_WC, 0);
  216. wrmsr(MSR_MTRR_PHYSMASK0 + 2 * use, physmask, 0);
  217. serial_puts(SERIAL_COM1, "mtrr: AMD LFB WC [");
  218. serial_puthex(SERIAL_COM1, base);
  219. serial_puts(SERIAL_COM1, " +");
  220. serial_puthex(SERIAL_COM1, region_size);
  221. serial_puts(SERIAL_COM1, "]\n");
  222. if (wc_base)
  223. *wc_base = base;
  224. if (wc_size)
  225. *wc_size = region_size;
  226. return 1;
  227. }
  228. int mtrr_set_wc(unsigned int base, unsigned int size,
  229. unsigned int *wc_base, unsigned int *wc_size)
  230. {
  231. unsigned int lo, hi, mlo;
  232. unsigned int def_type;
  233. unsigned int vcnt;
  234. unsigned int i, use = ~0u;
  235. unsigned int region_size, align, physmask;
  236. if (!cpu_has_mtrr() || size == 0)
  237. return 0;
  238. /* AMD's MTRR register layout and BIOS setup differ enough from the
  239. * Intel path that the latter reset the 970A-DS3P. Use the separate
  240. * free-entry/zero-high-mask path. */
  241. if (cpu_is_amd())
  242. return mtrr_set_wc_amd(base, size, wc_base, wc_size);
  243. rdmsr(MSR_MTRRCAP, &lo, &hi);
  244. vcnt = lo & 0xFF;
  245. if (vcnt > 16)
  246. vcnt = 16; /* sanity cap; CPUs ship 8..10 variable ranges */
  247. if (vcnt == 0)
  248. return 0;
  249. /* Dump the first 8 at most; the search below still covers every
  250. * variable range. */
  251. dump_mtrrs(vcnt > 8 ? 8 : vcnt);
  252. /* Round size up to the next power of two (min 4 KB). */
  253. region_size = 0x1000;
  254. while (region_size < size) {
  255. region_size <<= 1;
  256. if (region_size == 0)
  257. return 0; /* overflow (size > 2 GB) -- give up */
  258. }
  259. /* A variable-range MTRR must be aligned to its own size. The largest
  260. * region we may use is therefore the base address's own power-of-two
  261. * alignment. */
  262. align = base & (~base + 1u);
  263. if (align == 0 || align > 0x80000000u)
  264. align = 0x80000000u;
  265. if (region_size > align)
  266. region_size = align;
  267. if (region_size < 0x1000)
  268. return 0;
  269. base &= ~(region_size - 1);
  270. /* A kernel reload does not reset the MTRRs: reuse the WC entry an
  271. * earlier boot left for this base instead of burning another variable
  272. * slot (the kexec dev loop would otherwise exhaust the entries). */
  273. for (i = 0; i < vcnt; i++) {
  274. unsigned int esize;
  275. rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
  276. rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &hi);
  277. if (!(mlo & MTRR_VALID) || (lo & 0xFF) != MTRR_TYPE_WC ||
  278. (lo & 0xFFFFF000u) != base)
  279. continue;
  280. esize = ((~mlo) & 0xFFFFF000u) + 0x1000;
  281. if (esize < region_size)
  282. continue;
  283. serial_puts(SERIAL_COM1, "mtrr: LFB already write-combining\n");
  284. if (wc_base)
  285. *wc_base = base;
  286. if (wc_size)
  287. *wc_size = esize;
  288. return 1;
  289. }
  290. /* Only ever use a free variable entry. BIOS-populated entries may
  291. * keep chipset MMIO uncached and must not be disturbed. */
  292. for (i = 0; i < vcnt; i++) {
  293. rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &hi);
  294. if (!(mlo & MTRR_VALID)) {
  295. use = i;
  296. break;
  297. }
  298. }
  299. if (use == ~0u) {
  300. serial_puts(SERIAL_COM1, "mtrr: no free variable entry; skipping WC\n");
  301. return 0;
  302. }
  303. /* PhysMask bits [35:12] are compare bits: 1 = the address bit must
  304. * match PHYSBASE. For a 2^n-sized region, bits [n-1:12] are
  305. * don't-care (0) and every higher bit must match (1), i.e. the
  306. * complement of (size - 1). */
  307. physmask = (~(region_size - 1) & 0xFFFFF000u) | MTRR_VALID;
  308. /* Update the entry with the global enable temporarily cleared so an
  309. * in-progress region never exposes a bogus memory type. */
  310. rdmsr(MSR_MTRR_DEF_TYPE, &def_type, &hi);
  311. wrmsr(MSR_MTRR_DEF_TYPE, def_type & ~MTRR_VALID, 0);
  312. wrmsr(MSR_MTRR_PHYSBASE0 + 2 * use, base | MTRR_TYPE_WC, 0);
  313. wrmsr(MSR_MTRR_PHYSMASK0 + 2 * use, physmask, mtrr_hi_mask());
  314. wrmsr(MSR_MTRR_DEF_TYPE, def_type, 0);
  315. serial_puts(SERIAL_COM1, "mtrr: LFB WC [");
  316. serial_puthex(SERIAL_COM1, base);
  317. serial_puts(SERIAL_COM1, " +");
  318. serial_puthex(SERIAL_COM1, region_size);
  319. serial_puts(SERIAL_COM1, "]\n");
  320. if (wc_base)
  321. *wc_base = base;
  322. if (wc_size)
  323. *wc_size = region_size;
  324. return 1;
  325. }