| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407 |
- /*
- * MTRR write-combining.
- *
- * VRAM writes (the shadow flush) used to be separate uncached PCI
- * transactions: ~33 MB/s on the AT3IONT-I, so a full screen took 150 ms.
- * Marking the LFB range write-combining (WC) via an MTRR lets the CPU batch
- * those writes into bursts -- but only if the *pages* allow caching too:
- * paging.c used to map the LFB with PCD=1, which is UC regardless of the
- * MTRR, so the WC entry alone did nothing. paging.c now installs the MTRR
- * first (through mtrr_set_wc(), which reports whether it covered the range)
- * and maps only the covered pages cacheable, leaving the rest UC.
- *
- * Two hard rules (the first key press froze an ASUS AT3IONT-I; these were
- * the only places where the kernel touched cacheability behind the BIOS's
- * back, and overwriting the BIOS's MMIO-cacheability entries is the prime
- * suspect, so both are fixed to be safe regardless):
- *
- * 1. NEVER overwrite an enabled BIOS MTRR entry. Only ever add the WC
- * range to a free variable entry; if none is free, skip WC entirely.
- * 2. Get the physmask direction right: mask bits are "compare" bits and
- * are therefore the bitwise COMPLEMENT of (size - 1). (The previous
- * version had them inverted, so e.g. a 16 MB request marked the first
- * 4 KB of every 16 MB block as WC instead of the framebuffer range.)
- *
- * The MTRR layout found at boot and the result are dumped to serial, so this
- * can be inspected on machines without a VGA console.
- */
- #include "mtrr.h"
- #include "serial.h"
- #define MSR_MTRRCAP 0x0FE
- #define MSR_MTRR_DEF_TYPE 0x2FF
- #define MSR_MTRR_PHYSBASE0 0x200
- #define MSR_MTRR_PHYSMASK0 0x201
- #define MTRR_TYPE_WC 0x01
- #define MTRR_VALID 0x800
- static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
- {
- __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
- }
- static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
- {
- __asm__ volatile ("wrmsr" : : "a"(lo), "d"(hi), "c"(msr));
- }
- static int cpu_has_mtrr(void)
- {
- unsigned int eax, ebx, ecx, edx;
- __asm__ volatile ("cpuid"
- : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
- : "a"(1), "c"(0));
- return (edx & (1u << 12)) != 0;
- }
- static int cpu_is_amd(void)
- {
- unsigned int eax = 0, ebx, ecx, edx;
- __asm__ volatile ("cpuid"
- : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx));
- /* "AuthenticAMD" */
- return ebx == 0x68747541u && edx == 0x69746E65u &&
- ecx == 0x444D4163u;
- }
- /* Mask bits for physical address bits 32..35+. Those are reserved (writing
- * 1 to an unimplemented bit faults) so ask CPUID for the real width. All
- * ones for the implemented high bits means "must match PHYSBASE", which is
- * zero, so addresses >= 4 GB never match the WC entry. */
- static unsigned int mtrr_hi_mask(void)
- {
- unsigned int eax, ebx, ecx, edx;
- unsigned int bits, n;
- __asm__ volatile ("cpuid"
- : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
- : "a"(0x80000000u), "c"(0));
- if (eax < 0x80000008u)
- return 0;
- __asm__ volatile ("cpuid"
- : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
- : "a"(0x80000008u), "c"(0));
- bits = eax & 0xFF;
- if (bits <= 32)
- return 0;
- n = bits - 32;
- if (n >= 32)
- return 0xFFFFFFFFu;
- return (1u << n) - 1;
- }
- static char *app(char *p, const char *s)
- {
- while (*s)
- *p++ = *s++;
- return p;
- }
- static char *hex8(char *p, unsigned int v)
- {
- static const char h[] = "0123456789ABCDEF";
- int i;
- for (i = 7; i >= 0; i--)
- *p++ = h[(v >> (i * 4)) & 0xF];
- return p;
- }
- static const char *type_name(unsigned int t)
- {
- switch (t & 7) {
- case 0: return "UC";
- case 1: return "WC";
- case 4: return "WT";
- case 5: return "WP";
- case 6: return "WB";
- default: return "??";
- }
- }
- /* Effective variable-range MTRR memory type of one physical address below
- * 4 GB (above the fixed ranges): UC wins over everything, WT over WB, and
- * MTRRdefType applies where no entry matches. -1 when the CPU has no MTRRs
- * or they are disabled (then nothing can be learned from them). */
- int mtrr_mem_type(unsigned int phys)
- {
- unsigned int lo, hi, mlo, mhi, def, vcnt, i;
- int type = -1;
- if (!cpu_has_mtrr())
- return -1;
- rdmsr(MSR_MTRR_DEF_TYPE, &def, &hi);
- if (!(def & MTRR_VALID))
- return -1;
- rdmsr(MSR_MTRRCAP, &vcnt, &hi);
- vcnt &= 0xFF;
- if (vcnt > 16)
- vcnt = 16;
- for (i = 0; i < vcnt; i++) {
- int t;
- rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
- if (!(mlo & MTRR_VALID))
- continue;
- /* The address's bits 32+ are zero: a base with high bits
- * set under the mask never matches. */
- if ((hi & mhi) != 0 ||
- ((phys ^ lo) & mlo & 0xFFFFF000u) != 0)
- continue;
- t = (int)(lo & 0xFF);
- if (t == 0)
- return 0;
- if (type < 0 || (type == 6 && t == 4))
- type = t;
- }
- return type < 0 ? (int)(def & 0xFF) : type;
- }
- /* Dump MTRRdefType and the variable entries to serial. */
- static void dump_mtrrs(unsigned int vcnt)
- {
- char buf[56];
- char *p;
- unsigned int i, lo, hi, mlo, mhi;
- rdmsr(MSR_MTRR_DEF_TYPE, &lo, &hi);
- p = buf;
- p = app(p, "MTRRdef ");
- p = hex8(p, lo);
- p = app(p, " vcnt ");
- *p++ = (char)('0' + (vcnt > 9 ? 9 : vcnt));
- *p = '\0';
- serial_puts(SERIAL_COM1, buf);
- serial_puts(SERIAL_COM1, "\n");
- for (i = 0; i < vcnt && i < 8; i++) {
- rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
- p = buf;
- *p++ = 'M';
- *p++ = (char)('0' + i);
- *p++ = ' ';
- if (!(mlo & MTRR_VALID)) {
- p = app(p, "-");
- } else {
- p = app(p, "b=");
- p = hex8(p, lo & 0xFFFFF000u);
- p = app(p, " m=");
- p = hex8(p, mlo & 0xFFFFF000u);
- *p++ = ' ';
- p = app(p, type_name(lo));
- }
- *p = '\0';
- serial_puts(SERIAL_COM1, buf);
- serial_puts(SERIAL_COM1, "\n");
- }
- }
- /* AMD family 15h WC path. The Intel path's read-modify-write of
- * MSR_MTRRdefType is what reset the 970A-DS3P during the first paging init.
- * AMD only has eight variable MTRRs, and the LFB is below 4 GB, so this path
- * never touches MTRRdefType and writes a zero high mask (all 32..63 address
- * bits are don't-care); that avoids every reserved-bit write while still
- * covering the framebuffer range. Only a free entry is used. */
- static int mtrr_set_wc_amd(unsigned int base, unsigned int size,
- unsigned int *wc_base, unsigned int *wc_size)
- {
- unsigned int lo, hi, mlo, mhi;
- unsigned int vcnt, i, use = ~0u;
- unsigned int region_size, align, physmask;
- rdmsr(MSR_MTRRCAP, &lo, &hi);
- vcnt = lo & 0xFF;
- if (vcnt > 8)
- vcnt = 8;
- if (vcnt == 0)
- return 0;
- rdmsr(MSR_MTRR_DEF_TYPE, &lo, &hi);
- if (!(lo & MTRR_VALID)) {
- serial_puts(SERIAL_COM1, "mtrr: AMD MTRR disabled by BIOS\n");
- return 0;
- }
- region_size = 0x1000;
- while (region_size < size) {
- region_size <<= 1;
- if (region_size == 0)
- return 0;
- }
- align = base & (~base + 1u);
- if (align == 0 || align > 0x80000000u)
- align = 0x80000000u;
- if (region_size > align)
- region_size = align;
- if (region_size < 0x1000)
- return 0;
- base &= ~(region_size - 1);
- for (i = 0; i < vcnt; i++) {
- unsigned int esize;
- rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
- if (!(mlo & MTRR_VALID) || (lo & 0xFF) != MTRR_TYPE_WC ||
- (lo & 0xFFFFF000u) != base)
- continue;
- esize = ((~mlo) & 0xFFFFF000u) + 0x1000;
- if (esize < region_size)
- continue;
- serial_puts(SERIAL_COM1,
- "mtrr: AMD LFB already write-combining\n");
- if (wc_base)
- *wc_base = base;
- if (wc_size)
- *wc_size = esize;
- return 1;
- }
- for (i = 0; i < vcnt; i++) {
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &mhi);
- if (!(mlo & MTRR_VALID)) {
- use = i;
- break;
- }
- }
- if (use == ~0u) {
- serial_puts(SERIAL_COM1,
- "mtrr: AMD no free variable entry; skipping WC\n");
- return 0;
- }
- physmask = (~(region_size - 1) & 0xFFFFF000u) | MTRR_VALID;
- wrmsr(MSR_MTRR_PHYSBASE0 + 2 * use, base | MTRR_TYPE_WC, 0);
- wrmsr(MSR_MTRR_PHYSMASK0 + 2 * use, physmask, 0);
- serial_puts(SERIAL_COM1, "mtrr: AMD LFB WC [");
- serial_puthex(SERIAL_COM1, base);
- serial_puts(SERIAL_COM1, " +");
- serial_puthex(SERIAL_COM1, region_size);
- serial_puts(SERIAL_COM1, "]\n");
- if (wc_base)
- *wc_base = base;
- if (wc_size)
- *wc_size = region_size;
- return 1;
- }
- int mtrr_set_wc(unsigned int base, unsigned int size,
- unsigned int *wc_base, unsigned int *wc_size)
- {
- unsigned int lo, hi, mlo;
- unsigned int def_type;
- unsigned int vcnt;
- unsigned int i, use = ~0u;
- unsigned int region_size, align, physmask;
- if (!cpu_has_mtrr() || size == 0)
- return 0;
- /* AMD's MTRR register layout and BIOS setup differ enough from the
- * Intel path that the latter reset the 970A-DS3P. Use the separate
- * free-entry/zero-high-mask path. */
- if (cpu_is_amd())
- return mtrr_set_wc_amd(base, size, wc_base, wc_size);
- rdmsr(MSR_MTRRCAP, &lo, &hi);
- vcnt = lo & 0xFF;
- if (vcnt > 16)
- vcnt = 16; /* sanity cap; CPUs ship 8..10 variable ranges */
- if (vcnt == 0)
- return 0;
- /* Dump the first 8 at most; the search below still covers every
- * variable range. */
- dump_mtrrs(vcnt > 8 ? 8 : vcnt);
- /* Round size up to the next power of two (min 4 KB). */
- region_size = 0x1000;
- while (region_size < size) {
- region_size <<= 1;
- if (region_size == 0)
- return 0; /* overflow (size > 2 GB) -- give up */
- }
- /* A variable-range MTRR must be aligned to its own size. The largest
- * region we may use is therefore the base address's own power-of-two
- * alignment. */
- align = base & (~base + 1u);
- if (align == 0 || align > 0x80000000u)
- align = 0x80000000u;
- if (region_size > align)
- region_size = align;
- if (region_size < 0x1000)
- return 0;
- base &= ~(region_size - 1);
- /* A kernel reload does not reset the MTRRs: reuse the WC entry an
- * earlier boot left for this base instead of burning another variable
- * slot (the kexec dev loop would otherwise exhaust the entries). */
- for (i = 0; i < vcnt; i++) {
- unsigned int esize;
- rdmsr(MSR_MTRR_PHYSBASE0 + 2 * i, &lo, &hi);
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &hi);
- if (!(mlo & MTRR_VALID) || (lo & 0xFF) != MTRR_TYPE_WC ||
- (lo & 0xFFFFF000u) != base)
- continue;
- esize = ((~mlo) & 0xFFFFF000u) + 0x1000;
- if (esize < region_size)
- continue;
- serial_puts(SERIAL_COM1, "mtrr: LFB already write-combining\n");
- if (wc_base)
- *wc_base = base;
- if (wc_size)
- *wc_size = esize;
- return 1;
- }
- /* Only ever use a free variable entry. BIOS-populated entries may
- * keep chipset MMIO uncached and must not be disturbed. */
- for (i = 0; i < vcnt; i++) {
- rdmsr(MSR_MTRR_PHYSMASK0 + 2 * i, &mlo, &hi);
- if (!(mlo & MTRR_VALID)) {
- use = i;
- break;
- }
- }
- if (use == ~0u) {
- serial_puts(SERIAL_COM1, "mtrr: no free variable entry; skipping WC\n");
- return 0;
- }
- /* PhysMask bits [35:12] are compare bits: 1 = the address bit must
- * match PHYSBASE. For a 2^n-sized region, bits [n-1:12] are
- * don't-care (0) and every higher bit must match (1), i.e. the
- * complement of (size - 1). */
- physmask = (~(region_size - 1) & 0xFFFFF000u) | MTRR_VALID;
- /* Update the entry with the global enable temporarily cleared so an
- * in-progress region never exposes a bogus memory type. */
- rdmsr(MSR_MTRR_DEF_TYPE, &def_type, &hi);
- wrmsr(MSR_MTRR_DEF_TYPE, def_type & ~MTRR_VALID, 0);
- wrmsr(MSR_MTRR_PHYSBASE0 + 2 * use, base | MTRR_TYPE_WC, 0);
- wrmsr(MSR_MTRR_PHYSMASK0 + 2 * use, physmask, mtrr_hi_mask());
- wrmsr(MSR_MTRR_DEF_TYPE, def_type, 0);
- serial_puts(SERIAL_COM1, "mtrr: LFB WC [");
- serial_puthex(SERIAL_COM1, base);
- serial_puts(SERIAL_COM1, " +");
- serial_puthex(SERIAL_COM1, region_size);
- serial_puts(SERIAL_COM1, "]\n");
- if (wc_base)
- *wc_base = base;
- if (wc_size)
- *wc_size = region_size;
- return 1;
- }
|