power.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962
  1. /* Intel P-state and deep C-state management.
  2. *
  3. * Nehalem/Westmere (the X201) get the full race-to-idle P-state governor:
  4. * the module writes IA32_PERF_CTL (lowest ratio for idle CPUs, maximum
  5. * non-turbo ratio for a CPU that is running something, linear in between for
  6. * mixed windows) and idle.c uses C1E/C3/C6 MWAIT hints.
  7. *
  8. * Braswell/Airmont (the HP Chromebook, family 6 model 0x4C) reads the Atom
  9. * ratio/VID MSRs and MSR_FSB_FREQ, writes IA32_PERF_CTL with the pstate in
  10. * 15:8 and the Linux-interpolated VID in 7:0, and runs the same race-to-idle
  11. * governor. It enters at the maximum non-turbo pstate (pstate 20 = 1600 MHz
  12. * on the N3060) and drops to pstate 6 (480 MHz) when idle; the turbo pstate
  13. * is detected but not requested. The package/power-SKU and auto-demotion
  14. * MSRs are deliberately not touched (coreboot configures them, and an
  15. * unsupported rdmsr before the IDT exists triple-faults). APERF/MPERF are
  16. * used only when CPUID.06H:ECX[0] says they exist.
  17. *
  18. * The module only touches documented MSRs and is gated on the model list;
  19. * unknown CPUs keep the old HLT behaviour.
  20. */
  21. #include "power.h"
  22. #include "bootcon.h"
  23. #include "cpu.h"
  24. #include "idle.h"
  25. #include "io.h"
  26. #include "sched.h"
  27. #include "serial.h"
  28. #define MSR_IA32_PERF_STATUS 0x198
  29. #define MSR_IA32_PERF_CTL 0x199
  30. #define MSR_PLATFORM_INFO 0x0CE
  31. #define MSR_FSB_FREQ 0x0CD
  32. #define MSR_MISC_ENABLE 0x1A0
  33. #define MSR_MISC_PWR_MGMT 0x1AA
  34. #define MSR_POWER_CTL 0x1FC
  35. #define MSR_PKG_CST_CONFIG 0x0E2
  36. #define MSR_PKG_C3_RESIDENCY 0x3F8
  37. #define MSR_PKG_C6_RESIDENCY 0x3F9
  38. #define MSR_CORE_C3_RESIDENCY 0x3FC
  39. #define MSR_CORE_C6_RESIDENCY 0x3FD
  40. #define MSR_APERF 0x0E8
  41. #define MSR_MPERF 0x0E7
  42. #define MSR_TSC 0x010
  43. #define MSR_ATOM_CORE_RATIOS 0x66A
  44. #define MSR_ATOM_CORE_VIDS 0x66B
  45. #define MSR_ATOM_CORE_TURBO_RATIOS 0x66C
  46. #define MSR_ATOM_CORE_TURBO_VIDS 0x66D
  47. /* P-state ratios are the bus-ratio multipliers (133.33 MHz on Arrandale). */
  48. #define BCLK_MHZ 133u
  49. #define PLAT_NEHALEM 1
  50. #define PLAT_AIRMONT 2
  51. /* Airmont MWAIT C6N hint from Linux's intel_idle cht_cstates table. */
  52. #define MWAIT_C6N 0x58
  53. struct power_cpu {
  54. /* Governor state (100 ms windows). */
  55. unsigned int div; /* ms in this window */
  56. unsigned int busy_ms; /* ms this CPU ran a non-idle proc */
  57. unsigned int streak_ms; /* consecutive busy ms */
  58. unsigned int req; /* last ratio written */
  59. /* Once-a-second statistics. */
  60. unsigned int stat_div;
  61. int have_last;
  62. unsigned long long aperf, mperf, c3, c6, tsc;
  63. unsigned int mhz;
  64. unsigned int ratio;
  65. unsigned int status; /* raw IA32_PERF_STATUS[15:0] */
  66. unsigned int pc3, pc6; /* permille */
  67. };
  68. static struct power_cpu pcs[MAX_CPUS];
  69. static int platform;
  70. static int pstate_writes;
  71. static int supported;
  72. static int has_arat;
  73. static int has_aperf;
  74. static int eist_on;
  75. static int turbo_disabled;
  76. static int c1e_on;
  77. static unsigned int cst_limit;
  78. static int cst_locked;
  79. static unsigned int min_ratio = 6;
  80. static unsigned int max_ratio;
  81. static unsigned int ratio_shift;
  82. static unsigned int nominal_mhz;
  83. static int cpufreq_mode = POWER_CPUFREQ_AUTO;
  84. static unsigned int fixed_ratio;
  85. static unsigned long long pkg_c3_last, pkg_c6_last;
  86. static unsigned int pkg_pc3, pkg_pc6;
  87. static int pkg_have_last;
  88. /* Airmont probe values (kept for the `power` output and the follow-up that
  89. * enables the governor). */
  90. static unsigned int fsb_khz;
  91. static unsigned int turbo_ratio;
  92. static unsigned int turbo_vid;
  93. static unsigned int atom_vid_min;
  94. static unsigned int atom_vid_max;
  95. static int atom_vid_valid;
  96. static unsigned int c6n_ok;
  97. static int c6n_enabled;
  98. static unsigned int plat_info_lo, plat_info_hi;
  99. static void cpuid_leaf(unsigned int leaf, unsigned int *a, unsigned int *b,
  100. unsigned int *c, unsigned int *d)
  101. {
  102. __asm__ volatile ("cpuid"
  103. : "=a"(*a), "=b"(*b), "=c"(*c), "=d"(*d)
  104. : "a"(leaf), "c"(0));
  105. }
  106. static void rdmsr(unsigned int msr, unsigned int *lo, unsigned int *hi)
  107. {
  108. __asm__ volatile ("rdmsr" : "=a"(*lo), "=d"(*hi) : "c"(msr));
  109. }
  110. static void wrmsr(unsigned int msr, unsigned int lo, unsigned int hi)
  111. {
  112. __asm__ volatile ("wrmsr" : : "c"(msr), "a"(lo), "d"(hi));
  113. }
  114. static unsigned long long rdmsr64(unsigned int msr)
  115. {
  116. unsigned int lo, hi;
  117. rdmsr(msr, &lo, &hi);
  118. return ((unsigned long long)hi << 32) | lo;
  119. }
  120. /* Model allowlist. Nehalem/Westmere implement the P-state/C-state MSRs
  121. * used here; Braswell is Airmont family 6 model 0x4C. Bonnell
  122. * (0x1C/0x1D) is deliberately absent. */
  123. static int power_platform(unsigned int model)
  124. {
  125. switch (model) {
  126. case 0x1A: /* Nehalem */
  127. case 0x1E: /* Lynnfield/Clarksfield */
  128. case 0x1F: /* Nehalem-EX */
  129. case 0x25: /* Arrandale/Clarkdale (X201 i5 M 520) */
  130. case 0x2C: /* Westmere-EP */
  131. case 0x2E: /* Nehalem-EX */
  132. case 0x2F: /* Westmere-EX */
  133. return PLAT_NEHALEM;
  134. case 0x4C: /* Airmont: Braswell / Cherry Trail */
  135. return PLAT_AIRMONT;
  136. default:
  137. return 0;
  138. }
  139. }
  140. /* Airmont P-state VID for a target pstate. Linux intel_pstate interpolates
  141. * linearly between the min/max VID from MSR_ATOM_CORE_VIDS; the P-unit uses
  142. * the low byte of IA32_PERF_CTL together with the ratio in 15:8. */
  143. static unsigned int atom_vid_for(unsigned int ratio)
  144. {
  145. unsigned int span = max_ratio - min_ratio;
  146. unsigned int v;
  147. if (!atom_vid_valid || !span)
  148. return 0;
  149. if (ratio <= min_ratio)
  150. return atom_vid_min & 0x7F;
  151. if (ratio >= max_ratio)
  152. return atom_vid_max & 0x7F;
  153. v = atom_vid_min +
  154. ((ratio - min_ratio) * (atom_vid_max - atom_vid_min) +
  155. span / 2) / span;
  156. return v & 0x7F;
  157. }
  158. /* Write one target ratio. Nehalem/Westmere use bits 15:8 of
  159. * IA32_PERF_CTL for the target ratio; the low byte is reserved and written
  160. * as zero (coreboot does the same). Airmont takes the ratio in 15:8 and the
  161. * interpolated VID in 7:0. */
  162. static void power_write_ratio(unsigned int ratio)
  163. {
  164. unsigned int lo, hi;
  165. if (ratio < min_ratio)
  166. ratio = min_ratio;
  167. if (ratio > max_ratio)
  168. ratio = max_ratio;
  169. rdmsr(MSR_IA32_PERF_CTL, &lo, &hi);
  170. if (platform == PLAT_AIRMONT) {
  171. lo = (lo & 0xFFFF0000u) | ((ratio & 0xFF) << 8) |
  172. atom_vid_for(ratio);
  173. } else if (ratio_shift) {
  174. /* Bits 15:8 carry the ratio and 7:0 are reserved (the old VID
  175. * slot); coreboot writes them as zero. Keeping the BIOS VID
  176. * byte here made the request silently ineffective. */
  177. lo = (lo & 0xFFFF0000u) | (ratio << 8);
  178. } else {
  179. lo = (lo & ~0xFFu) | ratio;
  180. }
  181. wrmsr(MSR_IA32_PERF_CTL, lo, hi);
  182. }
  183. static void power_apply(struct power_cpu *p, unsigned int ratio)
  184. {
  185. if (ratio < min_ratio)
  186. ratio = min_ratio;
  187. if (ratio > max_ratio)
  188. ratio = max_ratio;
  189. if (ratio == p->req)
  190. return;
  191. power_write_ratio(ratio);
  192. p->req = ratio;
  193. }
  194. /* Airmont FSB frequencies in kHz, SDM Table 35-10 (index in MSR_FSB_FREQ). */
  195. static unsigned int airmont_fsb(unsigned int index)
  196. {
  197. static const unsigned int table[9] = {
  198. 83300, 100000, 133300, 116700, 80000,
  199. 93300, 90000, 88900, 87500
  200. };
  201. if (index >= sizeof(table) / sizeof(table[0]))
  202. return 100000;
  203. return table[index];
  204. }
  205. static void power_init_airmont(unsigned int max_leaf)
  206. {
  207. unsigned int lo, hi, vlo, vhi, tlo, thi, slo, shi;
  208. unsigned int misc_lo, misc_hi;
  209. unsigned int idx, r, i;
  210. bootcon_puts("power: airmont init\n");
  211. rdmsr(MSR_PLATFORM_INFO, &plat_info_lo, &plat_info_hi);
  212. rdmsr(MSR_ATOM_CORE_RATIOS, &lo, &hi);
  213. min_ratio = (lo >> 8) & 0x7F;
  214. max_ratio = (lo >> 16) & 0x7F;
  215. /* If the Atom-specific MSR is not populated, fall back to the
  216. * architectural PLATFORM_INFO fields. */
  217. if (max_ratio < 5 || max_ratio > 60 || min_ratio < 3 ||
  218. min_ratio > max_ratio) {
  219. max_ratio = (plat_info_lo >> 8) & 0xFF;
  220. min_ratio = (plat_info_hi >> 8) & 0xFF;
  221. }
  222. if (max_ratio < 5 || max_ratio > 60 || min_ratio < 3 ||
  223. min_ratio > max_ratio) {
  224. serial_puts(SERIAL_COM1,
  225. "power: implausible Atom ratios, disabled\n");
  226. return;
  227. }
  228. bootcon_puts("power: ratios ok\n");
  229. rdmsr(MSR_FSB_FREQ, &lo, &hi);
  230. idx = lo & 0xF;
  231. fsb_khz = airmont_fsb(idx);
  232. nominal_mhz = (max_ratio * fsb_khz + 500) / 1000;
  233. if (!nominal_mhz) {
  234. serial_puts(SERIAL_COM1, "power: zero nominal frequency, disabled\n");
  235. return;
  236. }
  237. rdmsr(MSR_ATOM_CORE_VIDS, &vlo, &vhi);
  238. atom_vid_min = (vlo >> 8) & 0x7F;
  239. atom_vid_max = (vlo >> 16) & 0x7F;
  240. atom_vid_valid = (max_ratio > min_ratio &&
  241. atom_vid_min <= atom_vid_max);
  242. rdmsr(MSR_ATOM_CORE_TURBO_RATIOS, &tlo, &thi);
  243. turbo_ratio = tlo & 0x7F;
  244. rdmsr(MSR_ATOM_CORE_TURBO_VIDS, &slo, &shi);
  245. turbo_vid = slo & 0x7F;
  246. rdmsr(MSR_MISC_ENABLE, &misc_lo, &misc_hi);
  247. eist_on = (misc_lo >> 16) & 1;
  248. turbo_disabled = (misc_hi >> 6) & 1;
  249. rdmsr(MSR_IA32_PERF_STATUS, &lo, &hi);
  250. r = (lo >> 8) & 0xFF;
  251. if (r >= min_ratio && r <= max_ratio) {
  252. pcs[0].ratio = r;
  253. pcs[0].req = r;
  254. } else {
  255. pcs[0].ratio = max_ratio;
  256. pcs[0].req = max_ratio;
  257. }
  258. pcs[0].status = lo & 0xFFFF;
  259. bootcon_puts("power: msrs ok\n");
  260. has_arat = 0;
  261. c6n_ok = 0;
  262. if (max_leaf >= 5) {
  263. unsigned int a, b, c, d;
  264. cpuid_leaf(5, &a, &b, &c, &d);
  265. c6n_ok = ((d >> 24) & 0xF) ? 1 : 0;
  266. }
  267. if (max_leaf >= 6) {
  268. unsigned int a, b, c, d;
  269. cpuid_leaf(6, &a, &b, &c, &d);
  270. has_arat = (a & 4u) ? 1 : 0;
  271. has_aperf = (c & 1u) ? 1 : 0;
  272. }
  273. c1e_on = 0;
  274. cst_limit = 0;
  275. cst_locked = 0;
  276. pkg_have_last = 0;
  277. /* The governor starts in auto: enter at max non-turbo, then drop to
  278. * pstate 6 after the first idle window. C6N is enabled when CPUID
  279. * advertises the C6 substates. */
  280. cpufreq_mode = POWER_CPUFREQ_AUTO;
  281. supported = 1;
  282. pstate_writes = 1;
  283. for (i = 0; i < MAX_CPUS; i++) {
  284. pcs[i].req = 0;
  285. pcs[i].ratio = max_ratio;
  286. }
  287. c6n_enabled = c6n_ok ? 1 : 0;
  288. if (has_aperf) {
  289. pcs[0].aperf = rdmsr64(MSR_APERF);
  290. pcs[0].mperf = rdmsr64(MSR_MPERF);
  291. }
  292. pcs[0].have_last = 1;
  293. bootcon_puts("power: apis ok\n");
  294. /* Enter at the maximum non-turbo pstate; the race-to-idle governor
  295. * drops to the minimum after the first idle window. */
  296. pcs[0].req = max_ratio;
  297. power_write_ratio(max_ratio);
  298. bootcon_puts("power: pstate writes enabled\n");
  299. if (c6n_enabled)
  300. bootcon_puts("power: c6n enabled\n");
  301. serial_puts(SERIAL_COM1,
  302. "power: Airmont/Braswell, P-states auto");
  303. if (c6n_enabled)
  304. serial_puts(SERIAL_COM1, ", C6N enabled");
  305. else
  306. serial_puts(SERIAL_COM1, ", C6N disabled");
  307. serial_puts(SERIAL_COM1, "\n");
  308. serial_puts(SERIAL_COM1, "power: pstates ");
  309. serial_putdec(SERIAL_COM1, min_ratio);
  310. serial_puts(SERIAL_COM1, "..");
  311. serial_putdec(SERIAL_COM1, max_ratio);
  312. serial_puts(SERIAL_COM1, ", fsb ");
  313. serial_putdec(SERIAL_COM1, fsb_khz);
  314. serial_puts(SERIAL_COM1, " kHz, nominal ");
  315. serial_putdec(SERIAL_COM1, nominal_mhz);
  316. serial_puts(SERIAL_COM1, " MHz, vid ");
  317. serial_puthex(SERIAL_COM1, atom_vid_min);
  318. serial_puts(SERIAL_COM1, "..");
  319. serial_puthex(SERIAL_COM1, atom_vid_max);
  320. serial_puts(SERIAL_COM1, atom_vid_valid ? "" : " (invalid)");
  321. serial_puts(SERIAL_COM1, "\npower: turbo p");
  322. serial_putdec(SERIAL_COM1, turbo_ratio);
  323. serial_puts(SERIAL_COM1, " vid ");
  324. serial_puthex(SERIAL_COM1, turbo_vid);
  325. serial_puts(SERIAL_COM1, ", eist ");
  326. serial_puts(SERIAL_COM1, eist_on ? "on" : "off");
  327. serial_puts(SERIAL_COM1, ", turbo ");
  328. serial_puts(SERIAL_COM1, turbo_disabled ? "disabled" : "enabled");
  329. serial_puts(SERIAL_COM1, ", arat ");
  330. serial_puts(SERIAL_COM1, has_arat ? "yes" : "no");
  331. serial_puts(SERIAL_COM1, ", c6n ");
  332. serial_puts(SERIAL_COM1, c6n_ok ? "yes" : "no");
  333. serial_puts(SERIAL_COM1, "\npower: platform info ");
  334. serial_puthex(SERIAL_COM1, plat_info_hi);
  335. serial_puthex(SERIAL_COM1, plat_info_lo);
  336. serial_puts(SERIAL_COM1, ", perf status ");
  337. serial_puthex(SERIAL_COM1, pcs[0].status);
  338. serial_puts(SERIAL_COM1, ", perf ctl ");
  339. rdmsr(MSR_IA32_PERF_CTL, &lo, &hi);
  340. serial_puthex(SERIAL_COM1, lo);
  341. serial_puts(SERIAL_COM1, "\npower: misc enable ");
  342. serial_puthex(SERIAL_COM1, misc_lo);
  343. serial_puts(SERIAL_COM1, "\n");
  344. }
  345. void power_init(void)
  346. {
  347. unsigned int max_leaf, a, b, c, d;
  348. unsigned int family, model;
  349. unsigned int lo, hi;
  350. supported = 0;
  351. pstate_writes = 0;
  352. platform = 0;
  353. has_arat = 0;
  354. has_aperf = 0;
  355. eist_on = 0;
  356. turbo_disabled = 0;
  357. c1e_on = 0;
  358. cst_limit = 0;
  359. cst_locked = 0;
  360. min_ratio = 6;
  361. max_ratio = 0;
  362. ratio_shift = 0;
  363. nominal_mhz = 0;
  364. cpufreq_mode = POWER_CPUFREQ_AUTO;
  365. fixed_ratio = 0;
  366. fsb_khz = 0;
  367. turbo_ratio = 0;
  368. turbo_vid = 0;
  369. atom_vid_min = 0;
  370. atom_vid_max = 0;
  371. atom_vid_valid = 0;
  372. c6n_ok = 0;
  373. c6n_enabled = 0;
  374. plat_info_lo = 0;
  375. plat_info_hi = 0;
  376. pkg_have_last = 0;
  377. pkg_pc3 = 0;
  378. pkg_pc6 = 0;
  379. if (!cpu_has_cpuid()) {
  380. serial_puts(SERIAL_COM1, "power: no CPUID, disabled\n");
  381. return;
  382. }
  383. cpuid_leaf(0, &max_leaf, &b, &c, &d);
  384. if (max_leaf < 1) {
  385. serial_puts(SERIAL_COM1, "power: no CPUID leaf 1\n");
  386. return;
  387. }
  388. cpuid_leaf(1, &a, &b, &c, &d);
  389. family = (a >> 8) & 0xF;
  390. model = (a >> 4) & 0xF;
  391. if (family == 0xF)
  392. family += (a >> 20) & 0xFF;
  393. if (family == 0x6 || family == 0xF)
  394. model += ((a >> 16) & 0xF) << 4;
  395. platform = (family == 0x6) ? power_platform(model) : 0;
  396. if (!platform) {
  397. serial_puts(SERIAL_COM1, "power: CPU family ");
  398. serial_putdec(SERIAL_COM1, family);
  399. serial_puts(SERIAL_COM1, " model ");
  400. serial_putdec(SERIAL_COM1, model);
  401. serial_puts(SERIAL_COM1, ", no supported PM\n");
  402. return;
  403. }
  404. if (platform == PLAT_AIRMONT) {
  405. power_init_airmont(max_leaf);
  406. return;
  407. }
  408. rdmsr(MSR_PLATFORM_INFO, &lo, &hi);
  409. max_ratio = (lo >> 8) & 0xFF;
  410. min_ratio = (hi >> 8) & 0xFF;
  411. if (max_ratio < 5 || max_ratio > 60 || min_ratio < 3 ||
  412. min_ratio > max_ratio) {
  413. serial_puts(SERIAL_COM1, "power: implausible platform info, disabled\n");
  414. return;
  415. }
  416. nominal_mhz = max_ratio * BCLK_MHZ + max_ratio / 3;
  417. /* Nehalem/Westmere keep the target ratio in bits 15:8 of
  418. * IA32_PERF_CTL/STATUS; the low byte is the (now unused) VID field.
  419. * coreboot's set_max_ratio writes (ratio << 8) for this family, and
  420. * reading the low byte on the X201 gives VID-looking values (0x11,
  421. * 0x18) that would otherwise masquerade as ratios. */
  422. rdmsr(MSR_IA32_PERF_STATUS, &lo, &hi);
  423. ratio_shift = 8;
  424. if (max_leaf >= 6) {
  425. cpuid_leaf(6, &a, &b, &c, &d);
  426. has_arat = (a & 4u) ? 1 : 0;
  427. }
  428. rdmsr(MSR_MISC_ENABLE, &lo, &hi);
  429. eist_on = (lo >> 16) & 1;
  430. turbo_disabled = (hi >> 6) & 1;
  431. if (!eist_on) {
  432. wrmsr(MSR_MISC_ENABLE, lo | (1u << 16), hi);
  433. eist_on = 1;
  434. }
  435. rdmsr(MSR_POWER_CTL, &lo, &hi);
  436. c1e_on = (lo >> 1) & 1;
  437. if (!c1e_on) {
  438. wrmsr(MSR_POWER_CTL, lo | (1u << 1), hi);
  439. c1e_on = 1;
  440. }
  441. rdmsr(MSR_PKG_CST_CONFIG, &lo, &hi);
  442. cst_limit = lo & 7u;
  443. cst_locked = (lo >> 15) & 1;
  444. if (!cst_locked && cst_limit < 3) {
  445. wrmsr(MSR_PKG_CST_CONFIG, (lo & ~7u) | 3u, hi);
  446. cst_limit = 3;
  447. }
  448. rdmsr(MSR_MISC_PWR_MGMT, &lo, &hi);
  449. supported = 1;
  450. pstate_writes = 1;
  451. has_aperf = 1;
  452. /* Start at the maximum non-turbo ratio; the governor drops to the
  453. * minimum after the first idle window. */
  454. rdmsr(MSR_IA32_PERF_STATUS, &lo, &hi);
  455. serial_puts(SERIAL_COM1, "power: perf status ");
  456. serial_puthex(SERIAL_COM1, lo);
  457. pcs[0].req = max_ratio;
  458. power_write_ratio(max_ratio);
  459. rdmsr(MSR_IA32_PERF_CTL, &lo, &hi);
  460. serial_puts(SERIAL_COM1, ", perf ctl ");
  461. serial_puthex(SERIAL_COM1, lo);
  462. rdmsr(MSR_IA32_PERF_STATUS, &lo, &hi);
  463. serial_puts(SERIAL_COM1, ", status now ");
  464. serial_puthex(SERIAL_COM1, lo);
  465. serial_puts(SERIAL_COM1, "\n");
  466. pcs[0].aperf = rdmsr64(MSR_APERF);
  467. pcs[0].mperf = rdmsr64(MSR_MPERF);
  468. pcs[0].c3 = rdmsr64(MSR_CORE_C3_RESIDENCY);
  469. pcs[0].c6 = rdmsr64(MSR_CORE_C6_RESIDENCY);
  470. pcs[0].tsc = rdmsr64(MSR_TSC);
  471. pcs[0].have_last = 1;
  472. pkg_c3_last = rdmsr64(MSR_PKG_C3_RESIDENCY);
  473. pkg_c6_last = rdmsr64(MSR_PKG_C6_RESIDENCY);
  474. pkg_have_last = 1;
  475. serial_puts(SERIAL_COM1, "power: Nehalem/Westmere, ratios ");
  476. serial_putdec(SERIAL_COM1, min_ratio);
  477. serial_puts(SERIAL_COM1, "..");
  478. serial_putdec(SERIAL_COM1, max_ratio);
  479. serial_puts(SERIAL_COM1, " (");
  480. serial_putdec(SERIAL_COM1, nominal_mhz);
  481. serial_puts(SERIAL_COM1, " MHz), perfctl shift ");
  482. serial_putdec(SERIAL_COM1, ratio_shift);
  483. serial_puts(SERIAL_COM1, "\npower: eist ");
  484. serial_puts(SERIAL_COM1, eist_on ? "on" : "off");
  485. serial_puts(SERIAL_COM1, ", turbo ");
  486. serial_puts(SERIAL_COM1, turbo_disabled ? "disabled" : "enabled");
  487. serial_puts(SERIAL_COM1, ", c1e ");
  488. serial_puts(SERIAL_COM1, c1e_on ? "on" : "off");
  489. serial_puts(SERIAL_COM1, ", cst ");
  490. serial_putdec(SERIAL_COM1, cst_limit);
  491. serial_puts(SERIAL_COM1, cst_locked ? " locked" : " unlocked");
  492. serial_puts(SERIAL_COM1, ", arat ");
  493. serial_puts(SERIAL_COM1, has_arat ? "yes" : "no");
  494. serial_puts(SERIAL_COM1, ", pwr mgmt ");
  495. serial_puthex(SERIAL_COM1, lo);
  496. serial_puts(SERIAL_COM1, "\n");
  497. }
  498. int power_idle_hint(void)
  499. {
  500. if (!supported)
  501. return -1;
  502. if (platform == PLAT_AIRMONT) {
  503. /* C6N is reported by CPUID, but the hint stays disabled
  504. * until the read-only probe has booted cleanly on target. */
  505. if (!c6n_enabled || !c6n_ok)
  506. return -1;
  507. /* An AP without ARAT may lose its local APIC timer, so it
  508. * falls back to HLT (C1). */
  509. if (this_cpu()->id != 0 && !has_arat)
  510. return -1;
  511. return MWAIT_C6N;
  512. }
  513. /* An AP can only use C3/C6 when its local APIC timer keeps running
  514. * there (ARAT); otherwise C1E is the deepest safe state and the BSP
  515. * PIT still wakes it up via IPIs. */
  516. if (this_cpu()->id != 0 && !has_arat)
  517. return 0x01;
  518. return 0x20; /* C6 hint: the CPU uses the deepest state it has */
  519. }
  520. void power_tick(int idle, unsigned int ms)
  521. {
  522. struct cpu *c = this_cpu();
  523. struct power_cpu *p;
  524. unsigned int load, r;
  525. if (!supported || c->id >= MAX_CPUS || ms == 0)
  526. return;
  527. p = &pcs[c->id];
  528. /* ---- P-state governor (100 ms windows) ---- */
  529. if (pstate_writes && cpufreq_mode == POWER_CPUFREQ_AUTO) {
  530. if (idle) {
  531. p->streak_ms = 0;
  532. } else {
  533. p->busy_ms += ms;
  534. if (p->streak_ms < 1000)
  535. p->streak_ms += ms;
  536. /* A short busy streak means interactive work or a
  537. * command: jump to max immediately, then let the next
  538. * window decide. */
  539. if (p->streak_ms >= 4 && p->req != max_ratio)
  540. power_apply(p, max_ratio);
  541. }
  542. p->div += ms;
  543. if (p->div >= 100) {
  544. load = p->busy_ms * 100u / p->div;
  545. p->div = 0;
  546. p->busy_ms = 0;
  547. if (load < 15)
  548. r = min_ratio;
  549. else if (load > 80)
  550. r = max_ratio;
  551. else
  552. r = min_ratio +
  553. (max_ratio - min_ratio) * load / 100u;
  554. power_apply(p, r);
  555. }
  556. } else if (pstate_writes && cpufreq_mode != POWER_CPUFREQ_OFF) {
  557. if (cpufreq_mode == POWER_CPUFREQ_MIN)
  558. r = min_ratio;
  559. else if (cpufreq_mode == POWER_CPUFREQ_MAX)
  560. r = max_ratio;
  561. else
  562. r = fixed_ratio;
  563. power_apply(p, r);
  564. }
  565. /* ---- once-a-second clock statistics ---- */
  566. p->stat_div += ms;
  567. if (p->stat_div < 1000)
  568. return;
  569. p->stat_div = 0;
  570. if (platform == PLAT_AIRMONT) {
  571. unsigned long long aperf, mperf;
  572. unsigned int da, dm;
  573. if (!has_aperf) {
  574. /* The CLOCK_MODULATION/APERF capability is absent;
  575. * report the PERF_STATUS ratio only. */
  576. rdmsr(MSR_IA32_PERF_STATUS, &da, &dm);
  577. p->status = da & 0xFFFFu;
  578. r = (p->status >> 8) & 0xFFu;
  579. if (r >= min_ratio && r <= max_ratio)
  580. p->ratio = r;
  581. else
  582. p->ratio = p->req;
  583. return;
  584. }
  585. aperf = rdmsr64(MSR_APERF);
  586. mperf = rdmsr64(MSR_MPERF);
  587. if (!p->have_last) {
  588. p->aperf = aperf;
  589. p->mperf = mperf;
  590. p->have_last = 1;
  591. return;
  592. }
  593. da = (unsigned int)((aperf - p->aperf) >> 12);
  594. dm = (unsigned int)((mperf - p->mperf) >> 12);
  595. p->aperf = aperf;
  596. p->mperf = mperf;
  597. p->mhz = dm ? da * nominal_mhz / dm : 0;
  598. rdmsr(MSR_IA32_PERF_STATUS, &da, &dm);
  599. p->status = da & 0xFFFFu;
  600. r = (p->status >> 8) & 0xFFu;
  601. if (r >= min_ratio && r <= max_ratio)
  602. p->ratio = r;
  603. else
  604. p->ratio = p->req;
  605. return;
  606. }
  607. {
  608. unsigned long long aperf = rdmsr64(MSR_APERF);
  609. unsigned long long mperf = rdmsr64(MSR_MPERF);
  610. unsigned long long tsc = rdmsr64(MSR_TSC);
  611. unsigned long long c3 = rdmsr64(MSR_CORE_C3_RESIDENCY);
  612. unsigned long long c6 = rdmsr64(MSR_CORE_C6_RESIDENCY);
  613. unsigned int dtsc, d3, d6, da, dm;
  614. if (!p->have_last) {
  615. p->aperf = aperf;
  616. p->mperf = mperf;
  617. p->c3 = c3;
  618. p->c6 = c6;
  619. p->tsc = tsc;
  620. p->have_last = 1;
  621. return;
  622. }
  623. dtsc = (unsigned int)((tsc - p->tsc) >> 12);
  624. d3 = (unsigned int)((c3 - p->c3) >> 12);
  625. d6 = (unsigned int)((c6 - p->c6) >> 12);
  626. da = (unsigned int)((aperf - p->aperf) >> 12);
  627. dm = (unsigned int)((mperf - p->mperf) >> 12);
  628. p->aperf = aperf;
  629. p->mperf = mperf;
  630. p->c3 = c3;
  631. p->c6 = c6;
  632. p->tsc = tsc;
  633. p->pc3 = dtsc ? d3 * 1000u / dtsc : 0;
  634. p->pc6 = dtsc ? d6 * 1000u / dtsc : 0;
  635. if (p->pc3 > 1000)
  636. p->pc3 = 1000;
  637. if (p->pc6 > 1000)
  638. p->pc6 = 1000;
  639. p->mhz = dm ? da * nominal_mhz / dm : 0;
  640. /* IA32_PERF_STATUS reports the current ratio in bits 7:0
  641. * (the boot dump shows 0x12 = 18 at the max non-turbo ratio),
  642. * unlike PERF_CTL, which takes the target in bits 15:8. */
  643. rdmsr(MSR_IA32_PERF_STATUS, &da, &dm);
  644. p->status = da & 0xFFFFu;
  645. p->ratio = p->status & 0xFFu;
  646. if (c->id == 0) {
  647. unsigned long long pc3 = rdmsr64(MSR_PKG_C3_RESIDENCY);
  648. unsigned long long pc6 = rdmsr64(MSR_PKG_C6_RESIDENCY);
  649. if (pkg_have_last) {
  650. pkg_pc3 = dtsc ?
  651. (unsigned int)((pc3 - pkg_c3_last) >> 12) *
  652. 1000u / dtsc : 0;
  653. pkg_pc6 = dtsc ?
  654. (unsigned int)((pc6 - pkg_c6_last) >> 12) *
  655. 1000u / dtsc : 0;
  656. if (pkg_pc3 > 1000)
  657. pkg_pc3 = 1000;
  658. if (pkg_pc6 > 1000)
  659. pkg_pc6 = 1000;
  660. }
  661. pkg_c3_last = pc3;
  662. pkg_c6_last = pc6;
  663. pkg_have_last = 1;
  664. }
  665. }
  666. }
  667. const char *power_cpufreq_name(void)
  668. {
  669. switch (cpufreq_mode) {
  670. case POWER_CPUFREQ_MIN:
  671. return "min";
  672. case POWER_CPUFREQ_MAX:
  673. return "max";
  674. case POWER_CPUFREQ_OFF:
  675. return "off";
  676. case POWER_CPUFREQ_FIXED:
  677. return "fixed";
  678. default:
  679. return "auto";
  680. }
  681. }
  682. static void put_dec(unsigned int v)
  683. {
  684. char buf[12];
  685. unsigned int n = 0;
  686. if (!v) {
  687. io_putchar('0');
  688. return;
  689. }
  690. while (v && n < sizeof(buf)) {
  691. buf[n++] = (char)('0' + v % 10);
  692. v /= 10;
  693. }
  694. while (n)
  695. io_putchar(buf[--n]);
  696. }
  697. static void put_pct(unsigned int permille)
  698. {
  699. put_dec(permille / 10);
  700. io_putchar('.');
  701. io_putchar((char)('0' + permille % 10));
  702. io_putchar('%');
  703. }
  704. void power_print(void)
  705. {
  706. unsigned int i;
  707. unsigned int lo, hi;
  708. if (!supported) {
  709. io_putstr("power: not available on this CPU\n");
  710. return;
  711. }
  712. io_putstr("power: ");
  713. io_putstr(platform == PLAT_AIRMONT ? "airmont/braswell" :
  714. "nehalem/westmere");
  715. if (!pstate_writes)
  716. io_putstr(" (probe only)");
  717. io_putstr(", idle ");
  718. io_putstr(idle_mode_name(idle_mode()));
  719. if (platform == PLAT_AIRMONT)
  720. io_putstr((c6n_enabled && idle_mwait_supported()) ?
  721. " (mwait c6n)" : " (hlt)");
  722. io_putstr(", cpufreq ");
  723. if (!pstate_writes)
  724. io_putstr("probe");
  725. else {
  726. io_putstr(power_cpufreq_name());
  727. if (cpufreq_mode == POWER_CPUFREQ_FIXED) {
  728. io_putchar(' ');
  729. put_dec(fixed_ratio);
  730. }
  731. }
  732. io_putstr(" (pstates ");
  733. put_dec(min_ratio);
  734. io_putstr("..");
  735. put_dec(max_ratio);
  736. io_putstr(")\n");
  737. if (platform == PLAT_AIRMONT) {
  738. io_putstr("eist ");
  739. io_putstr(eist_on ? "on" : "off");
  740. io_putstr(", turbo ");
  741. io_putstr(turbo_disabled ? "disabled" : "enabled");
  742. io_putstr(", arat ");
  743. io_putstr(has_arat ? "yes" : "no");
  744. io_putstr(", c6n ");
  745. if (!c6n_ok)
  746. io_putstr("no");
  747. else if (c6n_enabled)
  748. io_putstr("yes");
  749. else
  750. io_putstr("yes (disabled)");
  751. io_putstr("\nfsb ");
  752. put_dec(fsb_khz);
  753. io_putstr(" kHz, nominal ");
  754. put_dec(nominal_mhz);
  755. io_putstr(" MHz, turbo p");
  756. put_dec(turbo_ratio);
  757. io_putstr(" vid 0x");
  758. io_puthex(turbo_vid);
  759. io_putstr(", vid 0x");
  760. io_puthex(atom_vid_min);
  761. io_putstr("..0x");
  762. io_puthex(atom_vid_max);
  763. io_putstr(atom_vid_valid ? "" : " (invalid)");
  764. rdmsr(MSR_IA32_PERF_CTL, &lo, &hi);
  765. io_putstr(", perf ctl 0x");
  766. io_puthex(lo);
  767. io_putchar('\n');
  768. for (i = 0; i < cpu_count && i < MAX_CPUS; i++) {
  769. struct power_cpu *p = &pcs[i];
  770. io_putstr("cpu");
  771. put_dec(i);
  772. io_putchar(' ');
  773. put_dec(p->mhz);
  774. io_putstr(" MHz cur r");
  775. if (p->ratio >= min_ratio && p->ratio <= max_ratio) {
  776. if (p->ratio < 10)
  777. io_putchar('0');
  778. put_dec(p->ratio);
  779. } else {
  780. io_putstr("--");
  781. }
  782. io_putstr(" req r");
  783. if (p->req < 10)
  784. io_putchar('0');
  785. put_dec(p->req);
  786. io_putchar('\n');
  787. }
  788. return;
  789. }
  790. io_putstr("cst ");
  791. put_dec(cst_limit);
  792. io_putstr(cst_locked ? " locked" : " unlocked");
  793. io_putstr(", c1e ");
  794. io_putstr(c1e_on ? "on" : "off");
  795. io_putstr(", arat ");
  796. io_putstr(has_arat ? "yes\n" : "no\n");
  797. io_putstr("pkg c3 ");
  798. put_pct(pkg_pc3);
  799. io_putstr(" c6 ");
  800. put_pct(pkg_pc6);
  801. io_putchar('\n');
  802. for (i = 0; i < cpu_count && i < MAX_CPUS; i++) {
  803. struct power_cpu *p = &pcs[i];
  804. io_putstr("cpu");
  805. put_dec(i);
  806. io_putchar(' ');
  807. put_dec(p->mhz);
  808. io_putstr(" MHz cur r");
  809. if (p->ratio >= min_ratio && p->ratio <= max_ratio + 8) {
  810. if (p->ratio < 10)
  811. io_putchar('0');
  812. put_dec(p->ratio);
  813. } else {
  814. io_putstr("--");
  815. }
  816. io_putstr(" req r");
  817. if (p->req < 10)
  818. io_putchar('0');
  819. put_dec(p->req);
  820. io_putstr(" c3 ");
  821. put_pct(p->pc3);
  822. io_putstr(" c6 ");
  823. put_pct(p->pc6);
  824. io_putchar('\n');
  825. }
  826. }
  827. int power_cpufreq_command(const char *arg)
  828. {
  829. unsigned int ratio = 0;
  830. int have_ratio = 0;
  831. while (*arg == ' ')
  832. arg++;
  833. if (!supported) {
  834. io_putstr("cpufreq: not available on this CPU\n");
  835. return -1;
  836. }
  837. if (!pstate_writes) {
  838. if (!*arg) {
  839. io_putstr("cpufreq probe (P-state writes disabled)\n");
  840. return 0;
  841. }
  842. io_putstr("cpufreq: P-state writes are disabled in this probe build\n");
  843. return -1;
  844. }
  845. if (!*arg) {
  846. io_putstr("cpufreq ");
  847. io_putstr(power_cpufreq_name());
  848. if (cpufreq_mode == POWER_CPUFREQ_FIXED) {
  849. io_putchar(' ');
  850. put_dec(fixed_ratio);
  851. }
  852. io_putchar('\n');
  853. return 0;
  854. }
  855. if (arg[0] == 'a' && arg[1] == 'u' && arg[2] == 't' &&
  856. arg[3] == 'o' && !arg[4]) {
  857. cpufreq_mode = POWER_CPUFREQ_AUTO;
  858. } else if (arg[0] == 'm' && arg[1] == 'i' && arg[2] == 'n' &&
  859. !arg[3]) {
  860. cpufreq_mode = POWER_CPUFREQ_MIN;
  861. } else if (arg[0] == 'm' && arg[1] == 'a' && arg[2] == 'x' &&
  862. !arg[3]) {
  863. cpufreq_mode = POWER_CPUFREQ_MAX;
  864. } else if (arg[0] == 'o' && arg[1] == 'f' && arg[2] == 'f' &&
  865. !arg[3]) {
  866. cpufreq_mode = POWER_CPUFREQ_OFF;
  867. } else {
  868. while (*arg >= '0' && *arg <= '9') {
  869. ratio = ratio * 10u + (unsigned int)(*arg - '0');
  870. arg++;
  871. have_ratio = 1;
  872. }
  873. if (!have_ratio || *arg || ratio < min_ratio ||
  874. ratio > max_ratio) {
  875. io_putstr("cpufreq: usage cpufreq [auto|min|max|off|<ratio>]\n");
  876. return -1;
  877. }
  878. fixed_ratio = ratio;
  879. cpufreq_mode = POWER_CPUFREQ_FIXED;
  880. }
  881. io_putstr("cpufreq ");
  882. io_putstr(power_cpufreq_name());
  883. if (cpufreq_mode == POWER_CPUFREQ_FIXED) {
  884. io_putchar(' ');
  885. put_dec(fixed_ratio);
  886. }
  887. io_putchar('\n');
  888. return 0;
  889. }