ahci.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507
  1. /* AHCI (SATA) driver: PCI discovery, port bring-up, IDENTIFY and DMA
  2. * read/write (READ/WRITE DMA EXT, LBA48), all polled (no IRQs).
  3. *
  4. * Target: the ION/AT3IONT-I MCP79 SATA controller in AHCI mode (the same
  5. * hardware family Linux drives with board_ahci_noncq), and QEMU's ich9-ahci
  6. * for testing. Only non-NCQ commands are used.
  7. *
  8. * All buffers (command list, FIS receive area, command table, DMA bounce
  9. * buffer) come from the page allocator -- the kernel lives under a tight
  10. * linker bound, so nothing large may go into BSS. */
  11. #include "ahci.h"
  12. #include "ata.h"
  13. #include "mm.h"
  14. #include "paging.h"
  15. #include "pci.h"
  16. #include "process.h"
  17. #include "serial.h"
  18. #include "timer.h"
  19. #define AHCI_DEBUG 0
  20. /* ---- generic host control registers (ABAR-relative) ---- */
  21. #define H_CAP 0x00
  22. #define H_GHC 0x04
  23. #define H_PI 0x0C
  24. #define GHC_AE 0x80000000u /* AHCI enable */
  25. /* ---- port registers (0x100 + 0x80 * port) ---- */
  26. #define P_CLB 0x00
  27. #define P_CLBU 0x04
  28. #define P_FB 0x08
  29. #define P_FBU 0x0C
  30. #define P_IS 0x10
  31. #define P_IE 0x14
  32. #define P_CMD 0x18
  33. #define P_TFD 0x20
  34. #define P_SIG 0x24
  35. #define P_SSTS 0x28
  36. #define P_SCTL 0x2C
  37. #define P_SERR 0x30
  38. #define P_CI 0x38
  39. #define CMD_ST 0x00000001u /* start */
  40. #define CMD_FRE 0x00000010u /* FIS receive enable */
  41. #define CMD_FR 0x00004000u /* FIS receive running */
  42. #define CMD_CR 0x00008000u /* command list running */
  43. #define TFD_ERR 0x01
  44. #define TFD_BSY 0x80
  45. #define TFD_DRQ 0x08
  46. #define TFD_ERR_BITS (TFD_ERR | TFD_BSY | TFD_DRQ)
  47. #define IS_TFES 0x40000000u /* task file error status */
  48. #define SIG_ATA 0x00000101u
  49. /* Register Host-to-Device FIS. */
  50. #define FIS_H2D 0x27
  51. #define FIS_C 0x80
  52. #define ATA_IDENTIFY 0xEC
  53. #define ATA_READ_DMA_EXT 0x25
  54. #define ATA_WRITE_DMA_EXT 0x35
  55. #define AHCI_CHUNK 255 /* sectors per command (128 KB-ish) */
  56. #define BOUNCE_PAGES 33 /* 255 * 512 = 130560 B < 33 * 4096 */
  57. static struct {
  58. int inited;
  59. int ok;
  60. unsigned int abar;
  61. unsigned int nports;
  62. unsigned int pi;
  63. unsigned int cl; /* command list (1 page) */
  64. unsigned int fis; /* FIS receive area (1 page) */
  65. unsigned int ct; /* command table (1 page) */
  66. unsigned int bounce; /* data bounce buffer (BOUNCE_PAGES) */
  67. } ahci;
  68. static unsigned int rd(unsigned int off)
  69. {
  70. return *(volatile unsigned int *)(ahci.abar + off);
  71. }
  72. static void wr(unsigned int off, unsigned int v)
  73. {
  74. *(volatile unsigned int *)(ahci.abar + off) = v;
  75. }
  76. static unsigned int prd(unsigned int port, unsigned int off)
  77. {
  78. return *(volatile unsigned int *)(ahci.abar + 0x100 + 0x80 * port + off);
  79. }
  80. static void pwr(unsigned int port, unsigned int off, unsigned int v)
  81. {
  82. *(volatile unsigned int *)(ahci.abar + 0x100 + 0x80 * port + off) = v;
  83. }
  84. static void mem_copy(unsigned char *dst, const unsigned char *src, unsigned int n)
  85. {
  86. while (n--)
  87. *dst++ = *src++;
  88. }
  89. static void delay_ms(unsigned int ms)
  90. {
  91. unsigned int t0 = get_ticks_ms();
  92. while (get_ticks_ms() - t0 < ms)
  93. ;
  94. }
  95. /* ---- PCI discovery ---- */
  96. static int ahci_find(unsigned int *out_bus, unsigned int *out_dev, unsigned int *out_func)
  97. {
  98. unsigned int bus, dev, func;
  99. for (bus = 0; bus < 256; bus++) {
  100. for (dev = 0; dev < 32; dev++) {
  101. unsigned int id = pci_cfg_read32(bus, dev, 0, 0x00);
  102. unsigned int htype, nfunc;
  103. if (id == 0xFFFFFFFFu || id == 0)
  104. continue;
  105. htype = (pci_cfg_read32(bus, dev, 0, 0x0C) >> 16) & 0xFFu;
  106. nfunc = (htype & 0x80u) ? 8u : 1u;
  107. for (func = 0; func < nfunc; func++) {
  108. unsigned int idf = pci_cfg_read32(bus, dev, func, 0x00);
  109. unsigned int crev, cls, sub, prog;
  110. if (idf == 0xFFFFFFFFu || idf == 0)
  111. continue;
  112. crev = pci_cfg_read32(bus, dev, func, 0x08);
  113. cls = (crev >> 24) & 0xFFu;
  114. sub = (crev >> 16) & 0xFFu;
  115. prog = (crev >> 8) & 0xFFu;
  116. if (cls != 0x01)
  117. continue;
  118. /* One line per storage function so a board whose
  119. * SATA mode does not match can be diagnosed from
  120. * the serial/netlog output (class CCDD, prog-if,
  121. * vendor:device). */
  122. serial_puts(SERIAL_COM1, "ahci: storage fn ");
  123. serial_putdec(SERIAL_COM1, bus);
  124. serial_putc(SERIAL_COM1, ':');
  125. serial_putdec(SERIAL_COM1, dev);
  126. serial_putc(SERIAL_COM1, '.');
  127. serial_putdec(SERIAL_COM1, func);
  128. serial_puts(SERIAL_COM1, " id ");
  129. serial_puthex(SERIAL_COM1, idf);
  130. serial_puts(SERIAL_COM1, " class ");
  131. serial_puthex(SERIAL_COM1, (cls << 8) | sub);
  132. serial_puts(SERIAL_COM1, " prog ");
  133. serial_puthex(SERIAL_COM1, prog);
  134. serial_putc(SERIAL_COM1, '\n');
  135. if (sub == 0x06 && prog == 0x01) {
  136. *out_bus = bus;
  137. *out_dev = dev;
  138. *out_func = func;
  139. return 0;
  140. }
  141. }
  142. }
  143. }
  144. return -1;
  145. }
  146. /* ---- port bring-up ---- */
  147. static void ahci_port_init(unsigned int port)
  148. {
  149. unsigned int t0, c;
  150. /* Stop the port and its FIS receive engine. */
  151. c = prd(port, P_CMD);
  152. if (c & CMD_ST) {
  153. pwr(port, P_CMD, c & ~CMD_ST);
  154. t0 = get_ticks_ms();
  155. while ((prd(port, P_CMD) & CMD_CR) && get_ticks_ms() - t0 < 500)
  156. ;
  157. }
  158. c = prd(port, P_CMD);
  159. if (c & CMD_FRE) {
  160. pwr(port, P_CMD, c & ~CMD_FRE);
  161. t0 = get_ticks_ms();
  162. while ((prd(port, P_CMD) & CMD_FR) && get_ticks_ms() - t0 < 500)
  163. ;
  164. }
  165. pwr(port, P_CLB, ahci.cl);
  166. pwr(port, P_CLBU, 0);
  167. pwr(port, P_FB, ahci.fis);
  168. pwr(port, P_FBU, 0);
  169. pwr(port, P_SERR, 0xFFFFFFFFu);
  170. pwr(port, P_IS, 0xFFFFFFFFu);
  171. pwr(port, P_IE, 0);
  172. pwr(port, P_CMD, prd(port, P_CMD) | CMD_FRE);
  173. t0 = get_ticks_ms();
  174. while (!(prd(port, P_CMD) & CMD_FR) && get_ticks_ms() - t0 < 500)
  175. ;
  176. /* Cold plug: reset the link if it is not established yet. */
  177. if ((prd(port, P_SSTS) & 0x0Fu) != 3) {
  178. pwr(port, P_SCTL, 1); /* DET = 1: initiate COMRESET */
  179. t0 = get_ticks_ms();
  180. while ((prd(port, P_SSTS) & 0x0Fu) != 3 && get_ticks_ms() - t0 < 1000)
  181. ;
  182. }
  183. pwr(port, P_CMD, prd(port, P_CMD) | CMD_ST);
  184. t0 = get_ticks_ms();
  185. while (!(prd(port, P_CMD) & CMD_CR) && get_ticks_ms() - t0 < 500)
  186. ;
  187. }
  188. int ahci_init(void)
  189. {
  190. unsigned int bus, dev, func, bar, cmd, cap, i;
  191. if (ahci.inited)
  192. return ahci.ok ? 0 : -1;
  193. ahci.inited = 1;
  194. if (ahci_find(&bus, &dev, &func) != 0) {
  195. serial_puts(SERIAL_COM1, "ahci: no matching controller (need 0106:01)\n");
  196. return -1;
  197. }
  198. bar = pci_cfg_read32(bus, dev, func, 0x24);
  199. if (bar & 1u) {
  200. serial_puts(SERIAL_COM1, "ahci: BAR5 is not MMIO\n");
  201. return -1;
  202. }
  203. bar &= 0xFFFFFFF0u;
  204. if (!bar || bar == 0xFFFFFFF0u) {
  205. serial_puts(SERIAL_COM1, "ahci: BAR5 is unassigned\n");
  206. return -1;
  207. }
  208. cmd = pci_cfg_read32(bus, dev, func, 0x04);
  209. pci_cfg_write32(bus, dev, func, 0x04, cmd | 0x0006u); /* mem + bus master */
  210. process_map_mmio_all(bar, 0x2000);
  211. ahci.abar = bar;
  212. cap = rd(H_CAP);
  213. if (cap == 0 || cap == 0xFFFFFFFFu) {
  214. serial_puts(SERIAL_COM1, "ahci: no register response\n");
  215. return -1;
  216. }
  217. ahci.cl = mm_alloc_page();
  218. ahci.fis = mm_alloc_page();
  219. ahci.ct = mm_alloc_page();
  220. ahci.bounce = mm_alloc_pages(BOUNCE_PAGES);
  221. if (!ahci.cl || !ahci.fis || !ahci.ct || !ahci.bounce) {
  222. serial_puts(SERIAL_COM1, "ahci: no memory for DMA buffers\n");
  223. return -1;
  224. }
  225. {
  226. unsigned int *p;
  227. unsigned int n;
  228. for (p = (unsigned int *)ahci.cl, n = 0; n < 0x400; n++) p[n] = 0;
  229. for (p = (unsigned int *)ahci.fis, n = 0; n < 0x400; n++) p[n] = 0;
  230. for (p = (unsigned int *)ahci.ct, n = 0; n < 0x400; n++) p[n] = 0;
  231. for (p = (unsigned int *)ahci.bounce, n = 0; n < BOUNCE_PAGES * 0x400; n++)
  232. p[n] = 0;
  233. }
  234. wr(H_GHC, rd(H_GHC) | GHC_AE);
  235. ahci.nports = (cap & 0x1Fu) + 1;
  236. if (ahci.nports > 32)
  237. ahci.nports = 32;
  238. ahci.pi = rd(H_PI);
  239. serial_puts(SERIAL_COM1, "ahci: ctrl ");
  240. serial_putdec(SERIAL_COM1, bus);
  241. serial_putc(SERIAL_COM1, ':');
  242. serial_putdec(SERIAL_COM1, dev);
  243. serial_putc(SERIAL_COM1, '.');
  244. serial_putdec(SERIAL_COM1, func);
  245. serial_puts(SERIAL_COM1, " bar=");
  246. serial_puthex(SERIAL_COM1, bar);
  247. serial_puts(SERIAL_COM1, " pi=");
  248. serial_puthex(SERIAL_COM1, ahci.pi);
  249. serial_putc(SERIAL_COM1, '\n');
  250. for (i = 0; i < ahci.nports; i++) {
  251. if (ahci.pi & (1u << i))
  252. ahci_port_init(i);
  253. }
  254. ahci.ok = 1;
  255. return 0;
  256. }
  257. int ahci_controller_present(void)
  258. {
  259. ahci_init();
  260. return ahci.ok;
  261. }
  262. /* ---- command issue (slot 0, polled) ---- */
  263. static int ahci_issue(unsigned int port, const unsigned char fis[20], int write,
  264. unsigned int dma_addr, unsigned int bytes)
  265. {
  266. unsigned int *hdr = (unsigned int *)ahci.cl;
  267. unsigned char *ct = (unsigned char *)ahci.ct;
  268. unsigned int t0, i, prdtl = bytes ? 1u : 0u;
  269. /* Wait for the port to be idle (previous command finished). */
  270. t0 = get_ticks_ms();
  271. while (prd(port, P_CI) & 1u) {
  272. if (get_ticks_ms() - t0 > 4000)
  273. return -1;
  274. }
  275. if (prd(port, P_TFD) & TFD_BSY)
  276. return -1;
  277. for (i = 0; i < 64; i++)
  278. ct[i] = 0;
  279. for (i = 0; i < 20; i++)
  280. ct[i] = fis[i];
  281. /* PRDT entry 0: data buffer + byte count (no data: PRDTL = 0). */
  282. if (prdtl) {
  283. ((unsigned int *)(ct + 0x80))[0] = dma_addr;
  284. ((unsigned int *)(ct + 0x80))[1] = 0;
  285. ((unsigned int *)(ct + 0x80))[2] = 0;
  286. ((unsigned int *)(ct + 0x80))[3] = ((bytes - 1) & 0x3FFFFFu) | 0x80000000u;
  287. }
  288. /* Command header: CFL=5 dwords, W (write), PRDTL. */
  289. hdr[0] = 5u | (write ? (1u << 6) : 0) | (prdtl << 16);
  290. hdr[1] = 0;
  291. hdr[2] = ahci.ct;
  292. hdr[3] = 0;
  293. pwr(port, P_IS, 0xFFFFFFFFu);
  294. pwr(port, P_CI, 1u);
  295. t0 = get_ticks_ms();
  296. while (prd(port, P_CI) & 1u) {
  297. if (get_ticks_ms() - t0 > 4000) {
  298. #if AHCI_DEBUG
  299. serial_puts(SERIAL_COM1, "ahci: command timeout\n");
  300. #endif
  301. return -1;
  302. }
  303. }
  304. if (prd(port, P_IS) & IS_TFES) {
  305. #if AHCI_DEBUG
  306. serial_puts(SERIAL_COM1, "ahci: task file error, tfd=");
  307. serial_puthex(SERIAL_COM1, prd(port, P_TFD));
  308. serial_putc(SERIAL_COM1, '\n');
  309. #endif
  310. return -1;
  311. }
  312. if (prd(port, P_TFD) & TFD_ERR_BITS)
  313. return -1;
  314. return 0;
  315. }
  316. static int ahci_identify_port(unsigned int port, unsigned int *sectors, char model[41])
  317. {
  318. unsigned char fis[20];
  319. unsigned int attempt, i;
  320. int r = -1;
  321. for (i = 0; i < 20; i++)
  322. fis[i] = 0;
  323. fis[0] = FIS_H2D;
  324. fis[1] = FIS_C;
  325. fis[2] = ATA_IDENTIFY;
  326. fis[7] = 0xA0; /* device 0 */
  327. for (attempt = 0; attempt < 3; attempt++) {
  328. r = ahci_issue(port, fis, 0, ahci.bounce, 512);
  329. if (r == 0)
  330. break;
  331. delay_ms(20);
  332. }
  333. if (r != 0)
  334. return -1;
  335. return ata_identify_parse((const unsigned short *)ahci.bounce, sectors, model);
  336. }
  337. static int ahci_xfer(unsigned int port, unsigned int lba, void *buf,
  338. unsigned int count, int write)
  339. {
  340. unsigned char *p = buf;
  341. unsigned char *bounce = (unsigned char *)ahci.bounce;
  342. unsigned int done = 0;
  343. if (!ahci.ok || count == 0)
  344. return -1;
  345. while (done < count) {
  346. unsigned int chunk = count - done;
  347. unsigned int cur = lba + done;
  348. unsigned char fis[20];
  349. unsigned int i;
  350. if (chunk > AHCI_CHUNK)
  351. chunk = AHCI_CHUNK;
  352. for (i = 0; i < 20; i++)
  353. fis[i] = 0;
  354. fis[0] = FIS_H2D;
  355. fis[1] = FIS_C;
  356. fis[2] = write ? ATA_WRITE_DMA_EXT : ATA_READ_DMA_EXT;
  357. fis[4] = (unsigned char)(cur & 0xFF);
  358. fis[5] = (unsigned char)((cur >> 8) & 0xFF);
  359. fis[6] = (unsigned char)((cur >> 16) & 0xFF);
  360. fis[7] = 0x40; /* LBA mode */
  361. fis[8] = (unsigned char)((cur >> 24) & 0xFF);
  362. fis[9] = 0; /* LBA4/LBA5: 32-bit LBA only */
  363. fis[10] = 0;
  364. fis[12] = (unsigned char)(chunk & 0xFF);
  365. fis[13] = (unsigned char)((chunk >> 8) & 0xFF);
  366. if (write)
  367. mem_copy(bounce, p + done * 512u, chunk * 512u);
  368. if (ahci_issue(port, fis, write, ahci.bounce, chunk * 512u) != 0) {
  369. serial_puts(SERIAL_COM1, "ahci: ");
  370. serial_puts(SERIAL_COM1, write ? "write" : "read");
  371. serial_puts(SERIAL_COM1, " failed port ");
  372. serial_putdec(SERIAL_COM1, port);
  373. serial_puts(SERIAL_COM1, " lba ");
  374. serial_putdec(SERIAL_COM1, cur);
  375. serial_puts(SERIAL_COM1, " count ");
  376. serial_putdec(SERIAL_COM1, chunk);
  377. serial_puts(SERIAL_COM1, " tfd ");
  378. serial_puthex(SERIAL_COM1, prd(port, P_TFD));
  379. serial_puts(SERIAL_COM1, " is ");
  380. serial_puthex(SERIAL_COM1, prd(port, P_IS));
  381. serial_putc(SERIAL_COM1, '\n');
  382. return -1;
  383. }
  384. if (!write)
  385. mem_copy(p + done * 512u, bounce, chunk * 512u);
  386. done += chunk;
  387. }
  388. return 0;
  389. }
  390. int ahci_read_sectors(unsigned int port, unsigned int lba, void *buf, unsigned int count)
  391. {
  392. return ahci_xfer(port, lba, buf, count, 0);
  393. }
  394. int ahci_write_sectors(unsigned int port, unsigned int lba, const void *buf,
  395. unsigned int count)
  396. {
  397. return ahci_xfer(port, lba, (void *)buf, count, 1);
  398. }
  399. int ahci_flush(unsigned int port)
  400. {
  401. unsigned char fis[20];
  402. unsigned int i;
  403. if (!ahci.ok || port >= ahci.nports)
  404. return -1;
  405. for (i = 0; i < 20; i++)
  406. fis[i] = 0;
  407. fis[0] = FIS_H2D;
  408. fis[1] = FIS_C;
  409. fis[2] = 0xEA; /* FLUSH CACHE EXT */
  410. return ahci_issue(port, fis, 0, 0, 0);
  411. }
  412. unsigned int ahci_scan(struct ahci_disk *out, unsigned int max)
  413. {
  414. unsigned int port, n = 0;
  415. if (ahci_init() != 0)
  416. return 0;
  417. for (port = 0; port < ahci.nports && n < max; port++) {
  418. if (!(ahci.pi & (1u << port)))
  419. continue;
  420. if ((prd(port, P_SSTS) & 0x0Fu) != 3)
  421. continue;
  422. if (prd(port, P_SIG) != SIG_ATA)
  423. continue; /* ATAPI / port multiplier / SEMB */
  424. if (ahci_identify_port(port, &out[n].sectors, out[n].model) != 0)
  425. continue;
  426. out[n].port = port;
  427. serial_puts(SERIAL_COM1, "ahci: port ");
  428. serial_putdec(SERIAL_COM1, port);
  429. serial_puts(SERIAL_COM1, " ");
  430. serial_puts(SERIAL_COM1, out[n].model);
  431. serial_puts(SERIAL_COM1, " ");
  432. serial_putdec(SERIAL_COM1, out[n].sectors);
  433. serial_puts(SERIAL_COM1, " sectors\n");
  434. n++;
  435. }
  436. return n;
  437. }