ehci.c 36 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432
  1. /* EHCI (USB 2.0) host controller driver.
  2. *
  3. * Scope: high-speed devices on the root ports. The EHCI controller is paired
  4. * with UHCI/OHCI companions on most chipsets; low/full-speed devices are left
  5. * to those controllers (the hardware routes them back when a port reset shows
  6. * a non-high-speed device). usb.c enumerates every controller type, so a
  7. * high-speed USB Ethernet dongle on EHCI and a full-speed mouse/keyboard on a
  8. * companion both work.
  9. *
  10. * The schedule is polled (no IRQs), like the UHCI/OHCI drivers. All control,
  11. * bulk and interrupt endpoints live in the asynchronous schedule: a circular
  12. * QH list with the control QH after the reclamation head. Interrupt endpoints
  13. * are retried every microframe rather than through the periodic list; for a
  14. * handful of boot HID devices the extra NAK traffic is negligible and it keeps
  15. * the schedule simple.
  16. *
  17. * Descriptors (frame list, QHs, qTDs, bounce buffers) come from the page
  18. * allocator, so the driver does not grow the kernel BSS. */
  19. #include "usb.h"
  20. #include "ehci.h"
  21. #include "io.h"
  22. #include "mm.h"
  23. #include "paging.h"
  24. #include "pci.h"
  25. #include "serial.h"
  26. #include "timer.h"
  27. #define EHCI_DEBUG 0
  28. /* ---- capability / operational register offsets (bytes) ----------------- */
  29. #define EHCI_CAP_CAPLENGTH 0x00
  30. #define EHCI_CAP_HCSPARAMS 0x04
  31. #define EHCI_CAP_HCCPARAMS 0x08
  32. #define EHCI_OP_USBCMD 0x00
  33. #define EHCI_OP_USBSTS 0x04
  34. #define EHCI_OP_USBINTR 0x08
  35. #define EHCI_OP_FRINDEX 0x0C
  36. #define EHCI_OP_CTRLDSSEGMENT 0x10
  37. #define EHCI_OP_PERIODICLIST 0x14
  38. #define EHCI_OP_ASYNCLIST 0x18
  39. #define EHCI_OP_CONFIGFLAG 0x40
  40. #define EHCI_OP_PORTSC 0x44
  41. /* USBCMD */
  42. #define CMD_RUN 0x00000001u
  43. #define CMD_RESET 0x00000002u
  44. #define CMD_PSE 0x00000010u
  45. #define CMD_ASE 0x00000020u
  46. #define CMD_IAAD 0x00000040u
  47. #define CMD_ITC_8 0x00080000u
  48. /* USBSTS */
  49. #define STS_INT 0x00000001u
  50. #define STS_ERR 0x00000002u
  51. #define STS_PCD 0x00000004u
  52. #define STS_FLR 0x00000008u
  53. #define STS_FATAL 0x00000010u
  54. #define STS_IAA 0x00000020u
  55. #define STS_HALT 0x00001000u
  56. #define STS_RECL 0x00002000u
  57. #define STS_PSS 0x00004000u
  58. #define STS_ASS 0x00008000u
  59. /* PORTSC */
  60. #define PORT_CONNECT 0x00000001u
  61. #define PORT_CSC 0x00000002u
  62. #define PORT_PE 0x00000004u
  63. #define PORT_PEC 0x00000008u
  64. #define PORT_OC 0x00000010u
  65. #define PORT_OCC 0x00000020u
  66. #define PORT_RESUME 0x00000040u
  67. #define PORT_SUSPEND 0x00000080u
  68. #define PORT_RESET 0x00000100u
  69. #define PORT_LS_MASK 0x00000C00u
  70. #define PORT_POWER 0x00001000u
  71. #define PORT_OWNER 0x00002000u
  72. #define PORT_RWC (PORT_CSC | PORT_PEC | PORT_OCC)
  73. #define PORT_USB11(ps) (((ps) & PORT_LS_MASK) == (1u << 10))
  74. /* qTD token bits (EHCI 3.5.3). */
  75. #define QTD_ACTIVE 0x00000080u
  76. #define QTD_HALT 0x00000040u
  77. #define QTD_DBE 0x00000020u
  78. #define QTD_BABBLE 0x00000010u
  79. #define QTD_XACT 0x00000008u
  80. #define QTD_MMF 0x00000004u
  81. #define QTD_PING 0x00000001u
  82. #define QTD_ERRORS (QTD_HALT | QTD_DBE | QTD_BABBLE | \
  83. QTD_XACT | QTD_MMF)
  84. #define QTD_TOGGLE 0x80000000u
  85. #define QTD_PID_OUT 0u
  86. #define QTD_PID_IN 1u
  87. #define QTD_PID_SETUP 2u
  88. #define QTD_CERR_3 (3u << 10)
  89. #define QTD_LEN(tok) (((tok) >> 16) & 0x7FFFu)
  90. /* QH endpoint characteristics (EHCI 3.6). */
  91. #define QH_DEVADDR(a) ((a) & 0x7Fu)
  92. #define QH_ENDPT(e) (((e) & 0x0Fu) << 8)
  93. #define QH_EPS_HIGH (2u << 12)
  94. #define QH_DTC (1u << 14)
  95. #define QH_HEAD (1u << 15)
  96. #define QH_MPS(m) (((m) & 0x7FFu) << 16)
  97. #define QH_CTRL_EP (1u << 27)
  98. #define QH_RL(n) (((n) & 7u) << 28)
  99. /* QH endpoint capabilities. */
  100. #define QH_MULT(n) (((n) & 3u) << 30)
  101. /* Link pointers: bit 0 terminates, bits 2:1 select the structure type. */
  102. #define QH_LIST_END 1u
  103. #define QH_NEXT(a) (((a) & ~0x1Fu) | 0x2u)
  104. #define EHCI_MAX_QH (4 + USB_HCD_INT_SLOTS)
  105. #define EHCI_QH_STRIDE 128
  106. #define EHCI_QTD_STRIDE 64
  107. struct ehci_qh {
  108. volatile unsigned int next;
  109. volatile unsigned int info1;
  110. volatile unsigned int info2;
  111. volatile unsigned int current;
  112. volatile unsigned int qtd_next;
  113. volatile unsigned int alt_next;
  114. volatile unsigned int token;
  115. volatile unsigned int buf[5];
  116. volatile unsigned int buf_hi[5]; /* 64-bit addressing extension */
  117. } __attribute__((aligned(32)));
  118. struct ehci_qtd {
  119. volatile unsigned int next;
  120. volatile unsigned int alt_next;
  121. volatile unsigned int token;
  122. volatile unsigned int buf[5];
  123. volatile unsigned int buf_hi[5]; /* 64-bit addressing extension */
  124. } __attribute__((aligned(32)));
  125. struct ehci_ctrl;
  126. struct ehci_bulk {
  127. struct ehci_qh *qh;
  128. struct ehci_qtd *qtd;
  129. struct ehci_qtd *zqtd; /* zero-length packet, bulk OUT only */
  130. unsigned int addr;
  131. unsigned int ep;
  132. unsigned int mps;
  133. unsigned int buf_len;
  134. unsigned int toggle;
  135. int armed;
  136. int sched;
  137. struct ehci_ctrl *owner;
  138. };
  139. struct ehci_int {
  140. struct ehci_qh *qh;
  141. struct ehci_qtd *qtd;
  142. unsigned char *buf;
  143. unsigned int mps;
  144. unsigned int buf_len;
  145. unsigned int toggle;
  146. int active;
  147. struct ehci_ctrl *owner;
  148. };
  149. struct ehci_ctrl {
  150. unsigned int base; /* kernel mapping of the register BAR */
  151. unsigned int phys; /* the BAR itself (reported by lsusb) */
  152. unsigned int cap_len;
  153. unsigned int bus, dev, func;
  154. unsigned int n_ports;
  155. int ppc;
  156. int last_port;
  157. unsigned int cc_ports; /* ports handed to a companion (bitmask) */
  158. unsigned int hcs;
  159. unsigned int hcc;
  160. unsigned int frame_list;
  161. unsigned int desc_page;
  162. unsigned int data_page;
  163. struct ehci_qh *head;
  164. struct ehci_qh *ctl_qh;
  165. struct ehci_qh *bin_qh;
  166. struct ehci_qh *bout_qh;
  167. struct ehci_qh *int_qh[USB_HCD_INT_SLOTS];
  168. struct ehci_qtd *ctl_qtd[3];
  169. struct ehci_qtd *bin_qtd;
  170. struct ehci_qtd *bout_qtd;
  171. struct ehci_qtd *bout_zqtd;
  172. struct ehci_qtd *int_qtd[USB_HCD_INT_SLOTS];
  173. unsigned char *setup;
  174. unsigned char *rx_buf;
  175. unsigned char *int_buf[USB_HCD_INT_SLOTS];
  176. struct ehci_bulk bin;
  177. struct ehci_bulk bout;
  178. struct ehci_int ints[USB_HCD_INT_SLOTS];
  179. int ctl_busy;
  180. int recovering;
  181. int sched_on;
  182. int sched_refs;
  183. unsigned char addr_port[128];
  184. };
  185. static struct ehci_ctrl ctrls[USB_HCD_MAX_CTRL];
  186. static unsigned int n_ctrls;
  187. static struct ehci_int *slot_int[USB_HCD_INT_SLOTS];
  188. static int ehci_start(struct ehci_ctrl *c);
  189. /* ---- debug ------------------------------------------------------------- */
  190. #if EHCI_DEBUG
  191. static void dbg(const char *s)
  192. {
  193. serial_puts(SERIAL_COM1, s);
  194. }
  195. #else
  196. static void dbg(const char *s) { (void)s; }
  197. #endif
  198. static unsigned int rd32(unsigned int addr)
  199. {
  200. return *(volatile unsigned int *)addr;
  201. }
  202. static void wr32(unsigned int addr, unsigned int val)
  203. {
  204. *(volatile unsigned int *)addr = val;
  205. }
  206. static unsigned int op_reg(struct ehci_ctrl *c, unsigned int off)
  207. {
  208. return c->base + c->cap_len + off;
  209. }
  210. /* The asynchronous schedule is only enabled while a transfer or an armed
  211. * interrupt endpoint needs it. With it off the controller stops fetching
  212. * QHs and qTDs, which lets the package reach deep C-states while the
  213. * machine is idle; the port hotplug watcher only reads PORTSC and does not
  214. * need the schedule. */
  215. static void ehci_sched_set(struct ehci_ctrl *c, int on)
  216. {
  217. unsigned int v, t0;
  218. if (on == c->sched_on)
  219. return;
  220. v = rd32(op_reg(c, EHCI_OP_USBCMD));
  221. if (on) {
  222. wr32(op_reg(c, EHCI_OP_USBCMD), v | CMD_ASE);
  223. t0 = get_ticks_ms();
  224. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS)) {
  225. if (get_ticks_ms() - t0 > 10)
  226. break;
  227. }
  228. } else {
  229. /* Clear ASE and let the controller retire the current
  230. * microframe; a pending IAA status must be acknowledged or the
  231. * next enable can stall. */
  232. wr32(op_reg(c, EHCI_OP_USBCMD), v & ~CMD_ASE);
  233. t0 = get_ticks_ms();
  234. while (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS) {
  235. if (get_ticks_ms() - t0 > 10)
  236. break;
  237. }
  238. wr32(op_reg(c, EHCI_OP_USBSTS), STS_IAA);
  239. }
  240. c->sched_on = on;
  241. }
  242. static void ehci_sched_get(struct ehci_ctrl *c)
  243. {
  244. c->sched_refs++;
  245. ehci_sched_set(c, 1);
  246. }
  247. static void ehci_sched_put(struct ehci_ctrl *c)
  248. {
  249. if (c->sched_refs > 0)
  250. c->sched_refs--;
  251. if (!c->sched_refs)
  252. ehci_sched_set(c, 0);
  253. }
  254. static unsigned int port_reg(struct ehci_ctrl *c, unsigned int port)
  255. {
  256. return c->base + c->cap_len + EHCI_OP_PORTSC + port * 4u;
  257. }
  258. static void delay_ms(unsigned int ms)
  259. {
  260. usb_delay_ms(ms); /* sleeps inside a sleepable scan */
  261. }
  262. static void zero_region(unsigned int addr, unsigned int len)
  263. {
  264. unsigned int *p = (unsigned int *)addr;
  265. while (len >= 4) {
  266. *p++ = 0;
  267. len -= 4;
  268. }
  269. }
  270. static void zero_ctrl(struct ehci_ctrl *c)
  271. {
  272. unsigned char *p = (unsigned char *)c;
  273. unsigned int i;
  274. for (i = 0; i < sizeof(*c); i++)
  275. p[i] = 0;
  276. }
  277. /* ---- descriptor helpers ------------------------------------------------ */
  278. static void qh_init(struct ehci_qh *qh)
  279. {
  280. unsigned int i;
  281. qh->info1 = 0;
  282. qh->info2 = 0;
  283. qh->current = 0;
  284. qh->qtd_next = QH_LIST_END;
  285. qh->alt_next = QH_LIST_END;
  286. qh->token = QTD_HALT;
  287. for (i = 0; i < 5; i++) {
  288. qh->buf[i] = 0;
  289. qh->buf_hi[i] = 0;
  290. }
  291. }
  292. static void qh_configure(struct ehci_qh *qh, unsigned int addr,
  293. unsigned int ep, unsigned int mps, int control)
  294. {
  295. qh->info1 = QH_DEVADDR(addr) | QH_ENDPT(ep) | QH_EPS_HIGH |
  296. QH_MPS(mps) | QH_RL(4);
  297. if (control)
  298. qh->info1 |= QH_CTRL_EP | QH_DTC;
  299. qh->info2 = QH_MULT(1);
  300. }
  301. static void qh_arm(struct ehci_qh *qh, struct ehci_qtd *qtd)
  302. {
  303. __asm__ volatile ("" ::: "memory");
  304. qh->qtd_next = (unsigned int)qtd;
  305. qh->alt_next = QH_LIST_END;
  306. qh->token &= (QTD_TOGGLE | QTD_PING);
  307. __asm__ volatile ("" ::: "memory");
  308. }
  309. static void qh_disarm(struct ehci_qh *qh)
  310. {
  311. qh->qtd_next = QH_LIST_END;
  312. qh->token = (qh->token & (QTD_TOGGLE | QTD_PING)) | QTD_HALT;
  313. __asm__ volatile ("" ::: "memory");
  314. }
  315. /* Fill one qTD. Up to five 4 KB pages are described (all current callers
  316. are well below that); the C_PAGE field starts at zero. */
  317. static void qtd_fill(struct ehci_qtd *qtd, const void *buf, unsigned int len,
  318. unsigned int pid, int toggle)
  319. {
  320. unsigned int addr = (unsigned int)buf;
  321. unsigned int left, chunk, i;
  322. qtd->next = QH_LIST_END;
  323. qtd->alt_next = QH_LIST_END;
  324. for (i = 0; i < 5; i++) {
  325. qtd->buf[i] = 0;
  326. qtd->buf_hi[i] = 0;
  327. }
  328. qtd->token = ((len & 0x7FFFu) << 16) |
  329. ((pid & 3u) << 8) | QTD_CERR_3 | QTD_ACTIVE |
  330. (toggle ? QTD_TOGGLE : 0);
  331. if (!len)
  332. return;
  333. qtd->buf[0] = addr;
  334. left = len;
  335. chunk = 0x1000u - (addr & 0xFFFu);
  336. if (chunk > left)
  337. chunk = left;
  338. left -= chunk;
  339. addr += chunk;
  340. i = 1;
  341. while (left && i < 5) {
  342. qtd->buf[i++] = addr & 0xFFFFF000u;
  343. chunk = (left > 0x1000u) ? 0x1000u : left;
  344. left -= chunk;
  345. addr += chunk;
  346. }
  347. }
  348. static int qtd_error(unsigned int token)
  349. {
  350. return (token & QTD_ERRORS) != 0;
  351. }
  352. /* Wait for the qTD to retire. If early is non-NULL and has halted, stop
  353. waiting immediately (a control data-stage STALL leaves the status qTD
  354. armed forever). */
  355. static int qtd_wait(struct ehci_qtd *qtd, struct ehci_qtd *early,
  356. unsigned int ms)
  357. {
  358. unsigned int t0 = get_ticks_ms();
  359. while (qtd->token & QTD_ACTIVE) {
  360. if (early && !(early->token & QTD_ACTIVE) &&
  361. qtd_error(early->token))
  362. return -1;
  363. if (get_ticks_ms() - t0 >= ms)
  364. return -1;
  365. usb_poll_relax(t0);
  366. __asm__ volatile ("pause");
  367. }
  368. return 0;
  369. }
  370. /* ---- bulk endpoints ---------------------------------------------------- */
  371. static void bulk_stop(struct ehci_bulk *b)
  372. {
  373. if (b->qh)
  374. qh_disarm(b->qh);
  375. if (b->qtd)
  376. b->qtd->token = 0;
  377. if (b->zqtd)
  378. b->zqtd->token = 0;
  379. b->armed = 0;
  380. if (b->sched && b->owner) {
  381. ehci_sched_put(b->owner);
  382. b->sched = 0;
  383. }
  384. }
  385. static void bulk_reset(struct ehci_bulk *b)
  386. {
  387. bulk_stop(b);
  388. b->addr = 0xFFFFFFFFu;
  389. b->ep = 0;
  390. b->mps = 0;
  391. b->buf_len = 0;
  392. b->toggle = 0;
  393. }
  394. static void bulk_configure(struct ehci_bulk *b, unsigned int addr,
  395. unsigned int ep, unsigned int mps,
  396. unsigned int buf_len)
  397. {
  398. if (b->addr == addr && b->ep == ep && b->mps == mps)
  399. return;
  400. bulk_stop(b);
  401. b->addr = addr;
  402. b->ep = ep;
  403. b->mps = mps;
  404. b->buf_len = buf_len;
  405. b->toggle = 0; /* first packet is DATA0 */
  406. qh_init(b->qh); /* keeps the link pointer out of it */
  407. qh_configure(b->qh, addr, ep, mps, 0);
  408. b->qh->token = 0; /* DATA0, not halted */
  409. b->armed = 0;
  410. }
  411. static void bulk_arm(struct ehci_bulk *b)
  412. {
  413. if (!b->sched && b->owner) {
  414. ehci_sched_get(b->owner);
  415. b->sched = 1;
  416. }
  417. b->qh->token = (b->qh->token & QTD_PING) |
  418. (b->toggle ? QTD_TOGGLE : 0);
  419. qh_arm(b->qh, b->qtd);
  420. b->armed = 1;
  421. }
  422. static int ehci_bulk_out(unsigned int ctrl, unsigned int addr,
  423. unsigned int ep, unsigned int mps,
  424. const unsigned char *buf, unsigned int len)
  425. {
  426. struct ehci_ctrl *c;
  427. struct ehci_bulk *b;
  428. struct ehci_qtd *qtd, *zqtd, *last;
  429. unsigned int token, ztoken = 0;
  430. int zlp;
  431. if (ctrl >= n_ctrls || !buf || !len || len > 0x7FFFu)
  432. return -1;
  433. c = &ctrls[ctrl];
  434. b = &c->bout;
  435. if (!b->qh || !b->qtd || !b->zqtd)
  436. return -1;
  437. if (mps < 1)
  438. mps = 512;
  439. if (mps > 1024)
  440. mps = 1024;
  441. bulk_configure(b, addr, ep, mps, 0);
  442. bulk_stop(b);
  443. qtd = b->qtd;
  444. zqtd = b->zqtd;
  445. qtd_fill(qtd, buf, len, QTD_PID_OUT, 0);
  446. /* A bulk transfer that ends on a max-packet boundary needs a
  447. zero-length packet to mark the end (USB 2.0 5.8.3). */
  448. zlp = (len % mps) == 0;
  449. if (zlp) {
  450. qtd_fill(zqtd, 0, 0, QTD_PID_OUT, 0);
  451. qtd->next = (unsigned int)zqtd;
  452. last = zqtd;
  453. } else {
  454. last = qtd;
  455. }
  456. bulk_arm(b);
  457. if (qtd_wait(last, qtd, 2000) < 0) {
  458. bulk_stop(b);
  459. return -1;
  460. }
  461. token = qtd->token;
  462. if (zlp)
  463. ztoken = last->token;
  464. bulk_stop(b);
  465. if ((token & QTD_ACTIVE) || qtd_error(token) ||
  466. (zlp && ((ztoken & QTD_ACTIVE) || qtd_error(ztoken))))
  467. return -1;
  468. {
  469. unsigned int packets = (len + mps - 1) / mps;
  470. if (zlp)
  471. packets++;
  472. b->toggle ^= packets & 1u;
  473. }
  474. return 0;
  475. }
  476. /* One page of bounce buffer; the caller's buffer stays stable while the
  477. completed transfer is consumed (see usb_hcd.h). */
  478. #define EHCI_RX_MAX 2048u
  479. static int ehci_bulk_in(unsigned int ctrl, unsigned int addr,
  480. unsigned int ep, unsigned int mps,
  481. unsigned char *buf, unsigned int len)
  482. {
  483. struct ehci_ctrl *c;
  484. struct ehci_bulk *b;
  485. unsigned int token, raw, packets, actual, i;
  486. if (ctrl >= n_ctrls || !buf || !len)
  487. return -1;
  488. c = &ctrls[ctrl];
  489. b = &c->bin;
  490. if (!b->qh || !b->qtd || !c->rx_buf)
  491. return -1;
  492. if (mps < 1)
  493. mps = 512;
  494. if (mps > 1024)
  495. mps = 1024;
  496. bulk_configure(b, addr, ep, mps, EHCI_RX_MAX);
  497. if (!b->armed) {
  498. qtd_fill(b->qtd, c->rx_buf, b->buf_len, QTD_PID_IN, 0);
  499. bulk_arm(b);
  500. return 0;
  501. }
  502. token = b->qtd->token;
  503. if (token & QTD_ACTIVE)
  504. return 0;
  505. b->armed = 0;
  506. if (qtd_error(token)) {
  507. bulk_stop(b);
  508. return -1;
  509. }
  510. raw = QTD_LEN(token) <= b->buf_len ?
  511. b->buf_len - QTD_LEN(token) : 0;
  512. actual = raw;
  513. if (actual > len)
  514. actual = len;
  515. for (i = 0; i < actual; i++)
  516. buf[i] = c->rx_buf[i];
  517. /* A bulk IN transfer ends with a short packet; a zero-length
  518. packet also counts as one successful transaction. */
  519. packets = raw ? (raw + b->mps - 1) / b->mps : 1;
  520. b->toggle ^= packets & 1u;
  521. qh_disarm(b->qh);
  522. qtd_fill(b->qtd, c->rx_buf, b->buf_len, QTD_PID_IN, 0);
  523. bulk_arm(b);
  524. return (int)actual;
  525. }
  526. /* ---- interrupt-IN endpoints ------------------------------------------- */
  527. static void int_arm(struct ehci_int *in)
  528. {
  529. in->qh->token = (in->qh->token & QTD_PING) |
  530. (in->toggle ? QTD_TOGGLE : 0);
  531. qh_arm(in->qh, in->qtd);
  532. }
  533. static void ehci_int_in_stop(unsigned int slot)
  534. {
  535. struct ehci_int *in;
  536. if (slot >= USB_HCD_INT_SLOTS)
  537. return;
  538. in = slot_int[slot];
  539. if (!in)
  540. return;
  541. {
  542. int was_active = in->active;
  543. in->active = 0;
  544. if (in->qh)
  545. qh_disarm(in->qh);
  546. if (in->qtd)
  547. in->qtd->token = 0;
  548. slot_int[slot] = 0;
  549. if (was_active && in->owner)
  550. ehci_sched_put(in->owner);
  551. }
  552. }
  553. static int ehci_int_in_start(unsigned int ctrl, unsigned int slot,
  554. unsigned int addr, unsigned int ep,
  555. unsigned int mps, int low_speed)
  556. {
  557. struct ehci_ctrl *c;
  558. struct ehci_int *in;
  559. if (ctrl >= n_ctrls || slot >= USB_HCD_INT_SLOTS || low_speed)
  560. return -1;
  561. c = &ctrls[ctrl];
  562. in = &c->ints[slot];
  563. if (!in->qh || !in->qtd || !in->buf)
  564. return -1;
  565. ehci_int_in_stop(slot);
  566. in->mps = mps ? mps : 8;
  567. if (in->mps > 1024)
  568. in->mps = 1024;
  569. in->buf_len = in->mps > 64 ? 64 : in->mps;
  570. qh_init(in->qh);
  571. in->qh->info1 = QH_DEVADDR(addr) | QH_ENDPT(ep) | QH_EPS_HIGH |
  572. QH_MPS(in->mps) | QH_RL(4);
  573. in->qh->info2 = QH_MULT(1);
  574. in->qh->token = 0;
  575. in->toggle = 0; /* first report is DATA0 */
  576. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  577. int_arm(in);
  578. in->active = 1;
  579. slot_int[slot] = in;
  580. ehci_sched_get(c);
  581. return 0;
  582. }
  583. static int ehci_int_in_poll(unsigned int slot, unsigned char *buf,
  584. unsigned int len)
  585. {
  586. struct ehci_int *in;
  587. unsigned int token, raw, packets, actual, i;
  588. if (slot >= USB_HCD_INT_SLOTS)
  589. return 0;
  590. in = slot_int[slot];
  591. if (!in || !in->active)
  592. return 0;
  593. token = in->qtd->token;
  594. if (token & QTD_ACTIVE)
  595. return 0;
  596. if (qtd_error(token)) {
  597. qh_disarm(in->qh);
  598. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  599. int_arm(in);
  600. return -1;
  601. }
  602. raw = QTD_LEN(token) <= in->buf_len ?
  603. in->buf_len - QTD_LEN(token) : 0;
  604. actual = raw;
  605. if (actual > len)
  606. actual = len;
  607. for (i = 0; i < actual; i++)
  608. buf[i] = in->buf[i];
  609. packets = raw ? (raw + in->mps - 1) / in->mps : 1;
  610. in->toggle ^= packets & 1u;
  611. qh_disarm(in->qh);
  612. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  613. int_arm(in);
  614. return (int)actual;
  615. }
  616. /* ---- control transfers ------------------------------------------------- */
  617. static int ehci_control(struct ehci_ctrl *c, unsigned int addr,
  618. unsigned int mps, int low_speed,
  619. const unsigned char *setup, unsigned char *data,
  620. unsigned int len, int dir_in)
  621. {
  622. struct ehci_qh *qh;
  623. struct ehci_qtd *q0, *q1, *q2, *last, *early;
  624. unsigned int i, actual = 0, residual;
  625. (void)mps;
  626. if (!c->ctl_qh || !c->setup || !setup || c->ctl_busy || low_speed)
  627. return -1;
  628. if (len && !data)
  629. return -1;
  630. qh = c->ctl_qh;
  631. q0 = c->ctl_qtd[0];
  632. q1 = c->ctl_qtd[1];
  633. q2 = c->ctl_qtd[2];
  634. ehci_sched_get(c);
  635. c->ctl_busy = 1;
  636. for (i = 0; i < 8; i++)
  637. c->setup[i] = setup[i];
  638. qh_init(qh);
  639. qh_configure(qh, addr, 0, 64, 1);
  640. qh->token = 0; /* control toggles come from the qTDs */
  641. qtd_fill(q0, c->setup, 8, QTD_PID_SETUP, 0);
  642. if (len) {
  643. qtd_fill(q1, data, len,
  644. dir_in ? QTD_PID_IN : QTD_PID_OUT, 1);
  645. qtd_fill(q2, 0, 0,
  646. dir_in ? QTD_PID_OUT : QTD_PID_IN, 1);
  647. q0->next = (unsigned int)q1;
  648. q1->next = (unsigned int)q2;
  649. last = q2;
  650. early = q1;
  651. } else {
  652. qtd_fill(q2, 0, 0, QTD_PID_IN, 1);
  653. q0->next = (unsigned int)q2;
  654. last = q2;
  655. early = q0;
  656. }
  657. q2->next = QH_LIST_END;
  658. qh_arm(qh, q0);
  659. if (qtd_wait(last, early, 2000) < 0) {
  660. qh_disarm(qh);
  661. delay_ms(1);
  662. c->ctl_busy = 0;
  663. ehci_sched_put(c);
  664. return -1;
  665. }
  666. qh_disarm(qh);
  667. if (qtd_error(q0->token) || qtd_error(q2->token) ||
  668. (len && qtd_error(q1->token))) {
  669. c->ctl_busy = 0;
  670. ehci_sched_put(c);
  671. return -1;
  672. }
  673. if (len && dir_in) {
  674. residual = QTD_LEN(q1->token);
  675. actual = residual >= len ? 0 : len - residual;
  676. } else if (len) {
  677. actual = len;
  678. }
  679. /* The device changes address after the status stage. Remember which
  680. root port it came from so a port reset can drop this controller's
  681. bulk state without touching an unrelated port. */
  682. if (addr == 0 && setup[0] == 0x00 && setup[1] == 0x05) {
  683. unsigned int new_addr = (unsigned int)setup[2] |
  684. ((unsigned int)setup[3] << 8);
  685. if (new_addr > 0 && new_addr < 128 && c->last_port >= 0)
  686. c->addr_port[new_addr] = (unsigned char)c->last_port;
  687. }
  688. c->ctl_busy = 0;
  689. ehci_sched_put(c);
  690. return (int)actual;
  691. }
  692. static int ehci_control_transfer(unsigned int ctrl, unsigned int addr,
  693. unsigned int mps, int low_speed,
  694. const unsigned char *setup,
  695. unsigned char *data, unsigned int len,
  696. int dir_in)
  697. {
  698. if (ctrl >= n_ctrls)
  699. return -1;
  700. return ehci_control(&ctrls[ctrl], addr, mps, low_speed, setup, data,
  701. len, dir_in);
  702. }
  703. /* ---- root hub ports ---------------------------------------------------- */
  704. static void port_endpoints_reset(struct ehci_ctrl *c, unsigned int port)
  705. {
  706. unsigned int i;
  707. if (c->bin.addr < 128 && c->addr_port[c->bin.addr] == port)
  708. bulk_reset(&c->bin);
  709. if (c->bout.addr < 128 && c->addr_port[c->bout.addr] == port)
  710. bulk_reset(&c->bout);
  711. for (i = 0; i < 128; i++)
  712. if (c->addr_port[i] == port)
  713. c->addr_port[i] = 0xFF;
  714. }
  715. static unsigned int ehci_port_count(unsigned int ctrl)
  716. {
  717. return ctrl < n_ctrls ? ctrls[ctrl].n_ports : 0;
  718. }
  719. static int ehci_port_connected(unsigned int ctrl, unsigned int port)
  720. {
  721. struct ehci_ctrl *c;
  722. unsigned int ps;
  723. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  724. return 0;
  725. c = &ctrls[ctrl];
  726. ps = rd32(port_reg(c, port));
  727. return (ps & PORT_CONNECT) ? 1 : 0;
  728. }
  729. static int ehci_port_low_speed(unsigned int ctrl, unsigned int port)
  730. {
  731. (void)ctrl;
  732. (void)port;
  733. return 0; /* EHCI only drives high-speed devices */
  734. }
  735. static int ehci_port_take_change(unsigned int ctrl, unsigned int port)
  736. {
  737. struct ehci_ctrl *c;
  738. unsigned int ps;
  739. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  740. return 0;
  741. c = &ctrls[ctrl];
  742. ps = rd32(port_reg(c, port));
  743. if (!(ps & PORT_CSC))
  744. return 0;
  745. wr32(port_reg(c, port),
  746. (ps & ~PORT_RWC) | PORT_CSC | PORT_PEC | PORT_OCC);
  747. return 1;
  748. }
  749. /* HCSPARAMS N_CC: the number of companion controllers. */
  750. static unsigned int n_companions(const struct ehci_ctrl *c)
  751. {
  752. return (c->hcs >> 12) & 0x0Fu;
  753. }
  754. /* Give a non-high-speed device to the companion controller. */
  755. static void port_to_companion(struct ehci_ctrl *c, unsigned int port,
  756. unsigned int ps)
  757. {
  758. wr32(port_reg(c, port),
  759. (ps & ~(PORT_RWC | PORT_PE | PORT_RESET)) | PORT_OWNER |
  760. PORT_CSC | PORT_PEC | PORT_OCC);
  761. if (!(c->cc_ports & (1u << port))) {
  762. c->cc_ports |= 1u << port;
  763. serial_puts(SERIAL_COM1, "ehci: port ");
  764. serial_putdec(SERIAL_COM1, port + 1);
  765. serial_puts(SERIAL_COM1, " not high-speed, handed to companion\n");
  766. }
  767. }
  768. static int ehci_port_reset_once(unsigned int ctrl, unsigned int port)
  769. {
  770. struct ehci_ctrl *c;
  771. unsigned int ps, t0, try;
  772. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  773. return -1;
  774. c = &ctrls[ctrl];
  775. /* A halted controller ignores port commands; restart it first. */
  776. if (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT) {
  777. serial_puts(SERIAL_COM1, "ehci: controller halted, restarting\n");
  778. if (ehci_start(c) < 0)
  779. return -1;
  780. }
  781. ps = rd32(port_reg(c, port));
  782. if (!(ps & PORT_CONNECT))
  783. return -1;
  784. /* Drop any reset/suspend state the firmware left behind. The port
  785. can be stuck with PR set; a single clear write is not enough on
  786. some PCHs. */
  787. for (try = 0; try < 10 && (ps & PORT_RESET); try++) {
  788. wr32(port_reg(c, port),
  789. (ps & ~(PORT_RWC | PORT_RESET | PORT_PE |
  790. PORT_SUSPEND | PORT_RESUME)) |
  791. PORT_CSC | PORT_PEC | PORT_OCC);
  792. delay_ms(5);
  793. ps = rd32(port_reg(c, port));
  794. }
  795. if ((ps & PORT_RESET) || !(ps & PORT_CONNECT))
  796. return -2;
  797. /* The port may still be owned by the companion controller. A
  798. low/full-speed device belongs there; a high-speed one can be taken
  799. over by clearing PORT_OWNER (the controller may need more than one
  800. write while the port is enabled). */
  801. /* The hardware returns a port to the EHCI when its device goes away. */
  802. if (!(ps & PORT_OWNER))
  803. c->cc_ports &= ~(1u << port);
  804. if (ps & PORT_OWNER) {
  805. if (PORT_USB11(ps) || (c->cc_ports & (1u << port)))
  806. return -1;
  807. for (try = 0; try < 8 && (ps & PORT_OWNER); try++) {
  808. wr32(port_reg(c, port),
  809. (ps & ~(PORT_RWC | PORT_OWNER | PORT_RESET |
  810. PORT_PE | PORT_SUSPEND | PORT_RESUME)) |
  811. PORT_CSC | PORT_PEC | PORT_OCC);
  812. delay_ms(5);
  813. ps = rd32(port_reg(c, port));
  814. }
  815. if ((ps & PORT_OWNER) || !(ps & PORT_CONNECT))
  816. return -1;
  817. }
  818. /* A low-speed device (K state on an unenabled port) is never reset by
  819. the EHCI: hand it straight to the companion (EHCI 4.2.2). */
  820. if (!(ps & PORT_PE) && PORT_USB11(ps) && n_companions(c)) {
  821. port_to_companion(c, port, ps);
  822. return -1;
  823. }
  824. port_endpoints_reset(c, port);
  825. c->last_port = (int)port;
  826. /* Clear stale change bits, then assert reset. PED is cleared while
  827. reset is asserted (like Linux's SetPortFeature(RESET)). */
  828. wr32(port_reg(c, port),
  829. (ps & ~(PORT_RWC | PORT_PE | PORT_SUSPEND | PORT_RESUME)) |
  830. PORT_CSC | PORT_PEC | PORT_OCC | PORT_RESET);
  831. delay_ms(50);
  832. /* EHCI expects software to end the reset, but some controllers hold
  833. PR set while the device is still chirping. Give it up to 200 ms to
  834. self-clear before forcing the write. */
  835. t0 = get_ticks_ms();
  836. while (get_ticks_ms() - t0 < 200) {
  837. if (!(rd32(port_reg(c, port)) & PORT_RESET))
  838. break;
  839. delay_ms(1);
  840. }
  841. /* Force completion if the controller still holds PR; retry because
  842. the first write can land while the reset is still in progress. */
  843. for (try = 0; try < 20; try++) {
  844. ps = rd32(port_reg(c, port));
  845. if (!(ps & PORT_RESET))
  846. break;
  847. wr32(port_reg(c, port),
  848. (ps & ~(PORT_RWC | PORT_RESET | PORT_PE |
  849. PORT_SUSPEND | PORT_RESUME)) |
  850. PORT_CSC | PORT_PEC | PORT_OCC);
  851. delay_ms(10);
  852. }
  853. if (ps & PORT_RESET) {
  854. serial_puts(SERIAL_COM1, "ehci: reset stuck ");
  855. serial_putdec(SERIAL_COM1, c->bus);
  856. serial_putc(SERIAL_COM1, ':');
  857. serial_putdec(SERIAL_COM1, c->dev);
  858. serial_putc(SERIAL_COM1, '.');
  859. serial_putdec(SERIAL_COM1, c->func);
  860. serial_puts(SERIAL_COM1, " port ");
  861. serial_putdec(SERIAL_COM1, port + 1);
  862. serial_puts(SERIAL_COM1, " ps ");
  863. serial_puthex(SERIAL_COM1, ps);
  864. serial_putc(SERIAL_COM1, '\n');
  865. return -2;
  866. }
  867. t0 = get_ticks_ms();
  868. while (!(rd32(port_reg(c, port)) & PORT_PE)) {
  869. if (get_ticks_ms() - t0 > 200)
  870. break;
  871. usb_poll_relax(t0);
  872. }
  873. ps = rd32(port_reg(c, port));
  874. if (!(ps & PORT_PE)) {
  875. if (ps & PORT_OWNER)
  876. return -1;
  877. /* Not enabled after the reset: the device is not high-speed
  878. (EHCI 4.2.2), whatever the line state reads -- a full-speed
  879. receiver on the MCP79 reports SE0 here. It belongs to the
  880. companion. Only a controller without companions (Intel PCH
  881. rate-matching hubs) can have a genuinely stuck HS reset. */
  882. if (n_companions(c) || PORT_USB11(ps)) {
  883. port_to_companion(c, port, ps);
  884. return -1;
  885. }
  886. serial_puts(SERIAL_COM1, "ehci: reset no PED ");
  887. serial_puthex(SERIAL_COM1, ps);
  888. serial_putc(SERIAL_COM1, '\n');
  889. return -2;
  890. }
  891. if (ps & PORT_OWNER)
  892. return -1;
  893. return 0;
  894. }
  895. /* A stuck HS port reset on some Intel PCHs only completes after the whole
  896. EHCI controller is reset. Retry the port reset once in that case. */
  897. static int ehci_port_reset(unsigned int ctrl, unsigned int port)
  898. {
  899. struct ehci_ctrl *c;
  900. int r;
  901. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  902. return -1;
  903. c = &ctrls[ctrl];
  904. r = ehci_port_reset_once(ctrl, port);
  905. if (r == -2 && !c->recovering) {
  906. c->recovering = 1;
  907. serial_puts(SERIAL_COM1,
  908. "ehci: port reset retry after controller reset\n");
  909. if (ehci_start(c) == 0)
  910. r = ehci_port_reset_once(ctrl, port);
  911. else
  912. r = -1;
  913. c->recovering = 0;
  914. }
  915. return r < 0 ? -1 : 0;
  916. }
  917. /* ---- schedule / controller setup -------------------------------------- */
  918. static int ehci_sched_init(struct ehci_ctrl *c)
  919. {
  920. unsigned char *desc = (unsigned char *)c->desc_page;
  921. unsigned int qtd_base = c->desc_page + EHCI_MAX_QH * EHCI_QH_STRIDE;
  922. struct ehci_qh *list[EHCI_MAX_QH];
  923. unsigned int i;
  924. c->head = (struct ehci_qh *)(desc + 0 * EHCI_QH_STRIDE);
  925. c->ctl_qh = (struct ehci_qh *)(desc + 1 * EHCI_QH_STRIDE);
  926. c->bin_qh = (struct ehci_qh *)(desc + 2 * EHCI_QH_STRIDE);
  927. c->bout_qh = (struct ehci_qh *)(desc + 3 * EHCI_QH_STRIDE);
  928. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  929. c->int_qh[i] = (struct ehci_qh *)(desc +
  930. (4 + i) * EHCI_QH_STRIDE);
  931. c->ctl_qtd[0] = (struct ehci_qtd *)(qtd_base + 0 * EHCI_QTD_STRIDE);
  932. c->ctl_qtd[1] = (struct ehci_qtd *)(qtd_base + 1 * EHCI_QTD_STRIDE);
  933. c->ctl_qtd[2] = (struct ehci_qtd *)(qtd_base + 2 * EHCI_QTD_STRIDE);
  934. c->bin_qtd = (struct ehci_qtd *)(qtd_base + 3 * EHCI_QTD_STRIDE);
  935. c->bout_qtd = (struct ehci_qtd *)(qtd_base + 4 * EHCI_QTD_STRIDE);
  936. c->bout_zqtd = (struct ehci_qtd *)(qtd_base + 11 * EHCI_QTD_STRIDE);
  937. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  938. c->int_qtd[i] = (struct ehci_qtd *)(qtd_base +
  939. (5 + i) * EHCI_QTD_STRIDE);
  940. c->rx_buf = (unsigned char *)c->data_page;
  941. c->setup = (unsigned char *)(c->data_page + 2048);
  942. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  943. c->int_buf[i] = (unsigned char *)(c->data_page + 2112 +
  944. i * 64);
  945. for (i = 0; i < 1024; i++)
  946. wr32(c->frame_list + i * 4, QH_LIST_END);
  947. list[0] = c->head;
  948. list[1] = c->ctl_qh;
  949. list[2] = c->bin_qh;
  950. list[3] = c->bout_qh;
  951. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  952. list[4 + i] = c->int_qh[i];
  953. for (i = 0; i < EHCI_MAX_QH; i++) {
  954. qh_init(list[i]);
  955. list[i]->next = QH_NEXT((unsigned int)
  956. list[(i + 1) % EHCI_MAX_QH]);
  957. }
  958. c->head->info1 = QH_HEAD;
  959. c->head->token = QTD_HALT;
  960. c->bin.owner = c;
  961. c->bin.sched = 0;
  962. c->bout.owner = c;
  963. c->bout.sched = 0;
  964. c->bin.qh = c->bin_qh;
  965. c->bin.qtd = c->bin_qtd;
  966. c->bin.zqtd = 0;
  967. c->bin.addr = 0xFFFFFFFFu;
  968. c->bout.qh = c->bout_qh;
  969. c->bout.qtd = c->bout_qtd;
  970. c->bout.zqtd = c->bout_zqtd;
  971. c->bout.addr = 0xFFFFFFFFu;
  972. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  973. c->ints[i].qh = c->int_qh[i];
  974. c->ints[i].qtd = c->int_qtd[i];
  975. c->ints[i].buf = c->int_buf[i];
  976. c->ints[i].active = 0;
  977. c->ints[i].owner = c;
  978. }
  979. for (i = 0; i < 128; i++)
  980. c->addr_port[i] = 0xFF;
  981. c->last_port = -1;
  982. return 0;
  983. }
  984. static void ehci_halt(struct ehci_ctrl *c)
  985. {
  986. unsigned int t0;
  987. wr32(op_reg(c, EHCI_OP_USBINTR), 0);
  988. wr32(op_reg(c, EHCI_OP_USBCMD), 0);
  989. t0 = get_ticks_ms();
  990. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT)) {
  991. if (get_ticks_ms() - t0 > 100)
  992. break;
  993. }
  994. }
  995. static int ehci_start(struct ehci_ctrl *c)
  996. {
  997. unsigned int v, t0, i;
  998. /* Stop first, then reset. HCRESET self-clears. */
  999. ehci_halt(c);
  1000. wr32(op_reg(c, EHCI_OP_USBCMD), CMD_RESET);
  1001. t0 = get_ticks_ms();
  1002. while (rd32(op_reg(c, EHCI_OP_USBCMD)) & CMD_RESET) {
  1003. if (get_ticks_ms() - t0 > 250) {
  1004. dbg("ehci: reset timeout\n");
  1005. return -1;
  1006. }
  1007. }
  1008. delay_ms(2);
  1009. if (c->hcc & 1u) /* 64-bit addressing capable */
  1010. wr32(op_reg(c, EHCI_OP_CTRLDSSEGMENT), 0);
  1011. wr32(op_reg(c, EHCI_OP_PERIODICLIST), c->frame_list);
  1012. wr32(op_reg(c, EHCI_OP_ASYNCLIST), (unsigned int)c->head);
  1013. wr32(op_reg(c, EHCI_OP_USBINTR), 0);
  1014. wr32(op_reg(c, EHCI_OP_USBSTS), 0x3Fu);
  1015. c->sched_on = 0;
  1016. c->sched_refs = 0;
  1017. wr32(op_reg(c, EHCI_OP_USBCMD), CMD_RUN | CMD_ITC_8);
  1018. __asm__ volatile ("" ::: "memory");
  1019. t0 = get_ticks_ms();
  1020. while (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT) {
  1021. if (get_ticks_ms() - t0 > 100) {
  1022. dbg("ehci: controller did not start\n");
  1023. return -1;
  1024. }
  1025. }
  1026. /* Take the root ports over from the companion controllers. */
  1027. wr32(op_reg(c, EHCI_OP_CONFIGFLAG), 1);
  1028. v = rd32(op_reg(c, EHCI_OP_CONFIGFLAG));
  1029. if (!(v & 1u))
  1030. serial_puts(SERIAL_COM1, "ehci: CONFIGFLAG did not stick\n");
  1031. delay_ms(5);
  1032. if (c->ppc) {
  1033. for (i = 0; i < c->n_ports; i++) {
  1034. v = rd32(port_reg(c, i));
  1035. if (!(v & PORT_POWER))
  1036. wr32(port_reg(c, i),
  1037. (v & ~PORT_RWC) | PORT_POWER);
  1038. }
  1039. delay_ms(100);
  1040. }
  1041. t0 = get_ticks_ms();
  1042. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS)) {
  1043. if (get_ticks_ms() - t0 > 100)
  1044. break;
  1045. }
  1046. if (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS))
  1047. dbg("ehci: async schedule not active\n");
  1048. return 0;
  1049. }
  1050. /* Best-effort EHCI legacy-support handshake: stop the firmware's SMIs from
  1051. firing on port changes now that the kernel owns the controller. */
  1052. static void legacy_smi_disable(unsigned int bus, unsigned int dev,
  1053. unsigned int func, unsigned int hcc)
  1054. {
  1055. unsigned int eecp = (hcc >> 8) & 0xFFu;
  1056. unsigned int guard = 0;
  1057. while (eecp >= 0x40 && guard++ < 16) {
  1058. unsigned int cap = pci_cfg_read32(bus, dev, func, eecp);
  1059. unsigned int id = cap & 0xFFu;
  1060. unsigned int next = (cap >> 8) & 0xFFu;
  1061. if (id == 1) { /* EHCI Legacy Support */
  1062. pci_cfg_write32(bus, dev, func, eecp,
  1063. (cap | (1u << 24)) & ~(1u << 31));
  1064. cap = pci_cfg_read32(bus, dev, func, eecp + 4);
  1065. pci_cfg_write32(bus, dev, func, eecp + 4,
  1066. cap & 0xFFFF0000u);
  1067. return;
  1068. }
  1069. if (!next || next == eecp)
  1070. return;
  1071. eecp = next;
  1072. }
  1073. }
  1074. static int ehci_setup_one(unsigned int bus, unsigned int dev,
  1075. unsigned int func, unsigned int bar)
  1076. {
  1077. struct ehci_ctrl *c;
  1078. unsigned int cap_len, hcs, hcc;
  1079. if (n_ctrls >= USB_HCD_MAX_CTRL)
  1080. return -1;
  1081. c = &ctrls[n_ctrls];
  1082. zero_ctrl(c);
  1083. c->phys = bar;
  1084. c->bus = bus;
  1085. c->dev = dev;
  1086. c->func = func;
  1087. bar = (unsigned int)ioremap(bar, 0x1000, IOREMAP_MMIO);
  1088. if (!bar)
  1089. return -1;
  1090. c->base = bar;
  1091. if (rd32(bar) == 0xFFFFFFFFu)
  1092. return -1;
  1093. cap_len = rd32(bar + EHCI_CAP_CAPLENGTH) & 0xFFu;
  1094. if (cap_len < 0x20 || cap_len > 0x100)
  1095. return -1;
  1096. hcs = rd32(bar + EHCI_CAP_HCSPARAMS);
  1097. if (hcs == 0xFFFFFFFFu)
  1098. return -1;
  1099. c->cap_len = cap_len;
  1100. c->n_ports = hcs & 0x0Fu;
  1101. c->ppc = (hcs & 0x10u) ? 1 : 0;
  1102. c->hcs = hcs;
  1103. if (!c->n_ports) {
  1104. dbg("ehci: no root ports\n");
  1105. return -1;
  1106. }
  1107. hcc = rd32(bar + EHCI_CAP_HCCPARAMS);
  1108. c->hcc = hcc;
  1109. legacy_smi_disable(bus, dev, func, hcc);
  1110. c->frame_list = mm_alloc_page();
  1111. c->desc_page = mm_alloc_page();
  1112. c->data_page = mm_alloc_page();
  1113. if (!c->frame_list || !c->desc_page || !c->data_page) {
  1114. dbg("ehci: out of pages\n");
  1115. goto fail;
  1116. }
  1117. zero_region(c->frame_list, 4096);
  1118. zero_region(c->desc_page, 4096);
  1119. zero_region(c->data_page, 4096);
  1120. ehci_sched_init(c);
  1121. if (ehci_start(c) < 0)
  1122. goto fail;
  1123. serial_puts(SERIAL_COM1, "ehci: controller at ");
  1124. serial_putdec(SERIAL_COM1, bus);
  1125. serial_puts(SERIAL_COM1, ":");
  1126. serial_putdec(SERIAL_COM1, dev);
  1127. serial_puts(SERIAL_COM1, ".");
  1128. serial_putdec(SERIAL_COM1, func);
  1129. serial_puts(SERIAL_COM1, " ports ");
  1130. serial_putdec(SERIAL_COM1, c->n_ports);
  1131. serial_puts(SERIAL_COM1, "\n");
  1132. n_ctrls++;
  1133. return 0;
  1134. fail:
  1135. ehci_halt(c);
  1136. /* Give the shared ports back to the companion controllers. */
  1137. wr32(op_reg(c, EHCI_OP_CONFIGFLAG), 0);
  1138. if (c->frame_list)
  1139. mm_free_page(c->frame_list);
  1140. if (c->desc_page)
  1141. mm_free_page(c->desc_page);
  1142. if (c->data_page)
  1143. mm_free_page(c->data_page);
  1144. zero_ctrl(c);
  1145. return -1;
  1146. }
  1147. static int ehci_init(void)
  1148. {
  1149. unsigned int bus, dev, func;
  1150. if (n_ctrls)
  1151. return (int)n_ctrls;
  1152. for (bus = 0; bus < 256 && n_ctrls < USB_HCD_MAX_CTRL; bus++) {
  1153. for (dev = 0; dev < 32; dev++) {
  1154. for (func = 0; func < 8; func++) {
  1155. unsigned int id = pci_cfg_read32(bus, dev, func,
  1156. 0x00);
  1157. unsigned int class_rev, bar, bar_hi, cmd;
  1158. if (id == 0xFFFFFFFFu || id == 0)
  1159. continue;
  1160. class_rev = pci_cfg_read32(bus, dev, func,
  1161. 0x08);
  1162. if ((class_rev >> 16) != 0x0C03u)
  1163. continue;
  1164. if (((class_rev >> 8) & 0xFFu) != 0x20u)
  1165. continue; /* not EHCI */
  1166. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1167. if (bar & 1u)
  1168. continue; /* register block is MMIO */
  1169. bar_hi = 0;
  1170. if (((bar >> 1) & 3u) == 2u)
  1171. bar_hi = pci_cfg_read32(bus, dev, func,
  1172. 0x14);
  1173. bar &= 0xFFFFFFF0u;
  1174. if (!bar || bar_hi)
  1175. continue;
  1176. cmd = pci_cfg_read32(bus, dev, func, 0x04);
  1177. pci_cfg_write32(bus, dev, func, 0x04,
  1178. cmd | 0x0006u);
  1179. ehci_setup_one(bus, dev, func, bar);
  1180. }
  1181. }
  1182. }
  1183. return (int)n_ctrls;
  1184. }
  1185. static unsigned int ehci_count(void)
  1186. {
  1187. return n_ctrls;
  1188. }
  1189. static void ehci_info(unsigned int ctrl, unsigned int *bus, unsigned int *dev,
  1190. unsigned int *func, unsigned int *base, int *mmio)
  1191. {
  1192. struct ehci_ctrl *c = &ctrls[ctrl];
  1193. *bus = c->bus;
  1194. *dev = c->dev;
  1195. *func = c->func;
  1196. *base = c->phys;
  1197. *mmio = 1;
  1198. }
  1199. /* Raw PORTSC for `lsusb`; useful when a port is connected but owned by a
  1200. companion (bit 13) or not powered (bit 12). */
  1201. static void ehci_print_ports(unsigned int ctrl)
  1202. {
  1203. struct ehci_ctrl *c;
  1204. unsigned int i, ps, eecp;
  1205. if (ctrl >= n_ctrls)
  1206. return;
  1207. c = &ctrls[ctrl];
  1208. io_putstr(" CMD 0x");
  1209. io_puthex(rd32(op_reg(c, EHCI_OP_USBCMD)));
  1210. io_putstr(" STS 0x");
  1211. io_puthex(rd32(op_reg(c, EHCI_OP_USBSTS)));
  1212. io_putchar('\n');
  1213. io_putstr(" CF 0x");
  1214. io_puthex(rd32(op_reg(c, EHCI_OP_CONFIGFLAG)));
  1215. io_putstr(" HCC 0x");
  1216. io_puthex(c->hcc);
  1217. io_putstr(" HCS 0x");
  1218. io_puthex(c->hcs);
  1219. io_putchar('\n');
  1220. eecp = (c->hcc >> 8) & 0xFFu;
  1221. if (eecp >= 0x40) {
  1222. unsigned int leg = pci_cfg_read32(c->bus, c->dev, c->func,
  1223. eecp);
  1224. if ((leg & 0xFFu) == 1u) {
  1225. io_putstr(" LEGSUP 0x");
  1226. io_puthex(leg);
  1227. io_putchar('\n');
  1228. }
  1229. }
  1230. for (i = 0; i < c->n_ports; i++) {
  1231. ps = rd32(port_reg(c, i));
  1232. io_putstr(" Port ");
  1233. if (i < 9)
  1234. io_putchar((char)('1' + i));
  1235. else
  1236. io_putchar((char)('A' + i - 9));
  1237. io_putstr(": 0x");
  1238. io_puthex(ps);
  1239. io_putchar('\n');
  1240. }
  1241. }
  1242. const struct usb_hcd_ops usb_ehci_ops = {
  1243. "EHCI",
  1244. ehci_init,
  1245. ehci_count,
  1246. ehci_info,
  1247. ehci_control_transfer,
  1248. ehci_port_count,
  1249. ehci_port_connected,
  1250. ehci_port_low_speed,
  1251. ehci_port_reset,
  1252. ehci_port_take_change,
  1253. ehci_int_in_start,
  1254. ehci_int_in_poll,
  1255. ehci_int_in_stop,
  1256. ehci_bulk_out,
  1257. ehci_bulk_in,
  1258. ehci_print_ports,
  1259. };