ehci.c 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559
  1. /* EHCI (USB 2.0) host controller driver.
  2. *
  3. * Scope: high-speed devices on the root ports. The EHCI controller is paired
  4. * with UHCI/OHCI companions on most chipsets; low/full-speed devices are left
  5. * to those controllers (the hardware routes them back when a port reset shows
  6. * a non-high-speed device). usb.c enumerates every controller type, so a
  7. * high-speed USB Ethernet dongle on EHCI and a full-speed mouse/keyboard on a
  8. * companion both work.
  9. *
  10. * The schedule is polled (no IRQs), like the UHCI/OHCI drivers. All control,
  11. * bulk and interrupt endpoints live in the asynchronous schedule: a circular
  12. * QH list with the control QH after the reclamation head. Interrupt endpoints
  13. * are retried every microframe rather than through the periodic list; for a
  14. * handful of boot HID devices the extra NAK traffic is negligible and it keeps
  15. * the schedule simple.
  16. *
  17. * Descriptors (frame list, QHs, qTDs, bounce buffers) come from the page
  18. * allocator, so the driver does not grow the kernel BSS. */
  19. #include "usb.h"
  20. #include "ehci.h"
  21. #include "io.h"
  22. #include "mm.h"
  23. #include "paging.h"
  24. #include "pci.h"
  25. #include "serial.h"
  26. #include "timer.h"
  27. #define EHCI_DEBUG 0
  28. /* ---- capability / operational register offsets (bytes) ----------------- */
  29. #define EHCI_CAP_CAPLENGTH 0x00
  30. #define EHCI_CAP_HCSPARAMS 0x04
  31. #define EHCI_CAP_HCCPARAMS 0x08
  32. #define EHCI_OP_USBCMD 0x00
  33. #define EHCI_OP_USBSTS 0x04
  34. #define EHCI_OP_USBINTR 0x08
  35. #define EHCI_OP_FRINDEX 0x0C
  36. #define EHCI_OP_CTRLDSSEGMENT 0x10
  37. #define EHCI_OP_PERIODICLIST 0x14
  38. #define EHCI_OP_ASYNCLIST 0x18
  39. #define EHCI_OP_CONFIGFLAG 0x40
  40. #define EHCI_OP_PORTSC 0x44
  41. /* USBCMD */
  42. #define CMD_RUN 0x00000001u
  43. #define CMD_RESET 0x00000002u
  44. #define CMD_PSE 0x00000010u
  45. #define CMD_ASE 0x00000020u
  46. #define CMD_IAAD 0x00000040u
  47. #define CMD_ITC_8 0x00080000u
  48. /* USBSTS */
  49. #define STS_INT 0x00000001u
  50. #define STS_ERR 0x00000002u
  51. #define STS_PCD 0x00000004u
  52. #define STS_FLR 0x00000008u
  53. #define STS_FATAL 0x00000010u
  54. #define STS_IAA 0x00000020u
  55. #define STS_HALT 0x00001000u
  56. #define STS_RECL 0x00002000u
  57. #define STS_PSS 0x00004000u
  58. #define STS_ASS 0x00008000u
  59. /* PORTSC */
  60. #define PORT_CONNECT 0x00000001u
  61. #define PORT_CSC 0x00000002u
  62. #define PORT_PE 0x00000004u
  63. #define PORT_PEC 0x00000008u
  64. #define PORT_OC 0x00000010u
  65. #define PORT_OCC 0x00000020u
  66. #define PORT_RESUME 0x00000040u
  67. #define PORT_SUSPEND 0x00000080u
  68. #define PORT_RESET 0x00000100u
  69. #define PORT_LS_MASK 0x00000C00u
  70. #define PORT_POWER 0x00001000u
  71. #define PORT_OWNER 0x00002000u
  72. #define PORT_RWC (PORT_CSC | PORT_PEC | PORT_OCC)
  73. #define PORT_USB11(ps) (((ps) & PORT_LS_MASK) == (1u << 10))
  74. /* qTD token bits (EHCI 3.5.3). */
  75. #define QTD_ACTIVE 0x00000080u
  76. #define QTD_HALT 0x00000040u
  77. #define QTD_DBE 0x00000020u
  78. #define QTD_BABBLE 0x00000010u
  79. #define QTD_XACT 0x00000008u
  80. #define QTD_MMF 0x00000004u
  81. #define QTD_PING 0x00000001u
  82. #define QTD_ERRORS (QTD_HALT | QTD_DBE | QTD_BABBLE | \
  83. QTD_XACT | QTD_MMF)
  84. #define QTD_TOGGLE 0x80000000u
  85. #define QTD_PID_OUT 0u
  86. #define QTD_PID_IN 1u
  87. #define QTD_PID_SETUP 2u
  88. #define QTD_CERR_3 (3u << 10)
  89. #define QTD_LEN(tok) (((tok) >> 16) & 0x7FFFu)
  90. /* QH endpoint characteristics (EHCI 3.6). */
  91. #define QH_DEVADDR(a) ((a) & 0x7Fu)
  92. #define QH_ENDPT(e) (((e) & 0x0Fu) << 8)
  93. #define QH_EPS_HIGH (2u << 12)
  94. #define QH_DTC (1u << 14)
  95. #define QH_HEAD (1u << 15)
  96. #define QH_MPS(m) (((m) & 0x7FFu) << 16)
  97. #define QH_CTRL_EP (1u << 27)
  98. #define QH_RL(n) (((n) & 7u) << 28)
  99. /* QH endpoint capabilities. */
  100. #define QH_MULT(n) (((n) & 3u) << 30)
  101. /* Link pointers: bit 0 terminates, bits 2:1 select the structure type. */
  102. #define QH_LIST_END 1u
  103. #define QH_NEXT(a) (((a) & ~0x1Fu) | 0x2u)
  104. #define EHCI_MAX_QH (4 + USB_HCD_INT_SLOTS)
  105. #define EHCI_QH_STRIDE 128
  106. #define EHCI_QTD_STRIDE 64
  107. struct ehci_qh {
  108. volatile unsigned int next;
  109. volatile unsigned int info1;
  110. volatile unsigned int info2;
  111. volatile unsigned int current;
  112. volatile unsigned int qtd_next;
  113. volatile unsigned int alt_next;
  114. volatile unsigned int token;
  115. volatile unsigned int buf[5];
  116. volatile unsigned int buf_hi[5]; /* 64-bit addressing extension */
  117. } __attribute__((aligned(32)));
  118. struct ehci_qtd {
  119. volatile unsigned int next;
  120. volatile unsigned int alt_next;
  121. volatile unsigned int token;
  122. volatile unsigned int buf[5];
  123. volatile unsigned int buf_hi[5]; /* 64-bit addressing extension */
  124. } __attribute__((aligned(32)));
  125. struct ehci_ctrl;
  126. struct ehci_bulk {
  127. struct ehci_qh *qh;
  128. struct ehci_qtd *qtd;
  129. struct ehci_qtd *zqtd; /* zero-length packet, bulk OUT only */
  130. unsigned int addr;
  131. unsigned int ep;
  132. unsigned int mps;
  133. unsigned int buf_len; /* capacity of the HCD's IN bounce buffer */
  134. unsigned int armed_len; /* length the current IN qTD was armed with */
  135. unsigned int toggle;
  136. unsigned int arm_ms;
  137. int armed;
  138. int sched;
  139. struct ehci_ctrl *owner;
  140. };
  141. struct ehci_int {
  142. struct ehci_qh *qh;
  143. struct ehci_qtd *qtd;
  144. unsigned char *buf;
  145. unsigned int mps;
  146. unsigned int buf_len;
  147. unsigned int toggle;
  148. int active;
  149. struct ehci_ctrl *owner;
  150. };
  151. struct ehci_ctrl {
  152. unsigned int base; /* kernel mapping of the register BAR */
  153. unsigned int phys; /* the BAR itself (reported by lsusb) */
  154. unsigned int cap_len;
  155. unsigned int bus, dev, func;
  156. unsigned int n_ports;
  157. int ppc;
  158. int last_port;
  159. unsigned int cc_ports; /* ports handed to a companion (bitmask) */
  160. unsigned int hcs;
  161. unsigned int hcc;
  162. unsigned int frame_list;
  163. unsigned int desc_page;
  164. unsigned int data_page;
  165. struct ehci_qh *head;
  166. struct ehci_qh *ctl_qh;
  167. struct ehci_qh *bin_qh;
  168. struct ehci_qh *bout_qh;
  169. struct ehci_qh *int_qh[USB_HCD_INT_SLOTS];
  170. struct ehci_qtd *ctl_qtd[3];
  171. struct ehci_qtd *bin_qtd;
  172. struct ehci_qtd *bout_qtd;
  173. struct ehci_qtd *bout_zqtd;
  174. struct ehci_qtd *int_qtd[USB_HCD_INT_SLOTS];
  175. unsigned char *setup;
  176. unsigned char *rx_buf;
  177. unsigned char *int_buf[USB_HCD_INT_SLOTS];
  178. struct ehci_bulk bin;
  179. struct ehci_bulk bout;
  180. struct ehci_int ints[USB_HCD_INT_SLOTS];
  181. int ctl_busy;
  182. int recovering;
  183. int sched_on;
  184. int sched_refs;
  185. unsigned int last_async_ms; /* last async transfer, for idle IAA */
  186. unsigned char addr_port[128];
  187. };
  188. static struct ehci_ctrl ctrls[USB_HCD_MAX_CTRL];
  189. static unsigned int n_ctrls;
  190. static struct ehci_int *slot_int[USB_HCD_INT_SLOTS];
  191. static int ehci_start(struct ehci_ctrl *c);
  192. /* ---- debug ------------------------------------------------------------- */
  193. #if EHCI_DEBUG
  194. static void dbg(const char *s)
  195. {
  196. serial_puts(SERIAL_COM1, s);
  197. }
  198. #else
  199. static void dbg(const char *s) { (void)s; }
  200. #endif
  201. static unsigned int rd32(unsigned int addr)
  202. {
  203. return *(volatile unsigned int *)addr;
  204. }
  205. static void wr32(unsigned int addr, unsigned int val)
  206. {
  207. *(volatile unsigned int *)addr = val;
  208. }
  209. static unsigned int op_reg(struct ehci_ctrl *c, unsigned int off)
  210. {
  211. return c->base + c->cap_len + off;
  212. }
  213. /* The asynchronous schedule is only enabled while a transfer or an armed
  214. * interrupt endpoint needs it. With it off the controller stops fetching
  215. * QHs and qTDs, which lets the package reach deep C-states while the
  216. * machine is idle; the port hotplug watcher only reads PORTSC and does not
  217. * need the schedule. */
  218. static void ehci_sched_set(struct ehci_ctrl *c, int on)
  219. {
  220. unsigned int v, t0;
  221. if (on == c->sched_on) {
  222. /* The software flag can be stale if the controller lost ASE
  223. * (an error or an unexpected reset); re-check the hardware
  224. * before assuming the async schedule is running. */
  225. if (!on || (rd32(op_reg(c, EHCI_OP_USBCMD)) & CMD_ASE))
  226. return;
  227. }
  228. v = rd32(op_reg(c, EHCI_OP_USBCMD));
  229. if (on) {
  230. wr32(op_reg(c, EHCI_OP_USBCMD), v | CMD_ASE);
  231. t0 = get_ticks_ms();
  232. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS)) {
  233. if (get_ticks_ms() - t0 > 10)
  234. break;
  235. }
  236. } else {
  237. /* Clear ASE and let the controller retire the current
  238. * microframe; a pending IAA status must be acknowledged or the
  239. * next enable can stall. */
  240. wr32(op_reg(c, EHCI_OP_USBCMD), v & ~CMD_ASE);
  241. t0 = get_ticks_ms();
  242. while (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS) {
  243. if (get_ticks_ms() - t0 > 10)
  244. break;
  245. }
  246. wr32(op_reg(c, EHCI_OP_USBSTS), STS_IAA);
  247. }
  248. c->sched_on = on;
  249. }
  250. static void ehci_sched_get(struct ehci_ctrl *c)
  251. {
  252. c->sched_refs++;
  253. ehci_sched_set(c, 1);
  254. }
  255. static void ehci_sched_put(struct ehci_ctrl *c)
  256. {
  257. if (c->sched_refs > 0)
  258. c->sched_refs--;
  259. if (!c->sched_refs)
  260. ehci_sched_set(c, 0);
  261. }
  262. static unsigned int port_reg(struct ehci_ctrl *c, unsigned int port)
  263. {
  264. return c->base + c->cap_len + EHCI_OP_PORTSC + port * 4u;
  265. }
  266. static void delay_ms(unsigned int ms)
  267. {
  268. usb_delay_ms(ms); /* sleeps inside a sleepable scan */
  269. }
  270. static void zero_region(unsigned int addr, unsigned int len)
  271. {
  272. unsigned int *p = (unsigned int *)addr;
  273. while (len >= 4) {
  274. *p++ = 0;
  275. len -= 4;
  276. }
  277. }
  278. static void zero_ctrl(struct ehci_ctrl *c)
  279. {
  280. unsigned char *p = (unsigned char *)c;
  281. unsigned int i;
  282. for (i = 0; i < sizeof(*c); i++)
  283. p[i] = 0;
  284. }
  285. /* ---- descriptor helpers ------------------------------------------------ */
  286. static void qh_init(struct ehci_qh *qh)
  287. {
  288. unsigned int i;
  289. qh->info1 = 0;
  290. qh->info2 = 0;
  291. qh->current = 0;
  292. qh->qtd_next = QH_LIST_END;
  293. qh->alt_next = QH_LIST_END;
  294. qh->token = QTD_HALT;
  295. for (i = 0; i < 5; i++) {
  296. qh->buf[i] = 0;
  297. qh->buf_hi[i] = 0;
  298. }
  299. }
  300. static void qh_configure(struct ehci_qh *qh, unsigned int addr,
  301. unsigned int ep, unsigned int mps, int control)
  302. {
  303. qh->info1 = QH_DEVADDR(addr) | QH_ENDPT(ep) | QH_EPS_HIGH |
  304. QH_MPS(mps) | QH_RL(4);
  305. if (control)
  306. qh->info1 |= QH_CTRL_EP | QH_DTC;
  307. qh->info2 = QH_MULT(1);
  308. }
  309. static void qh_arm(struct ehci_qh *qh, struct ehci_qtd *qtd)
  310. {
  311. __asm__ volatile ("" ::: "memory");
  312. qh->qtd_next = (unsigned int)qtd;
  313. qh->alt_next = QH_LIST_END;
  314. qh->token &= (QTD_TOGGLE | QTD_PING);
  315. __asm__ volatile ("" ::: "memory");
  316. }
  317. static void qh_disarm(struct ehci_qh *qh)
  318. {
  319. qh->qtd_next = QH_LIST_END;
  320. qh->token = (qh->token & (QTD_TOGGLE | QTD_PING)) | QTD_HALT;
  321. __asm__ volatile ("" ::: "memory");
  322. }
  323. /* Fill one qTD. Up to five 4 KB pages are described (all current callers
  324. are well below that); the C_PAGE field starts at zero. */
  325. static void qtd_fill(struct ehci_qtd *qtd, const void *buf, unsigned int len,
  326. unsigned int pid, int toggle)
  327. {
  328. unsigned int addr = (unsigned int)buf;
  329. unsigned int left, chunk, i;
  330. qtd->next = QH_LIST_END;
  331. qtd->alt_next = QH_LIST_END;
  332. for (i = 0; i < 5; i++) {
  333. qtd->buf[i] = 0;
  334. qtd->buf_hi[i] = 0;
  335. }
  336. qtd->token = ((len & 0x7FFFu) << 16) |
  337. ((pid & 3u) << 8) | QTD_CERR_3 | QTD_ACTIVE |
  338. (toggle ? QTD_TOGGLE : 0);
  339. if (!len)
  340. return;
  341. qtd->buf[0] = addr;
  342. left = len;
  343. chunk = 0x1000u - (addr & 0xFFFu);
  344. if (chunk > left)
  345. chunk = left;
  346. left -= chunk;
  347. addr += chunk;
  348. i = 1;
  349. while (left && i < 5) {
  350. qtd->buf[i++] = addr & 0xFFFFF000u;
  351. chunk = (left > 0x1000u) ? 0x1000u : left;
  352. left -= chunk;
  353. addr += chunk;
  354. }
  355. }
  356. static int qtd_error(unsigned int token)
  357. {
  358. return (token & QTD_ERRORS) != 0;
  359. }
  360. /* Wait for the qTD to retire. If early is non-NULL and has halted, stop
  361. waiting immediately (a control data-stage STALL leaves the status qTD
  362. armed forever). */
  363. static int qtd_wait(struct ehci_qtd *qtd, struct ehci_qtd *early,
  364. unsigned int ms)
  365. {
  366. unsigned int t0 = get_ticks_ms();
  367. while (qtd->token & QTD_ACTIVE) {
  368. if (early && !(early->token & QTD_ACTIVE) &&
  369. qtd_error(early->token))
  370. return -1;
  371. if (get_ticks_ms() - t0 >= ms)
  372. return -1;
  373. usb_poll_relax(t0);
  374. __asm__ volatile ("pause");
  375. }
  376. return 0;
  377. }
  378. /* ---- bulk endpoints ---------------------------------------------------- */
  379. static void bulk_stop(struct ehci_bulk *b)
  380. {
  381. if (b->qh)
  382. qh_disarm(b->qh);
  383. if (b->qtd)
  384. b->qtd->token = 0;
  385. if (b->zqtd)
  386. b->zqtd->token = 0;
  387. b->armed = 0;
  388. if (b->sched && b->owner) {
  389. ehci_sched_put(b->owner);
  390. b->sched = 0;
  391. }
  392. }
  393. static void bulk_reset(struct ehci_bulk *b)
  394. {
  395. bulk_stop(b);
  396. b->addr = 0xFFFFFFFFu;
  397. b->ep = 0;
  398. b->mps = 0;
  399. b->buf_len = 0;
  400. b->armed_len = 0;
  401. b->toggle = 0;
  402. }
  403. static void bulk_configure(struct ehci_bulk *b, unsigned int addr,
  404. unsigned int ep, unsigned int mps,
  405. unsigned int buf_len)
  406. {
  407. if (b->addr == addr && b->ep == ep && b->mps == mps)
  408. return;
  409. bulk_stop(b);
  410. b->addr = addr;
  411. b->ep = ep;
  412. b->mps = mps;
  413. b->buf_len = buf_len;
  414. b->armed_len = 0;
  415. b->toggle = 0; /* first packet is DATA0 */
  416. qh_init(b->qh); /* keeps the link pointer out of it */
  417. qh_configure(b->qh, addr, ep, mps, 0);
  418. b->qh->token = 0; /* DATA0, not halted */
  419. b->armed = 0;
  420. }
  421. /* Ask the controller to release the async-schedule state it has cached.
  422. * NVIDIA/MCP79 can keep a halted QH's old qTD cached and never fetch the
  423. * re-armed one, so the first transfer after an idle gap (a USB disk's first
  424. * read after enumeration) stays ACTIVE and times out forever. Linux and
  425. * U-Boot work around this with the IAAD doorbell. */
  426. static void ehci_iaa_wait(struct ehci_ctrl *c)
  427. {
  428. unsigned int v, t0;
  429. if (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT) {
  430. serial_puts(SERIAL_COM1, "ehci: controller halted\n");
  431. return;
  432. }
  433. /* Drop a stale acknowledgement before ringing the doorbell. */
  434. wr32(op_reg(c, EHCI_OP_USBSTS), STS_IAA);
  435. v = rd32(op_reg(c, EHCI_OP_USBCMD));
  436. wr32(op_reg(c, EHCI_OP_USBCMD), v | CMD_IAAD);
  437. t0 = get_ticks_ms();
  438. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_IAA)) {
  439. if (get_ticks_ms() - t0 > 10) {
  440. serial_puts(SERIAL_COM1, "ehci: IAA timeout\n");
  441. break;
  442. }
  443. usb_poll_relax(t0);
  444. }
  445. wr32(op_reg(c, EHCI_OP_USBSTS), STS_IAA);
  446. }
  447. static void bulk_arm(struct ehci_bulk *b)
  448. {
  449. unsigned int now;
  450. if (!b->sched && b->owner) {
  451. ehci_sched_get(b->owner);
  452. b->sched = 1;
  453. }
  454. now = get_ticks_ms();
  455. if (b->owner && b->owner->last_async_ms &&
  456. now - b->owner->last_async_ms >= 100)
  457. ehci_iaa_wait(b->owner);
  458. if (b->owner)
  459. b->owner->last_async_ms = now;
  460. b->qh->token = (b->qh->token & QTD_PING) |
  461. (b->toggle ? QTD_TOGGLE : 0);
  462. qh_arm(b->qh, b->qtd);
  463. b->armed = 1;
  464. b->arm_ms = now;
  465. }
  466. static int ehci_bulk_out(unsigned int ctrl, unsigned int addr,
  467. unsigned int ep, unsigned int mps,
  468. const unsigned char *buf, unsigned int len)
  469. {
  470. struct ehci_ctrl *c;
  471. struct ehci_bulk *b;
  472. struct ehci_qtd *qtd, *zqtd, *last;
  473. unsigned int token, ztoken = 0;
  474. int zlp;
  475. if (ctrl >= n_ctrls || !buf || !len || len > 0x7FFFu)
  476. return -1;
  477. c = &ctrls[ctrl];
  478. b = &c->bout;
  479. if (!b->qh || !b->qtd || !b->zqtd)
  480. return -1;
  481. if (mps < 1)
  482. mps = 512;
  483. if (mps > 1024)
  484. mps = 1024;
  485. bulk_configure(b, addr, ep, mps, 0);
  486. bulk_stop(b);
  487. qtd = b->qtd;
  488. zqtd = b->zqtd;
  489. qtd_fill(qtd, buf, len, QTD_PID_OUT, 0);
  490. /* A bulk transfer that ends on a max-packet boundary needs a
  491. zero-length packet to mark the end (USB 2.0 5.8.3). */
  492. zlp = (len % mps) == 0;
  493. if (zlp) {
  494. qtd_fill(zqtd, 0, 0, QTD_PID_OUT, 0);
  495. qtd->next = (unsigned int)zqtd;
  496. last = zqtd;
  497. } else {
  498. last = qtd;
  499. }
  500. bulk_arm(b);
  501. if (qtd_wait(last, qtd, 2000) < 0) {
  502. bulk_reset(b);
  503. return -1;
  504. }
  505. token = qtd->token;
  506. if (zlp)
  507. ztoken = last->token;
  508. bulk_stop(b);
  509. if ((token & QTD_ACTIVE) || qtd_error(token) ||
  510. (zlp && ((ztoken & QTD_ACTIVE) || qtd_error(ztoken)))) {
  511. /* Clear the data toggle too: a failed transfer may have
  512. * advanced it or left the endpoint out of sync. */
  513. bulk_reset(b);
  514. return -1;
  515. }
  516. {
  517. unsigned int packets = (len + mps - 1) / mps;
  518. if (zlp)
  519. packets++;
  520. b->toggle ^= packets & 1u;
  521. }
  522. return 0;
  523. }
  524. /* One page of bounce buffer; the caller's buffer stays stable while the
  525. completed transfer is consumed (see usb_hcd.h). */
  526. #define EHCI_RX_MAX 2048u
  527. static int ehci_bulk_in(unsigned int ctrl, unsigned int addr,
  528. unsigned int ep, unsigned int mps,
  529. unsigned char *buf, unsigned int len)
  530. {
  531. struct ehci_ctrl *c;
  532. struct ehci_bulk *b;
  533. unsigned int token, raw, packets, actual, i, alen;
  534. if (ctrl >= n_ctrls || !buf || !len)
  535. return -1;
  536. c = &ctrls[ctrl];
  537. b = &c->bin;
  538. if (!b->qh || !b->qtd || !c->rx_buf)
  539. return -1;
  540. if (mps < 1)
  541. mps = 512;
  542. if (mps > 1024)
  543. mps = 1024;
  544. bulk_configure(b, addr, ep, mps, EHCI_RX_MAX);
  545. if (!b->armed) {
  546. /* Arm exactly the length this call is waiting for. A device
  547. * that ends a transfer which is an exact multiple of the max
  548. * packet size without a zero-length packet leaves a larger
  549. * qTD active forever (the ION's SanDisk does this for every
  550. * 512-byte sector read). Do not pre-arm the next transfer:
  551. * the next call's length can be smaller, because a BOT data
  552. * stage's final chunk need not fill the whole bounce buffer. */
  553. alen = len > EHCI_RX_MAX ? EHCI_RX_MAX : len;
  554. b->armed_len = alen;
  555. qtd_fill(b->qtd, c->rx_buf, alen, QTD_PID_IN, 0);
  556. bulk_arm(b);
  557. return 0;
  558. }
  559. token = b->qtd->token;
  560. if (token & QTD_ACTIVE)
  561. return 0;
  562. b->armed = 0;
  563. if (qtd_error(token)) {
  564. /* A qTD error also leaves the endpoint's data toggle
  565. * unreliable; drop the state so the next transfer starts at
  566. * DATA0 with a fresh QH/qTD. */
  567. bulk_reset(b);
  568. return -1;
  569. }
  570. raw = QTD_LEN(token) <= b->armed_len ?
  571. b->armed_len - QTD_LEN(token) : 0;
  572. actual = raw;
  573. if (actual > len)
  574. actual = len;
  575. for (i = 0; i < actual; i++)
  576. buf[i] = c->rx_buf[i];
  577. /* A bulk IN transfer ends with a short packet; a zero-length
  578. packet also counts as one successful transaction. */
  579. packets = raw ? (raw + b->mps - 1) / b->mps : 1;
  580. b->toggle ^= packets & 1u;
  581. qh_disarm(b->qh);
  582. return (int)actual;
  583. }
  584. /* Cancel a bulk transfer and clear the endpoint state, including the data
  585. * toggle. A timed-out bulk-IN qTD otherwise stays armed forever and every
  586. * later transfer sees it as busy; a BOT reset also leaves the device at
  587. * DATA0 while the host would keep its old toggle. */
  588. static void ehci_bulk_reset(unsigned int ctrl, unsigned int addr,
  589. unsigned int ep, int dir_in)
  590. {
  591. struct ehci_ctrl *c;
  592. struct ehci_bulk *b;
  593. if (ctrl >= n_ctrls)
  594. return;
  595. c = &ctrls[ctrl];
  596. b = dir_in ? &c->bin : &c->bout;
  597. if (b->addr == addr && b->ep == ep) {
  598. unsigned int token = b->qtd->token;
  599. if ((token & QTD_ACTIVE) &&
  600. get_ticks_ms() - b->arm_ms > 100) {
  601. unsigned int port = b->addr < 128 ?
  602. c->addr_port[b->addr] : 0xFF;
  603. unsigned int ps = 0;
  604. if (port < c->n_ports)
  605. ps = rd32(port_reg(c, port));
  606. serial_puts(SERIAL_COM1, "ehci: stuck bulk ");
  607. serial_puts(SERIAL_COM1, dir_in ? "in" : "out");
  608. serial_puts(SERIAL_COM1, " sts ");
  609. serial_puthex(SERIAL_COM1,
  610. rd32(op_reg(c, EHCI_OP_USBSTS)));
  611. serial_puts(SERIAL_COM1, " cmd ");
  612. serial_puthex(SERIAL_COM1,
  613. rd32(op_reg(c, EHCI_OP_USBCMD)));
  614. serial_puts(SERIAL_COM1, " qtd ");
  615. serial_puthex(SERIAL_COM1, token);
  616. serial_puts(SERIAL_COM1, " qh ");
  617. serial_puthex(SERIAL_COM1, b->qh->token);
  618. serial_puts(SERIAL_COM1, " cur ");
  619. serial_puthex(SERIAL_COM1, b->qh->current);
  620. serial_puts(SERIAL_COM1, " port ");
  621. serial_putdec(SERIAL_COM1, port + 1);
  622. serial_puts(SERIAL_COM1, " ps ");
  623. serial_puthex(SERIAL_COM1, ps);
  624. serial_puts(SERIAL_COM1, " addr ");
  625. serial_putdec(SERIAL_COM1, b->addr);
  626. serial_puts(SERIAL_COM1, " ep ");
  627. serial_putdec(SERIAL_COM1, b->ep);
  628. serial_puts(SERIAL_COM1, " tgl ");
  629. serial_putdec(SERIAL_COM1, b->toggle);
  630. serial_putc(SERIAL_COM1, '\n');
  631. }
  632. if (c->sched_on && (token & QTD_ACTIVE))
  633. ehci_iaa_wait(c);
  634. bulk_reset(b);
  635. }
  636. }
  637. /* ---- interrupt-IN endpoints ------------------------------------------- */
  638. static void int_arm(struct ehci_int *in)
  639. {
  640. in->qh->token = (in->qh->token & QTD_PING) |
  641. (in->toggle ? QTD_TOGGLE : 0);
  642. qh_arm(in->qh, in->qtd);
  643. }
  644. static void ehci_int_in_stop(unsigned int slot)
  645. {
  646. struct ehci_int *in;
  647. if (slot >= USB_HCD_INT_SLOTS)
  648. return;
  649. in = slot_int[slot];
  650. if (!in)
  651. return;
  652. {
  653. int was_active = in->active;
  654. in->active = 0;
  655. if (in->qh)
  656. qh_disarm(in->qh);
  657. if (in->qtd)
  658. in->qtd->token = 0;
  659. slot_int[slot] = 0;
  660. if (was_active && in->owner)
  661. ehci_sched_put(in->owner);
  662. }
  663. }
  664. static int ehci_int_in_start(unsigned int ctrl, unsigned int slot,
  665. unsigned int addr, unsigned int ep,
  666. unsigned int mps, int low_speed)
  667. {
  668. struct ehci_ctrl *c;
  669. struct ehci_int *in;
  670. if (ctrl >= n_ctrls || slot >= USB_HCD_INT_SLOTS || low_speed)
  671. return -1;
  672. c = &ctrls[ctrl];
  673. in = &c->ints[slot];
  674. if (!in->qh || !in->qtd || !in->buf)
  675. return -1;
  676. ehci_int_in_stop(slot);
  677. in->mps = mps ? mps : 8;
  678. if (in->mps > 1024)
  679. in->mps = 1024;
  680. in->buf_len = in->mps > 64 ? 64 : in->mps;
  681. qh_init(in->qh);
  682. in->qh->info1 = QH_DEVADDR(addr) | QH_ENDPT(ep) | QH_EPS_HIGH |
  683. QH_MPS(in->mps) | QH_RL(4);
  684. in->qh->info2 = QH_MULT(1);
  685. in->qh->token = 0;
  686. in->toggle = 0; /* first report is DATA0 */
  687. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  688. int_arm(in);
  689. in->active = 1;
  690. slot_int[slot] = in;
  691. ehci_sched_get(c);
  692. return 0;
  693. }
  694. static int ehci_int_in_poll(unsigned int slot, unsigned char *buf,
  695. unsigned int len)
  696. {
  697. struct ehci_int *in;
  698. unsigned int token, raw, packets, actual, i;
  699. if (slot >= USB_HCD_INT_SLOTS)
  700. return 0;
  701. in = slot_int[slot];
  702. if (!in || !in->active)
  703. return 0;
  704. token = in->qtd->token;
  705. if (token & QTD_ACTIVE)
  706. return 0;
  707. if (qtd_error(token)) {
  708. qh_disarm(in->qh);
  709. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  710. int_arm(in);
  711. return -1;
  712. }
  713. raw = QTD_LEN(token) <= in->buf_len ?
  714. in->buf_len - QTD_LEN(token) : 0;
  715. actual = raw;
  716. if (actual > len)
  717. actual = len;
  718. for (i = 0; i < actual; i++)
  719. buf[i] = in->buf[i];
  720. packets = raw ? (raw + in->mps - 1) / in->mps : 1;
  721. in->toggle ^= packets & 1u;
  722. qh_disarm(in->qh);
  723. qtd_fill(in->qtd, in->buf, in->buf_len, QTD_PID_IN, 0);
  724. int_arm(in);
  725. return (int)actual;
  726. }
  727. /* ---- control transfers ------------------------------------------------- */
  728. static int ehci_control(struct ehci_ctrl *c, unsigned int addr,
  729. unsigned int mps, int low_speed,
  730. const unsigned char *setup, unsigned char *data,
  731. unsigned int len, int dir_in)
  732. {
  733. struct ehci_qh *qh;
  734. struct ehci_qtd *q0, *q1, *q2, *last, *early;
  735. unsigned int i, actual = 0, residual;
  736. (void)mps;
  737. if (!c->ctl_qh || !c->setup || !setup || c->ctl_busy || low_speed)
  738. return -1;
  739. if (len && !data)
  740. return -1;
  741. qh = c->ctl_qh;
  742. q0 = c->ctl_qtd[0];
  743. q1 = c->ctl_qtd[1];
  744. q2 = c->ctl_qtd[2];
  745. ehci_sched_get(c);
  746. c->ctl_busy = 1;
  747. {
  748. unsigned int now = get_ticks_ms();
  749. if (c->last_async_ms && now - c->last_async_ms >= 100)
  750. ehci_iaa_wait(c);
  751. c->last_async_ms = now;
  752. }
  753. for (i = 0; i < 8; i++)
  754. c->setup[i] = setup[i];
  755. qh_init(qh);
  756. qh_configure(qh, addr, 0, 64, 1);
  757. qh->token = 0; /* control toggles come from the qTDs */
  758. qtd_fill(q0, c->setup, 8, QTD_PID_SETUP, 0);
  759. if (len) {
  760. qtd_fill(q1, data, len,
  761. dir_in ? QTD_PID_IN : QTD_PID_OUT, 1);
  762. qtd_fill(q2, 0, 0,
  763. dir_in ? QTD_PID_OUT : QTD_PID_IN, 1);
  764. q0->next = (unsigned int)q1;
  765. q1->next = (unsigned int)q2;
  766. last = q2;
  767. early = q1;
  768. } else {
  769. qtd_fill(q2, 0, 0, QTD_PID_IN, 1);
  770. q0->next = (unsigned int)q2;
  771. last = q2;
  772. early = q0;
  773. }
  774. q2->next = QH_LIST_END;
  775. qh_arm(qh, q0);
  776. if (qtd_wait(last, early, 2000) < 0) {
  777. qh_disarm(qh);
  778. delay_ms(1);
  779. c->ctl_busy = 0;
  780. ehci_sched_put(c);
  781. return -1;
  782. }
  783. qh_disarm(qh);
  784. if (qtd_error(q0->token) || qtd_error(q2->token) ||
  785. (len && qtd_error(q1->token))) {
  786. c->ctl_busy = 0;
  787. ehci_sched_put(c);
  788. return -1;
  789. }
  790. if (len && dir_in) {
  791. residual = QTD_LEN(q1->token);
  792. actual = residual >= len ? 0 : len - residual;
  793. } else if (len) {
  794. actual = len;
  795. }
  796. /* The device changes address after the status stage. Remember which
  797. root port it came from so a port reset can drop this controller's
  798. bulk state without touching an unrelated port. */
  799. if (addr == 0 && setup[0] == 0x00 && setup[1] == 0x05) {
  800. unsigned int new_addr = (unsigned int)setup[2] |
  801. ((unsigned int)setup[3] << 8);
  802. if (new_addr > 0 && new_addr < 128 && c->last_port >= 0)
  803. c->addr_port[new_addr] = (unsigned char)c->last_port;
  804. }
  805. c->ctl_busy = 0;
  806. ehci_sched_put(c);
  807. return (int)actual;
  808. }
  809. static int ehci_control_transfer(unsigned int ctrl, unsigned int addr,
  810. unsigned int mps, int low_speed,
  811. const unsigned char *setup,
  812. unsigned char *data, unsigned int len,
  813. int dir_in)
  814. {
  815. if (ctrl >= n_ctrls)
  816. return -1;
  817. return ehci_control(&ctrls[ctrl], addr, mps, low_speed, setup, data,
  818. len, dir_in);
  819. }
  820. /* ---- root hub ports ---------------------------------------------------- */
  821. static void port_endpoints_reset(struct ehci_ctrl *c, unsigned int port)
  822. {
  823. unsigned int i;
  824. if (c->bin.addr < 128 && c->addr_port[c->bin.addr] == port)
  825. bulk_reset(&c->bin);
  826. if (c->bout.addr < 128 && c->addr_port[c->bout.addr] == port)
  827. bulk_reset(&c->bout);
  828. for (i = 0; i < 128; i++)
  829. if (c->addr_port[i] == port)
  830. c->addr_port[i] = 0xFF;
  831. }
  832. static unsigned int ehci_port_count(unsigned int ctrl)
  833. {
  834. return ctrl < n_ctrls ? ctrls[ctrl].n_ports : 0;
  835. }
  836. static int ehci_port_connected(unsigned int ctrl, unsigned int port)
  837. {
  838. struct ehci_ctrl *c;
  839. unsigned int ps;
  840. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  841. return 0;
  842. c = &ctrls[ctrl];
  843. ps = rd32(port_reg(c, port));
  844. return (ps & PORT_CONNECT) ? 1 : 0;
  845. }
  846. static int ehci_port_low_speed(unsigned int ctrl, unsigned int port)
  847. {
  848. (void)ctrl;
  849. (void)port;
  850. return 0; /* EHCI only drives high-speed devices */
  851. }
  852. static int ehci_port_take_change(unsigned int ctrl, unsigned int port)
  853. {
  854. struct ehci_ctrl *c;
  855. unsigned int ps;
  856. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  857. return 0;
  858. c = &ctrls[ctrl];
  859. ps = rd32(port_reg(c, port));
  860. if (!(ps & PORT_CSC))
  861. return 0;
  862. wr32(port_reg(c, port),
  863. (ps & ~PORT_RWC) | PORT_CSC | PORT_PEC | PORT_OCC);
  864. return 1;
  865. }
  866. /* HCSPARAMS N_CC: the number of companion controllers. */
  867. static unsigned int n_companions(const struct ehci_ctrl *c)
  868. {
  869. return (c->hcs >> 12) & 0x0Fu;
  870. }
  871. /* Give a non-high-speed device to the companion controller. */
  872. static void port_to_companion(struct ehci_ctrl *c, unsigned int port,
  873. unsigned int ps)
  874. {
  875. wr32(port_reg(c, port),
  876. (ps & ~(PORT_RWC | PORT_PE | PORT_RESET)) | PORT_OWNER |
  877. PORT_CSC | PORT_PEC | PORT_OCC);
  878. if (!(c->cc_ports & (1u << port))) {
  879. c->cc_ports |= 1u << port;
  880. serial_puts(SERIAL_COM1, "ehci: port ");
  881. serial_putdec(SERIAL_COM1, port + 1);
  882. serial_puts(SERIAL_COM1, " not high-speed, handed to companion\n");
  883. }
  884. }
  885. static int ehci_port_reset_once(unsigned int ctrl, unsigned int port)
  886. {
  887. struct ehci_ctrl *c;
  888. unsigned int ps, t0, try;
  889. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  890. return -1;
  891. c = &ctrls[ctrl];
  892. /* A halted controller ignores port commands; restart it first. */
  893. if (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT) {
  894. serial_puts(SERIAL_COM1, "ehci: controller halted, restarting\n");
  895. if (ehci_start(c) < 0)
  896. return -1;
  897. }
  898. ps = rd32(port_reg(c, port));
  899. if (!(ps & PORT_CONNECT))
  900. return -1;
  901. /* Drop any reset/suspend state the firmware left behind. The port
  902. can be stuck with PR set; a single clear write is not enough on
  903. some PCHs. */
  904. for (try = 0; try < 10 && (ps & PORT_RESET); try++) {
  905. wr32(port_reg(c, port),
  906. (ps & ~(PORT_RWC | PORT_RESET | PORT_PE |
  907. PORT_SUSPEND | PORT_RESUME)) |
  908. PORT_CSC | PORT_PEC | PORT_OCC);
  909. delay_ms(5);
  910. ps = rd32(port_reg(c, port));
  911. }
  912. if ((ps & PORT_RESET) || !(ps & PORT_CONNECT))
  913. return -2;
  914. /* The port may still be owned by the companion controller. A
  915. low/full-speed device belongs there; a high-speed one can be taken
  916. over by clearing PORT_OWNER (the controller may need more than one
  917. write while the port is enabled). */
  918. /* The hardware returns a port to the EHCI when its device goes away. */
  919. if (!(ps & PORT_OWNER))
  920. c->cc_ports &= ~(1u << port);
  921. if (ps & PORT_OWNER) {
  922. if (PORT_USB11(ps) || (c->cc_ports & (1u << port)))
  923. return -1;
  924. for (try = 0; try < 8 && (ps & PORT_OWNER); try++) {
  925. wr32(port_reg(c, port),
  926. (ps & ~(PORT_RWC | PORT_OWNER | PORT_RESET |
  927. PORT_PE | PORT_SUSPEND | PORT_RESUME)) |
  928. PORT_CSC | PORT_PEC | PORT_OCC);
  929. delay_ms(5);
  930. ps = rd32(port_reg(c, port));
  931. }
  932. if ((ps & PORT_OWNER) || !(ps & PORT_CONNECT))
  933. return -1;
  934. }
  935. /* A low-speed device (K state on an unenabled port) is never reset by
  936. the EHCI: hand it straight to the companion (EHCI 4.2.2). */
  937. if (!(ps & PORT_PE) && PORT_USB11(ps) && n_companions(c)) {
  938. port_to_companion(c, port, ps);
  939. return -1;
  940. }
  941. port_endpoints_reset(c, port);
  942. c->last_port = (int)port;
  943. /* Clear stale change bits, then assert reset. PED is cleared while
  944. reset is asserted (like Linux's SetPortFeature(RESET)). */
  945. wr32(port_reg(c, port),
  946. (ps & ~(PORT_RWC | PORT_PE | PORT_SUSPEND | PORT_RESUME)) |
  947. PORT_CSC | PORT_PEC | PORT_OCC | PORT_RESET);
  948. delay_ms(50);
  949. /* EHCI expects software to end the reset, but some controllers hold
  950. PR set while the device is still chirping. Give it up to 200 ms to
  951. self-clear before forcing the write. */
  952. t0 = get_ticks_ms();
  953. while (get_ticks_ms() - t0 < 200) {
  954. if (!(rd32(port_reg(c, port)) & PORT_RESET))
  955. break;
  956. delay_ms(1);
  957. }
  958. /* Force completion if the controller still holds PR; retry because
  959. the first write can land while the reset is still in progress. */
  960. for (try = 0; try < 20; try++) {
  961. ps = rd32(port_reg(c, port));
  962. if (!(ps & PORT_RESET))
  963. break;
  964. wr32(port_reg(c, port),
  965. (ps & ~(PORT_RWC | PORT_RESET | PORT_PE |
  966. PORT_SUSPEND | PORT_RESUME)) |
  967. PORT_CSC | PORT_PEC | PORT_OCC);
  968. delay_ms(10);
  969. }
  970. if (ps & PORT_RESET) {
  971. serial_puts(SERIAL_COM1, "ehci: reset stuck ");
  972. serial_putdec(SERIAL_COM1, c->bus);
  973. serial_putc(SERIAL_COM1, ':');
  974. serial_putdec(SERIAL_COM1, c->dev);
  975. serial_putc(SERIAL_COM1, '.');
  976. serial_putdec(SERIAL_COM1, c->func);
  977. serial_puts(SERIAL_COM1, " port ");
  978. serial_putdec(SERIAL_COM1, port + 1);
  979. serial_puts(SERIAL_COM1, " ps ");
  980. serial_puthex(SERIAL_COM1, ps);
  981. serial_putc(SERIAL_COM1, '\n');
  982. return -2;
  983. }
  984. t0 = get_ticks_ms();
  985. while (!(rd32(port_reg(c, port)) & PORT_PE)) {
  986. if (get_ticks_ms() - t0 > 200)
  987. break;
  988. usb_poll_relax(t0);
  989. }
  990. ps = rd32(port_reg(c, port));
  991. if (!(ps & PORT_PE)) {
  992. if (ps & PORT_OWNER)
  993. return -1;
  994. /* Not enabled after the reset: the device is not high-speed
  995. (EHCI 4.2.2), whatever the line state reads -- a full-speed
  996. receiver on the MCP79 reports SE0 here. It belongs to the
  997. companion. Only a controller without companions (Intel PCH
  998. rate-matching hubs) can have a genuinely stuck HS reset. */
  999. if (n_companions(c) || PORT_USB11(ps)) {
  1000. port_to_companion(c, port, ps);
  1001. return -1;
  1002. }
  1003. serial_puts(SERIAL_COM1, "ehci: reset no PED ");
  1004. serial_puthex(SERIAL_COM1, ps);
  1005. serial_putc(SERIAL_COM1, '\n');
  1006. return -2;
  1007. }
  1008. if (ps & PORT_OWNER)
  1009. return -1;
  1010. return 0;
  1011. }
  1012. /* A stuck HS port reset on some Intel PCHs only completes after the whole
  1013. EHCI controller is reset. Retry the port reset once in that case. */
  1014. static int ehci_port_reset(unsigned int ctrl, unsigned int port)
  1015. {
  1016. struct ehci_ctrl *c;
  1017. int r;
  1018. if (ctrl >= n_ctrls || port >= ctrls[ctrl].n_ports)
  1019. return -1;
  1020. c = &ctrls[ctrl];
  1021. r = ehci_port_reset_once(ctrl, port);
  1022. if (r == -2 && !c->recovering) {
  1023. c->recovering = 1;
  1024. serial_puts(SERIAL_COM1,
  1025. "ehci: port reset retry after controller reset\n");
  1026. if (ehci_start(c) == 0)
  1027. r = ehci_port_reset_once(ctrl, port);
  1028. else
  1029. r = -1;
  1030. c->recovering = 0;
  1031. }
  1032. return r < 0 ? -1 : 0;
  1033. }
  1034. /* ---- schedule / controller setup -------------------------------------- */
  1035. static int ehci_sched_init(struct ehci_ctrl *c)
  1036. {
  1037. unsigned char *desc = (unsigned char *)c->desc_page;
  1038. unsigned int qtd_base = c->desc_page + EHCI_MAX_QH * EHCI_QH_STRIDE;
  1039. struct ehci_qh *list[EHCI_MAX_QH];
  1040. unsigned int i;
  1041. c->head = (struct ehci_qh *)(desc + 0 * EHCI_QH_STRIDE);
  1042. c->ctl_qh = (struct ehci_qh *)(desc + 1 * EHCI_QH_STRIDE);
  1043. c->bin_qh = (struct ehci_qh *)(desc + 2 * EHCI_QH_STRIDE);
  1044. c->bout_qh = (struct ehci_qh *)(desc + 3 * EHCI_QH_STRIDE);
  1045. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  1046. c->int_qh[i] = (struct ehci_qh *)(desc +
  1047. (4 + i) * EHCI_QH_STRIDE);
  1048. c->ctl_qtd[0] = (struct ehci_qtd *)(qtd_base + 0 * EHCI_QTD_STRIDE);
  1049. c->ctl_qtd[1] = (struct ehci_qtd *)(qtd_base + 1 * EHCI_QTD_STRIDE);
  1050. c->ctl_qtd[2] = (struct ehci_qtd *)(qtd_base + 2 * EHCI_QTD_STRIDE);
  1051. c->bin_qtd = (struct ehci_qtd *)(qtd_base + 3 * EHCI_QTD_STRIDE);
  1052. c->bout_qtd = (struct ehci_qtd *)(qtd_base + 4 * EHCI_QTD_STRIDE);
  1053. c->bout_zqtd = (struct ehci_qtd *)(qtd_base + 11 * EHCI_QTD_STRIDE);
  1054. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  1055. c->int_qtd[i] = (struct ehci_qtd *)(qtd_base +
  1056. (5 + i) * EHCI_QTD_STRIDE);
  1057. c->rx_buf = (unsigned char *)c->data_page;
  1058. c->setup = (unsigned char *)(c->data_page + 2048);
  1059. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  1060. c->int_buf[i] = (unsigned char *)(c->data_page + 2112 +
  1061. i * 64);
  1062. for (i = 0; i < 1024; i++)
  1063. wr32(c->frame_list + i * 4, QH_LIST_END);
  1064. list[0] = c->head;
  1065. list[1] = c->ctl_qh;
  1066. list[2] = c->bin_qh;
  1067. list[3] = c->bout_qh;
  1068. for (i = 0; i < USB_HCD_INT_SLOTS; i++)
  1069. list[4 + i] = c->int_qh[i];
  1070. for (i = 0; i < EHCI_MAX_QH; i++) {
  1071. qh_init(list[i]);
  1072. list[i]->next = QH_NEXT((unsigned int)
  1073. list[(i + 1) % EHCI_MAX_QH]);
  1074. }
  1075. c->head->info1 = QH_HEAD;
  1076. c->head->token = QTD_HALT;
  1077. c->bin.owner = c;
  1078. c->bin.sched = 0;
  1079. c->bout.owner = c;
  1080. c->bout.sched = 0;
  1081. c->bin.qh = c->bin_qh;
  1082. c->bin.qtd = c->bin_qtd;
  1083. c->bin.zqtd = 0;
  1084. c->bin.addr = 0xFFFFFFFFu;
  1085. c->bout.qh = c->bout_qh;
  1086. c->bout.qtd = c->bout_qtd;
  1087. c->bout.zqtd = c->bout_zqtd;
  1088. c->bout.addr = 0xFFFFFFFFu;
  1089. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  1090. c->ints[i].qh = c->int_qh[i];
  1091. c->ints[i].qtd = c->int_qtd[i];
  1092. c->ints[i].buf = c->int_buf[i];
  1093. c->ints[i].active = 0;
  1094. c->ints[i].owner = c;
  1095. }
  1096. for (i = 0; i < 128; i++)
  1097. c->addr_port[i] = 0xFF;
  1098. c->last_port = -1;
  1099. return 0;
  1100. }
  1101. static void ehci_halt(struct ehci_ctrl *c)
  1102. {
  1103. unsigned int t0;
  1104. wr32(op_reg(c, EHCI_OP_USBINTR), 0);
  1105. wr32(op_reg(c, EHCI_OP_USBCMD), 0);
  1106. t0 = get_ticks_ms();
  1107. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT)) {
  1108. if (get_ticks_ms() - t0 > 100)
  1109. break;
  1110. }
  1111. }
  1112. static int ehci_start(struct ehci_ctrl *c)
  1113. {
  1114. unsigned int v, t0, i;
  1115. /* Stop first, then reset. HCRESET self-clears. */
  1116. ehci_halt(c);
  1117. wr32(op_reg(c, EHCI_OP_USBCMD), CMD_RESET);
  1118. t0 = get_ticks_ms();
  1119. while (rd32(op_reg(c, EHCI_OP_USBCMD)) & CMD_RESET) {
  1120. if (get_ticks_ms() - t0 > 250) {
  1121. dbg("ehci: reset timeout\n");
  1122. return -1;
  1123. }
  1124. }
  1125. delay_ms(2);
  1126. if (c->hcc & 1u) /* 64-bit addressing capable */
  1127. wr32(op_reg(c, EHCI_OP_CTRLDSSEGMENT), 0);
  1128. wr32(op_reg(c, EHCI_OP_PERIODICLIST), c->frame_list);
  1129. wr32(op_reg(c, EHCI_OP_ASYNCLIST), (unsigned int)c->head);
  1130. wr32(op_reg(c, EHCI_OP_USBINTR), 0);
  1131. wr32(op_reg(c, EHCI_OP_USBSTS), 0x3Fu);
  1132. c->sched_on = 0;
  1133. c->sched_refs = 0;
  1134. wr32(op_reg(c, EHCI_OP_USBCMD), CMD_RUN | CMD_ITC_8);
  1135. __asm__ volatile ("" ::: "memory");
  1136. t0 = get_ticks_ms();
  1137. while (rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_HALT) {
  1138. if (get_ticks_ms() - t0 > 100) {
  1139. dbg("ehci: controller did not start\n");
  1140. return -1;
  1141. }
  1142. }
  1143. /* Take the root ports over from the companion controllers. */
  1144. wr32(op_reg(c, EHCI_OP_CONFIGFLAG), 1);
  1145. v = rd32(op_reg(c, EHCI_OP_CONFIGFLAG));
  1146. if (!(v & 1u))
  1147. serial_puts(SERIAL_COM1, "ehci: CONFIGFLAG did not stick\n");
  1148. delay_ms(5);
  1149. if (c->ppc) {
  1150. for (i = 0; i < c->n_ports; i++) {
  1151. v = rd32(port_reg(c, i));
  1152. if (!(v & PORT_POWER))
  1153. wr32(port_reg(c, i),
  1154. (v & ~PORT_RWC) | PORT_POWER);
  1155. }
  1156. delay_ms(100);
  1157. }
  1158. t0 = get_ticks_ms();
  1159. while (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS)) {
  1160. if (get_ticks_ms() - t0 > 100)
  1161. break;
  1162. }
  1163. if (!(rd32(op_reg(c, EHCI_OP_USBSTS)) & STS_ASS))
  1164. dbg("ehci: async schedule not active\n");
  1165. return 0;
  1166. }
  1167. /* Best-effort EHCI legacy-support handshake: stop the firmware's SMIs from
  1168. firing on port changes now that the kernel owns the controller. */
  1169. static void legacy_smi_disable(unsigned int bus, unsigned int dev,
  1170. unsigned int func, unsigned int hcc)
  1171. {
  1172. unsigned int eecp = (hcc >> 8) & 0xFFu;
  1173. unsigned int guard = 0;
  1174. while (eecp >= 0x40 && guard++ < 16) {
  1175. unsigned int cap = pci_cfg_read32(bus, dev, func, eecp);
  1176. unsigned int id = cap & 0xFFu;
  1177. unsigned int next = (cap >> 8) & 0xFFu;
  1178. if (id == 1) { /* EHCI Legacy Support */
  1179. pci_cfg_write32(bus, dev, func, eecp,
  1180. (cap | (1u << 24)) & ~(1u << 31));
  1181. cap = pci_cfg_read32(bus, dev, func, eecp + 4);
  1182. pci_cfg_write32(bus, dev, func, eecp + 4,
  1183. cap & 0xFFFF0000u);
  1184. return;
  1185. }
  1186. if (!next || next == eecp)
  1187. return;
  1188. eecp = next;
  1189. }
  1190. }
  1191. static int ehci_setup_one(unsigned int bus, unsigned int dev,
  1192. unsigned int func, unsigned int bar)
  1193. {
  1194. struct ehci_ctrl *c;
  1195. unsigned int cap_len, hcs, hcc;
  1196. if (n_ctrls >= USB_HCD_MAX_CTRL)
  1197. return -1;
  1198. c = &ctrls[n_ctrls];
  1199. zero_ctrl(c);
  1200. c->phys = bar;
  1201. c->bus = bus;
  1202. c->dev = dev;
  1203. c->func = func;
  1204. bar = (unsigned int)ioremap(bar, 0x1000, IOREMAP_MMIO);
  1205. if (!bar)
  1206. return -1;
  1207. c->base = bar;
  1208. if (rd32(bar) == 0xFFFFFFFFu)
  1209. return -1;
  1210. cap_len = rd32(bar + EHCI_CAP_CAPLENGTH) & 0xFFu;
  1211. if (cap_len < 0x20 || cap_len > 0x100)
  1212. return -1;
  1213. hcs = rd32(bar + EHCI_CAP_HCSPARAMS);
  1214. if (hcs == 0xFFFFFFFFu)
  1215. return -1;
  1216. c->cap_len = cap_len;
  1217. c->n_ports = hcs & 0x0Fu;
  1218. c->ppc = (hcs & 0x10u) ? 1 : 0;
  1219. c->hcs = hcs;
  1220. if (!c->n_ports) {
  1221. dbg("ehci: no root ports\n");
  1222. return -1;
  1223. }
  1224. hcc = rd32(bar + EHCI_CAP_HCCPARAMS);
  1225. c->hcc = hcc;
  1226. legacy_smi_disable(bus, dev, func, hcc);
  1227. c->frame_list = mm_alloc_page();
  1228. c->desc_page = mm_alloc_page();
  1229. c->data_page = mm_alloc_page();
  1230. if (!c->frame_list || !c->desc_page || !c->data_page) {
  1231. dbg("ehci: out of pages\n");
  1232. goto fail;
  1233. }
  1234. zero_region(c->frame_list, 4096);
  1235. zero_region(c->desc_page, 4096);
  1236. zero_region(c->data_page, 4096);
  1237. ehci_sched_init(c);
  1238. if (ehci_start(c) < 0)
  1239. goto fail;
  1240. serial_puts(SERIAL_COM1, "ehci: controller at ");
  1241. serial_putdec(SERIAL_COM1, bus);
  1242. serial_puts(SERIAL_COM1, ":");
  1243. serial_putdec(SERIAL_COM1, dev);
  1244. serial_puts(SERIAL_COM1, ".");
  1245. serial_putdec(SERIAL_COM1, func);
  1246. serial_puts(SERIAL_COM1, " ports ");
  1247. serial_putdec(SERIAL_COM1, c->n_ports);
  1248. serial_puts(SERIAL_COM1, "\n");
  1249. n_ctrls++;
  1250. return 0;
  1251. fail:
  1252. ehci_halt(c);
  1253. /* Give the shared ports back to the companion controllers. */
  1254. wr32(op_reg(c, EHCI_OP_CONFIGFLAG), 0);
  1255. if (c->frame_list)
  1256. mm_free_page(c->frame_list);
  1257. if (c->desc_page)
  1258. mm_free_page(c->desc_page);
  1259. if (c->data_page)
  1260. mm_free_page(c->data_page);
  1261. zero_ctrl(c);
  1262. return -1;
  1263. }
  1264. static int ehci_init(void)
  1265. {
  1266. unsigned int bus, dev, func;
  1267. if (n_ctrls)
  1268. return (int)n_ctrls;
  1269. for (bus = 0; bus < 256 && n_ctrls < USB_HCD_MAX_CTRL; bus++) {
  1270. for (dev = 0; dev < 32; dev++) {
  1271. for (func = 0; func < 8; func++) {
  1272. unsigned int id = pci_cfg_read32(bus, dev, func,
  1273. 0x00);
  1274. unsigned int class_rev, bar, bar_hi, cmd;
  1275. if (id == 0xFFFFFFFFu || id == 0)
  1276. continue;
  1277. class_rev = pci_cfg_read32(bus, dev, func,
  1278. 0x08);
  1279. if ((class_rev >> 16) != 0x0C03u)
  1280. continue;
  1281. if (((class_rev >> 8) & 0xFFu) != 0x20u)
  1282. continue; /* not EHCI */
  1283. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1284. if (bar & 1u)
  1285. continue; /* register block is MMIO */
  1286. bar_hi = 0;
  1287. if (((bar >> 1) & 3u) == 2u)
  1288. bar_hi = pci_cfg_read32(bus, dev, func,
  1289. 0x14);
  1290. bar &= 0xFFFFFFF0u;
  1291. if (!bar || bar_hi)
  1292. continue;
  1293. cmd = pci_cfg_read32(bus, dev, func, 0x04);
  1294. pci_cfg_write32(bus, dev, func, 0x04,
  1295. cmd | 0x0006u);
  1296. ehci_setup_one(bus, dev, func, bar);
  1297. }
  1298. }
  1299. }
  1300. return (int)n_ctrls;
  1301. }
  1302. static unsigned int ehci_count(void)
  1303. {
  1304. return n_ctrls;
  1305. }
  1306. static void ehci_info(unsigned int ctrl, unsigned int *bus, unsigned int *dev,
  1307. unsigned int *func, unsigned int *base, int *mmio)
  1308. {
  1309. struct ehci_ctrl *c = &ctrls[ctrl];
  1310. *bus = c->bus;
  1311. *dev = c->dev;
  1312. *func = c->func;
  1313. *base = c->phys;
  1314. *mmio = 1;
  1315. }
  1316. /* Raw PORTSC for `lsusb`; useful when a port is connected but owned by a
  1317. companion (bit 13) or not powered (bit 12). */
  1318. static void ehci_print_ports(unsigned int ctrl)
  1319. {
  1320. struct ehci_ctrl *c;
  1321. unsigned int i, ps, eecp;
  1322. if (ctrl >= n_ctrls)
  1323. return;
  1324. c = &ctrls[ctrl];
  1325. io_putstr(" CMD 0x");
  1326. io_puthex(rd32(op_reg(c, EHCI_OP_USBCMD)));
  1327. io_putstr(" STS 0x");
  1328. io_puthex(rd32(op_reg(c, EHCI_OP_USBSTS)));
  1329. io_putchar('\n');
  1330. io_putstr(" CF 0x");
  1331. io_puthex(rd32(op_reg(c, EHCI_OP_CONFIGFLAG)));
  1332. io_putstr(" HCC 0x");
  1333. io_puthex(c->hcc);
  1334. io_putstr(" HCS 0x");
  1335. io_puthex(c->hcs);
  1336. io_putchar('\n');
  1337. eecp = (c->hcc >> 8) & 0xFFu;
  1338. if (eecp >= 0x40) {
  1339. unsigned int leg = pci_cfg_read32(c->bus, c->dev, c->func,
  1340. eecp);
  1341. if ((leg & 0xFFu) == 1u) {
  1342. io_putstr(" LEGSUP 0x");
  1343. io_puthex(leg);
  1344. io_putchar('\n');
  1345. }
  1346. }
  1347. for (i = 0; i < c->n_ports; i++) {
  1348. ps = rd32(port_reg(c, i));
  1349. io_putstr(" Port ");
  1350. if (i < 9)
  1351. io_putchar((char)('1' + i));
  1352. else
  1353. io_putchar((char)('A' + i - 9));
  1354. io_putstr(": 0x");
  1355. io_puthex(ps);
  1356. io_putchar('\n');
  1357. }
  1358. }
  1359. const struct usb_hcd_ops usb_ehci_ops = {
  1360. "EHCI",
  1361. ehci_init,
  1362. ehci_count,
  1363. ehci_info,
  1364. ehci_control_transfer,
  1365. ehci_port_count,
  1366. ehci_port_connected,
  1367. ehci_port_low_speed,
  1368. ehci_port_reset,
  1369. ehci_port_take_change,
  1370. ehci_int_in_start,
  1371. ehci_int_in_poll,
  1372. ehci_int_in_stop,
  1373. ehci_bulk_out,
  1374. ehci_bulk_in,
  1375. ehci_bulk_reset,
  1376. ehci_print_ports,
  1377. };