xhci.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495
  1. /* xHCI (USB 3.0) host controller driver.
  2. *
  3. * Scope: one controller, USB2 root ports and devices plugged directly into
  4. * them. The USB3 ports are left alone (devices there need SuperSpeed
  5. * resource/companion-descriptor handling that the HID stack does not need
  6. * yet). Control transfers and one interrupt-IN endpoint per HID interface
  7. * are implemented, which is exactly what usbhid.c uses.
  8. *
  9. * The schedule is polled, like the rest of the kernel. Rings and contexts
  10. * are page-allocated so the driver does not grow the kernel BSS into the
  11. * 0x7F000 wall. */
  12. #include "mm.h"
  13. #include "paging.h"
  14. #include "pci.h"
  15. #include "serial.h"
  16. #include "spinlock.h"
  17. #include "timer.h"
  18. #include "usb_hcd.h"
  19. #include "xhci.h"
  20. #define XHCI_DEBUG 0
  21. /* ---- controller limits ------------------------------------------------- */
  22. #define XHCI_MAX_SLOTS 4
  23. #define XHCI_MAX_PORTS 32
  24. #define XHCI_EVT_TRBS 256
  25. #define XHCI_CMD_TRBS 32
  26. #define XHCI_EP0_TRBS 16
  27. #define XHCI_INT_TRBS 16
  28. #define XHCI_BULK_TRBS 32
  29. /* ---- capability / operational / runtime offsets (bytes) --------------- */
  30. #define XHCI_CAP_HCSPARAMS1 0x04
  31. #define XHCI_CAP_HCSPARAMS2 0x08
  32. #define XHCI_CAP_HCCPARAMS1 0x10
  33. #define XHCI_CAP_DBOFF 0x14
  34. #define XHCI_CAP_RTSOFF 0x18
  35. #define XHCI_OP_USBCMD 0x00
  36. #define XHCI_OP_USBSTS 0x04
  37. #define XHCI_OP_PAGESIZE 0x08
  38. #define XHCI_OP_CRCR 0x18
  39. #define XHCI_OP_DCBAAP 0x30
  40. #define XHCI_OP_CONFIG 0x38
  41. #define XHCI_OP_PORTSC 0x400
  42. #define XHCI_RT_IMAN 0x20
  43. #define XHCI_RT_IMOD 0x24
  44. #define XHCI_RT_ERSTSZ 0x28
  45. #define XHCI_RT_ERSTBA 0x30
  46. #define XHCI_RT_ERDP 0x38
  47. /* USBCMD / USBSTS */
  48. #define XHCI_CMD_RS 0x0001u
  49. #define XHCI_CMD_HCRST 0x0002u
  50. #define XHCI_STS_HCH 0x0001u
  51. #define XHCI_STS_CNR 0x0800u
  52. #define XHCI_STS_HCE 0x1000u
  53. /* PORTSC */
  54. #define PORTSC_CCS 0x00000001u
  55. #define PORTSC_PED 0x00000002u
  56. #define PORTSC_PR 0x00000010u
  57. #define PORTSC_PP 0x00000200u
  58. #define PORTSC_SPEED_SHIFT 10
  59. #define PORTSC_SPEED_MASK 0xFu
  60. #define PORTSC_CSC 0x00020000u
  61. #define PORTSC_PEC 0x00040000u
  62. #define PORTSC_WRC 0x00080000u
  63. #define PORTSC_OCC 0x00100000u
  64. #define PORTSC_PRC 0x00200000u
  65. #define PORTSC_PLC 0x00400000u
  66. #define PORTSC_CEC 0x00800000u
  67. #define PORTSC_RWC (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
  68. PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
  69. PORTSC_CEC | PORTSC_PED)
  70. /* TRB types */
  71. #define TRB_NORMAL 1
  72. #define TRB_SETUP 2
  73. #define TRB_DATA 3
  74. #define TRB_STATUS 4
  75. #define TRB_LINK 6
  76. #define TRB_ENABLE_SLOT 9
  77. #define TRB_DISABLE_SLOT 10
  78. #define TRB_ADDR_DEV 11
  79. #define TRB_CONFIG_EP 12
  80. #define TRB_TRANSFER 32
  81. #define TRB_COMPLETION 33
  82. /* TRB control bits */
  83. #define TRB_CYCLE 0x00000001u
  84. #define TRB_TC 0x00000002u
  85. #define TRB_ISP 0x00000004u
  86. #define TRB_CHAIN 0x00000010u
  87. #define TRB_IOC 0x00000020u
  88. #define TRB_IDT 0x00000040u
  89. #define TRB_BSR 0x00000200u
  90. #define TRB_DIR_IN 0x00010000u
  91. #define TRB_TYPE(t) ((t) << 10)
  92. /* completion codes */
  93. #define CC_SUCCESS 1
  94. #define CC_SHORT_PACKET 13
  95. /* slot/EP context flags and endpoint types */
  96. #define SLOT_FLAG 0x00000001u
  97. #define EP0_FLAG 0x00000002u
  98. #define EP_CTRL 4
  99. #define EP_BULK_OUT 2
  100. #define EP_BULK_IN 6
  101. #define EP_INT_IN 7
  102. #define XHCI_CTRL_MPS 8
  103. struct xhci_trb {
  104. unsigned int p0;
  105. unsigned int p1;
  106. unsigned int status;
  107. unsigned int control;
  108. };
  109. struct xhci_slot {
  110. int used;
  111. unsigned int slot_id;
  112. unsigned int addr;
  113. unsigned int port;
  114. unsigned int speed;
  115. unsigned int dc; /* device context page */
  116. unsigned int ep0; /* EP0 transfer ring page */
  117. unsigned int ep0_enq;
  118. unsigned int ep0_cycle;
  119. unsigned int ep_mask; /* configured endpoint flags */
  120. };
  121. struct xhci_int {
  122. int active;
  123. unsigned int slot; /* hardware slot id */
  124. unsigned int ep_id; /* xHCI endpoint id (2n+dir) */
  125. unsigned int mps;
  126. unsigned int ring;
  127. unsigned int enq;
  128. unsigned int cycle;
  129. unsigned int trb_len;
  130. unsigned int pending_len;
  131. int pending;
  132. int error;
  133. unsigned char buf[8] __attribute__((aligned(16)));
  134. };
  135. struct xhci_bulk {
  136. int active;
  137. int armed;
  138. unsigned int slot;
  139. unsigned int ep; /* endpoint number */
  140. unsigned int ep_id;
  141. unsigned int mps;
  142. unsigned int ring;
  143. unsigned int enq;
  144. unsigned int cycle;
  145. unsigned int buf;
  146. unsigned int len;
  147. unsigned int trb_len;
  148. unsigned int actual;
  149. int pending;
  150. int error;
  151. };
  152. static unsigned int mmio_base;
  153. static unsigned int cap_len;
  154. static unsigned int rt_off;
  155. static unsigned int db_off;
  156. static unsigned int max_slots;
  157. static unsigned int max_ports;
  158. static unsigned int ctx_size;
  159. static unsigned int usb2_mask;
  160. static unsigned int hcc_params;
  161. static unsigned int pci_bus, pci_dev, pci_func;
  162. static int found;
  163. static int last_port = -1;
  164. static unsigned int dcbaa;
  165. static unsigned int input_ctx;
  166. static unsigned int cmd_ring;
  167. static unsigned int evt_ring;
  168. static unsigned int erst;
  169. static unsigned int cmd_enq;
  170. static unsigned int cmd_cycle;
  171. static unsigned int evt_idx;
  172. static unsigned int evt_cycle;
  173. static unsigned int cmd_busy;
  174. static unsigned int cmd_done;
  175. static unsigned int cmd_status;
  176. static unsigned int cmd_slot;
  177. static unsigned int cmd_trb_addr;
  178. static unsigned int ctl_busy;
  179. static unsigned int ctl_done;
  180. static unsigned int ctl_slot;
  181. static unsigned int ctl_status;
  182. static unsigned int ctl_req_len;
  183. static unsigned int ctl_actual;
  184. static unsigned int ctl_data_seen;
  185. static unsigned int ctl_setup_trb;
  186. static unsigned int ctl_data_trb;
  187. static unsigned int ctl_status_trb;
  188. static struct xhci_bulk bulk_in;
  189. static struct xhci_bulk bulk_out;
  190. static unsigned int out_busy;
  191. static unsigned int out_done;
  192. static unsigned int out_status;
  193. static unsigned int out_slot;
  194. static unsigned int out_trb;
  195. static struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
  196. static unsigned int addr_slot[128];
  197. static unsigned int port_slot[XHCI_MAX_PORTS];
  198. static struct xhci_int ints[USB_HCD_INT_SLOTS];
  199. static spinlock_t xhci_lock;
  200. /* ---- debug ------------------------------------------------------------- */
  201. #if XHCI_DEBUG
  202. static void dbg(const char *s)
  203. {
  204. serial_puts(SERIAL_COM1, s);
  205. }
  206. static void dbg_hex(const char *s, unsigned int v)
  207. {
  208. serial_puts(SERIAL_COM1, s);
  209. serial_puthex(SERIAL_COM1, v);
  210. serial_putc(SERIAL_COM1, '\n');
  211. }
  212. static void dbg_dec(const char *s, unsigned int v)
  213. {
  214. serial_puts(SERIAL_COM1, s);
  215. serial_putdec(SERIAL_COM1, v);
  216. serial_putc(SERIAL_COM1, '\n');
  217. }
  218. #else
  219. static void dbg(const char *s) { (void)s; }
  220. static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
  221. static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
  222. #endif
  223. /* ---- MMIO helpers ------------------------------------------------------ */
  224. static unsigned int mmio_rd(unsigned int off)
  225. {
  226. return *(volatile unsigned int *)(mmio_base + off);
  227. }
  228. static void mmio_wr(unsigned int off, unsigned int val)
  229. {
  230. *(volatile unsigned int *)(mmio_base + off) = val;
  231. }
  232. static void mmio_wr64(unsigned int off, unsigned int lo)
  233. {
  234. mmio_wr(off, lo);
  235. mmio_wr(off + 4, 0);
  236. }
  237. /* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
  238. static void ram_wr64(unsigned int addr, unsigned int lo)
  239. {
  240. *(volatile unsigned int *)addr = lo;
  241. *(volatile unsigned int *)(addr + 4) = 0;
  242. }
  243. static void zero_page(unsigned int addr, unsigned int len)
  244. {
  245. unsigned int *p = (unsigned int *)addr;
  246. while (len >= 4) {
  247. *p++ = 0;
  248. len -= 4;
  249. }
  250. }
  251. static unsigned int op(unsigned int reg)
  252. {
  253. return cap_len + reg;
  254. }
  255. static unsigned int rt(unsigned int reg)
  256. {
  257. return rt_off + reg;
  258. }
  259. /* ---- rings ------------------------------------------------------------- */
  260. static void ring_init(unsigned int base, unsigned int size,
  261. unsigned int *enq, unsigned int *cycle)
  262. {
  263. struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
  264. link->p0 = base;
  265. link->p1 = 0;
  266. link->status = 0;
  267. link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  268. *enq = 0;
  269. *cycle = 1;
  270. }
  271. static void ring_queue(unsigned int base, unsigned int size,
  272. unsigned int *enq, unsigned int *cycle,
  273. unsigned int p0, unsigned int p1,
  274. unsigned int status, unsigned int control)
  275. {
  276. struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
  277. trb->p0 = p0;
  278. trb->p1 = p1;
  279. trb->status = status;
  280. __asm__ volatile ("" ::: "memory");
  281. trb->control = control | *cycle;
  282. __asm__ volatile ("" ::: "memory");
  283. (*enq)++;
  284. if (*enq == size - 1) {
  285. struct xhci_trb *link =
  286. (struct xhci_trb *)(base + (size - 1) * 16);
  287. __asm__ volatile ("" ::: "memory");
  288. link->control ^= TRB_CYCLE;
  289. *enq = 0;
  290. *cycle ^= 1;
  291. }
  292. }
  293. /* ---- event handling ---------------------------------------------------- */
  294. static void xhci_poll_locked(void);
  295. static void stop_int(unsigned int slot);
  296. static void bulk_reset(struct xhci_bulk *b);
  297. static void xhci_int_event(unsigned int slot, unsigned int ep,
  298. unsigned int code, unsigned int residual)
  299. {
  300. unsigned int i;
  301. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  302. if (!ints[i].active || ints[i].slot != slot ||
  303. ints[i].ep_id != ep)
  304. continue;
  305. if (!ints[i].pending) {
  306. if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
  307. ints[i].pending_len =
  308. (ints[i].trb_len >= residual) ?
  309. ints[i].trb_len - residual : 0;
  310. ints[i].pending = 1;
  311. } else {
  312. ints[i].pending = 1;
  313. ints[i].error = 1;
  314. }
  315. }
  316. break;
  317. }
  318. }
  319. static void xhci_poll_locked(void)
  320. {
  321. unsigned int n = 0;
  322. while (n++ < 512) {
  323. struct xhci_trb *ev =
  324. (struct xhci_trb *)(evt_ring + evt_idx * 16);
  325. unsigned int type;
  326. if ((ev->control & TRB_CYCLE) != evt_cycle)
  327. break;
  328. type = (ev->control >> 10) & 0x3F;
  329. if (type == TRB_COMPLETION) {
  330. unsigned int code = (ev->status >> 24) & 0xFF;
  331. unsigned int slot = (ev->control >> 24) & 0xFF;
  332. if (cmd_busy && ev->p0 == cmd_trb_addr) {
  333. cmd_status = code;
  334. cmd_slot = slot;
  335. cmd_done = 1;
  336. }
  337. } else if (type == TRB_TRANSFER) {
  338. unsigned int code = (ev->status >> 24) & 0xFF;
  339. unsigned int residual = ev->status & 0xFFFFFF;
  340. unsigned int slot = (ev->control >> 24) & 0xFF;
  341. unsigned int ep = (ev->control >> 16) & 0x1F;
  342. if (ctl_busy && slot == ctl_slot && ep == 1) {
  343. if (ev->p0 == ctl_data_trb) {
  344. ctl_actual = (ctl_req_len >= residual) ?
  345. ctl_req_len - residual : 0;
  346. ctl_data_seen = 1;
  347. } else if (ev->p0 == ctl_status_trb) {
  348. if (!ctl_data_seen)
  349. ctl_actual =
  350. (code == CC_SUCCESS) ?
  351. ctl_req_len : 0;
  352. ctl_status = code;
  353. ctl_done = 1;
  354. }
  355. } else if (out_busy && slot == out_slot &&
  356. ep == bulk_out.ep_id &&
  357. ev->p0 == out_trb) {
  358. out_status = code;
  359. out_done = 1;
  360. } else if (bulk_in.active && slot == bulk_in.slot &&
  361. ep == bulk_in.ep_id) {
  362. bulk_in.armed = 0;
  363. if (!bulk_in.pending) {
  364. if (code == CC_SUCCESS ||
  365. code == CC_SHORT_PACKET) {
  366. bulk_in.actual =
  367. (bulk_in.trb_len >=
  368. residual) ?
  369. bulk_in.trb_len -
  370. residual : 0;
  371. bulk_in.pending = 1;
  372. } else {
  373. bulk_in.pending = 1;
  374. bulk_in.error = 1;
  375. }
  376. }
  377. } else {
  378. xhci_int_event(slot, ep, code, residual);
  379. }
  380. }
  381. evt_idx++;
  382. if (evt_idx >= XHCI_EVT_TRBS) {
  383. evt_idx = 0;
  384. evt_cycle ^= 1;
  385. }
  386. }
  387. mmio_wr64(rt(XHCI_RT_ERDP),
  388. (evt_ring + evt_idx * 16) | 8);
  389. }
  390. /* ---- commands ---------------------------------------------------------- */
  391. static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
  392. unsigned int *out_slot)
  393. {
  394. unsigned int flags, t0, rc;
  395. if (!found)
  396. return -1;
  397. spin_lock_irqsave(&xhci_lock, &flags);
  398. if (cmd_busy) {
  399. spin_unlock_irqrestore(&xhci_lock, flags);
  400. return -1;
  401. }
  402. cmd_busy = 1;
  403. cmd_done = 0;
  404. cmd_status = 0;
  405. cmd_slot = 0;
  406. cmd_trb_addr = cmd_ring + cmd_enq * 16;
  407. ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
  408. p0, p1, 0, control);
  409. __asm__ volatile ("" ::: "memory");
  410. mmio_wr(db_off, 0);
  411. spin_unlock_irqrestore(&xhci_lock, flags);
  412. t0 = get_ticks_ms();
  413. while (get_ticks_ms() - t0 < 1000) {
  414. spin_lock_irqsave(&xhci_lock, &flags);
  415. xhci_poll_locked();
  416. rc = cmd_done;
  417. spin_unlock_irqrestore(&xhci_lock, flags);
  418. if (rc)
  419. break;
  420. __asm__ volatile ("pause");
  421. }
  422. spin_lock_irqsave(&xhci_lock, &flags);
  423. cmd_busy = 0;
  424. if (!cmd_done) {
  425. spin_unlock_irqrestore(&xhci_lock, flags);
  426. dbg("xhci: command timeout\n");
  427. return -1;
  428. }
  429. if (out_slot)
  430. *out_slot = cmd_slot;
  431. rc = cmd_status;
  432. spin_unlock_irqrestore(&xhci_lock, flags);
  433. if (rc != CC_SUCCESS) {
  434. dbg_hex("xhci: command status ", rc);
  435. return -1;
  436. }
  437. return 0;
  438. }
  439. /* ---- contexts ---------------------------------------------------------- */
  440. static void fill_slot_ctx(unsigned int addr, unsigned int port,
  441. unsigned int speed, unsigned int last_ctx)
  442. {
  443. unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
  444. p[0] = (speed << 20) | (last_ctx << 27);
  445. p[1] = (port + 1) << 16;
  446. p[2] = 0;
  447. p[3] = addr;
  448. }
  449. static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
  450. unsigned int enq, unsigned int cycle)
  451. {
  452. unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
  453. p[0] = 0;
  454. p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
  455. p[2] = (ring + enq * 16) | cycle;
  456. p[3] = 0;
  457. p[4] = 8;
  458. }
  459. static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
  460. unsigned int mps, unsigned int interval,
  461. unsigned int ring)
  462. {
  463. unsigned int *p =
  464. (unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
  465. p[0] = interval << 16;
  466. p[1] = (type << 3) | (3 << 1) | (mps << 16);
  467. p[2] = ring | 1;
  468. p[3] = 0;
  469. p[4] = mps;
  470. }
  471. static struct xhci_slot *slot_by_addr(unsigned int addr)
  472. {
  473. unsigned int id;
  474. if (addr > 127)
  475. return 0;
  476. id = addr_slot[addr];
  477. if (id < 1 || id > max_slots || !slots[id].used)
  478. return 0;
  479. return &slots[id];
  480. }
  481. static unsigned int port_speed(unsigned int port)
  482. {
  483. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  484. unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
  485. return sp ? sp : 1;
  486. }
  487. static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
  488. unsigned int type, unsigned int mps,
  489. unsigned int interval, unsigned int ring)
  490. {
  491. unsigned int ep_flag = 1u << (ep_index + 1);
  492. unsigned int last = ep_index + 1;
  493. unsigned int i, flags;
  494. int rc;
  495. for (i = 1; i < 32; i++)
  496. if (s->ep_mask & (1u << i))
  497. last = i;
  498. spin_lock_irqsave(&xhci_lock, &flags);
  499. zero_page(input_ctx, 4096);
  500. *(unsigned int *)input_ctx = 0;
  501. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
  502. fill_slot_ctx(s->addr, s->port, s->speed, last);
  503. fill_ep_ctx(ep_index, type, mps, interval, ring);
  504. spin_unlock_irqrestore(&xhci_lock, flags);
  505. rc = xhci_cmd(input_ctx, 0,
  506. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  507. if (rc == 0)
  508. s->ep_mask |= ep_flag;
  509. return rc;
  510. }
  511. /* ---- slots ------------------------------------------------------------- */
  512. static struct xhci_slot *new_slot(void)
  513. {
  514. struct xhci_slot *s;
  515. unsigned int slot_id, dc, ep0, flags;
  516. int rc;
  517. if (last_port < 0 || last_port >= (int)max_ports)
  518. return 0;
  519. if (port_slot[last_port])
  520. return 0;
  521. slot_id = 0;
  522. rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
  523. if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
  524. dbg("xhci: enable slot failed\n");
  525. return 0;
  526. }
  527. dc = mm_alloc_page();
  528. ep0 = mm_alloc_page();
  529. if (!dc || !ep0) {
  530. if (dc)
  531. mm_free_page(dc);
  532. if (ep0)
  533. mm_free_page(ep0);
  534. return 0;
  535. }
  536. zero_page(dc, 4096);
  537. zero_page(ep0, 4096);
  538. s = &slots[slot_id];
  539. s->used = 1;
  540. s->slot_id = slot_id;
  541. s->addr = 0;
  542. s->port = (unsigned int)last_port;
  543. s->speed = port_speed((unsigned int)last_port);
  544. s->dc = dc;
  545. s->ep0 = ep0;
  546. s->ep0_enq = 0;
  547. s->ep0_cycle = 1;
  548. s->ep_mask = 0;
  549. ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
  550. ram_wr64(dcbaa + slot_id * 8, dc);
  551. spin_lock_irqsave(&xhci_lock, &flags);
  552. zero_page(input_ctx, 4096);
  553. *(unsigned int *)input_ctx = 0;
  554. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  555. fill_slot_ctx(0, s->port, s->speed, 1);
  556. fill_ep0_ctx(XHCI_CTRL_MPS, ep0, s->ep0_enq, s->ep0_cycle);
  557. spin_unlock_irqrestore(&xhci_lock, flags);
  558. rc = xhci_cmd(input_ctx, 0,
  559. TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) | TRB_BSR, 0);
  560. if (rc != 0) {
  561. dbg("xhci: address default failed\n");
  562. mm_free_page(dc);
  563. mm_free_page(ep0);
  564. s->used = 0;
  565. return 0;
  566. }
  567. addr_slot[0] = slot_id;
  568. port_slot[s->port] = slot_id;
  569. return s;
  570. }
  571. static void free_slot(struct xhci_slot *s)
  572. {
  573. unsigned int i, flags;
  574. if (!s || !s->used)
  575. return;
  576. xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
  577. ram_wr64(dcbaa + s->slot_id * 8, 0);
  578. spin_lock_irqsave(&xhci_lock, &flags);
  579. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  580. if (ints[i].slot == s->slot_id) {
  581. ints[i].active = 0;
  582. ints[i].pending = 0;
  583. ints[i].error = 0;
  584. if (ints[i].ring) {
  585. mm_free_page(ints[i].ring);
  586. ints[i].ring = 0;
  587. }
  588. ints[i].slot = 0;
  589. }
  590. }
  591. if (s->dc) {
  592. mm_free_page(s->dc);
  593. s->dc = 0;
  594. }
  595. if (s->ep0) {
  596. mm_free_page(s->ep0);
  597. s->ep0 = 0;
  598. }
  599. if (s->addr < 128)
  600. addr_slot[s->addr] = 0;
  601. if (bulk_in.active && bulk_in.slot == s->slot_id)
  602. bulk_reset(&bulk_in);
  603. if (bulk_out.active && bulk_out.slot == s->slot_id)
  604. bulk_reset(&bulk_out);
  605. if (s->port < XHCI_MAX_PORTS)
  606. port_slot[s->port] = 0;
  607. s->used = 0;
  608. s->addr = 0;
  609. s->ep_mask = 0;
  610. spin_unlock_irqrestore(&xhci_lock, flags);
  611. }
  612. static int set_address(unsigned int new_addr, unsigned int mps)
  613. {
  614. struct xhci_slot *s = slot_by_addr(0);
  615. unsigned int flags;
  616. int rc;
  617. if (!s || new_addr == 0 || new_addr > 127)
  618. return -1;
  619. spin_lock_irqsave(&xhci_lock, &flags);
  620. zero_page(input_ctx, 4096);
  621. *(unsigned int *)input_ctx = 0;
  622. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  623. fill_slot_ctx(new_addr, s->port, s->speed, 1);
  624. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  625. spin_unlock_irqrestore(&xhci_lock, flags);
  626. rc = xhci_cmd(input_ctx, 0,
  627. TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
  628. if (rc != 0) {
  629. dbg("xhci: set address failed\n");
  630. return -1;
  631. }
  632. spin_lock_irqsave(&xhci_lock, &flags);
  633. addr_slot[0] = 0;
  634. s->addr = new_addr;
  635. addr_slot[new_addr] = s->slot_id;
  636. spin_unlock_irqrestore(&xhci_lock, flags);
  637. return 0;
  638. }
  639. /* ---- control transfers ------------------------------------------------- */
  640. static int control_transfer(unsigned int ctrl, unsigned int addr,
  641. unsigned int mps, int low_speed,
  642. const unsigned char *setup, unsigned char *data,
  643. unsigned int len, int dir_in)
  644. {
  645. struct xhci_slot *s;
  646. unsigned int flags, t0, rc, st;
  647. unsigned int setup_c, data_c, status_c;
  648. (void)ctrl;
  649. (void)low_speed;
  650. if (!found)
  651. return -1;
  652. /* xHCI owns SET_ADDRESS: it is an Address Device command, not a
  653. * control request on the wire. */
  654. if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0)
  655. return set_address((unsigned int)setup[2] |
  656. ((unsigned int)setup[3] << 8), mps);
  657. s = slot_by_addr(addr);
  658. if (!s) {
  659. if (addr != 0)
  660. return -1;
  661. s = new_slot();
  662. if (!s)
  663. return -1;
  664. }
  665. spin_lock_irqsave(&xhci_lock, &flags);
  666. if (ctl_busy) {
  667. spin_unlock_irqrestore(&xhci_lock, flags);
  668. dbg("xhci: control busy\n");
  669. return -1;
  670. }
  671. ctl_busy = 1;
  672. ctl_done = 0;
  673. ctl_status = 0;
  674. ctl_actual = 0;
  675. ctl_data_seen = 0;
  676. ctl_slot = s->slot_id;
  677. ctl_req_len = len;
  678. setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_CHAIN;
  679. if (len)
  680. setup_c |= (dir_in ? 3u : 2u) << 16;
  681. ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
  682. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  683. (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  684. ((unsigned int)setup[2] << 16) |
  685. ((unsigned int)setup[3] << 24),
  686. (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
  687. ((unsigned int)setup[6] << 16) |
  688. ((unsigned int)setup[7] << 24),
  689. 8, setup_c);
  690. ctl_data_trb = 0;
  691. if (len) {
  692. ctl_data_trb = s->ep0 + s->ep0_enq * 16;
  693. data_c = TRB_TYPE(TRB_DATA) | TRB_CHAIN;
  694. if (dir_in)
  695. data_c |= TRB_DIR_IN | TRB_ISP;
  696. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
  697. &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
  698. }
  699. ctl_status_trb = s->ep0 + s->ep0_enq * 16;
  700. status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
  701. if (!len || !dir_in)
  702. status_c |= TRB_DIR_IN;
  703. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  704. 0, 0, 0, status_c);
  705. __asm__ volatile ("" ::: "memory");
  706. mmio_wr(db_off + 4 * s->slot_id, 1);
  707. spin_unlock_irqrestore(&xhci_lock, flags);
  708. t0 = get_ticks_ms();
  709. while (get_ticks_ms() - t0 < 1000) {
  710. spin_lock_irqsave(&xhci_lock, &flags);
  711. xhci_poll_locked();
  712. rc = ctl_done;
  713. spin_unlock_irqrestore(&xhci_lock, flags);
  714. if (rc)
  715. break;
  716. __asm__ volatile ("pause");
  717. }
  718. spin_lock_irqsave(&xhci_lock, &flags);
  719. if (!ctl_done) {
  720. ctl_busy = 0;
  721. spin_unlock_irqrestore(&xhci_lock, flags);
  722. dbg("xhci: control timeout\n");
  723. return -1;
  724. }
  725. rc = ctl_actual;
  726. st = ctl_status;
  727. ctl_busy = 0;
  728. spin_unlock_irqrestore(&xhci_lock, flags);
  729. if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
  730. dbg_hex("xhci: control status ", st);
  731. return -1;
  732. }
  733. return (int)rc;
  734. }
  735. /* ---- HID interrupt endpoint -------------------------------------------- */
  736. static int int_in_start(unsigned int ctrl, unsigned int slot,
  737. unsigned int addr, unsigned int ep,
  738. unsigned int mps, int low_speed)
  739. {
  740. struct xhci_slot *s = slot_by_addr(addr);
  741. struct xhci_int *in;
  742. unsigned int ep_index, ep_id, ring, flags;
  743. int rc;
  744. (void)ctrl;
  745. (void)low_speed;
  746. if (!found || !s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
  747. return -1;
  748. if (mps < 1)
  749. mps = 8;
  750. if (mps > 8)
  751. mps = 8;
  752. if (ints[slot].active)
  753. stop_int(slot);
  754. ep_index = (ep - 1) * 2 + 2; /* EP index 2 for EP1 IN, 4 for EP2 IN */
  755. ep_id = 2 * ep + 1;
  756. ring = mm_alloc_page();
  757. if (!ring)
  758. return -1;
  759. zero_page(ring, 4096);
  760. in = &ints[slot];
  761. in->slot = s->slot_id;
  762. in->ep_id = ep_id;
  763. in->mps = mps;
  764. in->ring = ring;
  765. in->enq = 0;
  766. in->cycle = 1;
  767. in->trb_len = mps;
  768. in->pending = 0;
  769. in->error = 0;
  770. in->active = 0;
  771. ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
  772. rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
  773. if (rc != 0) {
  774. dbg_hex("xhci: configure int ep failed ", ep_index);
  775. mm_free_page(ring);
  776. in->ring = 0;
  777. return -1;
  778. }
  779. in->active = 1;
  780. spin_lock_irqsave(&xhci_lock, &flags);
  781. ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  782. (unsigned int)in->buf, 0, in->trb_len,
  783. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  784. __asm__ volatile ("" ::: "memory");
  785. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  786. spin_unlock_irqrestore(&xhci_lock, flags);
  787. return 0;
  788. }
  789. static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
  790. {
  791. struct xhci_int *in;
  792. unsigned int flags, n, i;
  793. if (!found || slot >= USB_HCD_INT_SLOTS)
  794. return 0;
  795. in = &ints[slot];
  796. if (!in->active)
  797. return 0;
  798. spin_lock_irqsave(&xhci_lock, &flags);
  799. xhci_poll_locked();
  800. spin_unlock_irqrestore(&xhci_lock, flags);
  801. if (!in->pending)
  802. return 0;
  803. if (in->error) {
  804. in->pending = 0;
  805. in->error = 0;
  806. return -1;
  807. }
  808. n = in->pending_len;
  809. if (n > len)
  810. n = len;
  811. for (i = 0; i < n; i++)
  812. buf[i] = in->buf[i];
  813. in->pending = 0;
  814. spin_lock_irqsave(&xhci_lock, &flags);
  815. if (in->active) {
  816. ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  817. (unsigned int)in->buf, 0, in->trb_len,
  818. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  819. __asm__ volatile ("" ::: "memory");
  820. mmio_wr(db_off + 4 * in->slot, in->ep_id);
  821. }
  822. spin_unlock_irqrestore(&xhci_lock, flags);
  823. return (int)in->pending_len;
  824. }
  825. static void stop_int(unsigned int slot)
  826. {
  827. struct xhci_int *in;
  828. unsigned int flags;
  829. if (slot >= USB_HCD_INT_SLOTS)
  830. return;
  831. in = &ints[slot];
  832. spin_lock_irqsave(&xhci_lock, &flags);
  833. in->active = 0;
  834. in->pending = 0;
  835. in->error = 0;
  836. spin_unlock_irqrestore(&xhci_lock, flags);
  837. }
  838. /* ---- bulk transfers ---------------------------------------------------- */
  839. static void bulk_reset(struct xhci_bulk *b)
  840. {
  841. if (b->ring) {
  842. mm_free_page(b->ring);
  843. b->ring = 0;
  844. }
  845. b->active = 0;
  846. b->armed = 0;
  847. b->slot = 0;
  848. b->pending = 0;
  849. b->error = 0;
  850. b->actual = 0;
  851. }
  852. static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
  853. unsigned int mps, unsigned char *buf, unsigned int len)
  854. {
  855. struct xhci_slot *s = slot_by_addr(addr);
  856. struct xhci_bulk *b = &bulk_in;
  857. unsigned int ep_index, ep_id, ring, flags, n, err;
  858. int rc;
  859. (void)ctrl;
  860. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  861. return -1;
  862. if (mps < 1)
  863. mps = 512;
  864. ep_index = (ep - 1) * 2 + 2;
  865. ep_id = 2 * ep + 1;
  866. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  867. ring = mm_alloc_page();
  868. if (!ring)
  869. return -1;
  870. zero_page(ring, 4096);
  871. bulk_reset(b);
  872. b->slot = s->slot_id;
  873. b->ep = ep;
  874. b->ep_id = ep_id;
  875. b->mps = mps;
  876. b->ring = ring;
  877. b->enq = 0;
  878. b->cycle = 1;
  879. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  880. rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
  881. if (rc != 0) {
  882. bulk_reset(b);
  883. return -1;
  884. }
  885. b->active = 1;
  886. }
  887. b->buf = (unsigned int)buf;
  888. b->len = len;
  889. spin_lock_irqsave(&xhci_lock, &flags);
  890. xhci_poll_locked();
  891. if (b->pending) {
  892. n = b->actual;
  893. err = (unsigned int)b->error;
  894. b->pending = 0;
  895. b->error = 0;
  896. if (!err && b->active) {
  897. b->trb_len = len;
  898. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq,
  899. &b->cycle, (unsigned int)buf, 0, len,
  900. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  901. b->armed = 1;
  902. __asm__ volatile ("" ::: "memory");
  903. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  904. }
  905. spin_unlock_irqrestore(&xhci_lock, flags);
  906. return err ? -1 : (int)n;
  907. }
  908. if (!b->armed) {
  909. b->trb_len = len;
  910. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  911. (unsigned int)buf, 0, len,
  912. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  913. b->armed = 1;
  914. __asm__ volatile ("" ::: "memory");
  915. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  916. }
  917. spin_unlock_irqrestore(&xhci_lock, flags);
  918. return 0;
  919. }
  920. static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
  921. unsigned int mps, const unsigned char *buf,
  922. unsigned int len)
  923. {
  924. struct xhci_slot *s = slot_by_addr(addr);
  925. struct xhci_bulk *b = &bulk_out;
  926. unsigned int ep_index, ep_id, ring, flags, t0, done, status;
  927. int rc;
  928. (void)ctrl;
  929. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  930. return -1;
  931. if (mps < 1)
  932. mps = 512;
  933. ep_index = (ep - 1) * 2 + 1;
  934. ep_id = 2 * ep;
  935. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  936. ring = mm_alloc_page();
  937. if (!ring)
  938. return -1;
  939. zero_page(ring, 4096);
  940. bulk_reset(b);
  941. b->slot = s->slot_id;
  942. b->ep = ep;
  943. b->ep_id = ep_id;
  944. b->mps = mps;
  945. b->ring = ring;
  946. b->enq = 0;
  947. b->cycle = 1;
  948. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  949. rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
  950. if (rc != 0) {
  951. bulk_reset(b);
  952. return -1;
  953. }
  954. b->active = 1;
  955. }
  956. spin_lock_irqsave(&xhci_lock, &flags);
  957. if (out_busy) {
  958. spin_unlock_irqrestore(&xhci_lock, flags);
  959. return -1;
  960. }
  961. out_busy = 1;
  962. out_done = 0;
  963. out_status = 0;
  964. out_slot = s->slot_id;
  965. out_trb = b->ring + b->enq * 16;
  966. b->trb_len = len;
  967. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  968. (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  969. __asm__ volatile ("" ::: "memory");
  970. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  971. spin_unlock_irqrestore(&xhci_lock, flags);
  972. t0 = get_ticks_ms();
  973. while (get_ticks_ms() - t0 < 1000) {
  974. spin_lock_irqsave(&xhci_lock, &flags);
  975. xhci_poll_locked();
  976. done = out_done;
  977. spin_unlock_irqrestore(&xhci_lock, flags);
  978. if (done)
  979. break;
  980. __asm__ volatile ("pause");
  981. }
  982. spin_lock_irqsave(&xhci_lock, &flags);
  983. done = out_done;
  984. status = out_status;
  985. out_busy = 0;
  986. spin_unlock_irqrestore(&xhci_lock, flags);
  987. if (!done) {
  988. dbg("xhci: bulk out timeout\n");
  989. return -1;
  990. }
  991. if (status != CC_SUCCESS) {
  992. dbg_hex("xhci: bulk out status ", status);
  993. return -1;
  994. }
  995. return 0;
  996. }
  997. /* ---- root hub ports ---------------------------------------------------- */
  998. static int port_reset(unsigned int ctrl, unsigned int port)
  999. {
  1000. unsigned int ps, t0;
  1001. (void)ctrl;
  1002. if (!found || port >= max_ports ||
  1003. !(usb2_mask & (1u << port)))
  1004. return -1;
  1005. if (port_slot[port])
  1006. free_slot(&slots[port_slot[port]]);
  1007. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1008. if (!(ps & PORTSC_CCS))
  1009. return -1;
  1010. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1011. (ps & ~PORTSC_RWC) | PORTSC_PR);
  1012. t0 = get_ticks_ms();
  1013. while (get_ticks_ms() - t0 < 200) {
  1014. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1015. if (ps & PORTSC_PRC)
  1016. break;
  1017. }
  1018. if (!(ps & PORTSC_PRC)) {
  1019. dbg_hex("xhci: port reset timeout ", port);
  1020. return -1;
  1021. }
  1022. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1023. (ps & ~PORTSC_RWC) | PORTSC_PRC);
  1024. t0 = get_ticks_ms();
  1025. while (get_ticks_ms() - t0 < 100) {
  1026. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1027. if (ps & PORTSC_PED)
  1028. break;
  1029. }
  1030. if (!(ps & PORTSC_PED)) {
  1031. dbg_hex("xhci: port not enabled ", port);
  1032. return -1;
  1033. }
  1034. last_port = (int)port;
  1035. return 0;
  1036. }
  1037. static int port_connected(unsigned int ctrl, unsigned int port)
  1038. {
  1039. unsigned int ps;
  1040. (void)ctrl;
  1041. if (!found || port >= max_ports ||
  1042. !(usb2_mask & (1u << port)))
  1043. return 0;
  1044. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1045. return (ps & PORTSC_CCS) ? 1 : 0;
  1046. }
  1047. static int port_low_speed(unsigned int ctrl, unsigned int port)
  1048. {
  1049. unsigned int ps;
  1050. (void)ctrl;
  1051. if (!found || port >= max_ports)
  1052. return 0;
  1053. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1054. return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
  1055. }
  1056. static int port_take_change(unsigned int ctrl, unsigned int port)
  1057. {
  1058. unsigned int ps;
  1059. (void)ctrl;
  1060. if (!found || port >= max_ports)
  1061. return 0;
  1062. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1063. if (!(ps & PORTSC_CSC))
  1064. return 0;
  1065. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1066. (ps & ~PORTSC_RWC) | PORTSC_CSC);
  1067. return 1;
  1068. }
  1069. /* ---- controller discovery ---------------------------------------------- */
  1070. /* xHCI USB Legacy Support: stop the firmware's SMI from firing on port
  1071. * changes once we own the controller (the same idea as the PCI 0xC0 write
  1072. * in uhci.c/ohci.c, but xHCI keeps it in an extended capability). */
  1073. static void legacy_smi_disable(unsigned int off)
  1074. {
  1075. unsigned int sup = mmio_rd(off);
  1076. unsigned int ctl;
  1077. if ((sup & 0xFF) != 1)
  1078. return;
  1079. if (sup & (1u << 16)) /* BIOS still owns it */
  1080. mmio_wr(off, sup | (1u << 24));
  1081. ctl = mmio_rd(off + 4);
  1082. ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
  1083. ctl |= 0x7u << 29;
  1084. mmio_wr(off + 4, ctl);
  1085. }
  1086. static void scan_protocols(void)
  1087. {
  1088. unsigned int off, guard = 0;
  1089. usb2_mask = 0;
  1090. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1091. while (off && guard++ < 32) {
  1092. unsigned int cap = mmio_rd(off);
  1093. unsigned int id = cap & 0xFF;
  1094. unsigned int next = (cap >> 8) & 0xFF;
  1095. if (id == 1) {
  1096. legacy_smi_disable(off);
  1097. } else if (id == 2) { /* Supported Protocol */
  1098. unsigned int major = (cap >> 24) & 0xFF;
  1099. unsigned int pi = mmio_rd(off + 8);
  1100. unsigned int po = pi & 0xFF;
  1101. unsigned int pc = (pi >> 8) & 0xFF;
  1102. unsigned int i;
  1103. if (major <= 2 && po >= 1) {
  1104. for (i = 0; i < pc && po + i <= 32; i++)
  1105. usb2_mask |= 1u << (po - 1 + i);
  1106. }
  1107. }
  1108. if (!next)
  1109. break;
  1110. off += next << 2;
  1111. }
  1112. if (!usb2_mask)
  1113. usb2_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
  1114. ((1u << max_ports) - 1);
  1115. }
  1116. static int xhci_init(void)
  1117. {
  1118. unsigned int bus, dev, func, id, class_rev, bar, bar_hi, cmd;
  1119. unsigned int hcs1, hcs2, t0, i, n, scratch;
  1120. if (found)
  1121. return 1;
  1122. for (bus = 0; bus < 256; bus++) {
  1123. for (dev = 0; dev < 32; dev++) {
  1124. for (func = 0; func < 8; func++) {
  1125. id = pci_cfg_read32(bus, dev, func, 0x00);
  1126. if (id == 0xFFFFFFFFu || id == 0)
  1127. continue;
  1128. class_rev = pci_cfg_read32(bus, dev, func, 0x08);
  1129. if ((class_rev >> 16) != 0x0C03)
  1130. continue;
  1131. if (((class_rev >> 8) & 0xFF) != 0x30)
  1132. continue;
  1133. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1134. if (bar & 1)
  1135. continue;
  1136. bar_hi = 0;
  1137. if ((bar & 0x6) == 0x4)
  1138. bar_hi = pci_cfg_read32(bus, dev,
  1139. func, 0x14);
  1140. bar &= 0xFFFFFFF0u;
  1141. if (!bar || bar_hi)
  1142. continue;
  1143. pci_bus = bus;
  1144. pci_dev = dev;
  1145. pci_func = func;
  1146. goto found_xhci;
  1147. }
  1148. }
  1149. }
  1150. return 0;
  1151. found_xhci:
  1152. cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
  1153. pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
  1154. mmio_base = bar;
  1155. paging_map_mmio(mmio_base, 0x10000);
  1156. cap_len = mmio_rd(0) & 0xFF;
  1157. hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
  1158. hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
  1159. hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
  1160. rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
  1161. db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
  1162. max_slots = hcs1 & 0xFF;
  1163. if (max_slots > XHCI_MAX_SLOTS)
  1164. max_slots = XHCI_MAX_SLOTS;
  1165. if (!max_slots)
  1166. max_slots = 1;
  1167. max_ports = (hcs1 >> 24) & 0xFF;
  1168. if (max_ports > XHCI_MAX_PORTS)
  1169. max_ports = XHCI_MAX_PORTS;
  1170. if (!max_ports)
  1171. max_ports = 1;
  1172. ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
  1173. dbg("xhci: found\n");
  1174. dbg_hex("xhci: mmio ", mmio_base);
  1175. dbg_dec("xhci: slots ", max_slots);
  1176. dbg_dec("xhci: ports ", max_ports);
  1177. dbg_dec("xhci: cap len ", cap_len);
  1178. dcbaa = mm_alloc_page();
  1179. input_ctx = mm_alloc_page();
  1180. cmd_ring = mm_alloc_page();
  1181. evt_ring = mm_alloc_page();
  1182. erst = mm_alloc_page();
  1183. if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
  1184. dbg("xhci: out of pages\n");
  1185. return 0;
  1186. }
  1187. zero_page(dcbaa, 4096);
  1188. zero_page(input_ctx, 4096);
  1189. zero_page(cmd_ring, 4096);
  1190. zero_page(evt_ring, 4096);
  1191. zero_page(erst, 4096);
  1192. /* Reset. HCRST self-clears; wait for CNR to drop. */
  1193. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1194. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
  1195. t0 = get_ticks_ms();
  1196. while (get_ticks_ms() - t0 < 1000) {
  1197. if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
  1198. break;
  1199. }
  1200. t0 = get_ticks_ms();
  1201. while (get_ticks_ms() - t0 < 1000) {
  1202. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1203. break;
  1204. }
  1205. if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
  1206. dbg_hex("xhci: reset failed ",
  1207. mmio_rd(op(XHCI_OP_USBSTS)));
  1208. return 0;
  1209. }
  1210. mmio_wr(op(XHCI_OP_PAGESIZE), 1);
  1211. mmio_wr(op(XHCI_OP_CONFIG), max_slots);
  1212. mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
  1213. /* Scratchpad buffers, if the controller asks for any. */
  1214. n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
  1215. if (n) {
  1216. unsigned int arr;
  1217. if (n > 512)
  1218. n = 512;
  1219. arr = mm_alloc_page();
  1220. if (!arr) {
  1221. dbg("xhci: no scratchpad array\n");
  1222. return 0;
  1223. }
  1224. zero_page(arr, 4096);
  1225. for (i = 0; i < n; i++) {
  1226. scratch = mm_alloc_page();
  1227. if (!scratch) {
  1228. dbg("xhci: no scratchpad page\n");
  1229. return 0;
  1230. }
  1231. zero_page(scratch, 4096);
  1232. ram_wr64(arr + i * 8, scratch);
  1233. }
  1234. ram_wr64(dcbaa, arr);
  1235. dbg_dec("xhci: scratchpads ", n);
  1236. }
  1237. /* Command ring and event ring. */
  1238. ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
  1239. mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
  1240. evt_idx = 0;
  1241. evt_cycle = 1;
  1242. *(volatile unsigned int *)(erst + 0) = evt_ring;
  1243. *(volatile unsigned int *)(erst + 4) = 0;
  1244. *(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
  1245. *(volatile unsigned int *)(erst + 12) = 0;
  1246. mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
  1247. mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
  1248. mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
  1249. mmio_wr(rt(XHCI_RT_IMAN), 0);
  1250. mmio_wr(rt(XHCI_RT_IMOD), 0);
  1251. /* Run. */
  1252. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
  1253. t0 = get_ticks_ms();
  1254. while (get_ticks_ms() - t0 < 1000) {
  1255. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1256. break;
  1257. }
  1258. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
  1259. dbg("xhci: controller did not start\n");
  1260. return 0;
  1261. }
  1262. scan_protocols();
  1263. dbg_hex("xhci: usb2 ports ", usb2_mask);
  1264. /* Port power, where the controller has per-port switching. */
  1265. if (hcc_params & 0x0008u) {
  1266. for (i = 0; i < max_ports; i++) {
  1267. unsigned int ps;
  1268. if (!(usb2_mask & (1u << i)))
  1269. continue;
  1270. ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
  1271. if (!(ps & PORTSC_PP))
  1272. mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
  1273. (ps & ~PORTSC_RWC) | PORTSC_PP);
  1274. }
  1275. }
  1276. found = 1;
  1277. return 1;
  1278. }
  1279. static unsigned int xhci_count(void)
  1280. {
  1281. return found ? 1u : 0u;
  1282. }
  1283. static void xhci_info(unsigned int ctrl, unsigned int *bus,
  1284. unsigned int *dev, unsigned int *func,
  1285. unsigned int *base, int *mmio)
  1286. {
  1287. (void)ctrl;
  1288. *bus = pci_bus;
  1289. *dev = pci_dev;
  1290. *func = pci_func;
  1291. *base = mmio_base;
  1292. *mmio = 1;
  1293. }
  1294. static unsigned int xhci_port_count(unsigned int ctrl)
  1295. {
  1296. (void)ctrl;
  1297. return found ? max_ports : 0;
  1298. }
  1299. const struct usb_hcd_ops usb_xhci_ops = {
  1300. "xHCI",
  1301. xhci_init,
  1302. xhci_count,
  1303. xhci_info,
  1304. control_transfer,
  1305. xhci_port_count,
  1306. port_connected,
  1307. port_low_speed,
  1308. port_reset,
  1309. port_take_change,
  1310. int_in_start,
  1311. int_in_poll,
  1312. stop_int,
  1313. xhci_bulk_out,
  1314. xhci_bulk_in,
  1315. };