xhci.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495
  1. /* xHCI (USB 3.0) host controller driver.
  2. *
  3. * Scope: one controller, USB2 root ports and devices plugged directly into
  4. * them. The USB3 ports are left alone (devices there need SuperSpeed
  5. * resource/companion-descriptor handling that the HID stack does not need
  6. * yet). Control transfers and one interrupt-IN endpoint per HID interface
  7. * are implemented, which is exactly what usbhid.c uses.
  8. *
  9. * The schedule is polled, like the rest of the kernel. Rings and contexts
  10. * are page-allocated so the driver does not grow the kernel BSS. */
  11. #include "mm.h"
  12. #include "paging.h"
  13. #include "pci.h"
  14. #include "serial.h"
  15. #include "spinlock.h"
  16. #include "timer.h"
  17. #include "usb_hcd.h"
  18. #include "xhci.h"
  19. #define XHCI_DEBUG 0
  20. /* ---- controller limits ------------------------------------------------- */
  21. #define XHCI_MAX_SLOTS 4
  22. #define XHCI_MAX_PORTS 32
  23. #define XHCI_EVT_TRBS 256
  24. #define XHCI_CMD_TRBS 32
  25. #define XHCI_EP0_TRBS 16
  26. #define XHCI_INT_TRBS 16
  27. #define XHCI_BULK_TRBS 32
  28. /* ---- capability / operational / runtime offsets (bytes) --------------- */
  29. #define XHCI_CAP_HCSPARAMS1 0x04
  30. #define XHCI_CAP_HCSPARAMS2 0x08
  31. #define XHCI_CAP_HCCPARAMS1 0x10
  32. #define XHCI_CAP_DBOFF 0x14
  33. #define XHCI_CAP_RTSOFF 0x18
  34. #define XHCI_OP_USBCMD 0x00
  35. #define XHCI_OP_USBSTS 0x04
  36. #define XHCI_OP_PAGESIZE 0x08
  37. #define XHCI_OP_CRCR 0x18
  38. #define XHCI_OP_DCBAAP 0x30
  39. #define XHCI_OP_CONFIG 0x38
  40. #define XHCI_OP_PORTSC 0x400
  41. #define XHCI_RT_IMAN 0x20
  42. #define XHCI_RT_IMOD 0x24
  43. #define XHCI_RT_ERSTSZ 0x28
  44. #define XHCI_RT_ERSTBA 0x30
  45. #define XHCI_RT_ERDP 0x38
  46. /* USBCMD / USBSTS */
  47. #define XHCI_CMD_RS 0x0001u
  48. #define XHCI_CMD_HCRST 0x0002u
  49. #define XHCI_STS_HCH 0x0001u
  50. #define XHCI_STS_CNR 0x0800u
  51. #define XHCI_STS_HCE 0x1000u
  52. /* PORTSC */
  53. #define PORTSC_CCS 0x00000001u
  54. #define PORTSC_PED 0x00000002u
  55. #define PORTSC_PR 0x00000010u
  56. #define PORTSC_PP 0x00000200u
  57. #define PORTSC_SPEED_SHIFT 10
  58. #define PORTSC_SPEED_MASK 0xFu
  59. #define PORTSC_CSC 0x00020000u
  60. #define PORTSC_PEC 0x00040000u
  61. #define PORTSC_WRC 0x00080000u
  62. #define PORTSC_OCC 0x00100000u
  63. #define PORTSC_PRC 0x00200000u
  64. #define PORTSC_PLC 0x00400000u
  65. #define PORTSC_CEC 0x00800000u
  66. #define PORTSC_RWC (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
  67. PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
  68. PORTSC_CEC | PORTSC_PED)
  69. /* TRB types */
  70. #define TRB_NORMAL 1
  71. #define TRB_SETUP 2
  72. #define TRB_DATA 3
  73. #define TRB_STATUS 4
  74. #define TRB_LINK 6
  75. #define TRB_ENABLE_SLOT 9
  76. #define TRB_DISABLE_SLOT 10
  77. #define TRB_ADDR_DEV 11
  78. #define TRB_CONFIG_EP 12
  79. #define TRB_TRANSFER 32
  80. #define TRB_COMPLETION 33
  81. /* TRB control bits */
  82. #define TRB_CYCLE 0x00000001u
  83. #define TRB_TC 0x00000002u
  84. #define TRB_ISP 0x00000004u
  85. #define TRB_CHAIN 0x00000010u
  86. #define TRB_IOC 0x00000020u
  87. #define TRB_IDT 0x00000040u
  88. #define TRB_BSR 0x00000200u
  89. #define TRB_DIR_IN 0x00010000u
  90. #define TRB_TYPE(t) ((t) << 10)
  91. /* completion codes */
  92. #define CC_SUCCESS 1
  93. #define CC_SHORT_PACKET 13
  94. /* slot/EP context flags and endpoint types */
  95. #define SLOT_FLAG 0x00000001u
  96. #define EP0_FLAG 0x00000002u
  97. #define EP_CTRL 4
  98. #define EP_BULK_OUT 2
  99. #define EP_BULK_IN 6
  100. #define EP_INT_IN 7
  101. #define XHCI_CTRL_MPS 8
  102. struct xhci_trb {
  103. unsigned int p0;
  104. unsigned int p1;
  105. unsigned int status;
  106. unsigned int control;
  107. };
  108. struct xhci_slot {
  109. int used;
  110. unsigned int slot_id;
  111. unsigned int addr;
  112. unsigned int port;
  113. unsigned int speed;
  114. unsigned int dc; /* device context page */
  115. unsigned int ep0; /* EP0 transfer ring page */
  116. unsigned int ep0_enq;
  117. unsigned int ep0_cycle;
  118. unsigned int ep_mask; /* configured endpoint flags */
  119. };
  120. struct xhci_int {
  121. int active;
  122. unsigned int slot; /* hardware slot id */
  123. unsigned int ep_id; /* xHCI endpoint id (2n+dir) */
  124. unsigned int mps;
  125. unsigned int ring;
  126. unsigned int enq;
  127. unsigned int cycle;
  128. unsigned int trb_len;
  129. unsigned int pending_len;
  130. int pending;
  131. int error;
  132. unsigned char buf[8] __attribute__((aligned(16)));
  133. };
  134. struct xhci_bulk {
  135. int active;
  136. int armed;
  137. unsigned int slot;
  138. unsigned int ep; /* endpoint number */
  139. unsigned int ep_id;
  140. unsigned int mps;
  141. unsigned int ring;
  142. unsigned int enq;
  143. unsigned int cycle;
  144. unsigned int buf;
  145. unsigned int len;
  146. unsigned int trb_len;
  147. unsigned int actual;
  148. int pending;
  149. int error;
  150. };
  151. static unsigned int mmio_base;
  152. static unsigned int cap_len;
  153. static unsigned int rt_off;
  154. static unsigned int db_off;
  155. static unsigned int max_slots;
  156. static unsigned int max_ports;
  157. static unsigned int ctx_size;
  158. static unsigned int usb2_mask;
  159. static unsigned int hcc_params;
  160. static unsigned int pci_bus, pci_dev, pci_func;
  161. static int found;
  162. static int last_port = -1;
  163. static unsigned int dcbaa;
  164. static unsigned int input_ctx;
  165. static unsigned int cmd_ring;
  166. static unsigned int evt_ring;
  167. static unsigned int erst;
  168. static unsigned int cmd_enq;
  169. static unsigned int cmd_cycle;
  170. static unsigned int evt_idx;
  171. static unsigned int evt_cycle;
  172. static unsigned int cmd_busy;
  173. static unsigned int cmd_done;
  174. static unsigned int cmd_status;
  175. static unsigned int cmd_slot;
  176. static unsigned int cmd_trb_addr;
  177. static unsigned int ctl_busy;
  178. static unsigned int ctl_done;
  179. static unsigned int ctl_slot;
  180. static unsigned int ctl_status;
  181. static unsigned int ctl_req_len;
  182. static unsigned int ctl_actual;
  183. static unsigned int ctl_data_seen;
  184. static unsigned int ctl_setup_trb;
  185. static unsigned int ctl_data_trb;
  186. static unsigned int ctl_status_trb;
  187. static struct xhci_bulk bulk_in;
  188. static struct xhci_bulk bulk_out;
  189. static unsigned int out_busy;
  190. static unsigned int out_done;
  191. static unsigned int out_status;
  192. static unsigned int out_slot;
  193. static unsigned int out_trb;
  194. static struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
  195. static unsigned int addr_slot[128];
  196. static unsigned int port_slot[XHCI_MAX_PORTS];
  197. static struct xhci_int ints[USB_HCD_INT_SLOTS];
  198. static spinlock_t xhci_lock;
  199. /* ---- debug ------------------------------------------------------------- */
  200. #if XHCI_DEBUG
  201. static void dbg(const char *s)
  202. {
  203. serial_puts(SERIAL_COM1, s);
  204. }
  205. static void dbg_hex(const char *s, unsigned int v)
  206. {
  207. serial_puts(SERIAL_COM1, s);
  208. serial_puthex(SERIAL_COM1, v);
  209. serial_putc(SERIAL_COM1, '\n');
  210. }
  211. static void dbg_dec(const char *s, unsigned int v)
  212. {
  213. serial_puts(SERIAL_COM1, s);
  214. serial_putdec(SERIAL_COM1, v);
  215. serial_putc(SERIAL_COM1, '\n');
  216. }
  217. #else
  218. static void dbg(const char *s) { (void)s; }
  219. static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
  220. static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
  221. #endif
  222. /* ---- MMIO helpers ------------------------------------------------------ */
  223. static unsigned int mmio_rd(unsigned int off)
  224. {
  225. return *(volatile unsigned int *)(mmio_base + off);
  226. }
  227. static void mmio_wr(unsigned int off, unsigned int val)
  228. {
  229. *(volatile unsigned int *)(mmio_base + off) = val;
  230. }
  231. static void mmio_wr64(unsigned int off, unsigned int lo)
  232. {
  233. mmio_wr(off, lo);
  234. mmio_wr(off + 4, 0);
  235. }
  236. /* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
  237. static void ram_wr64(unsigned int addr, unsigned int lo)
  238. {
  239. *(volatile unsigned int *)addr = lo;
  240. *(volatile unsigned int *)(addr + 4) = 0;
  241. }
  242. static void zero_page(unsigned int addr, unsigned int len)
  243. {
  244. unsigned int *p = (unsigned int *)addr;
  245. while (len >= 4) {
  246. *p++ = 0;
  247. len -= 4;
  248. }
  249. }
  250. static unsigned int op(unsigned int reg)
  251. {
  252. return cap_len + reg;
  253. }
  254. static unsigned int rt(unsigned int reg)
  255. {
  256. return rt_off + reg;
  257. }
  258. /* ---- rings ------------------------------------------------------------- */
  259. static void ring_init(unsigned int base, unsigned int size,
  260. unsigned int *enq, unsigned int *cycle)
  261. {
  262. struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
  263. link->p0 = base;
  264. link->p1 = 0;
  265. link->status = 0;
  266. link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  267. *enq = 0;
  268. *cycle = 1;
  269. }
  270. static void ring_queue(unsigned int base, unsigned int size,
  271. unsigned int *enq, unsigned int *cycle,
  272. unsigned int p0, unsigned int p1,
  273. unsigned int status, unsigned int control)
  274. {
  275. struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
  276. trb->p0 = p0;
  277. trb->p1 = p1;
  278. trb->status = status;
  279. __asm__ volatile ("" ::: "memory");
  280. trb->control = control | *cycle;
  281. __asm__ volatile ("" ::: "memory");
  282. (*enq)++;
  283. if (*enq == size - 1) {
  284. struct xhci_trb *link =
  285. (struct xhci_trb *)(base + (size - 1) * 16);
  286. __asm__ volatile ("" ::: "memory");
  287. link->control ^= TRB_CYCLE;
  288. *enq = 0;
  289. *cycle ^= 1;
  290. }
  291. }
  292. /* ---- event handling ---------------------------------------------------- */
  293. static void xhci_poll_locked(void);
  294. static void stop_int(unsigned int slot);
  295. static void bulk_reset(struct xhci_bulk *b);
  296. static void xhci_int_event(unsigned int slot, unsigned int ep,
  297. unsigned int code, unsigned int residual)
  298. {
  299. unsigned int i;
  300. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  301. if (!ints[i].active || ints[i].slot != slot ||
  302. ints[i].ep_id != ep)
  303. continue;
  304. if (!ints[i].pending) {
  305. if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
  306. ints[i].pending_len =
  307. (ints[i].trb_len >= residual) ?
  308. ints[i].trb_len - residual : 0;
  309. ints[i].pending = 1;
  310. } else {
  311. ints[i].pending = 1;
  312. ints[i].error = 1;
  313. }
  314. }
  315. break;
  316. }
  317. }
  318. static void xhci_poll_locked(void)
  319. {
  320. unsigned int n = 0;
  321. while (n++ < 512) {
  322. struct xhci_trb *ev =
  323. (struct xhci_trb *)(evt_ring + evt_idx * 16);
  324. unsigned int type;
  325. if ((ev->control & TRB_CYCLE) != evt_cycle)
  326. break;
  327. type = (ev->control >> 10) & 0x3F;
  328. if (type == TRB_COMPLETION) {
  329. unsigned int code = (ev->status >> 24) & 0xFF;
  330. unsigned int slot = (ev->control >> 24) & 0xFF;
  331. if (cmd_busy && ev->p0 == cmd_trb_addr) {
  332. cmd_status = code;
  333. cmd_slot = slot;
  334. cmd_done = 1;
  335. }
  336. } else if (type == TRB_TRANSFER) {
  337. unsigned int code = (ev->status >> 24) & 0xFF;
  338. unsigned int residual = ev->status & 0xFFFFFF;
  339. unsigned int slot = (ev->control >> 24) & 0xFF;
  340. unsigned int ep = (ev->control >> 16) & 0x1F;
  341. if (ctl_busy && slot == ctl_slot && ep == 1) {
  342. if (ev->p0 == ctl_data_trb) {
  343. ctl_actual = (ctl_req_len >= residual) ?
  344. ctl_req_len - residual : 0;
  345. ctl_data_seen = 1;
  346. } else if (ev->p0 == ctl_status_trb) {
  347. if (!ctl_data_seen)
  348. ctl_actual =
  349. (code == CC_SUCCESS) ?
  350. ctl_req_len : 0;
  351. ctl_status = code;
  352. ctl_done = 1;
  353. }
  354. } else if (out_busy && slot == out_slot &&
  355. ep == bulk_out.ep_id &&
  356. ev->p0 == out_trb) {
  357. out_status = code;
  358. out_done = 1;
  359. } else if (bulk_in.active && slot == bulk_in.slot &&
  360. ep == bulk_in.ep_id) {
  361. bulk_in.armed = 0;
  362. if (!bulk_in.pending) {
  363. if (code == CC_SUCCESS ||
  364. code == CC_SHORT_PACKET) {
  365. bulk_in.actual =
  366. (bulk_in.trb_len >=
  367. residual) ?
  368. bulk_in.trb_len -
  369. residual : 0;
  370. bulk_in.pending = 1;
  371. } else {
  372. bulk_in.pending = 1;
  373. bulk_in.error = 1;
  374. }
  375. }
  376. } else {
  377. xhci_int_event(slot, ep, code, residual);
  378. }
  379. }
  380. evt_idx++;
  381. if (evt_idx >= XHCI_EVT_TRBS) {
  382. evt_idx = 0;
  383. evt_cycle ^= 1;
  384. }
  385. }
  386. mmio_wr64(rt(XHCI_RT_ERDP),
  387. (evt_ring + evt_idx * 16) | 8);
  388. }
  389. /* ---- commands ---------------------------------------------------------- */
  390. static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
  391. unsigned int *out_slot)
  392. {
  393. unsigned int flags, t0, rc;
  394. if (!found)
  395. return -1;
  396. spin_lock_irqsave(&xhci_lock, &flags);
  397. if (cmd_busy) {
  398. spin_unlock_irqrestore(&xhci_lock, flags);
  399. return -1;
  400. }
  401. cmd_busy = 1;
  402. cmd_done = 0;
  403. cmd_status = 0;
  404. cmd_slot = 0;
  405. cmd_trb_addr = cmd_ring + cmd_enq * 16;
  406. ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
  407. p0, p1, 0, control);
  408. __asm__ volatile ("" ::: "memory");
  409. mmio_wr(db_off, 0);
  410. spin_unlock_irqrestore(&xhci_lock, flags);
  411. t0 = get_ticks_ms();
  412. while (get_ticks_ms() - t0 < 1000) {
  413. spin_lock_irqsave(&xhci_lock, &flags);
  414. xhci_poll_locked();
  415. rc = cmd_done;
  416. spin_unlock_irqrestore(&xhci_lock, flags);
  417. if (rc)
  418. break;
  419. __asm__ volatile ("pause");
  420. }
  421. spin_lock_irqsave(&xhci_lock, &flags);
  422. cmd_busy = 0;
  423. if (!cmd_done) {
  424. spin_unlock_irqrestore(&xhci_lock, flags);
  425. dbg("xhci: command timeout\n");
  426. return -1;
  427. }
  428. if (out_slot)
  429. *out_slot = cmd_slot;
  430. rc = cmd_status;
  431. spin_unlock_irqrestore(&xhci_lock, flags);
  432. if (rc != CC_SUCCESS) {
  433. dbg_hex("xhci: command status ", rc);
  434. return -1;
  435. }
  436. return 0;
  437. }
  438. /* ---- contexts ---------------------------------------------------------- */
  439. static void fill_slot_ctx(unsigned int addr, unsigned int port,
  440. unsigned int speed, unsigned int last_ctx)
  441. {
  442. unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
  443. p[0] = (speed << 20) | (last_ctx << 27);
  444. p[1] = (port + 1) << 16;
  445. p[2] = 0;
  446. p[3] = addr;
  447. }
  448. static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
  449. unsigned int enq, unsigned int cycle)
  450. {
  451. unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
  452. p[0] = 0;
  453. p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
  454. p[2] = (ring + enq * 16) | cycle;
  455. p[3] = 0;
  456. p[4] = 8;
  457. }
  458. static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
  459. unsigned int mps, unsigned int interval,
  460. unsigned int ring)
  461. {
  462. unsigned int *p =
  463. (unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
  464. p[0] = interval << 16;
  465. p[1] = (type << 3) | (3 << 1) | (mps << 16);
  466. p[2] = ring | 1;
  467. p[3] = 0;
  468. p[4] = mps;
  469. }
  470. static struct xhci_slot *slot_by_addr(unsigned int addr)
  471. {
  472. unsigned int id;
  473. if (addr > 127)
  474. return 0;
  475. id = addr_slot[addr];
  476. if (id < 1 || id > max_slots || !slots[id].used)
  477. return 0;
  478. return &slots[id];
  479. }
  480. static unsigned int port_speed(unsigned int port)
  481. {
  482. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  483. unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
  484. return sp ? sp : 1;
  485. }
  486. static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
  487. unsigned int type, unsigned int mps,
  488. unsigned int interval, unsigned int ring)
  489. {
  490. unsigned int ep_flag = 1u << (ep_index + 1);
  491. unsigned int last = ep_index + 1;
  492. unsigned int i, flags;
  493. int rc;
  494. for (i = 1; i < 32; i++)
  495. if (s->ep_mask & (1u << i))
  496. last = i;
  497. spin_lock_irqsave(&xhci_lock, &flags);
  498. zero_page(input_ctx, 4096);
  499. *(unsigned int *)input_ctx = 0;
  500. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
  501. fill_slot_ctx(s->addr, s->port, s->speed, last);
  502. fill_ep_ctx(ep_index, type, mps, interval, ring);
  503. spin_unlock_irqrestore(&xhci_lock, flags);
  504. rc = xhci_cmd(input_ctx, 0,
  505. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  506. if (rc == 0)
  507. s->ep_mask |= ep_flag;
  508. return rc;
  509. }
  510. /* ---- slots ------------------------------------------------------------- */
  511. static struct xhci_slot *new_slot(void)
  512. {
  513. struct xhci_slot *s;
  514. unsigned int slot_id, dc, ep0, flags;
  515. int rc;
  516. if (last_port < 0 || last_port >= (int)max_ports)
  517. return 0;
  518. if (port_slot[last_port])
  519. return 0;
  520. slot_id = 0;
  521. rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
  522. if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
  523. dbg("xhci: enable slot failed\n");
  524. return 0;
  525. }
  526. dc = mm_alloc_page();
  527. ep0 = mm_alloc_page();
  528. if (!dc || !ep0) {
  529. if (dc)
  530. mm_free_page(dc);
  531. if (ep0)
  532. mm_free_page(ep0);
  533. return 0;
  534. }
  535. zero_page(dc, 4096);
  536. zero_page(ep0, 4096);
  537. s = &slots[slot_id];
  538. s->used = 1;
  539. s->slot_id = slot_id;
  540. s->addr = 0;
  541. s->port = (unsigned int)last_port;
  542. s->speed = port_speed((unsigned int)last_port);
  543. s->dc = dc;
  544. s->ep0 = ep0;
  545. s->ep0_enq = 0;
  546. s->ep0_cycle = 1;
  547. s->ep_mask = 0;
  548. ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
  549. ram_wr64(dcbaa + slot_id * 8, dc);
  550. spin_lock_irqsave(&xhci_lock, &flags);
  551. zero_page(input_ctx, 4096);
  552. *(unsigned int *)input_ctx = 0;
  553. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  554. fill_slot_ctx(0, s->port, s->speed, 1);
  555. fill_ep0_ctx(XHCI_CTRL_MPS, ep0, s->ep0_enq, s->ep0_cycle);
  556. spin_unlock_irqrestore(&xhci_lock, flags);
  557. rc = xhci_cmd(input_ctx, 0,
  558. TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) | TRB_BSR, 0);
  559. if (rc != 0) {
  560. dbg("xhci: address default failed\n");
  561. mm_free_page(dc);
  562. mm_free_page(ep0);
  563. s->used = 0;
  564. return 0;
  565. }
  566. addr_slot[0] = slot_id;
  567. port_slot[s->port] = slot_id;
  568. return s;
  569. }
  570. static void free_slot(struct xhci_slot *s)
  571. {
  572. unsigned int i, flags;
  573. if (!s || !s->used)
  574. return;
  575. xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
  576. ram_wr64(dcbaa + s->slot_id * 8, 0);
  577. spin_lock_irqsave(&xhci_lock, &flags);
  578. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  579. if (ints[i].slot == s->slot_id) {
  580. ints[i].active = 0;
  581. ints[i].pending = 0;
  582. ints[i].error = 0;
  583. if (ints[i].ring) {
  584. mm_free_page(ints[i].ring);
  585. ints[i].ring = 0;
  586. }
  587. ints[i].slot = 0;
  588. }
  589. }
  590. if (s->dc) {
  591. mm_free_page(s->dc);
  592. s->dc = 0;
  593. }
  594. if (s->ep0) {
  595. mm_free_page(s->ep0);
  596. s->ep0 = 0;
  597. }
  598. if (s->addr < 128)
  599. addr_slot[s->addr] = 0;
  600. if (bulk_in.active && bulk_in.slot == s->slot_id)
  601. bulk_reset(&bulk_in);
  602. if (bulk_out.active && bulk_out.slot == s->slot_id)
  603. bulk_reset(&bulk_out);
  604. if (s->port < XHCI_MAX_PORTS)
  605. port_slot[s->port] = 0;
  606. s->used = 0;
  607. s->addr = 0;
  608. s->ep_mask = 0;
  609. spin_unlock_irqrestore(&xhci_lock, flags);
  610. }
  611. static int set_address(unsigned int new_addr, unsigned int mps)
  612. {
  613. struct xhci_slot *s = slot_by_addr(0);
  614. unsigned int flags;
  615. int rc;
  616. if (!s || new_addr == 0 || new_addr > 127)
  617. return -1;
  618. spin_lock_irqsave(&xhci_lock, &flags);
  619. zero_page(input_ctx, 4096);
  620. *(unsigned int *)input_ctx = 0;
  621. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  622. fill_slot_ctx(new_addr, s->port, s->speed, 1);
  623. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  624. spin_unlock_irqrestore(&xhci_lock, flags);
  625. rc = xhci_cmd(input_ctx, 0,
  626. TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
  627. if (rc != 0) {
  628. dbg("xhci: set address failed\n");
  629. return -1;
  630. }
  631. spin_lock_irqsave(&xhci_lock, &flags);
  632. addr_slot[0] = 0;
  633. s->addr = new_addr;
  634. addr_slot[new_addr] = s->slot_id;
  635. spin_unlock_irqrestore(&xhci_lock, flags);
  636. return 0;
  637. }
  638. /* ---- control transfers ------------------------------------------------- */
  639. static int control_transfer(unsigned int ctrl, unsigned int addr,
  640. unsigned int mps, int low_speed,
  641. const unsigned char *setup, unsigned char *data,
  642. unsigned int len, int dir_in)
  643. {
  644. struct xhci_slot *s;
  645. unsigned int flags, t0, rc, st;
  646. unsigned int setup_c, data_c, status_c;
  647. (void)ctrl;
  648. (void)low_speed;
  649. if (!found)
  650. return -1;
  651. /* xHCI owns SET_ADDRESS: it is an Address Device command, not a
  652. * control request on the wire. */
  653. if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0)
  654. return set_address((unsigned int)setup[2] |
  655. ((unsigned int)setup[3] << 8), mps);
  656. s = slot_by_addr(addr);
  657. if (!s) {
  658. if (addr != 0)
  659. return -1;
  660. s = new_slot();
  661. if (!s)
  662. return -1;
  663. }
  664. spin_lock_irqsave(&xhci_lock, &flags);
  665. if (ctl_busy) {
  666. spin_unlock_irqrestore(&xhci_lock, flags);
  667. dbg("xhci: control busy\n");
  668. return -1;
  669. }
  670. ctl_busy = 1;
  671. ctl_done = 0;
  672. ctl_status = 0;
  673. ctl_actual = 0;
  674. ctl_data_seen = 0;
  675. ctl_slot = s->slot_id;
  676. ctl_req_len = len;
  677. setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_CHAIN;
  678. if (len)
  679. setup_c |= (dir_in ? 3u : 2u) << 16;
  680. ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
  681. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  682. (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  683. ((unsigned int)setup[2] << 16) |
  684. ((unsigned int)setup[3] << 24),
  685. (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
  686. ((unsigned int)setup[6] << 16) |
  687. ((unsigned int)setup[7] << 24),
  688. 8, setup_c);
  689. ctl_data_trb = 0;
  690. if (len) {
  691. ctl_data_trb = s->ep0 + s->ep0_enq * 16;
  692. data_c = TRB_TYPE(TRB_DATA) | TRB_CHAIN;
  693. if (dir_in)
  694. data_c |= TRB_DIR_IN | TRB_ISP;
  695. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
  696. &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
  697. }
  698. ctl_status_trb = s->ep0 + s->ep0_enq * 16;
  699. status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
  700. if (!len || !dir_in)
  701. status_c |= TRB_DIR_IN;
  702. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  703. 0, 0, 0, status_c);
  704. __asm__ volatile ("" ::: "memory");
  705. mmio_wr(db_off + 4 * s->slot_id, 1);
  706. spin_unlock_irqrestore(&xhci_lock, flags);
  707. t0 = get_ticks_ms();
  708. while (get_ticks_ms() - t0 < 1000) {
  709. spin_lock_irqsave(&xhci_lock, &flags);
  710. xhci_poll_locked();
  711. rc = ctl_done;
  712. spin_unlock_irqrestore(&xhci_lock, flags);
  713. if (rc)
  714. break;
  715. __asm__ volatile ("pause");
  716. }
  717. spin_lock_irqsave(&xhci_lock, &flags);
  718. if (!ctl_done) {
  719. ctl_busy = 0;
  720. spin_unlock_irqrestore(&xhci_lock, flags);
  721. dbg("xhci: control timeout\n");
  722. return -1;
  723. }
  724. rc = ctl_actual;
  725. st = ctl_status;
  726. ctl_busy = 0;
  727. spin_unlock_irqrestore(&xhci_lock, flags);
  728. if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
  729. dbg_hex("xhci: control status ", st);
  730. return -1;
  731. }
  732. return (int)rc;
  733. }
  734. /* ---- HID interrupt endpoint -------------------------------------------- */
  735. static int int_in_start(unsigned int ctrl, unsigned int slot,
  736. unsigned int addr, unsigned int ep,
  737. unsigned int mps, int low_speed)
  738. {
  739. struct xhci_slot *s = slot_by_addr(addr);
  740. struct xhci_int *in;
  741. unsigned int ep_index, ep_id, ring, flags;
  742. int rc;
  743. (void)ctrl;
  744. (void)low_speed;
  745. if (!found || !s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
  746. return -1;
  747. if (mps < 1)
  748. mps = 8;
  749. if (mps > 8)
  750. mps = 8;
  751. if (ints[slot].active)
  752. stop_int(slot);
  753. ep_index = (ep - 1) * 2 + 2; /* EP index 2 for EP1 IN, 4 for EP2 IN */
  754. ep_id = 2 * ep + 1;
  755. ring = mm_alloc_page();
  756. if (!ring)
  757. return -1;
  758. zero_page(ring, 4096);
  759. in = &ints[slot];
  760. in->slot = s->slot_id;
  761. in->ep_id = ep_id;
  762. in->mps = mps;
  763. in->ring = ring;
  764. in->enq = 0;
  765. in->cycle = 1;
  766. in->trb_len = mps;
  767. in->pending = 0;
  768. in->error = 0;
  769. in->active = 0;
  770. ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
  771. rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
  772. if (rc != 0) {
  773. dbg_hex("xhci: configure int ep failed ", ep_index);
  774. mm_free_page(ring);
  775. in->ring = 0;
  776. return -1;
  777. }
  778. in->active = 1;
  779. spin_lock_irqsave(&xhci_lock, &flags);
  780. ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  781. (unsigned int)in->buf, 0, in->trb_len,
  782. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  783. __asm__ volatile ("" ::: "memory");
  784. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  785. spin_unlock_irqrestore(&xhci_lock, flags);
  786. return 0;
  787. }
  788. static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
  789. {
  790. struct xhci_int *in;
  791. unsigned int flags, n, i;
  792. if (!found || slot >= USB_HCD_INT_SLOTS)
  793. return 0;
  794. in = &ints[slot];
  795. if (!in->active)
  796. return 0;
  797. spin_lock_irqsave(&xhci_lock, &flags);
  798. xhci_poll_locked();
  799. spin_unlock_irqrestore(&xhci_lock, flags);
  800. if (!in->pending)
  801. return 0;
  802. if (in->error) {
  803. in->pending = 0;
  804. in->error = 0;
  805. return -1;
  806. }
  807. n = in->pending_len;
  808. if (n > len)
  809. n = len;
  810. for (i = 0; i < n; i++)
  811. buf[i] = in->buf[i];
  812. in->pending = 0;
  813. spin_lock_irqsave(&xhci_lock, &flags);
  814. if (in->active) {
  815. ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  816. (unsigned int)in->buf, 0, in->trb_len,
  817. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  818. __asm__ volatile ("" ::: "memory");
  819. mmio_wr(db_off + 4 * in->slot, in->ep_id);
  820. }
  821. spin_unlock_irqrestore(&xhci_lock, flags);
  822. return (int)in->pending_len;
  823. }
  824. static void stop_int(unsigned int slot)
  825. {
  826. struct xhci_int *in;
  827. unsigned int flags;
  828. if (slot >= USB_HCD_INT_SLOTS)
  829. return;
  830. in = &ints[slot];
  831. spin_lock_irqsave(&xhci_lock, &flags);
  832. in->active = 0;
  833. in->pending = 0;
  834. in->error = 0;
  835. spin_unlock_irqrestore(&xhci_lock, flags);
  836. }
  837. /* ---- bulk transfers ---------------------------------------------------- */
  838. static void bulk_reset(struct xhci_bulk *b)
  839. {
  840. if (b->ring) {
  841. mm_free_page(b->ring);
  842. b->ring = 0;
  843. }
  844. b->active = 0;
  845. b->armed = 0;
  846. b->slot = 0;
  847. b->pending = 0;
  848. b->error = 0;
  849. b->actual = 0;
  850. }
  851. static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
  852. unsigned int mps, unsigned char *buf, unsigned int len)
  853. {
  854. struct xhci_slot *s = slot_by_addr(addr);
  855. struct xhci_bulk *b = &bulk_in;
  856. unsigned int ep_index, ep_id, ring, flags, n, err;
  857. int rc;
  858. (void)ctrl;
  859. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  860. return -1;
  861. if (mps < 1)
  862. mps = 512;
  863. ep_index = (ep - 1) * 2 + 2;
  864. ep_id = 2 * ep + 1;
  865. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  866. ring = mm_alloc_page();
  867. if (!ring)
  868. return -1;
  869. zero_page(ring, 4096);
  870. bulk_reset(b);
  871. b->slot = s->slot_id;
  872. b->ep = ep;
  873. b->ep_id = ep_id;
  874. b->mps = mps;
  875. b->ring = ring;
  876. b->enq = 0;
  877. b->cycle = 1;
  878. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  879. rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
  880. if (rc != 0) {
  881. bulk_reset(b);
  882. return -1;
  883. }
  884. b->active = 1;
  885. }
  886. b->buf = (unsigned int)buf;
  887. b->len = len;
  888. spin_lock_irqsave(&xhci_lock, &flags);
  889. xhci_poll_locked();
  890. if (b->pending) {
  891. n = b->actual;
  892. err = (unsigned int)b->error;
  893. b->pending = 0;
  894. b->error = 0;
  895. if (!err && b->active) {
  896. b->trb_len = len;
  897. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq,
  898. &b->cycle, (unsigned int)buf, 0, len,
  899. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  900. b->armed = 1;
  901. __asm__ volatile ("" ::: "memory");
  902. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  903. }
  904. spin_unlock_irqrestore(&xhci_lock, flags);
  905. return err ? -1 : (int)n;
  906. }
  907. if (!b->armed) {
  908. b->trb_len = len;
  909. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  910. (unsigned int)buf, 0, len,
  911. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  912. b->armed = 1;
  913. __asm__ volatile ("" ::: "memory");
  914. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  915. }
  916. spin_unlock_irqrestore(&xhci_lock, flags);
  917. return 0;
  918. }
  919. static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
  920. unsigned int mps, const unsigned char *buf,
  921. unsigned int len)
  922. {
  923. struct xhci_slot *s = slot_by_addr(addr);
  924. struct xhci_bulk *b = &bulk_out;
  925. unsigned int ep_index, ep_id, ring, flags, t0, done, status;
  926. int rc;
  927. (void)ctrl;
  928. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  929. return -1;
  930. if (mps < 1)
  931. mps = 512;
  932. ep_index = (ep - 1) * 2 + 1;
  933. ep_id = 2 * ep;
  934. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  935. ring = mm_alloc_page();
  936. if (!ring)
  937. return -1;
  938. zero_page(ring, 4096);
  939. bulk_reset(b);
  940. b->slot = s->slot_id;
  941. b->ep = ep;
  942. b->ep_id = ep_id;
  943. b->mps = mps;
  944. b->ring = ring;
  945. b->enq = 0;
  946. b->cycle = 1;
  947. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  948. rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
  949. if (rc != 0) {
  950. bulk_reset(b);
  951. return -1;
  952. }
  953. b->active = 1;
  954. }
  955. spin_lock_irqsave(&xhci_lock, &flags);
  956. if (out_busy) {
  957. spin_unlock_irqrestore(&xhci_lock, flags);
  958. return -1;
  959. }
  960. out_busy = 1;
  961. out_done = 0;
  962. out_status = 0;
  963. out_slot = s->slot_id;
  964. out_trb = b->ring + b->enq * 16;
  965. b->trb_len = len;
  966. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  967. (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  968. __asm__ volatile ("" ::: "memory");
  969. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  970. spin_unlock_irqrestore(&xhci_lock, flags);
  971. t0 = get_ticks_ms();
  972. while (get_ticks_ms() - t0 < 1000) {
  973. spin_lock_irqsave(&xhci_lock, &flags);
  974. xhci_poll_locked();
  975. done = out_done;
  976. spin_unlock_irqrestore(&xhci_lock, flags);
  977. if (done)
  978. break;
  979. __asm__ volatile ("pause");
  980. }
  981. spin_lock_irqsave(&xhci_lock, &flags);
  982. done = out_done;
  983. status = out_status;
  984. out_busy = 0;
  985. spin_unlock_irqrestore(&xhci_lock, flags);
  986. if (!done) {
  987. dbg("xhci: bulk out timeout\n");
  988. return -1;
  989. }
  990. if (status != CC_SUCCESS) {
  991. dbg_hex("xhci: bulk out status ", status);
  992. return -1;
  993. }
  994. return 0;
  995. }
  996. /* ---- root hub ports ---------------------------------------------------- */
  997. static int port_reset(unsigned int ctrl, unsigned int port)
  998. {
  999. unsigned int ps, t0;
  1000. (void)ctrl;
  1001. if (!found || port >= max_ports ||
  1002. !(usb2_mask & (1u << port)))
  1003. return -1;
  1004. if (port_slot[port])
  1005. free_slot(&slots[port_slot[port]]);
  1006. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1007. if (!(ps & PORTSC_CCS))
  1008. return -1;
  1009. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1010. (ps & ~PORTSC_RWC) | PORTSC_PR);
  1011. t0 = get_ticks_ms();
  1012. while (get_ticks_ms() - t0 < 200) {
  1013. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1014. if (ps & PORTSC_PRC)
  1015. break;
  1016. }
  1017. if (!(ps & PORTSC_PRC)) {
  1018. dbg_hex("xhci: port reset timeout ", port);
  1019. return -1;
  1020. }
  1021. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1022. (ps & ~PORTSC_RWC) | PORTSC_PRC);
  1023. t0 = get_ticks_ms();
  1024. while (get_ticks_ms() - t0 < 100) {
  1025. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1026. if (ps & PORTSC_PED)
  1027. break;
  1028. }
  1029. if (!(ps & PORTSC_PED)) {
  1030. dbg_hex("xhci: port not enabled ", port);
  1031. return -1;
  1032. }
  1033. last_port = (int)port;
  1034. return 0;
  1035. }
  1036. static int port_connected(unsigned int ctrl, unsigned int port)
  1037. {
  1038. unsigned int ps;
  1039. (void)ctrl;
  1040. if (!found || port >= max_ports ||
  1041. !(usb2_mask & (1u << port)))
  1042. return 0;
  1043. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1044. return (ps & PORTSC_CCS) ? 1 : 0;
  1045. }
  1046. static int port_low_speed(unsigned int ctrl, unsigned int port)
  1047. {
  1048. unsigned int ps;
  1049. (void)ctrl;
  1050. if (!found || port >= max_ports)
  1051. return 0;
  1052. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1053. return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
  1054. }
  1055. static int port_take_change(unsigned int ctrl, unsigned int port)
  1056. {
  1057. unsigned int ps;
  1058. (void)ctrl;
  1059. if (!found || port >= max_ports)
  1060. return 0;
  1061. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1062. if (!(ps & PORTSC_CSC))
  1063. return 0;
  1064. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1065. (ps & ~PORTSC_RWC) | PORTSC_CSC);
  1066. return 1;
  1067. }
  1068. /* ---- controller discovery ---------------------------------------------- */
  1069. /* xHCI USB Legacy Support: stop the firmware's SMI from firing on port
  1070. * changes once we own the controller (the same idea as the PCI 0xC0 write
  1071. * in uhci.c/ohci.c, but xHCI keeps it in an extended capability). */
  1072. static void legacy_smi_disable(unsigned int off)
  1073. {
  1074. unsigned int sup = mmio_rd(off);
  1075. unsigned int ctl;
  1076. if ((sup & 0xFF) != 1)
  1077. return;
  1078. if (sup & (1u << 16)) /* BIOS still owns it */
  1079. mmio_wr(off, sup | (1u << 24));
  1080. ctl = mmio_rd(off + 4);
  1081. ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
  1082. ctl |= 0x7u << 29;
  1083. mmio_wr(off + 4, ctl);
  1084. }
  1085. static void scan_protocols(void)
  1086. {
  1087. unsigned int off, guard = 0;
  1088. usb2_mask = 0;
  1089. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1090. while (off && guard++ < 32) {
  1091. unsigned int cap = mmio_rd(off);
  1092. unsigned int id = cap & 0xFF;
  1093. unsigned int next = (cap >> 8) & 0xFF;
  1094. if (id == 1) {
  1095. legacy_smi_disable(off);
  1096. } else if (id == 2) { /* Supported Protocol */
  1097. unsigned int major = (cap >> 24) & 0xFF;
  1098. unsigned int pi = mmio_rd(off + 8);
  1099. unsigned int po = pi & 0xFF;
  1100. unsigned int pc = (pi >> 8) & 0xFF;
  1101. unsigned int i;
  1102. if (major <= 2 && po >= 1) {
  1103. for (i = 0; i < pc && po + i <= 32; i++)
  1104. usb2_mask |= 1u << (po - 1 + i);
  1105. }
  1106. }
  1107. if (!next)
  1108. break;
  1109. off += next << 2;
  1110. }
  1111. if (!usb2_mask)
  1112. usb2_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
  1113. ((1u << max_ports) - 1);
  1114. }
  1115. static int xhci_init(void)
  1116. {
  1117. unsigned int bus, dev, func, id, class_rev, bar, bar_hi, cmd;
  1118. unsigned int hcs1, hcs2, t0, i, n, scratch;
  1119. if (found)
  1120. return 1;
  1121. for (bus = 0; bus < 256; bus++) {
  1122. for (dev = 0; dev < 32; dev++) {
  1123. for (func = 0; func < 8; func++) {
  1124. id = pci_cfg_read32(bus, dev, func, 0x00);
  1125. if (id == 0xFFFFFFFFu || id == 0)
  1126. continue;
  1127. class_rev = pci_cfg_read32(bus, dev, func, 0x08);
  1128. if ((class_rev >> 16) != 0x0C03)
  1129. continue;
  1130. if (((class_rev >> 8) & 0xFF) != 0x30)
  1131. continue;
  1132. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1133. if (bar & 1)
  1134. continue;
  1135. bar_hi = 0;
  1136. if ((bar & 0x6) == 0x4)
  1137. bar_hi = pci_cfg_read32(bus, dev,
  1138. func, 0x14);
  1139. bar &= 0xFFFFFFF0u;
  1140. if (!bar || bar_hi)
  1141. continue;
  1142. pci_bus = bus;
  1143. pci_dev = dev;
  1144. pci_func = func;
  1145. goto found_xhci;
  1146. }
  1147. }
  1148. }
  1149. return 0;
  1150. found_xhci:
  1151. cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
  1152. pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
  1153. mmio_base = bar;
  1154. paging_map_mmio(mmio_base, 0x10000);
  1155. cap_len = mmio_rd(0) & 0xFF;
  1156. hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
  1157. hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
  1158. hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
  1159. rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
  1160. db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
  1161. max_slots = hcs1 & 0xFF;
  1162. if (max_slots > XHCI_MAX_SLOTS)
  1163. max_slots = XHCI_MAX_SLOTS;
  1164. if (!max_slots)
  1165. max_slots = 1;
  1166. max_ports = (hcs1 >> 24) & 0xFF;
  1167. if (max_ports > XHCI_MAX_PORTS)
  1168. max_ports = XHCI_MAX_PORTS;
  1169. if (!max_ports)
  1170. max_ports = 1;
  1171. ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
  1172. dbg("xhci: found\n");
  1173. dbg_hex("xhci: mmio ", mmio_base);
  1174. dbg_dec("xhci: slots ", max_slots);
  1175. dbg_dec("xhci: ports ", max_ports);
  1176. dbg_dec("xhci: cap len ", cap_len);
  1177. dcbaa = mm_alloc_page();
  1178. input_ctx = mm_alloc_page();
  1179. cmd_ring = mm_alloc_page();
  1180. evt_ring = mm_alloc_page();
  1181. erst = mm_alloc_page();
  1182. if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
  1183. dbg("xhci: out of pages\n");
  1184. return 0;
  1185. }
  1186. zero_page(dcbaa, 4096);
  1187. zero_page(input_ctx, 4096);
  1188. zero_page(cmd_ring, 4096);
  1189. zero_page(evt_ring, 4096);
  1190. zero_page(erst, 4096);
  1191. /* Reset. HCRST self-clears; wait for CNR to drop. */
  1192. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1193. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
  1194. t0 = get_ticks_ms();
  1195. while (get_ticks_ms() - t0 < 1000) {
  1196. if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
  1197. break;
  1198. }
  1199. t0 = get_ticks_ms();
  1200. while (get_ticks_ms() - t0 < 1000) {
  1201. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1202. break;
  1203. }
  1204. if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
  1205. dbg_hex("xhci: reset failed ",
  1206. mmio_rd(op(XHCI_OP_USBSTS)));
  1207. return 0;
  1208. }
  1209. mmio_wr(op(XHCI_OP_PAGESIZE), 1);
  1210. mmio_wr(op(XHCI_OP_CONFIG), max_slots);
  1211. mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
  1212. /* Scratchpad buffers, if the controller asks for any. */
  1213. n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
  1214. if (n) {
  1215. unsigned int arr;
  1216. if (n > 512)
  1217. n = 512;
  1218. arr = mm_alloc_page();
  1219. if (!arr) {
  1220. dbg("xhci: no scratchpad array\n");
  1221. return 0;
  1222. }
  1223. zero_page(arr, 4096);
  1224. for (i = 0; i < n; i++) {
  1225. scratch = mm_alloc_page();
  1226. if (!scratch) {
  1227. dbg("xhci: no scratchpad page\n");
  1228. return 0;
  1229. }
  1230. zero_page(scratch, 4096);
  1231. ram_wr64(arr + i * 8, scratch);
  1232. }
  1233. ram_wr64(dcbaa, arr);
  1234. dbg_dec("xhci: scratchpads ", n);
  1235. }
  1236. /* Command ring and event ring. */
  1237. ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
  1238. mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
  1239. evt_idx = 0;
  1240. evt_cycle = 1;
  1241. *(volatile unsigned int *)(erst + 0) = evt_ring;
  1242. *(volatile unsigned int *)(erst + 4) = 0;
  1243. *(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
  1244. *(volatile unsigned int *)(erst + 12) = 0;
  1245. mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
  1246. mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
  1247. mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
  1248. mmio_wr(rt(XHCI_RT_IMAN), 0);
  1249. mmio_wr(rt(XHCI_RT_IMOD), 0);
  1250. /* Run. */
  1251. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
  1252. t0 = get_ticks_ms();
  1253. while (get_ticks_ms() - t0 < 1000) {
  1254. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1255. break;
  1256. }
  1257. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
  1258. dbg("xhci: controller did not start\n");
  1259. return 0;
  1260. }
  1261. scan_protocols();
  1262. dbg_hex("xhci: usb2 ports ", usb2_mask);
  1263. /* Port power, where the controller has per-port switching. */
  1264. if (hcc_params & 0x0008u) {
  1265. for (i = 0; i < max_ports; i++) {
  1266. unsigned int ps;
  1267. if (!(usb2_mask & (1u << i)))
  1268. continue;
  1269. ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
  1270. if (!(ps & PORTSC_PP))
  1271. mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
  1272. (ps & ~PORTSC_RWC) | PORTSC_PP);
  1273. }
  1274. }
  1275. found = 1;
  1276. return 1;
  1277. }
  1278. static unsigned int xhci_count(void)
  1279. {
  1280. return found ? 1u : 0u;
  1281. }
  1282. static void xhci_info(unsigned int ctrl, unsigned int *bus,
  1283. unsigned int *dev, unsigned int *func,
  1284. unsigned int *base, int *mmio)
  1285. {
  1286. (void)ctrl;
  1287. *bus = pci_bus;
  1288. *dev = pci_dev;
  1289. *func = pci_func;
  1290. *base = mmio_base;
  1291. *mmio = 1;
  1292. }
  1293. static unsigned int xhci_port_count(unsigned int ctrl)
  1294. {
  1295. (void)ctrl;
  1296. return found ? max_ports : 0;
  1297. }
  1298. const struct usb_hcd_ops usb_xhci_ops = {
  1299. "xHCI",
  1300. xhci_init,
  1301. xhci_count,
  1302. xhci_info,
  1303. control_transfer,
  1304. xhci_port_count,
  1305. port_connected,
  1306. port_low_speed,
  1307. port_reset,
  1308. port_take_change,
  1309. int_in_start,
  1310. int_in_poll,
  1311. stop_int,
  1312. xhci_bulk_out,
  1313. xhci_bulk_in,
  1314. 0,
  1315. };