xhci.c 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714
  1. /* xHCI (USB 3.0) host controller driver.
  2. *
  3. * Scope: one controller, USB2 and USB3 root ports and devices plugged
  4. * directly into them. Control transfers, one interrupt-IN endpoint per HID
  5. * interface and the bulk endpoints used by ASIX and usbmsc are implemented.
  6. * SuperSpeed devices use a 512-byte EP0 and their endpoint descriptors may
  7. * carry a companion descriptor, which is ignored because only the bulk/HID
  8. * endpoint itself is needed.
  9. *
  10. * The schedule is polled, like the rest of the kernel. Rings and contexts
  11. * are page-allocated so the driver does not grow the kernel BSS. */
  12. #include "mm.h"
  13. #include "paging.h"
  14. #include "pci.h"
  15. #include "serial.h"
  16. #include "spinlock.h"
  17. #include "timer.h"
  18. #include "usb_hcd.h"
  19. #include "xhci.h"
  20. #define XHCI_DEBUG 0
  21. /* ---- controller limits ------------------------------------------------- */
  22. #define XHCI_MAX_SLOTS 4
  23. #define XHCI_MAX_PORTS 32
  24. #define XHCI_EVT_TRBS 256
  25. #define XHCI_CMD_TRBS 32
  26. #define XHCI_EP0_TRBS 16
  27. #define XHCI_INT_TRBS 16
  28. #define XHCI_BULK_TRBS 32
  29. /* ---- capability / operational / runtime offsets (bytes) --------------- */
  30. #define XHCI_CAP_HCSPARAMS1 0x04
  31. #define XHCI_CAP_HCSPARAMS2 0x08
  32. #define XHCI_CAP_HCCPARAMS1 0x10
  33. #define XHCI_CAP_DBOFF 0x14
  34. #define XHCI_CAP_RTSOFF 0x18
  35. #define XHCI_OP_USBCMD 0x00
  36. #define XHCI_OP_USBSTS 0x04
  37. #define XHCI_OP_PAGESIZE 0x08
  38. #define XHCI_OP_CRCR 0x18
  39. #define XHCI_OP_DCBAAP 0x30
  40. #define XHCI_OP_CONFIG 0x38
  41. #define XHCI_OP_PORTSC 0x400
  42. #define XHCI_RT_IMAN 0x20
  43. #define XHCI_RT_IMOD 0x24
  44. #define XHCI_RT_ERSTSZ 0x28
  45. #define XHCI_RT_ERSTBA 0x30
  46. #define XHCI_RT_ERDP 0x38
  47. /* USBCMD / USBSTS */
  48. #define XHCI_CMD_RS 0x0001u
  49. #define XHCI_CMD_HCRST 0x0002u
  50. #define XHCI_STS_HCH 0x0001u
  51. #define XHCI_STS_CNR 0x0800u
  52. #define XHCI_STS_HCE 0x1000u
  53. /* PORTSC */
  54. #define PORTSC_CCS 0x00000001u
  55. #define PORTSC_PED 0x00000002u
  56. #define PORTSC_PR 0x00000010u
  57. #define PORTSC_PP 0x00000200u
  58. #define PORTSC_SPEED_SHIFT 10
  59. #define PORTSC_SPEED_MASK 0xFu
  60. #define PORTSC_CSC 0x00020000u
  61. #define PORTSC_PEC 0x00040000u
  62. #define PORTSC_WRC 0x00080000u
  63. #define PORTSC_OCC 0x00100000u
  64. #define PORTSC_PRC 0x00200000u
  65. #define PORTSC_PLC 0x00400000u
  66. #define PORTSC_CEC 0x00800000u
  67. #define PORTSC_RWC (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
  68. PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
  69. PORTSC_CEC | PORTSC_PED)
  70. /* Bits that may be written back while preserving the port state:
  71. * read-only status (CCS, OCA, speed, device-removable) and the
  72. * read/write-state fields (PLS, PP, PIC, wake enables). Everything else
  73. * -- especially PR/WPR (RW1S) and PED (RW1CS) -- must be written as 0 or
  74. * the write changes the port state. Linux's
  75. * xhci_port_state_to_neutral() does the same. */
  76. #define PORTSC_RO_MASK (PORTSC_CCS | 0x00000008u | \
  77. (PORTSC_SPEED_MASK << PORTSC_SPEED_SHIFT) | \
  78. 0x40000000u)
  79. #define PORTSC_RWS_MASK (0x000001E0u | PORTSC_PP | 0x0000C000u | \
  80. 0x0E000000u)
  81. static unsigned int portsc_neutral(unsigned int ps)
  82. {
  83. return (ps & PORTSC_RO_MASK) | (ps & PORTSC_RWS_MASK);
  84. }
  85. /* TRB types */
  86. #define TRB_NORMAL 1
  87. #define TRB_SETUP 2
  88. #define TRB_DATA 3
  89. #define TRB_STATUS 4
  90. #define TRB_LINK 6
  91. #define TRB_ENABLE_SLOT 9
  92. #define TRB_DISABLE_SLOT 10
  93. #define TRB_ADDR_DEV 11
  94. #define TRB_CONFIG_EP 12
  95. #define TRB_TRANSFER 32
  96. #define TRB_COMPLETION 33
  97. /* TRB control bits */
  98. #define TRB_CYCLE 0x00000001u
  99. #define TRB_TC 0x00000002u
  100. #define TRB_ISP 0x00000004u
  101. #define TRB_CHAIN 0x00000010u
  102. #define TRB_IOC 0x00000020u
  103. #define TRB_IDT 0x00000040u
  104. #define TRB_BSR 0x00000200u
  105. #define TRB_DIR_IN 0x00010000u
  106. #define TRB_TYPE(t) ((t) << 10)
  107. /* completion codes */
  108. #define CC_SUCCESS 1
  109. #define CC_SHORT_PACKET 13
  110. /* slot/EP context flags and endpoint types */
  111. #define SLOT_FLAG 0x00000001u
  112. #define EP0_FLAG 0x00000002u
  113. #define EP_CTRL 4
  114. #define EP_BULK_OUT 2
  115. #define EP_BULK_IN 6
  116. #define EP_INT_IN 7
  117. struct xhci_trb {
  118. unsigned int p0;
  119. unsigned int p1;
  120. unsigned int status;
  121. unsigned int control;
  122. };
  123. struct xhci_slot {
  124. int used;
  125. unsigned int slot_id;
  126. unsigned int addr; /* logical address used by usb.c */
  127. unsigned int dev_addr; /* address assigned by the xHC */
  128. unsigned int port;
  129. unsigned int speed;
  130. unsigned int dc; /* device context page */
  131. unsigned int ep0; /* EP0 transfer ring page */
  132. unsigned int ep0_enq;
  133. unsigned int ep0_cycle;
  134. unsigned int ep_mask; /* configured endpoint flags */
  135. };
  136. struct xhci_int {
  137. int active;
  138. unsigned int slot; /* hardware slot id */
  139. unsigned int ep_id; /* xHCI endpoint id (2n+dir) */
  140. unsigned int mps;
  141. unsigned int ring;
  142. unsigned int enq;
  143. unsigned int cycle;
  144. unsigned int trb_len;
  145. unsigned int pending_len;
  146. int pending;
  147. int error;
  148. unsigned char buf[8] __attribute__((aligned(16)));
  149. };
  150. struct xhci_bulk {
  151. int active;
  152. int armed;
  153. unsigned int slot;
  154. unsigned int ep; /* endpoint number */
  155. unsigned int ep_id;
  156. unsigned int mps;
  157. unsigned int ring;
  158. unsigned int enq;
  159. unsigned int cycle;
  160. unsigned int buf;
  161. unsigned int len;
  162. unsigned int trb_len;
  163. unsigned int actual;
  164. int pending;
  165. int error;
  166. };
  167. static unsigned int mmio_base;
  168. static unsigned int cap_len;
  169. static unsigned int rt_off;
  170. static unsigned int db_off;
  171. static unsigned int max_slots;
  172. static unsigned int max_ports;
  173. static unsigned int ctx_size;
  174. static unsigned int port_mask;
  175. static unsigned int hcc_params;
  176. static unsigned int pci_bus, pci_dev, pci_func;
  177. static int found;
  178. static int last_port = -1;
  179. static unsigned int dcbaa;
  180. static unsigned int input_ctx;
  181. static unsigned int cmd_ring;
  182. static unsigned int evt_ring;
  183. static unsigned int erst;
  184. static unsigned int cmd_enq;
  185. static unsigned int cmd_cycle;
  186. static unsigned int evt_idx;
  187. static unsigned int evt_cycle;
  188. static unsigned int cmd_busy;
  189. static unsigned int cmd_done;
  190. static unsigned int cmd_status;
  191. static unsigned int cmd_slot;
  192. static unsigned int cmd_trb_addr;
  193. static unsigned int ctl_busy;
  194. static unsigned int ctl_done;
  195. static unsigned int ctl_slot;
  196. static unsigned int ctl_status;
  197. static unsigned int ctl_req_len;
  198. static unsigned int ctl_actual;
  199. static unsigned int ctl_data_seen;
  200. static unsigned int ctl_setup_trb;
  201. static unsigned int ctl_data_trb;
  202. static unsigned int ctl_status_trb;
  203. static struct xhci_bulk bulk_in;
  204. static struct xhci_bulk bulk_out;
  205. static unsigned int out_busy;
  206. static unsigned int out_done;
  207. static unsigned int out_status;
  208. static unsigned int out_slot;
  209. static unsigned int out_trb;
  210. static struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
  211. static unsigned int addr_slot[128];
  212. static unsigned int port_slot[XHCI_MAX_PORTS];
  213. static struct xhci_int ints[USB_HCD_INT_SLOTS];
  214. static spinlock_t xhci_lock;
  215. /* ---- debug ------------------------------------------------------------- */
  216. #if XHCI_DEBUG
  217. static void dbg(const char *s)
  218. {
  219. serial_puts(SERIAL_COM1, s);
  220. }
  221. static void dbg_hex(const char *s, unsigned int v)
  222. {
  223. serial_puts(SERIAL_COM1, s);
  224. serial_puthex(SERIAL_COM1, v);
  225. serial_putc(SERIAL_COM1, '\n');
  226. }
  227. static void dbg_dec(const char *s, unsigned int v)
  228. {
  229. serial_puts(SERIAL_COM1, s);
  230. serial_putdec(SERIAL_COM1, v);
  231. serial_putc(SERIAL_COM1, '\n');
  232. }
  233. #else
  234. static void dbg(const char *s) { (void)s; }
  235. static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
  236. static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
  237. #endif
  238. static void delay_ms(unsigned int ms)
  239. {
  240. unsigned int t0 = get_ticks_ms();
  241. while (get_ticks_ms() - t0 < ms)
  242. ;
  243. }
  244. /* ---- MMIO helpers ------------------------------------------------------ */
  245. static unsigned int mmio_rd(unsigned int off)
  246. {
  247. return *(volatile unsigned int *)(mmio_base + off);
  248. }
  249. static void mmio_wr(unsigned int off, unsigned int val)
  250. {
  251. *(volatile unsigned int *)(mmio_base + off) = val;
  252. }
  253. static void mmio_wr64(unsigned int off, unsigned int lo)
  254. {
  255. mmio_wr(off, lo);
  256. mmio_wr(off + 4, 0);
  257. }
  258. /* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
  259. static void ram_wr64(unsigned int addr, unsigned int lo)
  260. {
  261. *(volatile unsigned int *)addr = lo;
  262. *(volatile unsigned int *)(addr + 4) = 0;
  263. }
  264. static void zero_page(unsigned int addr, unsigned int len)
  265. {
  266. unsigned int *p = (unsigned int *)addr;
  267. while (len >= 4) {
  268. *p++ = 0;
  269. len -= 4;
  270. }
  271. }
  272. static unsigned int op(unsigned int reg)
  273. {
  274. return cap_len + reg;
  275. }
  276. static unsigned int rt(unsigned int reg)
  277. {
  278. return rt_off + reg;
  279. }
  280. /* ---- rings ------------------------------------------------------------- */
  281. static void ring_init(unsigned int base, unsigned int size,
  282. unsigned int *enq, unsigned int *cycle)
  283. {
  284. struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
  285. link->p0 = base;
  286. link->p1 = 0;
  287. link->status = 0;
  288. link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  289. *enq = 0;
  290. *cycle = 1;
  291. }
  292. static void ring_queue(unsigned int base, unsigned int size,
  293. unsigned int *enq, unsigned int *cycle,
  294. unsigned int p0, unsigned int p1,
  295. unsigned int status, unsigned int control)
  296. {
  297. struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
  298. trb->p0 = p0;
  299. trb->p1 = p1;
  300. trb->status = status;
  301. __asm__ volatile ("" ::: "memory");
  302. trb->control = control | *cycle;
  303. __asm__ volatile ("" ::: "memory");
  304. (*enq)++;
  305. if (*enq == size - 1) {
  306. struct xhci_trb *link =
  307. (struct xhci_trb *)(base + (size - 1) * 16);
  308. __asm__ volatile ("" ::: "memory");
  309. link->control ^= TRB_CYCLE;
  310. *enq = 0;
  311. *cycle ^= 1;
  312. }
  313. }
  314. /* ---- event handling ---------------------------------------------------- */
  315. static void xhci_poll_locked(void);
  316. static void stop_int(unsigned int slot);
  317. static void bulk_reset(struct xhci_bulk *b);
  318. static void xhci_int_event(unsigned int slot, unsigned int ep,
  319. unsigned int code, unsigned int residual)
  320. {
  321. unsigned int i;
  322. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  323. if (!ints[i].active || ints[i].slot != slot ||
  324. ints[i].ep_id != ep)
  325. continue;
  326. if (!ints[i].pending) {
  327. if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
  328. ints[i].pending_len =
  329. (ints[i].trb_len >= residual) ?
  330. ints[i].trb_len - residual : 0;
  331. ints[i].pending = 1;
  332. } else {
  333. ints[i].pending = 1;
  334. ints[i].error = 1;
  335. }
  336. }
  337. break;
  338. }
  339. }
  340. static void xhci_poll_locked(void)
  341. {
  342. unsigned int n = 0;
  343. while (n++ < 512) {
  344. struct xhci_trb *ev =
  345. (struct xhci_trb *)(evt_ring + evt_idx * 16);
  346. unsigned int type;
  347. if ((ev->control & TRB_CYCLE) != evt_cycle)
  348. break;
  349. type = (ev->control >> 10) & 0x3F;
  350. if (type == TRB_COMPLETION) {
  351. unsigned int code = (ev->status >> 24) & 0xFF;
  352. unsigned int slot = (ev->control >> 24) & 0xFF;
  353. if (cmd_busy && ev->p0 == cmd_trb_addr) {
  354. cmd_status = code;
  355. cmd_slot = slot;
  356. cmd_done = 1;
  357. }
  358. } else if (type == TRB_TRANSFER) {
  359. unsigned int code = (ev->status >> 24) & 0xFF;
  360. unsigned int residual = ev->status & 0xFFFFFF;
  361. unsigned int slot = (ev->control >> 24) & 0xFF;
  362. unsigned int ep = (ev->control >> 16) & 0x1F;
  363. if (ctl_busy && slot == ctl_slot && ep == 1) {
  364. if (ev->p0 == ctl_setup_trb) {
  365. /* The setup stage is its own TD and
  366. * completes first; only an error stops
  367. * the transfer. */
  368. if (code != CC_SUCCESS &&
  369. code != CC_SHORT_PACKET) {
  370. ctl_status = code;
  371. ctl_done = 1;
  372. }
  373. } else if (ev->p0 == ctl_data_trb) {
  374. ctl_actual = (ctl_req_len >= residual) ?
  375. ctl_req_len - residual : 0;
  376. ctl_data_seen = 1;
  377. if (code != CC_SUCCESS &&
  378. code != CC_SHORT_PACKET) {
  379. ctl_status = code;
  380. ctl_done = 1;
  381. }
  382. } else if (ev->p0 == ctl_status_trb) {
  383. if (!ctl_data_seen)
  384. ctl_actual =
  385. (code == CC_SUCCESS) ?
  386. ctl_req_len : 0;
  387. ctl_status = code;
  388. ctl_done = 1;
  389. }
  390. } else if (out_busy && slot == out_slot &&
  391. ep == bulk_out.ep_id &&
  392. ev->p0 == out_trb) {
  393. out_status = code;
  394. out_done = 1;
  395. } else if (bulk_in.active && slot == bulk_in.slot &&
  396. ep == bulk_in.ep_id) {
  397. bulk_in.armed = 0;
  398. if (!bulk_in.pending) {
  399. if (code == CC_SUCCESS ||
  400. code == CC_SHORT_PACKET) {
  401. bulk_in.actual =
  402. (bulk_in.trb_len >=
  403. residual) ?
  404. bulk_in.trb_len -
  405. residual : 0;
  406. bulk_in.pending = 1;
  407. } else {
  408. bulk_in.pending = 1;
  409. bulk_in.error = 1;
  410. }
  411. }
  412. } else {
  413. xhci_int_event(slot, ep, code, residual);
  414. }
  415. }
  416. evt_idx++;
  417. if (evt_idx >= XHCI_EVT_TRBS) {
  418. evt_idx = 0;
  419. evt_cycle ^= 1;
  420. }
  421. }
  422. mmio_wr64(rt(XHCI_RT_ERDP),
  423. (evt_ring + evt_idx * 16) | 8);
  424. }
  425. /* ---- commands ---------------------------------------------------------- */
  426. static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
  427. unsigned int *out_slot)
  428. {
  429. unsigned int flags, t0, rc;
  430. if (!found)
  431. return -1;
  432. spin_lock_irqsave(&xhci_lock, &flags);
  433. if (cmd_busy) {
  434. spin_unlock_irqrestore(&xhci_lock, flags);
  435. return -1;
  436. }
  437. cmd_busy = 1;
  438. cmd_done = 0;
  439. cmd_status = 0;
  440. cmd_slot = 0;
  441. cmd_trb_addr = cmd_ring + cmd_enq * 16;
  442. ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
  443. p0, p1, 0, control);
  444. __asm__ volatile ("" ::: "memory");
  445. mmio_wr(db_off, 0);
  446. spin_unlock_irqrestore(&xhci_lock, flags);
  447. t0 = get_ticks_ms();
  448. while (get_ticks_ms() - t0 < 1000) {
  449. spin_lock_irqsave(&xhci_lock, &flags);
  450. xhci_poll_locked();
  451. rc = cmd_done;
  452. spin_unlock_irqrestore(&xhci_lock, flags);
  453. if (rc)
  454. break;
  455. __asm__ volatile ("pause");
  456. }
  457. spin_lock_irqsave(&xhci_lock, &flags);
  458. cmd_busy = 0;
  459. if (!cmd_done) {
  460. spin_unlock_irqrestore(&xhci_lock, flags);
  461. dbg("xhci: command timeout\n");
  462. return -1;
  463. }
  464. if (out_slot)
  465. *out_slot = cmd_slot;
  466. rc = cmd_status;
  467. spin_unlock_irqrestore(&xhci_lock, flags);
  468. if (rc != CC_SUCCESS) {
  469. dbg_hex("xhci: command status ", rc);
  470. return -1;
  471. }
  472. return 0;
  473. }
  474. /* ---- contexts ---------------------------------------------------------- */
  475. static void fill_slot_ctx(unsigned int addr, unsigned int port,
  476. unsigned int speed, unsigned int last_ctx)
  477. {
  478. unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
  479. p[0] = (speed << 20) | (last_ctx << 27);
  480. p[1] = (port + 1) << 16;
  481. p[2] = 0;
  482. p[3] = addr;
  483. }
  484. static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
  485. unsigned int enq, unsigned int cycle)
  486. {
  487. unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
  488. p[0] = 0;
  489. p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
  490. p[2] = (ring + enq * 16) | cycle;
  491. p[3] = 0;
  492. p[4] = 8;
  493. }
  494. static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
  495. unsigned int mps, unsigned int interval,
  496. unsigned int ring)
  497. {
  498. unsigned int *p =
  499. (unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
  500. p[0] = interval << 16;
  501. p[1] = (type << 3) | (3 << 1) | (mps << 16);
  502. p[2] = ring | 1;
  503. p[3] = 0;
  504. p[4] = mps;
  505. }
  506. static struct xhci_slot *slot_by_addr(unsigned int addr)
  507. {
  508. unsigned int id;
  509. if (addr > 127)
  510. return 0;
  511. id = addr_slot[addr];
  512. if (id < 1 || id > max_slots || !slots[id].used)
  513. return 0;
  514. return &slots[id];
  515. }
  516. static unsigned int port_speed(unsigned int port)
  517. {
  518. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  519. unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
  520. return sp ? sp : 1;
  521. }
  522. /* Default control-pipe max packet size for the BSR Address Device command.
  523. * High speed must be 64; full speed uses the USB core's 64-byte guess (the
  524. * descriptor read then corrects it); low speed is 8. Using 8 for a
  525. * high/full-speed slot leaves EP0 unusable on real xHCs. */
  526. static unsigned int ep0_mps_for_speed(unsigned int speed)
  527. {
  528. if (speed == 2) /* low speed */
  529. return 8;
  530. if (speed >= 4) /* SuperSpeed / SuperSpeedPlus */
  531. return 512;
  532. return 64; /* full speed (guess) and high speed */
  533. }
  534. static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
  535. unsigned int type, unsigned int mps,
  536. unsigned int interval, unsigned int ring)
  537. {
  538. unsigned int ep_flag = 1u << (ep_index + 1);
  539. unsigned int last = ep_index + 1;
  540. unsigned int i, flags;
  541. int rc;
  542. for (i = 1; i < 32; i++)
  543. if (s->ep_mask & (1u << i))
  544. last = i;
  545. spin_lock_irqsave(&xhci_lock, &flags);
  546. zero_page(input_ctx, 4096);
  547. *(unsigned int *)input_ctx = 0;
  548. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
  549. fill_slot_ctx(s->dev_addr, s->port, s->speed, last);
  550. fill_ep_ctx(ep_index, type, mps, interval, ring);
  551. spin_unlock_irqrestore(&xhci_lock, flags);
  552. rc = xhci_cmd(input_ctx, 0,
  553. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  554. if (rc == 0)
  555. s->ep_mask |= ep_flag;
  556. return rc;
  557. }
  558. /* ---- slots ------------------------------------------------------------- */
  559. /* Create a slot. bsr=1: Address Device with Block Set Address Request,
  560. * leaving the slot in the Default state at device address 0 (new
  561. * enumeration scheme). bsr=0: the command carries target_addr and moves
  562. * Enabled -> Addressed directly (traditional scheme), which is what Linux
  563. * does when the port is addressed before the first descriptor read.
  564. * mps == 0 (or a SuperSpeed port) derives it from the port speed. */
  565. static struct xhci_slot *new_slot_common(unsigned int target_addr,
  566. unsigned int mps, int bsr)
  567. {
  568. struct xhci_slot *s;
  569. unsigned int slot_id, dc, ep0, flags;
  570. int rc;
  571. if (last_port < 0 || last_port >= (int)max_ports)
  572. return 0;
  573. if (port_slot[last_port])
  574. return 0;
  575. slot_id = 0;
  576. rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
  577. if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
  578. dbg("xhci: enable slot failed\n");
  579. return 0;
  580. }
  581. dc = mm_alloc_page();
  582. ep0 = mm_alloc_page();
  583. if (!dc || !ep0) {
  584. if (dc)
  585. mm_free_page(dc);
  586. if (ep0)
  587. mm_free_page(ep0);
  588. return 0;
  589. }
  590. zero_page(dc, 4096);
  591. zero_page(ep0, 4096);
  592. s = &slots[slot_id];
  593. s->used = 1;
  594. s->slot_id = slot_id;
  595. s->addr = 0;
  596. s->port = (unsigned int)last_port;
  597. s->speed = port_speed((unsigned int)last_port);
  598. s->dc = dc;
  599. s->ep0 = ep0;
  600. s->ep0_enq = 0;
  601. s->ep0_cycle = 1;
  602. s->ep_mask = 0;
  603. s->dev_addr = 0;
  604. ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
  605. if (!mps || s->speed >= 4)
  606. mps = ep0_mps_for_speed(s->speed);
  607. ram_wr64(dcbaa + slot_id * 8, dc);
  608. spin_lock_irqsave(&xhci_lock, &flags);
  609. zero_page(input_ctx, 4096);
  610. *(unsigned int *)input_ctx = 0;
  611. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  612. fill_slot_ctx(0, s->port, s->speed, 1);
  613. fill_ep0_ctx(mps, ep0, s->ep0_enq, s->ep0_cycle);
  614. spin_unlock_irqrestore(&xhci_lock, flags);
  615. rc = xhci_cmd(input_ctx, 0,
  616. TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) |
  617. (bsr ? TRB_BSR : 0), 0);
  618. if (rc != 0) {
  619. dbg(bsr ? "xhci: address default failed\n" :
  620. "xhci: address device failed\n");
  621. ram_wr64(dcbaa + slot_id * 8, 0);
  622. xhci_cmd(0, 0,
  623. TRB_TYPE(TRB_DISABLE_SLOT) | (slot_id << 24), 0);
  624. mm_free_page(dc);
  625. mm_free_page(ep0);
  626. s->used = 0;
  627. return 0;
  628. }
  629. /* The xHC assigns the USB device address (the input slot context
  630. * address is 0); keep it for later Configure Endpoint commands. */
  631. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  632. s->addr = bsr ? 0 : target_addr;
  633. addr_slot[s->addr] = slot_id;
  634. port_slot[s->port] = slot_id;
  635. return s;
  636. }
  637. /* New-scheme slot: Enabled -> Default, at device address 0. */
  638. static struct xhci_slot *new_slot(void)
  639. {
  640. return new_slot_common(0, 0, 1);
  641. }
  642. static void free_slot(struct xhci_slot *s)
  643. {
  644. unsigned int i, flags;
  645. if (!s || !s->used)
  646. return;
  647. xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
  648. ram_wr64(dcbaa + s->slot_id * 8, 0);
  649. spin_lock_irqsave(&xhci_lock, &flags);
  650. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  651. if (ints[i].slot == s->slot_id) {
  652. ints[i].active = 0;
  653. ints[i].pending = 0;
  654. ints[i].error = 0;
  655. if (ints[i].ring) {
  656. mm_free_page(ints[i].ring);
  657. ints[i].ring = 0;
  658. }
  659. ints[i].slot = 0;
  660. }
  661. }
  662. if (s->dc) {
  663. mm_free_page(s->dc);
  664. s->dc = 0;
  665. }
  666. if (s->ep0) {
  667. mm_free_page(s->ep0);
  668. s->ep0 = 0;
  669. }
  670. if (s->addr < 128)
  671. addr_slot[s->addr] = 0;
  672. if (bulk_in.active && bulk_in.slot == s->slot_id)
  673. bulk_reset(&bulk_in);
  674. if (bulk_out.active && bulk_out.slot == s->slot_id)
  675. bulk_reset(&bulk_out);
  676. if (s->port < XHCI_MAX_PORTS)
  677. port_slot[s->port] = 0;
  678. s->used = 0;
  679. s->addr = 0;
  680. s->ep_mask = 0;
  681. spin_unlock_irqrestore(&xhci_lock, flags);
  682. }
  683. static int set_address(unsigned int new_addr, unsigned int mps)
  684. {
  685. struct xhci_slot *s = slot_by_addr(0);
  686. unsigned int flags;
  687. int rc;
  688. if (!s || new_addr == 0 || new_addr > 127)
  689. return -1;
  690. /* The USB core's first SET_ADDRESS uses its high/full-speed 64-byte
  691. * guess; a SuperSpeed slot's EP0 is always 512 bytes. */
  692. if (s->speed >= 4)
  693. mps = ep0_mps_for_speed(s->speed);
  694. spin_lock_irqsave(&xhci_lock, &flags);
  695. zero_page(input_ctx, 4096);
  696. *(unsigned int *)input_ctx = 0;
  697. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  698. fill_slot_ctx(0, s->port, s->speed, 1);
  699. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  700. spin_unlock_irqrestore(&xhci_lock, flags);
  701. rc = xhci_cmd(input_ctx, 0,
  702. TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
  703. if (rc != 0) {
  704. dbg("xhci: set address failed\n");
  705. return -1;
  706. }
  707. spin_lock_irqsave(&xhci_lock, &flags);
  708. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  709. addr_slot[0] = 0;
  710. s->addr = new_addr;
  711. addr_slot[new_addr] = s->slot_id;
  712. spin_unlock_irqrestore(&xhci_lock, flags);
  713. return 0;
  714. }
  715. /* ---- control transfers ------------------------------------------------- */
  716. static int control_transfer(unsigned int ctrl, unsigned int addr,
  717. unsigned int mps, int low_speed,
  718. const unsigned char *setup, unsigned char *data,
  719. unsigned int len, int dir_in)
  720. {
  721. struct xhci_slot *s;
  722. unsigned int flags, t0, rc, st;
  723. unsigned int setup_c, data_c, status_c;
  724. unsigned int last_bell;
  725. (void)ctrl;
  726. (void)low_speed;
  727. if (!found)
  728. return -1;
  729. /* xHCI owns SET_ADDRESS: it is an Address Device command, not a
  730. * control request on the wire. If a Default-state slot exists (new
  731. * scheme), move it to Addressed. Otherwise this is the traditional
  732. * scheme: create the slot Enabled and address it in one command,
  733. * without ever using the Default state. */
  734. if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0) {
  735. unsigned int new_addr = (unsigned int)setup[2] |
  736. ((unsigned int)setup[3] << 8);
  737. s = slot_by_addr(0);
  738. if (s)
  739. return set_address(new_addr, mps);
  740. s = new_slot_common(new_addr, mps, 0);
  741. return s ? 0 : -1;
  742. }
  743. s = slot_by_addr(addr);
  744. if (s && addr == 0 && (int)s->port != last_port) {
  745. /* A failed enumeration on another port left a slot at
  746. * address 0; it must not be reused for this device. */
  747. free_slot(s);
  748. s = 0;
  749. }
  750. if (!s) {
  751. if (addr != 0)
  752. return -1;
  753. s = new_slot();
  754. if (!s)
  755. return -1;
  756. }
  757. spin_lock_irqsave(&xhci_lock, &flags);
  758. if (ctl_busy) {
  759. spin_unlock_irqrestore(&xhci_lock, flags);
  760. dbg("xhci: control busy\n");
  761. return -1;
  762. }
  763. ctl_busy = 1;
  764. ctl_done = 0;
  765. ctl_status = 0;
  766. ctl_actual = 0;
  767. ctl_data_seen = 0;
  768. ctl_slot = s->slot_id;
  769. ctl_req_len = len;
  770. /* Encode the three stages the way EDK2's XhciDxe does: every stage
  771. * is its own TD (no CHAIN) and carries IOC. This Braswell xHC never
  772. * completes a chained multi-TRB control TD: with Linux's chaining
  773. * (CHAIN on setup+data, IOC on status) the data stage hangs and no
  774. * transfer event is ever posted, while the per-stage encoding
  775. * completes. */
  776. setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_IOC;
  777. if (len)
  778. setup_c |= (dir_in ? 3u : 2u) << 16;
  779. ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
  780. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  781. (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  782. ((unsigned int)setup[2] << 16) |
  783. ((unsigned int)setup[3] << 24),
  784. (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
  785. ((unsigned int)setup[6] << 16) |
  786. ((unsigned int)setup[7] << 24),
  787. 8, setup_c);
  788. ctl_data_trb = 0;
  789. if (len) {
  790. ctl_data_trb = s->ep0 + s->ep0_enq * 16;
  791. data_c = TRB_TYPE(TRB_DATA) | TRB_IOC;
  792. if (dir_in)
  793. data_c |= TRB_DIR_IN | TRB_ISP;
  794. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
  795. &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
  796. }
  797. ctl_status_trb = s->ep0 + s->ep0_enq * 16;
  798. status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
  799. if (!len || !dir_in)
  800. status_c |= TRB_DIR_IN;
  801. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  802. 0, 0, 0, status_c);
  803. __asm__ volatile ("" ::: "memory");
  804. mmio_wr(db_off + 4 * s->slot_id, 1);
  805. spin_unlock_irqrestore(&xhci_lock, flags);
  806. t0 = get_ticks_ms();
  807. last_bell = t0;
  808. while (get_ticks_ms() - t0 < 1000) {
  809. spin_lock_irqsave(&xhci_lock, &flags);
  810. xhci_poll_locked();
  811. rc = ctl_done;
  812. if (!rc && get_ticks_ms() - last_bell >= 100) {
  813. /* Re-ring while the endpoint is still busy; a posted
  814. * doorbell can be lost on some real xHCs. */
  815. mmio_wr(db_off + 4 * s->slot_id, 1);
  816. last_bell = get_ticks_ms();
  817. }
  818. spin_unlock_irqrestore(&xhci_lock, flags);
  819. if (rc)
  820. break;
  821. __asm__ volatile ("pause");
  822. }
  823. spin_lock_irqsave(&xhci_lock, &flags);
  824. if (!ctl_done) {
  825. ctl_busy = 0;
  826. spin_unlock_irqrestore(&xhci_lock, flags);
  827. dbg("xhci: control timeout\n");
  828. return -1;
  829. }
  830. rc = ctl_actual;
  831. st = ctl_status;
  832. ctl_busy = 0;
  833. spin_unlock_irqrestore(&xhci_lock, flags);
  834. if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
  835. dbg_hex("xhci: control status ", st);
  836. return -1;
  837. }
  838. return (int)rc;
  839. }
  840. /* ---- HID interrupt endpoint -------------------------------------------- */
  841. static int int_in_start(unsigned int ctrl, unsigned int slot,
  842. unsigned int addr, unsigned int ep,
  843. unsigned int mps, int low_speed)
  844. {
  845. struct xhci_slot *s = slot_by_addr(addr);
  846. struct xhci_int *in;
  847. unsigned int ep_index, ep_id, ring, flags;
  848. int rc;
  849. (void)ctrl;
  850. (void)low_speed;
  851. if (!found || !s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
  852. return -1;
  853. if (mps < 1)
  854. mps = 8;
  855. if (mps > 8)
  856. mps = 8;
  857. if (ints[slot].active)
  858. stop_int(slot);
  859. ep_index = (ep - 1) * 2 + 2; /* EP index 2 for EP1 IN, 4 for EP2 IN */
  860. ep_id = 2 * ep + 1;
  861. ring = mm_alloc_page();
  862. if (!ring)
  863. return -1;
  864. zero_page(ring, 4096);
  865. in = &ints[slot];
  866. in->slot = s->slot_id;
  867. in->ep_id = ep_id;
  868. in->mps = mps;
  869. in->ring = ring;
  870. in->enq = 0;
  871. in->cycle = 1;
  872. in->trb_len = mps;
  873. in->pending = 0;
  874. in->error = 0;
  875. in->active = 0;
  876. ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
  877. rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
  878. if (rc != 0) {
  879. dbg_hex("xhci: configure int ep failed ", ep_index);
  880. mm_free_page(ring);
  881. in->ring = 0;
  882. return -1;
  883. }
  884. in->active = 1;
  885. spin_lock_irqsave(&xhci_lock, &flags);
  886. ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  887. (unsigned int)in->buf, 0, in->trb_len,
  888. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  889. __asm__ volatile ("" ::: "memory");
  890. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  891. spin_unlock_irqrestore(&xhci_lock, flags);
  892. return 0;
  893. }
  894. static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
  895. {
  896. struct xhci_int *in;
  897. unsigned int flags, n, i;
  898. if (!found || slot >= USB_HCD_INT_SLOTS)
  899. return 0;
  900. in = &ints[slot];
  901. if (!in->active)
  902. return 0;
  903. spin_lock_irqsave(&xhci_lock, &flags);
  904. xhci_poll_locked();
  905. spin_unlock_irqrestore(&xhci_lock, flags);
  906. if (!in->pending)
  907. return 0;
  908. if (in->error) {
  909. in->pending = 0;
  910. in->error = 0;
  911. return -1;
  912. }
  913. n = in->pending_len;
  914. if (n > len)
  915. n = len;
  916. for (i = 0; i < n; i++)
  917. buf[i] = in->buf[i];
  918. in->pending = 0;
  919. spin_lock_irqsave(&xhci_lock, &flags);
  920. if (in->active) {
  921. ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  922. (unsigned int)in->buf, 0, in->trb_len,
  923. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  924. __asm__ volatile ("" ::: "memory");
  925. mmio_wr(db_off + 4 * in->slot, in->ep_id);
  926. }
  927. spin_unlock_irqrestore(&xhci_lock, flags);
  928. return (int)in->pending_len;
  929. }
  930. static void stop_int(unsigned int slot)
  931. {
  932. struct xhci_int *in;
  933. unsigned int flags;
  934. if (slot >= USB_HCD_INT_SLOTS)
  935. return;
  936. in = &ints[slot];
  937. spin_lock_irqsave(&xhci_lock, &flags);
  938. in->active = 0;
  939. in->pending = 0;
  940. in->error = 0;
  941. spin_unlock_irqrestore(&xhci_lock, flags);
  942. }
  943. /* ---- bulk transfers ---------------------------------------------------- */
  944. static void bulk_reset(struct xhci_bulk *b)
  945. {
  946. if (b->ring) {
  947. mm_free_page(b->ring);
  948. b->ring = 0;
  949. }
  950. b->active = 0;
  951. b->armed = 0;
  952. b->slot = 0;
  953. b->pending = 0;
  954. b->error = 0;
  955. b->actual = 0;
  956. }
  957. static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
  958. unsigned int mps, unsigned char *buf, unsigned int len)
  959. {
  960. struct xhci_slot *s = slot_by_addr(addr);
  961. struct xhci_bulk *b = &bulk_in;
  962. unsigned int ep_index, ep_id, ring, flags, n, err;
  963. int rc;
  964. (void)ctrl;
  965. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  966. return -1;
  967. if (mps < 1)
  968. mps = 512;
  969. ep_index = (ep - 1) * 2 + 2;
  970. ep_id = 2 * ep + 1;
  971. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  972. ring = mm_alloc_page();
  973. if (!ring)
  974. return -1;
  975. zero_page(ring, 4096);
  976. bulk_reset(b);
  977. b->slot = s->slot_id;
  978. b->ep = ep;
  979. b->ep_id = ep_id;
  980. b->mps = mps;
  981. b->ring = ring;
  982. b->enq = 0;
  983. b->cycle = 1;
  984. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  985. rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
  986. if (rc != 0) {
  987. bulk_reset(b);
  988. return -1;
  989. }
  990. b->active = 1;
  991. }
  992. b->buf = (unsigned int)buf;
  993. b->len = len;
  994. spin_lock_irqsave(&xhci_lock, &flags);
  995. xhci_poll_locked();
  996. if (b->pending) {
  997. n = b->actual;
  998. err = (unsigned int)b->error;
  999. b->pending = 0;
  1000. b->error = 0;
  1001. /* Do not re-arm here: the next call may use a different
  1002. * buffer/length (the mass-storage driver alternates CBW,
  1003. * data and CSW buffers). The caller's next usb_bulk_in()
  1004. * arms the new transfer with its own buffer. */
  1005. b->armed = 0;
  1006. spin_unlock_irqrestore(&xhci_lock, flags);
  1007. return err ? -1 : (int)n;
  1008. }
  1009. if (!b->armed) {
  1010. b->trb_len = len;
  1011. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1012. (unsigned int)buf, 0, len,
  1013. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1014. b->armed = 1;
  1015. __asm__ volatile ("" ::: "memory");
  1016. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  1017. }
  1018. spin_unlock_irqrestore(&xhci_lock, flags);
  1019. return 0;
  1020. }
  1021. static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
  1022. unsigned int mps, const unsigned char *buf,
  1023. unsigned int len)
  1024. {
  1025. struct xhci_slot *s = slot_by_addr(addr);
  1026. struct xhci_bulk *b = &bulk_out;
  1027. unsigned int ep_index, ep_id, ring, flags, t0, done, status;
  1028. int rc;
  1029. (void)ctrl;
  1030. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  1031. return -1;
  1032. if (mps < 1)
  1033. mps = 512;
  1034. ep_index = (ep - 1) * 2 + 1;
  1035. ep_id = 2 * ep;
  1036. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1037. ring = mm_alloc_page();
  1038. if (!ring)
  1039. return -1;
  1040. zero_page(ring, 4096);
  1041. bulk_reset(b);
  1042. b->slot = s->slot_id;
  1043. b->ep = ep;
  1044. b->ep_id = ep_id;
  1045. b->mps = mps;
  1046. b->ring = ring;
  1047. b->enq = 0;
  1048. b->cycle = 1;
  1049. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  1050. rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
  1051. if (rc != 0) {
  1052. bulk_reset(b);
  1053. return -1;
  1054. }
  1055. b->active = 1;
  1056. }
  1057. spin_lock_irqsave(&xhci_lock, &flags);
  1058. if (out_busy) {
  1059. spin_unlock_irqrestore(&xhci_lock, flags);
  1060. return -1;
  1061. }
  1062. out_busy = 1;
  1063. out_done = 0;
  1064. out_status = 0;
  1065. out_slot = s->slot_id;
  1066. out_trb = b->ring + b->enq * 16;
  1067. b->trb_len = len;
  1068. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1069. (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1070. __asm__ volatile ("" ::: "memory");
  1071. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  1072. spin_unlock_irqrestore(&xhci_lock, flags);
  1073. t0 = get_ticks_ms();
  1074. while (get_ticks_ms() - t0 < 1000) {
  1075. spin_lock_irqsave(&xhci_lock, &flags);
  1076. xhci_poll_locked();
  1077. done = out_done;
  1078. spin_unlock_irqrestore(&xhci_lock, flags);
  1079. if (done)
  1080. break;
  1081. __asm__ volatile ("pause");
  1082. }
  1083. spin_lock_irqsave(&xhci_lock, &flags);
  1084. done = out_done;
  1085. status = out_status;
  1086. out_busy = 0;
  1087. spin_unlock_irqrestore(&xhci_lock, flags);
  1088. if (!done) {
  1089. dbg("xhci: bulk out timeout\n");
  1090. return -1;
  1091. }
  1092. if (status != CC_SUCCESS) {
  1093. dbg_hex("xhci: bulk out status ", status);
  1094. return -1;
  1095. }
  1096. return 0;
  1097. }
  1098. /* ---- root hub ports ---------------------------------------------------- */
  1099. static int port_reset(unsigned int ctrl, unsigned int port)
  1100. {
  1101. unsigned int ps, t0;
  1102. (void)ctrl;
  1103. if (!found || port >= max_ports ||
  1104. !(port_mask & (1u << port)))
  1105. return -1;
  1106. if (port_slot[port])
  1107. free_slot(&slots[port_slot[port]]);
  1108. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1109. if (!(ps & PORTSC_CCS))
  1110. return -1;
  1111. /* Force the port into the Disabled state first: some xHCs ignore
  1112. * PR while the port is already enabled, leaving the device at its
  1113. * firmware-assigned address instead of address 0. */
  1114. if (ps & PORTSC_PED) {
  1115. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1116. portsc_neutral(ps) | PORTSC_PED);
  1117. t0 = get_ticks_ms();
  1118. while (get_ticks_ms() - t0 < 100) {
  1119. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1120. if (!(ps & PORTSC_PED))
  1121. break;
  1122. }
  1123. }
  1124. /* Acknowledge stale change latches first; a leftover PRC would
  1125. * otherwise let the wait below succeed before this reset has even
  1126. * started (the firmware may have left one behind). */
  1127. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1128. portsc_neutral(ps) | PORTSC_CSC | PORTSC_PEC |
  1129. PORTSC_PRC | PORTSC_PLC);
  1130. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1131. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1132. portsc_neutral(ps) | PORTSC_PR);
  1133. t0 = get_ticks_ms();
  1134. while (get_ticks_ms() - t0 < 500) {
  1135. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1136. if (ps & PORTSC_PRC)
  1137. break;
  1138. }
  1139. if (!(ps & PORTSC_PRC)) {
  1140. dbg_hex("xhci: port reset timeout ", port);
  1141. return -1;
  1142. }
  1143. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1144. portsc_neutral(ps) | PORTSC_PRC);
  1145. t0 = get_ticks_ms();
  1146. while (get_ticks_ms() - t0 < 500) {
  1147. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1148. if (ps & PORTSC_PED)
  1149. break;
  1150. }
  1151. if (!(ps & PORTSC_PED)) {
  1152. dbg_hex("xhci: port not enabled ", port);
  1153. return -1;
  1154. }
  1155. /* Reset recovery: USB2 devices need TRSTRCY (10 ms) before the first
  1156. * control request; the delay is harmless for SuperSpeed. */
  1157. delay_ms(10);
  1158. last_port = (int)port;
  1159. return 0;
  1160. }
  1161. static int port_connected(unsigned int ctrl, unsigned int port)
  1162. {
  1163. unsigned int ps;
  1164. (void)ctrl;
  1165. if (!found || port >= max_ports ||
  1166. !(port_mask & (1u << port)))
  1167. return 0;
  1168. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1169. return (ps & PORTSC_CCS) ? 1 : 0;
  1170. }
  1171. static int port_low_speed(unsigned int ctrl, unsigned int port)
  1172. {
  1173. unsigned int ps;
  1174. (void)ctrl;
  1175. if (!found || port >= max_ports)
  1176. return 0;
  1177. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1178. return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
  1179. }
  1180. static int port_take_change(unsigned int ctrl, unsigned int port)
  1181. {
  1182. unsigned int ps;
  1183. (void)ctrl;
  1184. if (!found || port >= max_ports)
  1185. return 0;
  1186. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1187. if (!(ps & PORTSC_CSC))
  1188. return 0;
  1189. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1190. portsc_neutral(ps) | PORTSC_CSC);
  1191. return 1;
  1192. }
  1193. /* ---- controller discovery ---------------------------------------------- */
  1194. /* xHCI USB Legacy Support: take ownership from the firmware before the
  1195. * reset and stop its SMI from firing on port changes once we own the
  1196. * controller (the same idea as the PCI 0xC0 write in uhci.c/ohci.c, but
  1197. * xHCI keeps it in an extended capability).
  1198. *
  1199. * The BIOS-owned semaphore must actually clear before HCRST: while the
  1200. * firmware still owns the controller its SMI handler can interfere with
  1201. * the reset and the command ring. Linux waits for it too. */
  1202. static void legacy_handoff(unsigned int off)
  1203. {
  1204. unsigned int sup = mmio_rd(off);
  1205. unsigned int ctl, t0;
  1206. if ((sup & 0xFF) != 1)
  1207. return;
  1208. if (sup & (1u << 16)) { /* BIOS still owns it */
  1209. mmio_wr(off, sup | (1u << 24));
  1210. t0 = get_ticks_ms();
  1211. while (get_ticks_ms() - t0 < 1000) {
  1212. if (!(mmio_rd(off) & (1u << 16)))
  1213. break;
  1214. }
  1215. }
  1216. ctl = mmio_rd(off + 4);
  1217. ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
  1218. ctl |= 0x7u << 29;
  1219. mmio_wr(off + 4, ctl);
  1220. }
  1221. /* Walk the extended capabilities for the legacy handoff. Runs before the
  1222. * controller reset; scan_protocols() parses the port list afterwards. */
  1223. static void ext_caps_handoff(void)
  1224. {
  1225. unsigned int off, guard = 0;
  1226. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1227. while (off && guard++ < 32) {
  1228. unsigned int cap = mmio_rd(off);
  1229. unsigned int next = (cap >> 8) & 0xFF;
  1230. if ((cap & 0xFF) == 1)
  1231. legacy_handoff(off);
  1232. if (!next)
  1233. break;
  1234. off += next << 2;
  1235. }
  1236. }
  1237. static void scan_protocols(void)
  1238. {
  1239. unsigned int off, guard = 0;
  1240. port_mask = 0;
  1241. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1242. while (off && guard++ < 32) {
  1243. unsigned int cap = mmio_rd(off);
  1244. unsigned int id = cap & 0xFF;
  1245. unsigned int next = (cap >> 8) & 0xFF;
  1246. if (id == 2) { /* Supported Protocol */
  1247. unsigned int major = (cap >> 24) & 0xFF;
  1248. unsigned int pi = mmio_rd(off + 8);
  1249. unsigned int po = pi & 0xFF;
  1250. unsigned int pc = (pi >> 8) & 0xFF;
  1251. unsigned int i;
  1252. /* The controller may expose separate USB2 and USB3
  1253. * protocol entries for the same physical connectors;
  1254. * include both so a SuperSpeed device is not skipped. */
  1255. if ((major == 2 || major == 3) && po >= 1) {
  1256. for (i = 0; i < pc && po + i <= 32; i++)
  1257. port_mask |= 1u << (po - 1 + i);
  1258. }
  1259. }
  1260. if (!next)
  1261. break;
  1262. off += next << 2;
  1263. }
  1264. if (!port_mask)
  1265. port_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
  1266. ((1u << max_ports) - 1);
  1267. }
  1268. static int xhci_init(void)
  1269. {
  1270. unsigned int bus, dev, func, id, class_rev, bar, bar_hi, cmd;
  1271. unsigned int hcs1, hcs2, t0, i, n, scratch;
  1272. if (found)
  1273. return 1;
  1274. for (bus = 0; bus < 256; bus++) {
  1275. for (dev = 0; dev < 32; dev++) {
  1276. for (func = 0; func < 8; func++) {
  1277. id = pci_cfg_read32(bus, dev, func, 0x00);
  1278. if (id == 0xFFFFFFFFu || id == 0)
  1279. continue;
  1280. class_rev = pci_cfg_read32(bus, dev, func, 0x08);
  1281. if ((class_rev >> 16) != 0x0C03)
  1282. continue;
  1283. if (((class_rev >> 8) & 0xFF) != 0x30)
  1284. continue;
  1285. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1286. if (bar & 1)
  1287. continue;
  1288. bar_hi = 0;
  1289. if ((bar & 0x6) == 0x4)
  1290. bar_hi = pci_cfg_read32(bus, dev,
  1291. func, 0x14);
  1292. bar &= 0xFFFFFFF0u;
  1293. if (!bar || bar_hi)
  1294. continue;
  1295. pci_bus = bus;
  1296. pci_dev = dev;
  1297. pci_func = func;
  1298. goto found_xhci;
  1299. }
  1300. }
  1301. }
  1302. return 0;
  1303. found_xhci:
  1304. cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
  1305. pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
  1306. mmio_base = bar;
  1307. paging_map_mmio(mmio_base, 0x10000);
  1308. cap_len = mmio_rd(0) & 0xFF;
  1309. hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
  1310. hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
  1311. hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
  1312. rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
  1313. db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
  1314. max_slots = hcs1 & 0xFF;
  1315. if (max_slots > XHCI_MAX_SLOTS)
  1316. max_slots = XHCI_MAX_SLOTS;
  1317. if (!max_slots)
  1318. max_slots = 1;
  1319. max_ports = (hcs1 >> 24) & 0xFF;
  1320. if (max_ports > XHCI_MAX_PORTS)
  1321. max_ports = XHCI_MAX_PORTS;
  1322. if (!max_ports)
  1323. max_ports = 1;
  1324. ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
  1325. dbg("xhci: found\n");
  1326. dbg_hex("xhci: mmio ", mmio_base);
  1327. dbg_dec("xhci: slots ", max_slots);
  1328. dbg_dec("xhci: ports ", max_ports);
  1329. dbg_dec("xhci: cap len ", cap_len);
  1330. dcbaa = mm_alloc_page();
  1331. input_ctx = mm_alloc_page();
  1332. cmd_ring = mm_alloc_page();
  1333. evt_ring = mm_alloc_page();
  1334. erst = mm_alloc_page();
  1335. if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
  1336. dbg("xhci: out of pages\n");
  1337. return 0;
  1338. }
  1339. zero_page(dcbaa, 4096);
  1340. zero_page(input_ctx, 4096);
  1341. zero_page(cmd_ring, 4096);
  1342. zero_page(evt_ring, 4096);
  1343. zero_page(erst, 4096);
  1344. /* Firmware handoff before the reset: while the BIOS still owns the
  1345. * controller its SMI handler can interfere with HCRST and the
  1346. * command ring. */
  1347. ext_caps_handoff();
  1348. /* Wait for the controller to become ready before touching USBCMD. */
  1349. t0 = get_ticks_ms();
  1350. while (get_ticks_ms() - t0 < 1000) {
  1351. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1352. break;
  1353. }
  1354. /* Reset. The xHC may ignore HCRST while it is running, so halt it
  1355. * and wait for HCH first (xHCI 4.2; Linux does the same). QEMU
  1356. * halts instantly, real silicon does not. */
  1357. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1358. t0 = get_ticks_ms();
  1359. while (get_ticks_ms() - t0 < 100) {
  1360. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1361. break;
  1362. }
  1363. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1364. dbg("xhci: halt timeout\n");
  1365. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
  1366. /* Intel xHCI (CherryView/Braswell included) needs 1 ms after the
  1367. * HCRST write before any other HC register is accessed; polling
  1368. * USBCMD too early can leave the controller partially reset. */
  1369. delay_ms(2);
  1370. t0 = get_ticks_ms();
  1371. while (get_ticks_ms() - t0 < 1000) {
  1372. if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
  1373. break;
  1374. }
  1375. t0 = get_ticks_ms();
  1376. while (get_ticks_ms() - t0 < 1000) {
  1377. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1378. break;
  1379. }
  1380. if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
  1381. dbg_hex("xhci: reset failed ",
  1382. mmio_rd(op(XHCI_OP_USBSTS)));
  1383. return 0;
  1384. }
  1385. /* After HCRST the controller must be halted before the ring
  1386. * registers are written; otherwise the writes can be dropped and
  1387. * every command times out. */
  1388. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)) {
  1389. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1390. t0 = get_ticks_ms();
  1391. while (get_ticks_ms() - t0 < 100) {
  1392. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1393. break;
  1394. }
  1395. }
  1396. /* A working HCRST clears DCBAAP/CRCR/CONFIG. If firmware values
  1397. * survive, the reset did not take effect: the ports keep their old
  1398. * device addresses and every control transfer to address 0 fails. */
  1399. if (mmio_rd(op(XHCI_OP_DCBAAP)) != 0 ||
  1400. (mmio_rd(op(XHCI_OP_CRCR)) & ~0xFu) != 0 ||
  1401. (mmio_rd(op(XHCI_OP_CONFIG)) & 0xFFu) != 0)
  1402. dbg("xhci: hcreset not applied\n");
  1403. /* Clear stale status latches from the firmware's session. */
  1404. mmio_wr(op(XHCI_OP_USBSTS), (1u << 2) | (1u << 3) | (1u << 4));
  1405. mmio_wr(op(XHCI_OP_PAGESIZE), 1);
  1406. mmio_wr(op(XHCI_OP_CONFIG), max_slots);
  1407. mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
  1408. /* Scratchpad buffers, if the controller asks for any. */
  1409. n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
  1410. if (n) {
  1411. unsigned int arr;
  1412. if (n > 512)
  1413. n = 512;
  1414. arr = mm_alloc_page();
  1415. if (!arr) {
  1416. dbg("xhci: no scratchpad array\n");
  1417. return 0;
  1418. }
  1419. zero_page(arr, 4096);
  1420. for (i = 0; i < n; i++) {
  1421. scratch = mm_alloc_page();
  1422. if (!scratch) {
  1423. dbg("xhci: no scratchpad page\n");
  1424. return 0;
  1425. }
  1426. zero_page(scratch, 4096);
  1427. ram_wr64(arr + i * 8, scratch);
  1428. }
  1429. ram_wr64(dcbaa, arr);
  1430. dbg_dec("xhci: scratchpads ", n);
  1431. }
  1432. /* Command ring and event ring. */
  1433. ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
  1434. mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
  1435. evt_idx = 0;
  1436. evt_cycle = 1;
  1437. *(volatile unsigned int *)(erst + 0) = evt_ring;
  1438. *(volatile unsigned int *)(erst + 4) = 0;
  1439. *(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
  1440. *(volatile unsigned int *)(erst + 12) = 0;
  1441. mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
  1442. mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
  1443. mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
  1444. mmio_wr(rt(XHCI_RT_IMAN), 0);
  1445. mmio_wr(rt(XHCI_RT_IMOD), 0);
  1446. /* Run. */
  1447. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
  1448. t0 = get_ticks_ms();
  1449. while (get_ticks_ms() - t0 < 1000) {
  1450. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1451. break;
  1452. }
  1453. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
  1454. dbg("xhci: controller did not start\n");
  1455. return 0;
  1456. }
  1457. /* The first command completion (or timeout) tells whether the ring is
  1458. * actually serviced. Don't gate on CRR: QEMU leaves it clear even
  1459. * while commands complete normally. */
  1460. scan_protocols();
  1461. dbg_hex("xhci: ports ", port_mask);
  1462. /* Power every root port. VBus is supposed to come up after HCRST,
  1463. * but some firmware/PPC combinations leave PP clear; the USB3
  1464. * entries may share a switch with a USB2 connector, so write all of
  1465. * them, not just the USB2 range. */
  1466. for (i = 0; i < max_ports; i++) {
  1467. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
  1468. if (!(ps & PORTSC_PP))
  1469. mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
  1470. portsc_neutral(ps) | PORTSC_PP);
  1471. }
  1472. delay_ms(20);
  1473. found = 1;
  1474. return 1;
  1475. }
  1476. static unsigned int xhci_count(void)
  1477. {
  1478. return found ? 1u : 0u;
  1479. }
  1480. static void xhci_info(unsigned int ctrl, unsigned int *bus,
  1481. unsigned int *dev, unsigned int *func,
  1482. unsigned int *base, int *mmio)
  1483. {
  1484. (void)ctrl;
  1485. *bus = pci_bus;
  1486. *dev = pci_dev;
  1487. *func = pci_func;
  1488. *base = mmio_base;
  1489. *mmio = 1;
  1490. }
  1491. static unsigned int xhci_port_count(unsigned int ctrl)
  1492. {
  1493. (void)ctrl;
  1494. return found ? max_ports : 0;
  1495. }
  1496. const struct usb_hcd_ops usb_xhci_ops = {
  1497. "xHCI",
  1498. xhci_init,
  1499. xhci_count,
  1500. xhci_info,
  1501. control_transfer,
  1502. xhci_port_count,
  1503. port_connected,
  1504. port_low_speed,
  1505. port_reset,
  1506. port_take_change,
  1507. int_in_start,
  1508. int_in_poll,
  1509. stop_int,
  1510. xhci_bulk_out,
  1511. xhci_bulk_in,
  1512. 0,
  1513. };