xhci.c 49 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864
  1. /* xHCI (USB 3.0) host controller driver.
  2. *
  3. * Scope: one controller, USB2 and USB3 root ports and devices plugged
  4. * directly into them. Control transfers, one interrupt-IN endpoint per HID
  5. * interface and the bulk endpoints used by ASIX and usbmsc are implemented.
  6. * SuperSpeed devices use a 512-byte EP0 and their endpoint descriptors may
  7. * carry a companion descriptor, which is ignored because only the bulk/HID
  8. * endpoint itself is needed.
  9. *
  10. * The schedule is polled, like the rest of the kernel. Rings and contexts
  11. * are page-allocated so the driver does not grow the kernel BSS. */
  12. #include "usb.h"
  13. #include "mm.h"
  14. #include "paging.h"
  15. #include "pci.h"
  16. #include "serial.h"
  17. #include "spinlock.h"
  18. #include "timer.h"
  19. #include "usb_hcd.h"
  20. #include "xhci.h"
  21. #define XHCI_DEBUG 0
  22. /* ---- controller limits ------------------------------------------------- */
  23. #define XHCI_MAX_SLOTS 4
  24. #define XHCI_MAX_PORTS 32
  25. #define XHCI_EVT_TRBS 256
  26. #define XHCI_CMD_TRBS 32
  27. #define XHCI_EP0_TRBS 16
  28. #define XHCI_INT_TRBS 16
  29. #define XHCI_BULK_TRBS 32
  30. /* ---- capability / operational / runtime offsets (bytes) --------------- */
  31. #define XHCI_CAP_HCSPARAMS1 0x04
  32. #define XHCI_CAP_HCSPARAMS2 0x08
  33. #define XHCI_CAP_HCCPARAMS1 0x10
  34. #define XHCI_CAP_DBOFF 0x14
  35. #define XHCI_CAP_RTSOFF 0x18
  36. #define XHCI_OP_USBCMD 0x00
  37. #define XHCI_OP_USBSTS 0x04
  38. #define XHCI_OP_PAGESIZE 0x08
  39. #define XHCI_OP_CRCR 0x18
  40. #define XHCI_OP_DCBAAP 0x30
  41. #define XHCI_OP_CONFIG 0x38
  42. #define XHCI_OP_PORTSC 0x400
  43. #define XHCI_RT_IMAN 0x20
  44. #define XHCI_RT_IMOD 0x24
  45. #define XHCI_RT_ERSTSZ 0x28
  46. #define XHCI_RT_ERSTBA 0x30
  47. #define XHCI_RT_ERDP 0x38
  48. /* USBCMD / USBSTS */
  49. #define XHCI_CMD_RS 0x0001u
  50. #define XHCI_CMD_HCRST 0x0002u
  51. #define XHCI_STS_HCH 0x0001u
  52. #define XHCI_STS_CNR 0x0800u
  53. #define XHCI_STS_HCE 0x1000u
  54. /* PORTSC */
  55. #define PORTSC_CCS 0x00000001u
  56. #define PORTSC_PED 0x00000002u
  57. #define PORTSC_PR 0x00000010u
  58. #define PORTSC_PP 0x00000200u
  59. #define PORTSC_SPEED_SHIFT 10
  60. #define PORTSC_SPEED_MASK 0xFu
  61. #define PORTSC_CSC 0x00020000u
  62. #define PORTSC_PEC 0x00040000u
  63. #define PORTSC_WRC 0x00080000u
  64. #define PORTSC_OCC 0x00100000u
  65. #define PORTSC_PRC 0x00200000u
  66. #define PORTSC_PLC 0x00400000u
  67. #define PORTSC_CEC 0x00800000u
  68. #define PORTSC_RWC (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
  69. PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
  70. PORTSC_CEC | PORTSC_PED)
  71. /* Bits that may be written back while preserving the port state:
  72. * read-only status (CCS, OCA, speed, device-removable) and the
  73. * read/write-state fields (PLS, PP, PIC, wake enables). Everything else
  74. * -- especially PR/WPR (RW1S) and PED (RW1CS) -- must be written as 0 or
  75. * the write changes the port state. Linux's
  76. * xhci_port_state_to_neutral() does the same. */
  77. #define PORTSC_RO_MASK (PORTSC_CCS | 0x00000008u | \
  78. (PORTSC_SPEED_MASK << PORTSC_SPEED_SHIFT) | \
  79. 0x40000000u)
  80. #define PORTSC_RWS_MASK (0x000001E0u | PORTSC_PP | 0x0000C000u | \
  81. 0x0E000000u)
  82. static unsigned int portsc_neutral(unsigned int ps)
  83. {
  84. return (ps & PORTSC_RO_MASK) | (ps & PORTSC_RWS_MASK);
  85. }
  86. /* TRB types */
  87. #define TRB_NORMAL 1
  88. #define TRB_SETUP 2
  89. #define TRB_DATA 3
  90. #define TRB_STATUS 4
  91. #define TRB_LINK 6
  92. #define TRB_ENABLE_SLOT 9
  93. #define TRB_DISABLE_SLOT 10
  94. #define TRB_ADDR_DEV 11
  95. #define TRB_CONFIG_EP 12
  96. #define TRB_EVAL_CTX 13
  97. #define TRB_STOP_EP 15
  98. #define TRB_TRANSFER 32
  99. #define TRB_COMPLETION 33
  100. /* TRB control bits */
  101. #define TRB_CYCLE 0x00000001u
  102. #define TRB_TC 0x00000002u
  103. #define TRB_ISP 0x00000004u
  104. #define TRB_CHAIN 0x00000010u
  105. #define TRB_IOC 0x00000020u
  106. #define TRB_IDT 0x00000040u
  107. #define TRB_BSR 0x00000200u
  108. #define TRB_DIR_IN 0x00010000u
  109. #define TRB_TYPE(t) ((t) << 10)
  110. /* completion codes */
  111. #define CC_SUCCESS 1
  112. #define CC_SHORT_PACKET 13
  113. /* slot/EP context flags and endpoint types */
  114. #define SLOT_FLAG 0x00000001u
  115. #define EP0_FLAG 0x00000002u
  116. #define EP_CTRL 4
  117. #define EP_BULK_OUT 2
  118. #define EP_BULK_IN 6
  119. #define EP_INT_IN 7
  120. struct xhci_trb {
  121. unsigned int p0;
  122. unsigned int p1;
  123. unsigned int status;
  124. unsigned int control;
  125. };
  126. struct xhci_slot {
  127. int used;
  128. unsigned int slot_id;
  129. unsigned int addr; /* logical address used by usb.c */
  130. unsigned int dev_addr; /* address assigned by the xHC */
  131. unsigned int port;
  132. unsigned int speed;
  133. unsigned int dc; /* device context page */
  134. unsigned int ep0; /* EP0 transfer ring page */
  135. unsigned int ep0_enq;
  136. unsigned int ep0_cycle;
  137. unsigned int ep0_mps; /* max packet size programmed into EP0 */
  138. unsigned int ep_mask; /* configured endpoint flags */
  139. };
  140. struct xhci_int {
  141. int active;
  142. unsigned int slot; /* hardware slot id */
  143. unsigned int ep_id; /* xHCI endpoint id (2n+dir) */
  144. unsigned int mps;
  145. unsigned int ring;
  146. unsigned int enq;
  147. unsigned int cycle;
  148. unsigned int trb_len;
  149. unsigned int pending_len;
  150. int pending;
  151. int error;
  152. unsigned char buf[8] __attribute__((aligned(16)));
  153. };
  154. struct xhci_bulk {
  155. int active;
  156. int armed;
  157. unsigned int slot;
  158. unsigned int ep; /* endpoint number */
  159. unsigned int ep_id;
  160. unsigned int mps;
  161. unsigned int ring;
  162. unsigned int enq;
  163. unsigned int cycle;
  164. unsigned int buf;
  165. unsigned int len;
  166. unsigned int trb_len;
  167. unsigned int actual;
  168. int pending;
  169. int error;
  170. };
  171. static unsigned int mmio_base; /* kernel mapping of BAR0 */
  172. static unsigned int mmio_phys; /* BAR0 itself (logged, reported) */
  173. static unsigned int cap_len;
  174. static unsigned int rt_off;
  175. static unsigned int db_off;
  176. static unsigned int max_slots;
  177. static unsigned int max_ports;
  178. static unsigned int ctx_size;
  179. static unsigned int port_mask;
  180. static unsigned int hcc_params;
  181. static unsigned int pci_bus, pci_dev, pci_func;
  182. static unsigned int pci_id;
  183. static unsigned int hci_version;
  184. static int found;
  185. static int last_port = -1;
  186. static unsigned int dcbaa;
  187. static unsigned int input_ctx;
  188. static unsigned int cmd_ring;
  189. static unsigned int evt_ring;
  190. static unsigned int erst;
  191. static unsigned int cmd_enq;
  192. static unsigned int cmd_cycle;
  193. static unsigned int evt_idx;
  194. static unsigned int evt_cycle;
  195. static unsigned int cmd_busy;
  196. static unsigned int cmd_done;
  197. static unsigned int cmd_status;
  198. static unsigned int cmd_slot;
  199. static unsigned int cmd_trb_addr;
  200. static unsigned int ctl_busy;
  201. static unsigned int ctl_done;
  202. static unsigned int ctl_slot;
  203. static unsigned int ctl_status;
  204. static unsigned int ctl_req_len;
  205. static unsigned int ctl_actual;
  206. static unsigned int ctl_data_seen;
  207. static unsigned int ctl_setup_trb;
  208. static unsigned int ctl_data_trb;
  209. static unsigned int ctl_status_trb;
  210. static struct xhci_bulk bulk_in;
  211. static struct xhci_bulk bulk_out;
  212. static unsigned int out_busy;
  213. static unsigned int out_done;
  214. static unsigned int out_status;
  215. static unsigned int out_slot;
  216. static unsigned int out_trb;
  217. static struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
  218. static unsigned int addr_slot[128];
  219. static unsigned int port_slot[XHCI_MAX_PORTS];
  220. static struct xhci_int ints[USB_HCD_INT_SLOTS];
  221. static spinlock_t xhci_lock;
  222. /* ---- debug ------------------------------------------------------------- */
  223. #if XHCI_DEBUG
  224. static void dbg(const char *s)
  225. {
  226. serial_puts(SERIAL_COM1, s);
  227. }
  228. static void dbg_hex(const char *s, unsigned int v)
  229. {
  230. serial_puts(SERIAL_COM1, s);
  231. serial_puthex(SERIAL_COM1, v);
  232. serial_putc(SERIAL_COM1, '\n');
  233. }
  234. static void dbg_dec(const char *s, unsigned int v)
  235. {
  236. serial_puts(SERIAL_COM1, s);
  237. serial_putdec(SERIAL_COM1, v);
  238. serial_putc(SERIAL_COM1, '\n');
  239. }
  240. #else
  241. static void dbg(const char *s) { (void)s; }
  242. static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
  243. static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
  244. #endif
  245. /* Always-on one-line diagnostics (COM1, mirrored to netmon). Only used on
  246. * the init line and on failures, so a working machine stays quiet. */
  247. static void xlog_hex(const char *s, unsigned int v)
  248. {
  249. serial_puts(SERIAL_COM1, s);
  250. serial_puthex(SERIAL_COM1, v);
  251. }
  252. static void delay_ms(unsigned int ms)
  253. {
  254. usb_delay_ms(ms); /* sleeps inside a sleepable scan */
  255. }
  256. /* ---- MMIO helpers ------------------------------------------------------ */
  257. static unsigned int mmio_rd(unsigned int off)
  258. {
  259. return *(volatile unsigned int *)(mmio_base + off);
  260. }
  261. static void mmio_wr(unsigned int off, unsigned int val)
  262. {
  263. *(volatile unsigned int *)(mmio_base + off) = val;
  264. }
  265. static void mmio_wr64(unsigned int off, unsigned int lo)
  266. {
  267. mmio_wr(off, lo);
  268. mmio_wr(off + 4, 0);
  269. }
  270. /* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
  271. static void ram_wr64(unsigned int addr, unsigned int lo)
  272. {
  273. *(volatile unsigned int *)addr = lo;
  274. *(volatile unsigned int *)(addr + 4) = 0;
  275. }
  276. static void zero_page(unsigned int addr, unsigned int len)
  277. {
  278. unsigned int *p = (unsigned int *)addr;
  279. while (len >= 4) {
  280. *p++ = 0;
  281. len -= 4;
  282. }
  283. }
  284. static unsigned int op(unsigned int reg)
  285. {
  286. return cap_len + reg;
  287. }
  288. static unsigned int rt(unsigned int reg)
  289. {
  290. return rt_off + reg;
  291. }
  292. /* ---- rings ------------------------------------------------------------- */
  293. static void ring_init(unsigned int base, unsigned int size,
  294. unsigned int *enq, unsigned int *cycle)
  295. {
  296. struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
  297. link->p0 = base;
  298. link->p1 = 0;
  299. link->status = 0;
  300. link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  301. *enq = 0;
  302. *cycle = 1;
  303. }
  304. static void ring_queue(unsigned int base, unsigned int size,
  305. unsigned int *enq, unsigned int *cycle,
  306. unsigned int p0, unsigned int p1,
  307. unsigned int status, unsigned int control)
  308. {
  309. struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
  310. trb->p0 = p0;
  311. trb->p1 = p1;
  312. trb->status = status;
  313. __asm__ volatile ("" ::: "memory");
  314. trb->control = control | *cycle;
  315. __asm__ volatile ("" ::: "memory");
  316. (*enq)++;
  317. if (*enq == size - 1) {
  318. struct xhci_trb *link =
  319. (struct xhci_trb *)(base + (size - 1) * 16);
  320. __asm__ volatile ("" ::: "memory");
  321. link->control ^= TRB_CYCLE;
  322. *enq = 0;
  323. *cycle ^= 1;
  324. }
  325. }
  326. /* ---- event handling ---------------------------------------------------- */
  327. static void xhci_poll_locked(void);
  328. static void stop_int(unsigned int slot);
  329. static void bulk_reset(struct xhci_bulk *b);
  330. static void xhci_int_event(unsigned int slot, unsigned int ep,
  331. unsigned int code, unsigned int residual)
  332. {
  333. unsigned int i;
  334. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  335. if (!ints[i].active || ints[i].slot != slot ||
  336. ints[i].ep_id != ep)
  337. continue;
  338. if (!ints[i].pending) {
  339. if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
  340. ints[i].pending_len =
  341. (ints[i].trb_len >= residual) ?
  342. ints[i].trb_len - residual : 0;
  343. ints[i].pending = 1;
  344. } else {
  345. ints[i].pending = 1;
  346. ints[i].error = 1;
  347. }
  348. }
  349. break;
  350. }
  351. }
  352. static void xhci_poll_locked(void)
  353. {
  354. unsigned int n = 0;
  355. while (n++ < 512) {
  356. struct xhci_trb *ev =
  357. (struct xhci_trb *)(evt_ring + evt_idx * 16);
  358. unsigned int type;
  359. if ((ev->control & TRB_CYCLE) != evt_cycle)
  360. break;
  361. type = (ev->control >> 10) & 0x3F;
  362. if (type == TRB_COMPLETION) {
  363. unsigned int code = (ev->status >> 24) & 0xFF;
  364. unsigned int slot = (ev->control >> 24) & 0xFF;
  365. if (cmd_busy && ev->p0 == cmd_trb_addr) {
  366. cmd_status = code;
  367. cmd_slot = slot;
  368. cmd_done = 1;
  369. }
  370. } else if (type == TRB_TRANSFER) {
  371. unsigned int code = (ev->status >> 24) & 0xFF;
  372. unsigned int residual = ev->status & 0xFFFFFF;
  373. unsigned int slot = (ev->control >> 24) & 0xFF;
  374. unsigned int ep = (ev->control >> 16) & 0x1F;
  375. if (ctl_busy && slot == ctl_slot && ep == 1) {
  376. if (ev->p0 == ctl_setup_trb) {
  377. /* The setup stage is its own TD and
  378. * completes first; only an error stops
  379. * the transfer. */
  380. if (code != CC_SUCCESS &&
  381. code != CC_SHORT_PACKET) {
  382. ctl_status = code;
  383. ctl_done = 1;
  384. }
  385. } else if (ev->p0 == ctl_data_trb) {
  386. ctl_actual = (ctl_req_len >= residual) ?
  387. ctl_req_len - residual : 0;
  388. ctl_data_seen = 1;
  389. if (code != CC_SUCCESS &&
  390. code != CC_SHORT_PACKET) {
  391. ctl_status = code;
  392. ctl_done = 1;
  393. }
  394. } else if (ev->p0 == ctl_status_trb) {
  395. if (!ctl_data_seen)
  396. ctl_actual =
  397. (code == CC_SUCCESS) ?
  398. ctl_req_len : 0;
  399. ctl_status = code;
  400. ctl_done = 1;
  401. }
  402. } else if (out_busy && slot == out_slot &&
  403. ep == bulk_out.ep_id &&
  404. ev->p0 == out_trb) {
  405. out_status = code;
  406. out_done = 1;
  407. } else if (bulk_in.active && slot == bulk_in.slot &&
  408. ep == bulk_in.ep_id) {
  409. bulk_in.armed = 0;
  410. if (!bulk_in.pending) {
  411. if (code == CC_SUCCESS ||
  412. code == CC_SHORT_PACKET) {
  413. bulk_in.actual =
  414. (bulk_in.trb_len >=
  415. residual) ?
  416. bulk_in.trb_len -
  417. residual : 0;
  418. bulk_in.pending = 1;
  419. } else {
  420. bulk_in.pending = 1;
  421. bulk_in.error = 1;
  422. }
  423. }
  424. } else {
  425. xhci_int_event(slot, ep, code, residual);
  426. }
  427. }
  428. evt_idx++;
  429. if (evt_idx >= XHCI_EVT_TRBS) {
  430. evt_idx = 0;
  431. evt_cycle ^= 1;
  432. }
  433. }
  434. mmio_wr64(rt(XHCI_RT_ERDP),
  435. (evt_ring + evt_idx * 16) | 8);
  436. }
  437. /* ---- commands ---------------------------------------------------------- */
  438. static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
  439. unsigned int *out_slot)
  440. {
  441. unsigned int flags, t0, rc;
  442. if (!found)
  443. return -1;
  444. spin_lock_irqsave(&xhci_lock, &flags);
  445. if (cmd_busy) {
  446. spin_unlock_irqrestore(&xhci_lock, flags);
  447. return -1;
  448. }
  449. cmd_busy = 1;
  450. cmd_done = 0;
  451. cmd_status = 0;
  452. cmd_slot = 0;
  453. cmd_trb_addr = cmd_ring + cmd_enq * 16;
  454. ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
  455. p0, p1, 0, control);
  456. __asm__ volatile ("" ::: "memory");
  457. mmio_wr(db_off, 0);
  458. spin_unlock_irqrestore(&xhci_lock, flags);
  459. t0 = get_ticks_ms();
  460. while (get_ticks_ms() - t0 < 1000) {
  461. spin_lock_irqsave(&xhci_lock, &flags);
  462. xhci_poll_locked();
  463. rc = cmd_done;
  464. spin_unlock_irqrestore(&xhci_lock, flags);
  465. if (rc)
  466. break;
  467. usb_poll_relax(t0);
  468. __asm__ volatile ("pause");
  469. }
  470. spin_lock_irqsave(&xhci_lock, &flags);
  471. cmd_busy = 0;
  472. if (!cmd_done) {
  473. spin_unlock_irqrestore(&xhci_lock, flags);
  474. dbg("xhci: command timeout\n");
  475. return -1;
  476. }
  477. if (out_slot)
  478. *out_slot = cmd_slot;
  479. rc = cmd_status;
  480. spin_unlock_irqrestore(&xhci_lock, flags);
  481. if (rc != CC_SUCCESS) {
  482. dbg_hex("xhci: command status ", rc);
  483. return -1;
  484. }
  485. return 0;
  486. }
  487. /* ---- contexts ---------------------------------------------------------- */
  488. static void fill_slot_ctx(unsigned int addr, unsigned int port,
  489. unsigned int speed, unsigned int last_ctx)
  490. {
  491. unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
  492. p[0] = (speed << 20) | (last_ctx << 27);
  493. p[1] = (port + 1) << 16;
  494. p[2] = 0;
  495. p[3] = addr;
  496. }
  497. static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
  498. unsigned int enq, unsigned int cycle)
  499. {
  500. unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
  501. p[0] = 0;
  502. p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
  503. p[2] = (ring + enq * 16) | cycle;
  504. p[3] = 0;
  505. p[4] = 8;
  506. }
  507. static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
  508. unsigned int mps, unsigned int interval,
  509. unsigned int ring)
  510. {
  511. unsigned int *p =
  512. (unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
  513. p[0] = interval << 16;
  514. p[1] = (type << 3) | (3 << 1) | (mps << 16);
  515. p[2] = ring | 1;
  516. p[3] = 0;
  517. p[4] = mps;
  518. }
  519. static struct xhci_slot *slot_by_addr(unsigned int addr)
  520. {
  521. unsigned int id;
  522. if (addr > 127)
  523. return 0;
  524. id = addr_slot[addr];
  525. if (id < 1 || id > max_slots || !slots[id].used)
  526. return 0;
  527. return &slots[id];
  528. }
  529. static unsigned int port_speed(unsigned int port)
  530. {
  531. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  532. unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
  533. return sp ? sp : 1;
  534. }
  535. /* Default control-pipe max packet size for the BSR Address Device command.
  536. * High speed must be 64; full speed uses the USB core's 64-byte guess (the
  537. * descriptor read then corrects it); low speed is 8. Using 8 for a
  538. * high/full-speed slot leaves EP0 unusable on real xHCs. */
  539. static unsigned int ep0_mps_for_speed(unsigned int speed)
  540. {
  541. if (speed == 2) /* low speed */
  542. return 8;
  543. if (speed >= 4) /* SuperSpeed / SuperSpeedPlus */
  544. return 512;
  545. return 64; /* full speed (guess) and high speed */
  546. }
  547. /* EP0 max packet size for Address Device. Low, high and SuperSpeed have
  548. * one legal value, so the USB core's 8-byte first-read guess is replaced
  549. * by it; only full speed (8/16/32/64) takes the caller's value and is
  550. * corrected later by ep0_set_mps(). */
  551. static unsigned int ep0_mps_initial(unsigned int speed, unsigned int mps)
  552. {
  553. if (!mps || speed != 1)
  554. return ep0_mps_for_speed(speed);
  555. return mps;
  556. }
  557. /* Evaluate Context: tell the xHC the device's real bMaxPacketSize0 once
  558. * the first descriptor read has returned it. Without this the VL805 on
  559. * the 970A-DS3P kept the 8-byte EP0 that SET_ADDRESS created, and the
  560. * 18-byte device descriptor read of a high-speed stick failed with
  561. * Babble Detected (the device answers in one 18-byte packet). */
  562. static int ep0_set_mps(struct xhci_slot *s, unsigned int mps)
  563. {
  564. unsigned int flags;
  565. int rc;
  566. spin_lock_irqsave(&xhci_lock, &flags);
  567. zero_page(input_ctx, 4096);
  568. *(unsigned int *)input_ctx = 0;
  569. *(unsigned int *)(input_ctx + 4) = EP0_FLAG;
  570. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  571. spin_unlock_irqrestore(&xhci_lock, flags);
  572. rc = xhci_cmd(input_ctx, 0,
  573. TRB_TYPE(TRB_EVAL_CTX) | (s->slot_id << 24), 0);
  574. if (rc == 0) {
  575. s->ep0_mps = mps;
  576. } else {
  577. xlog_hex("xhci: evaluate context failed slot ", s->slot_id);
  578. xlog_hex(" mps ", mps);
  579. xlog_hex(" cc ", cmd_status);
  580. serial_putc(SERIAL_COM1, '\n');
  581. }
  582. return rc;
  583. }
  584. static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
  585. unsigned int type, unsigned int mps,
  586. unsigned int interval, unsigned int ring)
  587. {
  588. unsigned int ep_flag = 1u << (ep_index + 1);
  589. unsigned int last = ep_index + 1;
  590. unsigned int i, flags;
  591. int rc;
  592. for (i = 1; i < 32; i++)
  593. if (s->ep_mask & (1u << i))
  594. last = i;
  595. spin_lock_irqsave(&xhci_lock, &flags);
  596. zero_page(input_ctx, 4096);
  597. *(unsigned int *)input_ctx = 0;
  598. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
  599. fill_slot_ctx(s->dev_addr, s->port, s->speed, last);
  600. fill_ep_ctx(ep_index, type, mps, interval, ring);
  601. spin_unlock_irqrestore(&xhci_lock, flags);
  602. rc = xhci_cmd(input_ctx, 0,
  603. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  604. if (rc == 0)
  605. s->ep_mask |= ep_flag;
  606. return rc;
  607. }
  608. /* ---- slots ------------------------------------------------------------- */
  609. /* Create a slot. bsr=1: Address Device with Block Set Address Request,
  610. * leaving the slot in the Default state at device address 0 (new
  611. * enumeration scheme). bsr=0: the command carries target_addr and moves
  612. * Enabled -> Addressed directly (traditional scheme), which is what Linux
  613. * does when the port is addressed before the first descriptor read.
  614. * mps == 0 (or a SuperSpeed port) derives it from the port speed. */
  615. static struct xhci_slot *new_slot_common(unsigned int target_addr,
  616. unsigned int mps, int bsr)
  617. {
  618. struct xhci_slot *s;
  619. unsigned int slot_id, dc, ep0, flags;
  620. int rc;
  621. if (last_port < 0 || last_port >= (int)max_ports)
  622. return 0;
  623. if (port_slot[last_port])
  624. return 0;
  625. slot_id = 0;
  626. rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
  627. if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
  628. dbg("xhci: enable slot failed\n");
  629. return 0;
  630. }
  631. dc = mm_alloc_page();
  632. ep0 = mm_alloc_page();
  633. if (!dc || !ep0) {
  634. if (dc)
  635. mm_free_page(dc);
  636. if (ep0)
  637. mm_free_page(ep0);
  638. return 0;
  639. }
  640. zero_page(dc, 4096);
  641. zero_page(ep0, 4096);
  642. s = &slots[slot_id];
  643. s->used = 1;
  644. s->slot_id = slot_id;
  645. s->addr = 0;
  646. s->port = (unsigned int)last_port;
  647. s->speed = port_speed((unsigned int)last_port);
  648. s->dc = dc;
  649. s->ep0 = ep0;
  650. s->ep0_enq = 0;
  651. s->ep0_cycle = 1;
  652. s->ep_mask = 0;
  653. s->dev_addr = 0;
  654. ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
  655. mps = ep0_mps_initial(s->speed, mps);
  656. s->ep0_mps = mps;
  657. ram_wr64(dcbaa + slot_id * 8, dc);
  658. spin_lock_irqsave(&xhci_lock, &flags);
  659. zero_page(input_ctx, 4096);
  660. *(unsigned int *)input_ctx = 0;
  661. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  662. fill_slot_ctx(0, s->port, s->speed, 1);
  663. fill_ep0_ctx(mps, ep0, s->ep0_enq, s->ep0_cycle);
  664. spin_unlock_irqrestore(&xhci_lock, flags);
  665. rc = xhci_cmd(input_ctx, 0,
  666. TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) |
  667. (bsr ? TRB_BSR : 0), 0);
  668. if (rc != 0) {
  669. xlog_hex(bsr ? "xhci: address default failed port " :
  670. "xhci: address device failed port ", s->port);
  671. xlog_hex(" spd ", s->speed);
  672. xlog_hex(" cc ", cmd_status);
  673. xlog_hex(" portsc ",
  674. mmio_rd(op(XHCI_OP_PORTSC) + s->port * 0x10));
  675. serial_putc(SERIAL_COM1, '\n');
  676. ram_wr64(dcbaa + slot_id * 8, 0);
  677. xhci_cmd(0, 0,
  678. TRB_TYPE(TRB_DISABLE_SLOT) | (slot_id << 24), 0);
  679. mm_free_page(dc);
  680. mm_free_page(ep0);
  681. s->used = 0;
  682. return 0;
  683. }
  684. /* The xHC assigns the USB device address (the input slot context
  685. * address is 0); keep it for later Configure Endpoint commands. */
  686. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  687. s->addr = bsr ? 0 : target_addr;
  688. addr_slot[s->addr] = slot_id;
  689. port_slot[s->port] = slot_id;
  690. return s;
  691. }
  692. /* New-scheme slot: Enabled -> Default, at device address 0. */
  693. static struct xhci_slot *new_slot(void)
  694. {
  695. return new_slot_common(0, 0, 1);
  696. }
  697. static void free_slot(struct xhci_slot *s)
  698. {
  699. unsigned int i, flags;
  700. if (!s || !s->used)
  701. return;
  702. xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
  703. ram_wr64(dcbaa + s->slot_id * 8, 0);
  704. spin_lock_irqsave(&xhci_lock, &flags);
  705. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  706. if (ints[i].slot == s->slot_id) {
  707. ints[i].active = 0;
  708. ints[i].pending = 0;
  709. ints[i].error = 0;
  710. if (ints[i].ring) {
  711. mm_free_page(ints[i].ring);
  712. ints[i].ring = 0;
  713. }
  714. ints[i].slot = 0;
  715. }
  716. }
  717. if (s->dc) {
  718. mm_free_page(s->dc);
  719. s->dc = 0;
  720. }
  721. if (s->ep0) {
  722. mm_free_page(s->ep0);
  723. s->ep0 = 0;
  724. }
  725. if (s->addr < 128)
  726. addr_slot[s->addr] = 0;
  727. if (bulk_in.active && bulk_in.slot == s->slot_id)
  728. bulk_reset(&bulk_in);
  729. if (bulk_out.active && bulk_out.slot == s->slot_id)
  730. bulk_reset(&bulk_out);
  731. if (s->port < XHCI_MAX_PORTS)
  732. port_slot[s->port] = 0;
  733. s->used = 0;
  734. s->addr = 0;
  735. s->ep_mask = 0;
  736. spin_unlock_irqrestore(&xhci_lock, flags);
  737. }
  738. static int set_address(unsigned int new_addr, unsigned int mps)
  739. {
  740. struct xhci_slot *s = slot_by_addr(0);
  741. unsigned int flags;
  742. int rc;
  743. if (!s || new_addr == 0 || new_addr > 127)
  744. return -1;
  745. mps = ep0_mps_initial(s->speed, mps);
  746. spin_lock_irqsave(&xhci_lock, &flags);
  747. zero_page(input_ctx, 4096);
  748. *(unsigned int *)input_ctx = 0;
  749. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  750. fill_slot_ctx(0, s->port, s->speed, 1);
  751. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  752. spin_unlock_irqrestore(&xhci_lock, flags);
  753. rc = xhci_cmd(input_ctx, 0,
  754. TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
  755. if (rc != 0) {
  756. dbg("xhci: set address failed\n");
  757. return -1;
  758. }
  759. spin_lock_irqsave(&xhci_lock, &flags);
  760. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  761. s->ep0_mps = mps;
  762. addr_slot[0] = 0;
  763. s->addr = new_addr;
  764. addr_slot[new_addr] = s->slot_id;
  765. spin_unlock_irqrestore(&xhci_lock, flags);
  766. return 0;
  767. }
  768. /* ---- control transfers ------------------------------------------------- */
  769. /* One line per failed control transfer: the request, the completion code
  770. * (0 = no event within the timeout), the root port's PORTSC, and the slot /
  771. * EP0 states from the output device context (slot state 2 = Addressed,
  772. * EP state 2 = Halted, 4 = Error). */
  773. static void ctl_fail_log(const struct xhci_slot *s, const unsigned char *setup,
  774. unsigned int cc, unsigned int data_seen)
  775. {
  776. const unsigned int *dc = (const unsigned int *)s->dc;
  777. const unsigned int *ep0 = (const unsigned int *)(s->dc + ctx_size);
  778. serial_puts(SERIAL_COM1, cc ? "xhci: ctl err" : "xhci: ctl timeout");
  779. xlog_hex(" req ", (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  780. ((unsigned int)setup[2] << 16) | ((unsigned int)setup[3] << 24));
  781. xlog_hex(" cc ", cc);
  782. xlog_hex(" data ", data_seen);
  783. xlog_hex(" slot ", s->slot_id);
  784. xlog_hex(" spd ", s->speed);
  785. xlog_hex(" portsc ", mmio_rd(op(XHCI_OP_PORTSC) + s->port * 0x10));
  786. xlog_hex(" sstate ", dc[3] >> 27);
  787. xlog_hex(" dev ", dc[3] & 0xFF);
  788. xlog_hex(" ep0 ", ep0[0] & 7);
  789. xlog_hex(" mps ", ep0[1] >> 16);
  790. xlog_hex(" sts ", mmio_rd(op(XHCI_OP_USBSTS)));
  791. serial_putc(SERIAL_COM1, '\n');
  792. }
  793. static int control_transfer(unsigned int ctrl, unsigned int addr,
  794. unsigned int mps, int low_speed,
  795. const unsigned char *setup, unsigned char *data,
  796. unsigned int len, int dir_in)
  797. {
  798. struct xhci_slot *s;
  799. unsigned int flags, t0, rc, st;
  800. unsigned int setup_c, data_c, status_c;
  801. unsigned int last_bell;
  802. (void)ctrl;
  803. (void)low_speed;
  804. if (!found)
  805. return -1;
  806. /* xHCI owns SET_ADDRESS: it is an Address Device command, not a
  807. * control request on the wire. If a Default-state slot exists (new
  808. * scheme), move it to Addressed. Otherwise this is the traditional
  809. * scheme: create the slot Enabled and address it in one command,
  810. * without ever using the Default state. */
  811. if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0) {
  812. unsigned int new_addr = (unsigned int)setup[2] |
  813. ((unsigned int)setup[3] << 8);
  814. s = slot_by_addr(0);
  815. if (s)
  816. return set_address(new_addr, mps);
  817. s = new_slot_common(new_addr, mps, 0);
  818. return s ? 0 : -1;
  819. }
  820. s = slot_by_addr(addr);
  821. if (s && addr == 0 && (int)s->port != last_port) {
  822. /* A failed enumeration on another port left a slot at
  823. * address 0; it must not be reused for this device. */
  824. free_slot(s);
  825. s = 0;
  826. }
  827. if (!s) {
  828. if (addr != 0)
  829. return -1;
  830. s = new_slot();
  831. if (!s)
  832. return -1;
  833. }
  834. /* The core learned the real bMaxPacketSize0 from the first 8 bytes
  835. * of the device descriptor; reprogram EP0 before using it. Only a
  836. * full-speed EP0 can legally differ from the speed's fixed value. */
  837. if (s->speed == 1 && (mps == 8 || mps == 16 || mps == 32 ||
  838. mps == 64) && mps != s->ep0_mps &&
  839. ep0_set_mps(s, mps) != 0)
  840. return -1;
  841. spin_lock_irqsave(&xhci_lock, &flags);
  842. if (ctl_busy) {
  843. spin_unlock_irqrestore(&xhci_lock, flags);
  844. dbg("xhci: control busy\n");
  845. return -1;
  846. }
  847. ctl_busy = 1;
  848. ctl_done = 0;
  849. ctl_status = 0;
  850. ctl_actual = 0;
  851. ctl_data_seen = 0;
  852. ctl_slot = s->slot_id;
  853. ctl_req_len = len;
  854. /* Encode the three stages the way EDK2's XhciDxe does: every stage
  855. * is its own TD (no CHAIN) and carries IOC. This Braswell xHC never
  856. * completes a chained multi-TRB control TD: with Linux's chaining
  857. * (CHAIN on setup+data, IOC on status) the data stage hangs and no
  858. * transfer event is ever posted, while the per-stage encoding
  859. * completes. */
  860. setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_IOC;
  861. /* Transfer Type (TRT) only exists from xHCI 1.0 on; in a 0.96
  862. * controller (Etron EJ168, early VIA/Renesas parts) bits 17:16 are
  863. * reserved. Linux sets it only for hci_version >= 0x100 too. */
  864. if (len && hci_version >= 0x100)
  865. setup_c |= (dir_in ? 3u : 2u) << 16;
  866. ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
  867. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  868. (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  869. ((unsigned int)setup[2] << 16) |
  870. ((unsigned int)setup[3] << 24),
  871. (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
  872. ((unsigned int)setup[6] << 16) |
  873. ((unsigned int)setup[7] << 24),
  874. 8, setup_c);
  875. ctl_data_trb = 0;
  876. if (len) {
  877. ctl_data_trb = s->ep0 + s->ep0_enq * 16;
  878. data_c = TRB_TYPE(TRB_DATA) | TRB_IOC;
  879. if (dir_in)
  880. data_c |= TRB_DIR_IN | TRB_ISP;
  881. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
  882. &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
  883. }
  884. ctl_status_trb = s->ep0 + s->ep0_enq * 16;
  885. status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
  886. if (!len || !dir_in)
  887. status_c |= TRB_DIR_IN;
  888. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  889. 0, 0, 0, status_c);
  890. __asm__ volatile ("" ::: "memory");
  891. mmio_wr(db_off + 4 * s->slot_id, 1);
  892. spin_unlock_irqrestore(&xhci_lock, flags);
  893. t0 = get_ticks_ms();
  894. last_bell = t0;
  895. while (get_ticks_ms() - t0 < 1000) {
  896. spin_lock_irqsave(&xhci_lock, &flags);
  897. xhci_poll_locked();
  898. rc = ctl_done;
  899. if (!rc && get_ticks_ms() - last_bell >= 100) {
  900. /* Re-ring while the endpoint is still busy; a posted
  901. * doorbell can be lost on some real xHCs. */
  902. mmio_wr(db_off + 4 * s->slot_id, 1);
  903. last_bell = get_ticks_ms();
  904. }
  905. spin_unlock_irqrestore(&xhci_lock, flags);
  906. if (rc)
  907. break;
  908. usb_poll_relax(t0);
  909. __asm__ volatile ("pause");
  910. }
  911. spin_lock_irqsave(&xhci_lock, &flags);
  912. if (!ctl_done) {
  913. ctl_busy = 0;
  914. spin_unlock_irqrestore(&xhci_lock, flags);
  915. ctl_fail_log(s, setup, 0, ctl_data_seen);
  916. return -1;
  917. }
  918. rc = ctl_actual;
  919. st = ctl_status;
  920. ctl_busy = 0;
  921. spin_unlock_irqrestore(&xhci_lock, flags);
  922. if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
  923. ctl_fail_log(s, setup, st, ctl_data_seen);
  924. return -1;
  925. }
  926. return (int)rc;
  927. }
  928. /* ---- HID interrupt endpoint -------------------------------------------- */
  929. static int int_in_start(unsigned int ctrl, unsigned int slot,
  930. unsigned int addr, unsigned int ep,
  931. unsigned int mps, int low_speed)
  932. {
  933. struct xhci_slot *s = slot_by_addr(addr);
  934. struct xhci_int *in;
  935. unsigned int ep_index, ep_id, ring, flags;
  936. int rc;
  937. (void)ctrl;
  938. (void)low_speed;
  939. if (!found || !s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
  940. return -1;
  941. if (mps < 1)
  942. mps = 8;
  943. if (mps > 8)
  944. mps = 8;
  945. if (ints[slot].active)
  946. stop_int(slot);
  947. ep_index = (ep - 1) * 2 + 2; /* EP index 2 for EP1 IN, 4 for EP2 IN */
  948. ep_id = 2 * ep + 1;
  949. ring = mm_alloc_page();
  950. if (!ring)
  951. return -1;
  952. zero_page(ring, 4096);
  953. in = &ints[slot];
  954. in->slot = s->slot_id;
  955. in->ep_id = ep_id;
  956. in->mps = mps;
  957. in->ring = ring;
  958. in->enq = 0;
  959. in->cycle = 1;
  960. in->trb_len = mps;
  961. in->pending = 0;
  962. in->error = 0;
  963. in->active = 0;
  964. ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
  965. rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
  966. if (rc != 0) {
  967. dbg_hex("xhci: configure int ep failed ", ep_index);
  968. mm_free_page(ring);
  969. in->ring = 0;
  970. return -1;
  971. }
  972. in->active = 1;
  973. spin_lock_irqsave(&xhci_lock, &flags);
  974. ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  975. (unsigned int)in->buf, 0, in->trb_len,
  976. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  977. __asm__ volatile ("" ::: "memory");
  978. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  979. spin_unlock_irqrestore(&xhci_lock, flags);
  980. return 0;
  981. }
  982. static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
  983. {
  984. struct xhci_int *in;
  985. unsigned int flags, n, i;
  986. if (!found || slot >= USB_HCD_INT_SLOTS)
  987. return 0;
  988. in = &ints[slot];
  989. if (!in->active)
  990. return 0;
  991. spin_lock_irqsave(&xhci_lock, &flags);
  992. xhci_poll_locked();
  993. spin_unlock_irqrestore(&xhci_lock, flags);
  994. if (!in->pending)
  995. return 0;
  996. if (in->error) {
  997. in->pending = 0;
  998. in->error = 0;
  999. return -1;
  1000. }
  1001. n = in->pending_len;
  1002. if (n > len)
  1003. n = len;
  1004. for (i = 0; i < n; i++)
  1005. buf[i] = in->buf[i];
  1006. in->pending = 0;
  1007. spin_lock_irqsave(&xhci_lock, &flags);
  1008. if (in->active) {
  1009. ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  1010. (unsigned int)in->buf, 0, in->trb_len,
  1011. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1012. __asm__ volatile ("" ::: "memory");
  1013. mmio_wr(db_off + 4 * in->slot, in->ep_id);
  1014. }
  1015. spin_unlock_irqrestore(&xhci_lock, flags);
  1016. return (int)in->pending_len;
  1017. }
  1018. static void stop_int(unsigned int slot)
  1019. {
  1020. struct xhci_int *in;
  1021. unsigned int flags;
  1022. if (slot >= USB_HCD_INT_SLOTS)
  1023. return;
  1024. in = &ints[slot];
  1025. spin_lock_irqsave(&xhci_lock, &flags);
  1026. in->active = 0;
  1027. in->pending = 0;
  1028. in->error = 0;
  1029. spin_unlock_irqrestore(&xhci_lock, flags);
  1030. }
  1031. /* ---- bulk transfers ---------------------------------------------------- */
  1032. static void bulk_reset(struct xhci_bulk *b)
  1033. {
  1034. if (b->ring) {
  1035. mm_free_page(b->ring);
  1036. b->ring = 0;
  1037. }
  1038. b->active = 0;
  1039. b->armed = 0;
  1040. b->slot = 0;
  1041. b->pending = 0;
  1042. b->error = 0;
  1043. b->actual = 0;
  1044. }
  1045. static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
  1046. unsigned int mps, unsigned char *buf, unsigned int len)
  1047. {
  1048. struct xhci_slot *s = slot_by_addr(addr);
  1049. struct xhci_bulk *b = &bulk_in;
  1050. unsigned int ep_index, ep_id, ring, flags, n, err;
  1051. int rc;
  1052. (void)ctrl;
  1053. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  1054. return -1;
  1055. if (mps < 1)
  1056. mps = 512;
  1057. ep_index = (ep - 1) * 2 + 2;
  1058. ep_id = 2 * ep + 1;
  1059. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1060. ring = mm_alloc_page();
  1061. if (!ring)
  1062. return -1;
  1063. zero_page(ring, 4096);
  1064. bulk_reset(b);
  1065. b->slot = s->slot_id;
  1066. b->ep = ep;
  1067. b->ep_id = ep_id;
  1068. b->mps = mps;
  1069. b->ring = ring;
  1070. b->enq = 0;
  1071. b->cycle = 1;
  1072. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  1073. rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
  1074. if (rc != 0) {
  1075. bulk_reset(b);
  1076. return -1;
  1077. }
  1078. b->active = 1;
  1079. }
  1080. b->buf = (unsigned int)buf;
  1081. b->len = len;
  1082. spin_lock_irqsave(&xhci_lock, &flags);
  1083. xhci_poll_locked();
  1084. if (b->pending) {
  1085. n = b->actual;
  1086. err = (unsigned int)b->error;
  1087. b->pending = 0;
  1088. b->error = 0;
  1089. /* Do not re-arm here: the next call may use a different
  1090. * buffer/length (the mass-storage driver alternates CBW,
  1091. * data and CSW buffers). The caller's next usb_bulk_in()
  1092. * arms the new transfer with its own buffer. */
  1093. b->armed = 0;
  1094. spin_unlock_irqrestore(&xhci_lock, flags);
  1095. return err ? -1 : (int)n;
  1096. }
  1097. if (!b->armed) {
  1098. b->trb_len = len;
  1099. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1100. (unsigned int)buf, 0, len,
  1101. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1102. b->armed = 1;
  1103. __asm__ volatile ("" ::: "memory");
  1104. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  1105. }
  1106. spin_unlock_irqrestore(&xhci_lock, flags);
  1107. return 0;
  1108. }
  1109. /* Drop a bulk transfer after a timeout or a BOT reset. Stop the endpoint
  1110. * first so the xHC no longer references the old ring page, then clear the
  1111. * software state; the next bulk call configures a fresh ring. */
  1112. static void xhci_bulk_reset(unsigned int ctrl, unsigned int addr,
  1113. unsigned int ep, int dir_in)
  1114. {
  1115. struct xhci_slot *s = slot_by_addr(addr);
  1116. struct xhci_bulk *b = dir_in ? &bulk_in : &bulk_out;
  1117. unsigned int flags, slot, ep_id;
  1118. (void)ctrl;
  1119. if (!found || !s)
  1120. return;
  1121. spin_lock_irqsave(&xhci_lock, &flags);
  1122. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1123. spin_unlock_irqrestore(&xhci_lock, flags);
  1124. return;
  1125. }
  1126. slot = b->slot;
  1127. ep_id = b->ep_id;
  1128. spin_unlock_irqrestore(&xhci_lock, flags);
  1129. xhci_cmd(0, 0, TRB_TYPE(TRB_STOP_EP) | (slot << 24) | (ep_id << 16),
  1130. 0);
  1131. spin_lock_irqsave(&xhci_lock, &flags);
  1132. if (b->active && b->slot == slot && b->ep == ep)
  1133. bulk_reset(b);
  1134. spin_unlock_irqrestore(&xhci_lock, flags);
  1135. }
  1136. static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
  1137. unsigned int mps, const unsigned char *buf,
  1138. unsigned int len)
  1139. {
  1140. struct xhci_slot *s = slot_by_addr(addr);
  1141. struct xhci_bulk *b = &bulk_out;
  1142. unsigned int ep_index, ep_id, ring, flags, t0, done, status;
  1143. int rc;
  1144. (void)ctrl;
  1145. if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
  1146. return -1;
  1147. if (mps < 1)
  1148. mps = 512;
  1149. ep_index = (ep - 1) * 2 + 1;
  1150. ep_id = 2 * ep;
  1151. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1152. ring = mm_alloc_page();
  1153. if (!ring)
  1154. return -1;
  1155. zero_page(ring, 4096);
  1156. bulk_reset(b);
  1157. b->slot = s->slot_id;
  1158. b->ep = ep;
  1159. b->ep_id = ep_id;
  1160. b->mps = mps;
  1161. b->ring = ring;
  1162. b->enq = 0;
  1163. b->cycle = 1;
  1164. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  1165. rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
  1166. if (rc != 0) {
  1167. bulk_reset(b);
  1168. return -1;
  1169. }
  1170. b->active = 1;
  1171. }
  1172. spin_lock_irqsave(&xhci_lock, &flags);
  1173. if (out_busy) {
  1174. spin_unlock_irqrestore(&xhci_lock, flags);
  1175. return -1;
  1176. }
  1177. out_busy = 1;
  1178. out_done = 0;
  1179. out_status = 0;
  1180. out_slot = s->slot_id;
  1181. out_trb = b->ring + b->enq * 16;
  1182. b->trb_len = len;
  1183. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1184. (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1185. __asm__ volatile ("" ::: "memory");
  1186. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  1187. spin_unlock_irqrestore(&xhci_lock, flags);
  1188. t0 = get_ticks_ms();
  1189. while (get_ticks_ms() - t0 < 1000) {
  1190. spin_lock_irqsave(&xhci_lock, &flags);
  1191. xhci_poll_locked();
  1192. done = out_done;
  1193. spin_unlock_irqrestore(&xhci_lock, flags);
  1194. if (done)
  1195. break;
  1196. usb_poll_relax(t0);
  1197. __asm__ volatile ("pause");
  1198. }
  1199. spin_lock_irqsave(&xhci_lock, &flags);
  1200. done = out_done;
  1201. status = out_status;
  1202. out_busy = 0;
  1203. spin_unlock_irqrestore(&xhci_lock, flags);
  1204. if (!done) {
  1205. dbg("xhci: bulk out timeout\n");
  1206. return -1;
  1207. }
  1208. if (status != CC_SUCCESS) {
  1209. dbg_hex("xhci: bulk out status ", status);
  1210. return -1;
  1211. }
  1212. return 0;
  1213. }
  1214. /* ---- root hub ports ---------------------------------------------------- */
  1215. static int port_reset(unsigned int ctrl, unsigned int port)
  1216. {
  1217. unsigned int ps, t0;
  1218. (void)ctrl;
  1219. if (!found || port >= max_ports ||
  1220. !(port_mask & (1u << port)))
  1221. return -1;
  1222. if (port_slot[port])
  1223. free_slot(&slots[port_slot[port]]);
  1224. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1225. if (!(ps & PORTSC_CCS))
  1226. return -1;
  1227. /* Force the port into the Disabled state first: some xHCs ignore
  1228. * PR while the port is already enabled, leaving the device at its
  1229. * firmware-assigned address instead of address 0. USB2 ports only:
  1230. * on a SuperSpeed port PED=1 puts the link into SS.Disabled, which a
  1231. * hot reset (PR) does not leave -- software would have to write
  1232. * PLS=RxDetect -- so the BIOS-booted stick stayed unreachable. */
  1233. if ((ps & PORTSC_PED) &&
  1234. ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) < 4) {
  1235. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1236. portsc_neutral(ps) | PORTSC_PED);
  1237. t0 = get_ticks_ms();
  1238. while (get_ticks_ms() - t0 < 100) {
  1239. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1240. if (!(ps & PORTSC_PED))
  1241. break;
  1242. usb_poll_relax(t0);
  1243. }
  1244. }
  1245. /* Acknowledge stale change latches first; a leftover PRC would
  1246. * otherwise let the wait below succeed before this reset has even
  1247. * started (the firmware may have left one behind). */
  1248. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1249. portsc_neutral(ps) | PORTSC_CSC | PORTSC_PEC |
  1250. PORTSC_PRC | PORTSC_PLC);
  1251. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1252. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1253. portsc_neutral(ps) | PORTSC_PR);
  1254. t0 = get_ticks_ms();
  1255. while (get_ticks_ms() - t0 < 500) {
  1256. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1257. if (ps & PORTSC_PRC)
  1258. break;
  1259. usb_poll_relax(t0);
  1260. }
  1261. if (!(ps & PORTSC_PRC)) {
  1262. xlog_hex("xhci: port reset timeout port ", port);
  1263. xlog_hex(" portsc ", ps);
  1264. serial_putc(SERIAL_COM1, '\n');
  1265. return -1;
  1266. }
  1267. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1268. portsc_neutral(ps) | PORTSC_PRC);
  1269. t0 = get_ticks_ms();
  1270. while (get_ticks_ms() - t0 < 500) {
  1271. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1272. if (ps & PORTSC_PED)
  1273. break;
  1274. usb_poll_relax(t0);
  1275. }
  1276. if (!(ps & PORTSC_PED)) {
  1277. xlog_hex("xhci: port not enabled port ", port);
  1278. xlog_hex(" portsc ", ps);
  1279. serial_putc(SERIAL_COM1, '\n');
  1280. return -1;
  1281. }
  1282. /* Reset recovery: USB2 devices need TRSTRCY (10 ms) before the first
  1283. * control request; the delay is harmless for SuperSpeed. */
  1284. delay_ms(10);
  1285. last_port = (int)port;
  1286. return 0;
  1287. }
  1288. static int port_connected(unsigned int ctrl, unsigned int port)
  1289. {
  1290. unsigned int ps;
  1291. (void)ctrl;
  1292. if (!found || port >= max_ports ||
  1293. !(port_mask & (1u << port)))
  1294. return 0;
  1295. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1296. return (ps & PORTSC_CCS) ? 1 : 0;
  1297. }
  1298. static int port_low_speed(unsigned int ctrl, unsigned int port)
  1299. {
  1300. unsigned int ps;
  1301. (void)ctrl;
  1302. if (!found || port >= max_ports)
  1303. return 0;
  1304. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1305. return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
  1306. }
  1307. static int port_take_change(unsigned int ctrl, unsigned int port)
  1308. {
  1309. unsigned int ps;
  1310. (void)ctrl;
  1311. if (!found || port >= max_ports)
  1312. return 0;
  1313. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1314. if (!(ps & PORTSC_CSC))
  1315. return 0;
  1316. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1317. portsc_neutral(ps) | PORTSC_CSC);
  1318. return 1;
  1319. }
  1320. /* ---- controller discovery ---------------------------------------------- */
  1321. /* xHCI USB Legacy Support: take ownership from the firmware before the
  1322. * reset and stop its SMI from firing on port changes once we own the
  1323. * controller (the same idea as the PCI 0xC0 write in uhci.c/ohci.c, but
  1324. * xHCI keeps it in an extended capability).
  1325. *
  1326. * The BIOS-owned semaphore must actually clear before HCRST: while the
  1327. * firmware still owns the controller its SMI handler can interfere with
  1328. * the reset and the command ring. Linux waits for it too. */
  1329. static void legacy_handoff(unsigned int off)
  1330. {
  1331. unsigned int sup = mmio_rd(off);
  1332. unsigned int ctl, t0;
  1333. if ((sup & 0xFF) != 1)
  1334. return;
  1335. if (sup & (1u << 16)) { /* BIOS still owns it */
  1336. mmio_wr(off, sup | (1u << 24));
  1337. t0 = get_ticks_ms();
  1338. while (get_ticks_ms() - t0 < 1000) {
  1339. if (!(mmio_rd(off) & (1u << 16)))
  1340. break;
  1341. }
  1342. }
  1343. ctl = mmio_rd(off + 4);
  1344. ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
  1345. ctl |= 0x7u << 29;
  1346. mmio_wr(off + 4, ctl);
  1347. }
  1348. /* Walk the extended capabilities for the legacy handoff. Runs before the
  1349. * controller reset; scan_protocols() parses the port list afterwards. */
  1350. static void ext_caps_handoff(void)
  1351. {
  1352. unsigned int off, guard = 0;
  1353. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1354. while (off && guard++ < 32) {
  1355. unsigned int cap = mmio_rd(off);
  1356. unsigned int next = (cap >> 8) & 0xFF;
  1357. if ((cap & 0xFF) == 1)
  1358. legacy_handoff(off);
  1359. if (!next)
  1360. break;
  1361. off += next << 2;
  1362. }
  1363. }
  1364. static void scan_protocols(void)
  1365. {
  1366. unsigned int off, guard = 0;
  1367. port_mask = 0;
  1368. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1369. while (off && guard++ < 32) {
  1370. unsigned int cap = mmio_rd(off);
  1371. unsigned int id = cap & 0xFF;
  1372. unsigned int next = (cap >> 8) & 0xFF;
  1373. if (id == 2) { /* Supported Protocol */
  1374. unsigned int major = (cap >> 24) & 0xFF;
  1375. unsigned int pi = mmio_rd(off + 8);
  1376. unsigned int po = pi & 0xFF;
  1377. unsigned int pc = (pi >> 8) & 0xFF;
  1378. unsigned int i;
  1379. /* The controller may expose separate USB2 and USB3
  1380. * protocol entries for the same physical connectors;
  1381. * include both so a SuperSpeed device is not skipped. */
  1382. if ((major == 2 || major == 3) && po >= 1) {
  1383. for (i = 0; i < pc && po + i <= 32; i++)
  1384. port_mask |= 1u << (po - 1 + i);
  1385. }
  1386. }
  1387. if (!next)
  1388. break;
  1389. off += next << 2;
  1390. }
  1391. if (!port_mask)
  1392. port_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
  1393. ((1u << max_ports) - 1);
  1394. }
  1395. static int xhci_init(void)
  1396. {
  1397. unsigned int bus, dev, func, id, class_rev, bar, bar_hi, cmd;
  1398. unsigned int hcs1, hcs2, t0, i, n, scratch;
  1399. if (found)
  1400. return 1;
  1401. for (bus = 0; bus < 256; bus++) {
  1402. for (dev = 0; dev < 32; dev++) {
  1403. for (func = 0; func < 8; func++) {
  1404. id = pci_cfg_read32(bus, dev, func, 0x00);
  1405. if (id == 0xFFFFFFFFu || id == 0)
  1406. continue;
  1407. class_rev = pci_cfg_read32(bus, dev, func, 0x08);
  1408. if ((class_rev >> 16) != 0x0C03)
  1409. continue;
  1410. if (((class_rev >> 8) & 0xFF) != 0x30)
  1411. continue;
  1412. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1413. if (bar & 1)
  1414. continue;
  1415. bar_hi = 0;
  1416. if ((bar & 0x6) == 0x4)
  1417. bar_hi = pci_cfg_read32(bus, dev,
  1418. func, 0x14);
  1419. bar &= 0xFFFFFFF0u;
  1420. if (!bar || bar_hi)
  1421. continue;
  1422. pci_bus = bus;
  1423. pci_dev = dev;
  1424. pci_func = func;
  1425. goto found_xhci;
  1426. }
  1427. }
  1428. }
  1429. return 0;
  1430. found_xhci:
  1431. cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
  1432. pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
  1433. mmio_phys = bar;
  1434. mmio_base = (unsigned int)ioremap(bar, 0x10000, IOREMAP_MMIO);
  1435. if (!mmio_base)
  1436. return -1;
  1437. pci_id = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x00);
  1438. cap_len = mmio_rd(0) & 0xFF;
  1439. hci_version = mmio_rd(0) >> 16;
  1440. hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
  1441. hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
  1442. hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
  1443. rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
  1444. db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
  1445. max_slots = hcs1 & 0xFF;
  1446. if (max_slots > XHCI_MAX_SLOTS)
  1447. max_slots = XHCI_MAX_SLOTS;
  1448. if (!max_slots)
  1449. max_slots = 1;
  1450. max_ports = (hcs1 >> 24) & 0xFF;
  1451. if (max_ports > XHCI_MAX_PORTS)
  1452. max_ports = XHCI_MAX_PORTS;
  1453. if (!max_ports)
  1454. max_ports = 1;
  1455. ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
  1456. dbg("xhci: found\n");
  1457. dbg_hex("xhci: mmio ", mmio_phys);
  1458. dbg_dec("xhci: slots ", max_slots);
  1459. dbg_dec("xhci: ports ", max_ports);
  1460. dbg_dec("xhci: cap len ", cap_len);
  1461. dcbaa = mm_alloc_page();
  1462. input_ctx = mm_alloc_page();
  1463. cmd_ring = mm_alloc_page();
  1464. evt_ring = mm_alloc_page();
  1465. erst = mm_alloc_page();
  1466. if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
  1467. dbg("xhci: out of pages\n");
  1468. return 0;
  1469. }
  1470. zero_page(dcbaa, 4096);
  1471. zero_page(input_ctx, 4096);
  1472. zero_page(cmd_ring, 4096);
  1473. zero_page(evt_ring, 4096);
  1474. zero_page(erst, 4096);
  1475. /* Firmware handoff before the reset: while the BIOS still owns the
  1476. * controller its SMI handler can interfere with HCRST and the
  1477. * command ring. */
  1478. ext_caps_handoff();
  1479. /* Wait for the controller to become ready before touching USBCMD. */
  1480. t0 = get_ticks_ms();
  1481. while (get_ticks_ms() - t0 < 1000) {
  1482. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1483. break;
  1484. }
  1485. /* Reset. The xHC may ignore HCRST while it is running, so halt it
  1486. * and wait for HCH first (xHCI 4.2; Linux does the same). QEMU
  1487. * halts instantly, real silicon does not. */
  1488. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1489. t0 = get_ticks_ms();
  1490. while (get_ticks_ms() - t0 < 100) {
  1491. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1492. break;
  1493. }
  1494. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1495. dbg("xhci: halt timeout\n");
  1496. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
  1497. /* Intel xHCI (CherryView/Braswell included) needs 1 ms after the
  1498. * HCRST write before any other HC register is accessed; polling
  1499. * USBCMD too early can leave the controller partially reset. */
  1500. delay_ms(2);
  1501. t0 = get_ticks_ms();
  1502. while (get_ticks_ms() - t0 < 1000) {
  1503. if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
  1504. break;
  1505. }
  1506. t0 = get_ticks_ms();
  1507. while (get_ticks_ms() - t0 < 1000) {
  1508. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1509. break;
  1510. }
  1511. if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
  1512. dbg_hex("xhci: reset failed ",
  1513. mmio_rd(op(XHCI_OP_USBSTS)));
  1514. return 0;
  1515. }
  1516. /* After HCRST the controller must be halted before the ring
  1517. * registers are written; otherwise the writes can be dropped and
  1518. * every command times out. */
  1519. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)) {
  1520. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1521. t0 = get_ticks_ms();
  1522. while (get_ticks_ms() - t0 < 100) {
  1523. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1524. break;
  1525. }
  1526. }
  1527. /* A working HCRST clears DCBAAP/CRCR/CONFIG. If firmware values
  1528. * survive, the reset did not take effect: the ports keep their old
  1529. * device addresses and every control transfer to address 0 fails. */
  1530. if (mmio_rd(op(XHCI_OP_DCBAAP)) != 0 ||
  1531. (mmio_rd(op(XHCI_OP_CRCR)) & ~0xFu) != 0 ||
  1532. (mmio_rd(op(XHCI_OP_CONFIG)) & 0xFFu) != 0)
  1533. dbg("xhci: hcreset not applied\n");
  1534. /* Clear stale status latches from the firmware's session. */
  1535. mmio_wr(op(XHCI_OP_USBSTS), (1u << 2) | (1u << 3) | (1u << 4));
  1536. mmio_wr(op(XHCI_OP_PAGESIZE), 1);
  1537. mmio_wr(op(XHCI_OP_CONFIG), max_slots);
  1538. mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
  1539. /* Scratchpad buffers, if the controller asks for any. */
  1540. n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
  1541. if (n) {
  1542. unsigned int arr;
  1543. if (n > 512)
  1544. n = 512;
  1545. arr = mm_alloc_page();
  1546. if (!arr) {
  1547. dbg("xhci: no scratchpad array\n");
  1548. return 0;
  1549. }
  1550. zero_page(arr, 4096);
  1551. for (i = 0; i < n; i++) {
  1552. scratch = mm_alloc_page();
  1553. if (!scratch) {
  1554. dbg("xhci: no scratchpad page\n");
  1555. return 0;
  1556. }
  1557. zero_page(scratch, 4096);
  1558. ram_wr64(arr + i * 8, scratch);
  1559. }
  1560. ram_wr64(dcbaa, arr);
  1561. dbg_dec("xhci: scratchpads ", n);
  1562. }
  1563. /* Command ring and event ring. */
  1564. ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
  1565. mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
  1566. evt_idx = 0;
  1567. evt_cycle = 1;
  1568. *(volatile unsigned int *)(erst + 0) = evt_ring;
  1569. *(volatile unsigned int *)(erst + 4) = 0;
  1570. *(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
  1571. *(volatile unsigned int *)(erst + 12) = 0;
  1572. mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
  1573. mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
  1574. mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
  1575. mmio_wr(rt(XHCI_RT_IMAN), 0);
  1576. mmio_wr(rt(XHCI_RT_IMOD), 0);
  1577. /* Run. */
  1578. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
  1579. t0 = get_ticks_ms();
  1580. while (get_ticks_ms() - t0 < 1000) {
  1581. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1582. break;
  1583. }
  1584. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
  1585. dbg("xhci: controller did not start\n");
  1586. return 0;
  1587. }
  1588. /* The first command completion (or timeout) tells whether the ring is
  1589. * actually serviced. Don't gate on CRR: QEMU leaves it clear even
  1590. * while commands complete normally. */
  1591. scan_protocols();
  1592. xlog_hex("xhci: ", pci_id);
  1593. xlog_hex(" hci ", hci_version);
  1594. xlog_hex(" mmio ", mmio_phys);
  1595. xlog_hex(" ports ", port_mask);
  1596. xlog_hex(" csz ", ctx_size);
  1597. xlog_hex(" scratch ", n);
  1598. serial_putc(SERIAL_COM1, '\n');
  1599. /* Power every root port. VBus is supposed to come up after HCRST,
  1600. * but some firmware/PPC combinations leave PP clear; the USB3
  1601. * entries may share a switch with a USB2 connector, so write all of
  1602. * them, not just the USB2 range. */
  1603. for (i = 0; i < max_ports; i++) {
  1604. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
  1605. if (!(ps & PORTSC_PP))
  1606. mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
  1607. portsc_neutral(ps) | PORTSC_PP);
  1608. }
  1609. delay_ms(20);
  1610. found = 1;
  1611. return 1;
  1612. }
  1613. static unsigned int xhci_count(void)
  1614. {
  1615. return found ? 1u : 0u;
  1616. }
  1617. static void xhci_info(unsigned int ctrl, unsigned int *bus,
  1618. unsigned int *dev, unsigned int *func,
  1619. unsigned int *base, int *mmio)
  1620. {
  1621. (void)ctrl;
  1622. *bus = pci_bus;
  1623. *dev = pci_dev;
  1624. *func = pci_func;
  1625. *base = mmio_phys;
  1626. *mmio = 1;
  1627. }
  1628. static unsigned int xhci_port_count(unsigned int ctrl)
  1629. {
  1630. (void)ctrl;
  1631. return found ? max_ports : 0;
  1632. }
  1633. const struct usb_hcd_ops usb_xhci_ops = {
  1634. "xHCI",
  1635. xhci_init,
  1636. xhci_count,
  1637. xhci_info,
  1638. control_transfer,
  1639. xhci_port_count,
  1640. port_connected,
  1641. port_low_speed,
  1642. port_reset,
  1643. port_take_change,
  1644. int_in_start,
  1645. int_in_poll,
  1646. stop_int,
  1647. xhci_bulk_out,
  1648. xhci_bulk_in,
  1649. xhci_bulk_reset,
  1650. 0,
  1651. };