xhci.c 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138
  1. /* xHCI (USB 3.0) host controller driver.
  2. *
  3. * Scope: one controller, USB2 and USB3 root ports and devices plugged
  4. * directly into them. Control transfers, one interrupt-IN endpoint per HID
  5. * interface and the bulk endpoints used by ASIX and usbmsc are implemented.
  6. * SuperSpeed devices use a 512-byte EP0 and their endpoint descriptors may
  7. * carry a companion descriptor, which is ignored because only the bulk/HID
  8. * endpoint itself is needed.
  9. *
  10. * The schedule is polled, like the rest of the kernel. Rings and contexts
  11. * are page-allocated so the driver does not grow the kernel BSS. */
  12. #include "usb.h"
  13. #include "mm.h"
  14. #include "paging.h"
  15. #include "pci.h"
  16. #include "serial.h"
  17. #include "spinlock.h"
  18. #include "timer.h"
  19. #include "usb_hcd.h"
  20. #include "xhci.h"
  21. #define XHCI_DEBUG 0
  22. /* ---- controller limits ------------------------------------------------- */
  23. #define XHCI_MAX_SLOTS 8
  24. #define XHCI_MAX_PORTS 32
  25. #define XHCI_EVT_TRBS 256
  26. #define XHCI_CMD_TRBS 32
  27. #define XHCI_EP0_TRBS 16
  28. #define XHCI_INT_TRBS 16
  29. #define XHCI_BULK_TRBS 32
  30. /* ---- capability / operational / runtime offsets (bytes) --------------- */
  31. #define XHCI_CAP_HCSPARAMS1 0x04
  32. #define XHCI_CAP_HCSPARAMS2 0x08
  33. #define XHCI_CAP_HCCPARAMS1 0x10
  34. #define XHCI_CAP_DBOFF 0x14
  35. #define XHCI_CAP_RTSOFF 0x18
  36. #define XHCI_OP_USBCMD 0x00
  37. #define XHCI_OP_USBSTS 0x04
  38. #define XHCI_OP_PAGESIZE 0x08
  39. #define XHCI_OP_CRCR 0x18
  40. #define XHCI_OP_DCBAAP 0x30
  41. #define XHCI_OP_CONFIG 0x38
  42. #define XHCI_OP_PORTSC 0x400
  43. #define XHCI_RT_IMAN 0x20
  44. #define XHCI_RT_IMOD 0x24
  45. #define XHCI_RT_ERSTSZ 0x28
  46. #define XHCI_RT_ERSTBA 0x30
  47. #define XHCI_RT_ERDP 0x38
  48. /* USBCMD / USBSTS */
  49. #define XHCI_CMD_RS 0x0001u
  50. #define XHCI_CMD_HCRST 0x0002u
  51. #define XHCI_STS_HCH 0x0001u
  52. #define XHCI_STS_CNR 0x0800u
  53. #define XHCI_STS_HCE 0x1000u
  54. /* PORTSC */
  55. #define PORTSC_CCS 0x00000001u
  56. #define PORTSC_PED 0x00000002u
  57. #define PORTSC_PR 0x00000010u
  58. #define PORTSC_PP 0x00000200u
  59. #define PORTSC_SPEED_SHIFT 10
  60. #define PORTSC_SPEED_MASK 0xFu
  61. #define PORTSC_CSC 0x00020000u
  62. #define PORTSC_PEC 0x00040000u
  63. #define PORTSC_WRC 0x00080000u
  64. #define PORTSC_OCC 0x00100000u
  65. #define PORTSC_PRC 0x00200000u
  66. #define PORTSC_PLC 0x00400000u
  67. #define PORTSC_CEC 0x00800000u
  68. #define PORTSC_RWC (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
  69. PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
  70. PORTSC_CEC | PORTSC_PED)
  71. /* Bits that may be written back while preserving the port state:
  72. * read-only status (CCS, OCA, speed, device-removable) and the
  73. * read/write-state fields (PLS, PP, PIC, wake enables). Everything else
  74. * -- especially PR/WPR (RW1S) and PED (RW1CS) -- must be written as 0 or
  75. * the write changes the port state. Linux's
  76. * xhci_port_state_to_neutral() does the same. */
  77. #define PORTSC_RO_MASK (PORTSC_CCS | 0x00000008u | \
  78. (PORTSC_SPEED_MASK << PORTSC_SPEED_SHIFT) | \
  79. 0x40000000u)
  80. #define PORTSC_RWS_MASK (0x000001E0u | PORTSC_PP | 0x0000C000u | \
  81. 0x0E000000u)
  82. static unsigned int portsc_neutral(unsigned int ps)
  83. {
  84. return (ps & PORTSC_RO_MASK) | (ps & PORTSC_RWS_MASK);
  85. }
  86. /* TRB types */
  87. #define TRB_NORMAL 1
  88. #define TRB_SETUP 2
  89. #define TRB_DATA 3
  90. #define TRB_STATUS 4
  91. #define TRB_LINK 6
  92. #define TRB_ENABLE_SLOT 9
  93. #define TRB_DISABLE_SLOT 10
  94. #define TRB_ADDR_DEV 11
  95. #define TRB_CONFIG_EP 12
  96. #define TRB_EVAL_CTX 13
  97. #define TRB_STOP_EP 15
  98. #define TRB_TRANSFER 32
  99. #define TRB_COMPLETION 33
  100. /* TRB control bits */
  101. #define TRB_CYCLE 0x00000001u
  102. #define TRB_TC 0x00000002u
  103. #define TRB_ISP 0x00000004u
  104. #define TRB_CHAIN 0x00000010u
  105. #define TRB_IOC 0x00000020u
  106. #define TRB_IDT 0x00000040u
  107. #define TRB_BSR 0x00000200u
  108. #define TRB_DIR_IN 0x00010000u
  109. #define TRB_TYPE(t) ((t) << 10)
  110. /* completion codes */
  111. #define CC_SUCCESS 1
  112. #define CC_SHORT_PACKET 13
  113. /* slot/EP context flags and endpoint types */
  114. #define SLOT_FLAG 0x00000001u
  115. #define EP0_FLAG 0x00000002u
  116. #define EP_CTRL 4
  117. #define EP_BULK_OUT 2
  118. #define EP_BULK_IN 6
  119. #define EP_INT_IN 7
  120. struct xhci_trb {
  121. unsigned int p0;
  122. unsigned int p1;
  123. unsigned int status;
  124. unsigned int control;
  125. };
  126. struct xhci_slot {
  127. int used;
  128. unsigned int slot_id;
  129. unsigned int addr; /* logical address used by usb.c */
  130. unsigned int dev_addr; /* address assigned by the xHC */
  131. unsigned int port;
  132. unsigned int route; /* xHCI route string (0 for a root-port device) */
  133. unsigned int tt_slot; /* parent high-speed hub slot (TT) */
  134. unsigned int tt_port; /* downstream port on that hub */
  135. unsigned int speed;
  136. unsigned int dc; /* device context page */
  137. unsigned int ep0; /* EP0 transfer ring page */
  138. unsigned int ep0_enq;
  139. unsigned int ep0_cycle;
  140. unsigned int ep0_mps; /* max packet size programmed into EP0 */
  141. unsigned int ep_mask; /* configured endpoint flags */
  142. };
  143. struct xhci_int {
  144. int active;
  145. unsigned int slot; /* hardware slot id */
  146. unsigned int ep_id; /* xHCI endpoint id (2n+dir) */
  147. unsigned int mps;
  148. unsigned int ring;
  149. unsigned int enq;
  150. unsigned int cycle;
  151. unsigned int trb_len;
  152. unsigned int pending_len;
  153. int pending;
  154. int error;
  155. unsigned char buf[USB_INT_MAX] __attribute__((aligned(16)));
  156. };
  157. struct xhci_bulk {
  158. int active;
  159. int armed;
  160. unsigned int slot;
  161. unsigned int ep; /* endpoint number */
  162. unsigned int ep_id;
  163. unsigned int mps;
  164. unsigned int ring;
  165. unsigned int enq;
  166. unsigned int cycle;
  167. unsigned int buf;
  168. unsigned int len;
  169. unsigned int trb_len;
  170. unsigned int actual;
  171. int pending;
  172. int error;
  173. };
  174. #define XHCI_MAX_HCD 2
  175. /* One xHCI instance. The Yoga 720 has two: the Sunrise Point-LP PCH
  176. * controller (camera, Bluetooth, card reader) and the Alpine Ridge
  177. * USB-C/Thunderbolt controller that carries the Lenovo hub and the
  178. * Logitech receiver. The driver used to stop at the first one. */
  179. struct xhci_hcd {
  180. unsigned int mmio_base; /* kernel mapping of BAR0 */
  181. unsigned int mmio_phys; /* BAR0 itself (logged, reported) */
  182. unsigned int cap_len;
  183. unsigned int rt_off;
  184. unsigned int db_off;
  185. unsigned int max_slots;
  186. unsigned int max_ports;
  187. unsigned int ctx_size;
  188. unsigned int port_mask;
  189. unsigned int hcc_params;
  190. unsigned int bus_no, dev_no, func_no;
  191. unsigned int pci_id;
  192. unsigned int hci_version;
  193. int present;
  194. int last_port_num;
  195. unsigned int dcbaa;
  196. unsigned int input_ctx;
  197. unsigned int cmd_ring;
  198. unsigned int evt_ring;
  199. unsigned int erst;
  200. unsigned int cmd_enq;
  201. unsigned int cmd_cycle;
  202. unsigned int evt_idx;
  203. unsigned int evt_cycle;
  204. unsigned int cmd_busy;
  205. unsigned int cmd_done;
  206. unsigned int cmd_status;
  207. unsigned int cmd_slot;
  208. unsigned int cmd_trb_addr;
  209. unsigned int ctl_busy;
  210. unsigned int ctl_done;
  211. unsigned int ctl_slot;
  212. unsigned int ctl_status;
  213. unsigned int ctl_req_len;
  214. unsigned int ctl_actual;
  215. unsigned int ctl_data_seen;
  216. unsigned int ctl_setup_trb;
  217. unsigned int ctl_data_trb;
  218. unsigned int ctl_status_trb;
  219. struct xhci_bulk bulk_in;
  220. struct xhci_bulk bulk_out;
  221. unsigned int out_busy;
  222. unsigned int out_done;
  223. unsigned int out_status;
  224. unsigned int out_slot;
  225. unsigned int out_trb;
  226. struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
  227. unsigned int addr_slot[128];
  228. unsigned int port_slot[XHCI_MAX_PORTS];
  229. struct xhci_int ints[USB_HCD_INT_SLOTS];
  230. unsigned int enum_port;
  231. unsigned int enum_route;
  232. unsigned int enum_parent_addr;
  233. unsigned int enum_tt_slot;
  234. unsigned int enum_speed_hint;
  235. };
  236. /* Page-allocated: two full instances would otherwise add ~2 KB to the
  237. * kernel BSS, which is already close to the 0x90000 wall. */
  238. static struct xhci_hcd *xhci_hcds;
  239. static struct xhci_hcd *xh; /* instance of the current call */
  240. static unsigned int xhci_nctrl;
  241. /* int_in_poll()/stop() get only the slot, not the controller index. */
  242. static unsigned int int_ctrl[USB_HCD_INT_SLOTS];
  243. #define mmio_base (xh->mmio_base)
  244. #define mmio_phys (xh->mmio_phys)
  245. #define cap_len (xh->cap_len)
  246. #define rt_off (xh->rt_off)
  247. #define db_off (xh->db_off)
  248. #define max_slots (xh->max_slots)
  249. #define max_ports (xh->max_ports)
  250. #define ctx_size (xh->ctx_size)
  251. #define port_mask (xh->port_mask)
  252. #define hcc_params (xh->hcc_params)
  253. #define pci_bus (xh->bus_no)
  254. #define pci_dev (xh->dev_no)
  255. #define pci_func (xh->func_no)
  256. #define pci_id (xh->pci_id)
  257. #define hci_version (xh->hci_version)
  258. #define found (xh->present)
  259. #define last_port (xh->last_port_num)
  260. #define dcbaa (xh->dcbaa)
  261. #define input_ctx (xh->input_ctx)
  262. #define cmd_ring (xh->cmd_ring)
  263. #define evt_ring (xh->evt_ring)
  264. #define erst (xh->erst)
  265. #define cmd_enq (xh->cmd_enq)
  266. #define cmd_cycle (xh->cmd_cycle)
  267. #define evt_idx (xh->evt_idx)
  268. #define evt_cycle (xh->evt_cycle)
  269. #define cmd_busy (xh->cmd_busy)
  270. #define cmd_done (xh->cmd_done)
  271. #define cmd_status (xh->cmd_status)
  272. #define cmd_slot (xh->cmd_slot)
  273. #define cmd_trb_addr (xh->cmd_trb_addr)
  274. #define ctl_busy (xh->ctl_busy)
  275. #define ctl_done (xh->ctl_done)
  276. #define ctl_slot (xh->ctl_slot)
  277. #define ctl_status (xh->ctl_status)
  278. #define ctl_req_len (xh->ctl_req_len)
  279. #define ctl_actual (xh->ctl_actual)
  280. #define ctl_data_seen (xh->ctl_data_seen)
  281. #define ctl_setup_trb (xh->ctl_setup_trb)
  282. #define ctl_data_trb (xh->ctl_data_trb)
  283. #define ctl_status_trb (xh->ctl_status_trb)
  284. #define bulk_in (xh->bulk_in)
  285. #define bulk_out (xh->bulk_out)
  286. #define out_busy (xh->out_busy)
  287. #define out_done (xh->out_done)
  288. #define out_status (xh->out_status)
  289. #define out_slot (xh->out_slot)
  290. #define out_trb (xh->out_trb)
  291. #define slots (xh->slots)
  292. #define addr_slot (xh->addr_slot)
  293. #define port_slot (xh->port_slot)
  294. #define ints (xh->ints)
  295. #define enum_port (xh->enum_port)
  296. #define enum_route (xh->enum_route)
  297. #define enum_parent_addr (xh->enum_parent_addr)
  298. #define enum_tt_slot (xh->enum_tt_slot)
  299. #define enum_speed_hint (xh->enum_speed_hint)
  300. static spinlock_t xhci_lock;
  301. /* Select the instance for a vtable call (ctrl is the local index that
  302. * usb.c registered). */
  303. static int xhci_use(unsigned int ctrl)
  304. {
  305. if (!xhci_hcds || ctrl >= XHCI_MAX_HCD ||
  306. !xhci_hcds[ctrl].present)
  307. return 0;
  308. xh = &xhci_hcds[ctrl];
  309. return 1;
  310. }
  311. static int xhci_use_slot(unsigned int slot)
  312. {
  313. if (slot >= USB_HCD_INT_SLOTS || int_ctrl[slot] >= XHCI_MAX_HCD)
  314. return 0;
  315. return xhci_use(int_ctrl[slot]);
  316. }
  317. /* ---- debug ------------------------------------------------------------- */
  318. #if XHCI_DEBUG
  319. static void dbg(const char *s)
  320. {
  321. serial_puts(SERIAL_COM1, s);
  322. }
  323. static void dbg_hex(const char *s, unsigned int v)
  324. {
  325. serial_puts(SERIAL_COM1, s);
  326. serial_puthex(SERIAL_COM1, v);
  327. serial_putc(SERIAL_COM1, '\n');
  328. }
  329. static void dbg_dec(const char *s, unsigned int v)
  330. {
  331. serial_puts(SERIAL_COM1, s);
  332. serial_putdec(SERIAL_COM1, v);
  333. serial_putc(SERIAL_COM1, '\n');
  334. }
  335. #else
  336. static void dbg(const char *s) { (void)s; }
  337. static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
  338. static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
  339. #endif
  340. /* Always-on one-line diagnostics (COM1, mirrored to netmon). Only used on
  341. * the init line and on failures, so a working machine stays quiet. */
  342. static void xlog_hex(const char *s, unsigned int v)
  343. {
  344. serial_puts(SERIAL_COM1, s);
  345. serial_puthex(SERIAL_COM1, v);
  346. }
  347. static void delay_ms(unsigned int ms)
  348. {
  349. usb_delay_ms(ms); /* sleeps inside a sleepable scan */
  350. }
  351. /* ---- MMIO helpers ------------------------------------------------------ */
  352. static unsigned int mmio_rd(unsigned int off)
  353. {
  354. return *(volatile unsigned int *)(mmio_base + off);
  355. }
  356. static void mmio_wr(unsigned int off, unsigned int val)
  357. {
  358. *(volatile unsigned int *)(mmio_base + off) = val;
  359. }
  360. static void mmio_wr64(unsigned int off, unsigned int lo)
  361. {
  362. mmio_wr(off, lo);
  363. mmio_wr(off + 4, 0);
  364. }
  365. /* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
  366. static void ram_wr64(unsigned int addr, unsigned int lo)
  367. {
  368. *(volatile unsigned int *)addr = lo;
  369. *(volatile unsigned int *)(addr + 4) = 0;
  370. }
  371. static void zero_page(unsigned int addr, unsigned int len)
  372. {
  373. unsigned int *p = (unsigned int *)addr;
  374. while (len >= 4) {
  375. *p++ = 0;
  376. len -= 4;
  377. }
  378. }
  379. static unsigned int op(unsigned int reg)
  380. {
  381. return cap_len + reg;
  382. }
  383. static unsigned int rt(unsigned int reg)
  384. {
  385. return rt_off + reg;
  386. }
  387. /* ---- rings ------------------------------------------------------------- */
  388. static void ring_init(unsigned int base, unsigned int size,
  389. unsigned int *enq, unsigned int *cycle)
  390. {
  391. struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
  392. link->p0 = base;
  393. link->p1 = 0;
  394. link->status = 0;
  395. link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  396. *enq = 0;
  397. *cycle = 1;
  398. }
  399. static void ring_queue(unsigned int base, unsigned int size,
  400. unsigned int *enq, unsigned int *cycle,
  401. unsigned int p0, unsigned int p1,
  402. unsigned int status, unsigned int control)
  403. {
  404. struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
  405. trb->p0 = p0;
  406. trb->p1 = p1;
  407. trb->status = status;
  408. __asm__ volatile ("" ::: "memory");
  409. trb->control = control | *cycle;
  410. __asm__ volatile ("" ::: "memory");
  411. (*enq)++;
  412. if (*enq == size - 1) {
  413. struct xhci_trb *link =
  414. (struct xhci_trb *)(base + (size - 1) * 16);
  415. __asm__ volatile ("" ::: "memory");
  416. link->control ^= TRB_CYCLE;
  417. *enq = 0;
  418. *cycle ^= 1;
  419. }
  420. }
  421. /* ---- event handling ---------------------------------------------------- */
  422. static void xhci_poll_locked(void);
  423. static void stop_int(unsigned int slot);
  424. static void bulk_reset(struct xhci_bulk *b);
  425. static void xhci_int_event(unsigned int slot, unsigned int ep,
  426. unsigned int code, unsigned int residual)
  427. {
  428. unsigned int i;
  429. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  430. if (!ints[i].active || ints[i].slot != slot ||
  431. ints[i].ep_id != ep)
  432. continue;
  433. if (!ints[i].pending) {
  434. if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
  435. ints[i].pending_len =
  436. (ints[i].trb_len >= residual) ?
  437. ints[i].trb_len - residual : 0;
  438. ints[i].pending = 1;
  439. } else {
  440. ints[i].pending = 1;
  441. ints[i].error = 1;
  442. }
  443. }
  444. break;
  445. }
  446. }
  447. static void xhci_poll_locked(void)
  448. {
  449. unsigned int n = 0;
  450. while (n++ < 512) {
  451. struct xhci_trb *ev =
  452. (struct xhci_trb *)(evt_ring + evt_idx * 16);
  453. unsigned int type;
  454. if ((ev->control & TRB_CYCLE) != evt_cycle)
  455. break;
  456. type = (ev->control >> 10) & 0x3F;
  457. if (type == TRB_COMPLETION) {
  458. unsigned int code = (ev->status >> 24) & 0xFF;
  459. unsigned int slot = (ev->control >> 24) & 0xFF;
  460. if (cmd_busy && ev->p0 == cmd_trb_addr) {
  461. cmd_status = code;
  462. cmd_slot = slot;
  463. cmd_done = 1;
  464. }
  465. } else if (type == TRB_TRANSFER) {
  466. unsigned int code = (ev->status >> 24) & 0xFF;
  467. unsigned int residual = ev->status & 0xFFFFFF;
  468. unsigned int slot = (ev->control >> 24) & 0xFF;
  469. unsigned int ep = (ev->control >> 16) & 0x1F;
  470. if (ctl_busy && slot == ctl_slot && ep == 1) {
  471. if (ev->p0 == ctl_setup_trb) {
  472. /* The setup stage is its own TD and
  473. * completes first; only an error stops
  474. * the transfer. */
  475. if (code != CC_SUCCESS &&
  476. code != CC_SHORT_PACKET) {
  477. ctl_status = code;
  478. ctl_done = 1;
  479. }
  480. } else if (ev->p0 == ctl_data_trb) {
  481. ctl_actual = (ctl_req_len >= residual) ?
  482. ctl_req_len - residual : 0;
  483. ctl_data_seen = 1;
  484. if (code != CC_SUCCESS &&
  485. code != CC_SHORT_PACKET) {
  486. ctl_status = code;
  487. ctl_done = 1;
  488. }
  489. } else if (ev->p0 == ctl_status_trb) {
  490. if (!ctl_data_seen)
  491. ctl_actual =
  492. (code == CC_SUCCESS) ?
  493. ctl_req_len : 0;
  494. ctl_status = code;
  495. ctl_done = 1;
  496. }
  497. } else if (out_busy && slot == out_slot &&
  498. ep == bulk_out.ep_id &&
  499. ev->p0 == out_trb) {
  500. out_status = code;
  501. out_done = 1;
  502. } else if (bulk_in.active && slot == bulk_in.slot &&
  503. ep == bulk_in.ep_id) {
  504. bulk_in.armed = 0;
  505. if (!bulk_in.pending) {
  506. if (code == CC_SUCCESS ||
  507. code == CC_SHORT_PACKET) {
  508. bulk_in.actual =
  509. (bulk_in.trb_len >=
  510. residual) ?
  511. bulk_in.trb_len -
  512. residual : 0;
  513. bulk_in.pending = 1;
  514. } else {
  515. bulk_in.pending = 1;
  516. bulk_in.error = 1;
  517. }
  518. }
  519. } else {
  520. xhci_int_event(slot, ep, code, residual);
  521. }
  522. }
  523. evt_idx++;
  524. if (evt_idx >= XHCI_EVT_TRBS) {
  525. evt_idx = 0;
  526. evt_cycle ^= 1;
  527. }
  528. }
  529. mmio_wr64(rt(XHCI_RT_ERDP),
  530. (evt_ring + evt_idx * 16) | 8);
  531. }
  532. /* ---- commands ---------------------------------------------------------- */
  533. static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
  534. unsigned int *slot_out)
  535. {
  536. unsigned int flags, t0, rc;
  537. if (!found)
  538. return -1;
  539. spin_lock_irqsave(&xhci_lock, &flags);
  540. if (cmd_busy) {
  541. spin_unlock_irqrestore(&xhci_lock, flags);
  542. return -1;
  543. }
  544. cmd_busy = 1;
  545. cmd_done = 0;
  546. cmd_status = 0;
  547. cmd_slot = 0;
  548. cmd_trb_addr = cmd_ring + cmd_enq * 16;
  549. ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
  550. p0, p1, 0, control);
  551. __asm__ volatile ("" ::: "memory");
  552. mmio_wr(db_off, 0);
  553. spin_unlock_irqrestore(&xhci_lock, flags);
  554. t0 = get_ticks_ms();
  555. while (get_ticks_ms() - t0 < 1000) {
  556. spin_lock_irqsave(&xhci_lock, &flags);
  557. xhci_poll_locked();
  558. rc = cmd_done;
  559. spin_unlock_irqrestore(&xhci_lock, flags);
  560. if (rc)
  561. break;
  562. usb_poll_relax(t0);
  563. __asm__ volatile ("pause");
  564. }
  565. spin_lock_irqsave(&xhci_lock, &flags);
  566. cmd_busy = 0;
  567. if (!cmd_done) {
  568. spin_unlock_irqrestore(&xhci_lock, flags);
  569. dbg("xhci: command timeout\n");
  570. return -1;
  571. }
  572. if (slot_out)
  573. *slot_out = cmd_slot;
  574. rc = cmd_status;
  575. spin_unlock_irqrestore(&xhci_lock, flags);
  576. if (rc != CC_SUCCESS) {
  577. dbg_hex("xhci: command status ", rc);
  578. return -1;
  579. }
  580. return 0;
  581. }
  582. /* ---- contexts ---------------------------------------------------------- */
  583. static void fill_slot_ctx(unsigned int addr, unsigned int port,
  584. unsigned int route, unsigned int tt_slot,
  585. unsigned int tt_port, unsigned int speed,
  586. unsigned int last_ctx)
  587. {
  588. unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
  589. p[0] = (route & 0xFFFFFu) | (speed << 20) | (last_ctx << 27);
  590. p[1] = (port + 1) << 16; /* root hub port, max exit latency 0 */
  591. p[2] = (tt_slot & 0xFFu) | ((tt_port & 0xFFu) << 8); /* tt_info */
  592. p[3] = addr;
  593. }
  594. static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
  595. unsigned int enq, unsigned int cycle)
  596. {
  597. unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
  598. p[0] = 0;
  599. p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
  600. p[2] = (ring + enq * 16) | cycle;
  601. p[3] = 0;
  602. p[4] = 8;
  603. }
  604. static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
  605. unsigned int mps, unsigned int interval,
  606. unsigned int ring)
  607. {
  608. unsigned int *p =
  609. (unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
  610. p[0] = interval << 16;
  611. p[1] = (type << 3) | (3 << 1) | (mps << 16);
  612. p[2] = ring | 1;
  613. p[3] = 0;
  614. p[4] = mps;
  615. }
  616. static struct xhci_slot *slot_by_addr(unsigned int addr)
  617. {
  618. unsigned int id;
  619. if (addr > 127)
  620. return 0;
  621. id = addr_slot[addr];
  622. if (id < 1 || id > max_slots || !slots[id].used)
  623. return 0;
  624. return &slots[id];
  625. }
  626. static unsigned int port_speed(unsigned int port)
  627. {
  628. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  629. unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
  630. return sp ? sp : 1;
  631. }
  632. /* Default control-pipe max packet size for the BSR Address Device command.
  633. * High speed must be 64; full speed uses the USB core's 64-byte guess (the
  634. * descriptor read then corrects it); low speed is 8. Using 8 for a
  635. * high/full-speed slot leaves EP0 unusable on real xHCs. */
  636. static unsigned int ep0_mps_for_speed(unsigned int speed)
  637. {
  638. if (speed == 2) /* low speed */
  639. return 8;
  640. if (speed >= 4) /* SuperSpeed / SuperSpeedPlus */
  641. return 512;
  642. return 64; /* full speed (guess) and high speed */
  643. }
  644. /* EP0 max packet size for Address Device. Low, high and SuperSpeed have
  645. * one legal value, so the USB core's 8-byte first-read guess is replaced
  646. * by it; only full speed (8/16/32/64) takes the caller's value and is
  647. * corrected later by ep0_set_mps(). */
  648. static unsigned int ep0_mps_initial(unsigned int speed, unsigned int mps)
  649. {
  650. if (!mps || speed != 1)
  651. return ep0_mps_for_speed(speed);
  652. return mps;
  653. }
  654. /* Evaluate Context: tell the xHC the device's real bMaxPacketSize0 once
  655. * the first descriptor read has returned it. Without this the VL805 on
  656. * the 970A-DS3P kept the 8-byte EP0 that SET_ADDRESS created, and the
  657. * 18-byte device descriptor read of a high-speed stick failed with
  658. * Babble Detected (the device answers in one 18-byte packet). */
  659. static int ep0_set_mps(struct xhci_slot *s, unsigned int mps)
  660. {
  661. unsigned int flags;
  662. int rc;
  663. spin_lock_irqsave(&xhci_lock, &flags);
  664. zero_page(input_ctx, 4096);
  665. *(unsigned int *)input_ctx = 0;
  666. *(unsigned int *)(input_ctx + 4) = EP0_FLAG;
  667. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  668. spin_unlock_irqrestore(&xhci_lock, flags);
  669. rc = xhci_cmd(input_ctx, 0,
  670. TRB_TYPE(TRB_EVAL_CTX) | (s->slot_id << 24), 0);
  671. if (rc == 0) {
  672. s->ep0_mps = mps;
  673. } else {
  674. xlog_hex("xhci: evaluate context failed slot ", s->slot_id);
  675. xlog_hex(" mps ", mps);
  676. xlog_hex(" cc ", cmd_status);
  677. serial_putc(SERIAL_COM1, '\n');
  678. }
  679. return rc;
  680. }
  681. static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
  682. unsigned int type, unsigned int mps,
  683. unsigned int interval, unsigned int ring)
  684. {
  685. unsigned int ep_flag = 1u << (ep_index + 1);
  686. unsigned int last = ep_index + 1;
  687. unsigned int i, flags;
  688. int rc;
  689. /* Last Valid Endpoint is the highest DCI in the slot, including the
  690. * one being added now. Scanning only the existing mask and assigning
  691. * it unconditionally used to lower it again (EP1 IN = DCI 3 already
  692. * configured, then adding EP2 IN = DCI 5 sent LVE 3 and the xHC
  693. * rejected the command with Parameter Error). */
  694. for (i = 1; i < 32; i++)
  695. if ((s->ep_mask & (1u << i)) && i > last)
  696. last = i;
  697. spin_lock_irqsave(&xhci_lock, &flags);
  698. zero_page(input_ctx, 4096);
  699. *(unsigned int *)input_ctx = 0;
  700. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
  701. fill_slot_ctx(s->dev_addr, s->port, s->route, s->tt_slot,
  702. s->tt_port, s->speed, last);
  703. fill_ep_ctx(ep_index, type, mps, interval, ring);
  704. spin_unlock_irqrestore(&xhci_lock, flags);
  705. rc = xhci_cmd(input_ctx, 0,
  706. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  707. if (rc == 0) {
  708. s->ep_mask |= ep_flag;
  709. } else {
  710. xlog_hex("xhci: configure ep failed ep ", ep_index);
  711. xlog_hex(" cc ", cmd_status);
  712. xlog_hex(" slot ", s->slot_id);
  713. xlog_hex(" spd ", s->speed);
  714. xlog_hex(" route ", s->route);
  715. xlog_hex(" tt ", s->tt_slot);
  716. xlog_hex(" ttp ", s->tt_port);
  717. serial_putc(SERIAL_COM1, '\n');
  718. }
  719. return rc;
  720. }
  721. /* ---- slots ------------------------------------------------------------- */
  722. static void set_route(unsigned int ctrl, unsigned int port,
  723. unsigned int route, unsigned int parent_addr,
  724. unsigned int speed_hint)
  725. {
  726. struct xhci_slot *hs;
  727. if (!xhci_use(ctrl))
  728. return;
  729. enum_port = port;
  730. enum_route = route & 0xFFFFFu;
  731. enum_parent_addr = parent_addr;
  732. enum_speed_hint = speed_hint;
  733. enum_tt_slot = 0;
  734. if (parent_addr) {
  735. hs = slot_by_addr(parent_addr);
  736. if (hs && hs->speed == 3)
  737. enum_tt_slot = hs->slot_id;
  738. }
  739. }
  740. /* Move an Addressed slot to Configured. The xHC does not do this from the
  741. * USB SET_CONFIGURATION request; a Configure Endpoint command is required.
  742. * A high-speed hub used as a TT must be Configured before a downstream
  743. * Address Device, otherwise the xHC rejects the child with Parameter Error
  744. * (cc 0x11). Only the slot context is added; EP0 is already in the device
  745. * context. */
  746. static int xhci_configure_slot(struct xhci_slot *s)
  747. {
  748. unsigned int flags;
  749. int rc;
  750. spin_lock_irqsave(&xhci_lock, &flags);
  751. zero_page(input_ctx, 4096);
  752. *(unsigned int *)input_ctx = 0;
  753. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG;
  754. fill_slot_ctx(s->dev_addr, s->port, s->route, s->tt_slot,
  755. s->tt_port, s->speed, 1);
  756. spin_unlock_irqrestore(&xhci_lock, flags);
  757. rc = xhci_cmd(input_ctx, 0,
  758. TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
  759. if (rc != 0) {
  760. xlog_hex("xhci: configure slot failed cc ", cmd_status);
  761. xlog_hex(" slot ", s->slot_id);
  762. xlog_hex(" spd ", s->speed);
  763. serial_putc(SERIAL_COM1, '\n');
  764. return -1;
  765. }
  766. return 0;
  767. }
  768. /* Create a slot. bsr=1: Address Device with Block Set Address Request,
  769. * leaving the slot in the Default state at device address 0 (new
  770. * enumeration scheme). bsr=0: the command carries target_addr and moves
  771. * Enabled -> Addressed directly (traditional scheme), which is what Linux
  772. * does when the port is addressed before the first descriptor read.
  773. * mps == 0 (or a SuperSpeed port) derives it from the port speed. */
  774. static struct xhci_slot *new_slot_common(unsigned int target_addr,
  775. unsigned int mps, int bsr)
  776. {
  777. struct xhci_slot *s;
  778. unsigned int slot_id, dc, ep0, flags;
  779. int rc;
  780. if (enum_port >= max_ports)
  781. return 0;
  782. if (enum_route == 0 && port_slot[enum_port])
  783. return 0;
  784. slot_id = 0;
  785. rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
  786. if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
  787. xlog_hex("xhci: enable slot failed rc ", rc);
  788. xlog_hex(" cc ", cmd_status);
  789. xlog_hex(" slot ", slot_id);
  790. serial_putc(SERIAL_COM1, '\n');
  791. return 0;
  792. }
  793. dc = mm_alloc_page();
  794. ep0 = mm_alloc_page();
  795. if (!dc || !ep0) {
  796. if (dc)
  797. mm_free_page(dc);
  798. if (ep0)
  799. mm_free_page(ep0);
  800. return 0;
  801. }
  802. zero_page(dc, 4096);
  803. zero_page(ep0, 4096);
  804. s = &slots[slot_id];
  805. s->used = 1;
  806. s->slot_id = slot_id;
  807. s->addr = 0;
  808. s->port = enum_port;
  809. s->route = enum_route;
  810. s->speed = enum_speed_hint ? enum_speed_hint :
  811. port_speed(enum_port);
  812. if (enum_route && (s->speed == 1 || s->speed == 2)) {
  813. s->tt_slot = enum_tt_slot;
  814. s->tt_port = enum_route & 0xFu;
  815. } else {
  816. /* High/SuperSpeed devices do not go through a TT. */
  817. s->tt_slot = 0;
  818. s->tt_port = 0;
  819. }
  820. s->dc = dc;
  821. s->ep0 = ep0;
  822. s->ep0_enq = 0;
  823. s->ep0_cycle = 1;
  824. s->ep_mask = 0;
  825. s->dev_addr = 0;
  826. ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
  827. mps = ep0_mps_initial(s->speed, mps);
  828. s->ep0_mps = mps;
  829. ram_wr64(dcbaa + slot_id * 8, dc);
  830. spin_lock_irqsave(&xhci_lock, &flags);
  831. zero_page(input_ctx, 4096);
  832. *(unsigned int *)input_ctx = 0;
  833. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  834. fill_slot_ctx(0, s->port, s->route, s->tt_slot, s->tt_port,
  835. s->speed, 1);
  836. fill_ep0_ctx(mps, ep0, s->ep0_enq, s->ep0_cycle);
  837. spin_unlock_irqrestore(&xhci_lock, flags);
  838. rc = xhci_cmd(input_ctx, 0,
  839. TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) |
  840. (bsr ? TRB_BSR : 0), 0);
  841. if (rc != 0) {
  842. xlog_hex(bsr ? "xhci: address default failed port " :
  843. "xhci: address device failed port ", s->port);
  844. xlog_hex(" spd ", s->speed);
  845. xlog_hex(" route ", s->route);
  846. xlog_hex(" tt ", s->tt_slot);
  847. xlog_hex(" ttp ", s->tt_port);
  848. xlog_hex(" cc ", cmd_status);
  849. xlog_hex(" portsc ",
  850. mmio_rd(op(XHCI_OP_PORTSC) + s->port * 0x10));
  851. serial_putc(SERIAL_COM1, '\n');
  852. ram_wr64(dcbaa + slot_id * 8, 0);
  853. xhci_cmd(0, 0,
  854. TRB_TYPE(TRB_DISABLE_SLOT) | (slot_id << 24), 0);
  855. mm_free_page(dc);
  856. mm_free_page(ep0);
  857. s->used = 0;
  858. return 0;
  859. }
  860. /* The xHC assigns the USB device address (the input slot context
  861. * address is 0); keep it for later Configure Endpoint commands. */
  862. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  863. s->addr = bsr ? 0 : target_addr;
  864. addr_slot[s->addr] = slot_id;
  865. if (s->route == 0)
  866. port_slot[s->port] = slot_id;
  867. if (!bsr)
  868. (void)xhci_configure_slot(s);
  869. return s;
  870. }
  871. /* New-scheme slot: Enabled -> Default, at device address 0. */
  872. static struct xhci_slot *new_slot(void)
  873. {
  874. return new_slot_common(0, 0, 1);
  875. }
  876. static void free_slot(struct xhci_slot *s)
  877. {
  878. unsigned int i, flags;
  879. if (!s || !s->used)
  880. return;
  881. xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
  882. ram_wr64(dcbaa + s->slot_id * 8, 0);
  883. spin_lock_irqsave(&xhci_lock, &flags);
  884. for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
  885. if (ints[i].slot == s->slot_id) {
  886. ints[i].active = 0;
  887. ints[i].pending = 0;
  888. ints[i].error = 0;
  889. if (ints[i].ring) {
  890. mm_free_page(ints[i].ring);
  891. ints[i].ring = 0;
  892. }
  893. ints[i].slot = 0;
  894. }
  895. }
  896. if (s->dc) {
  897. mm_free_page(s->dc);
  898. s->dc = 0;
  899. }
  900. if (s->ep0) {
  901. mm_free_page(s->ep0);
  902. s->ep0 = 0;
  903. }
  904. if (s->addr < 128)
  905. addr_slot[s->addr] = 0;
  906. if (bulk_in.active && bulk_in.slot == s->slot_id)
  907. bulk_reset(&bulk_in);
  908. if (bulk_out.active && bulk_out.slot == s->slot_id)
  909. bulk_reset(&bulk_out);
  910. if (s->route == 0 && s->port < XHCI_MAX_PORTS)
  911. port_slot[s->port] = 0;
  912. s->used = 0;
  913. s->addr = 0;
  914. s->ep_mask = 0;
  915. spin_unlock_irqrestore(&xhci_lock, flags);
  916. }
  917. static int set_address(unsigned int new_addr, unsigned int mps)
  918. {
  919. struct xhci_slot *s = slot_by_addr(0);
  920. unsigned int flags;
  921. int rc;
  922. if (!s || new_addr == 0 || new_addr > 127)
  923. return -1;
  924. mps = ep0_mps_initial(s->speed, mps);
  925. spin_lock_irqsave(&xhci_lock, &flags);
  926. zero_page(input_ctx, 4096);
  927. *(unsigned int *)input_ctx = 0;
  928. *(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
  929. fill_slot_ctx(0, s->port, s->route, s->tt_slot, s->tt_port,
  930. s->speed, 1);
  931. fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
  932. spin_unlock_irqrestore(&xhci_lock, flags);
  933. rc = xhci_cmd(input_ctx, 0,
  934. TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
  935. if (rc != 0) {
  936. dbg("xhci: set address failed\n");
  937. return -1;
  938. }
  939. spin_lock_irqsave(&xhci_lock, &flags);
  940. s->dev_addr = ((unsigned int *)s->dc)[3] & 0xFF;
  941. s->ep0_mps = mps;
  942. addr_slot[0] = 0;
  943. s->addr = new_addr;
  944. addr_slot[new_addr] = s->slot_id;
  945. spin_unlock_irqrestore(&xhci_lock, flags);
  946. (void)xhci_configure_slot(s);
  947. return 0;
  948. }
  949. /* ---- control transfers ------------------------------------------------- */
  950. /* One line per failed control transfer: the request, the completion code
  951. * (0 = no event within the timeout), the root port's PORTSC, and the slot /
  952. * EP0 states from the output device context (slot state 2 = Addressed,
  953. * EP state 2 = Halted, 4 = Error). */
  954. static void ctl_fail_log(const struct xhci_slot *s, const unsigned char *setup,
  955. unsigned int cc, unsigned int data_seen)
  956. {
  957. const unsigned int *dc = (const unsigned int *)s->dc;
  958. const unsigned int *ep0 = (const unsigned int *)(s->dc + ctx_size);
  959. serial_puts(SERIAL_COM1, cc ? "xhci: ctl err" : "xhci: ctl timeout");
  960. xlog_hex(" req ", (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  961. ((unsigned int)setup[2] << 16) | ((unsigned int)setup[3] << 24));
  962. xlog_hex(" cc ", cc);
  963. xlog_hex(" data ", data_seen);
  964. xlog_hex(" slot ", s->slot_id);
  965. xlog_hex(" spd ", s->speed);
  966. xlog_hex(" portsc ", mmio_rd(op(XHCI_OP_PORTSC) + s->port * 0x10));
  967. xlog_hex(" sstate ", dc[3] >> 27);
  968. xlog_hex(" dev ", dc[3] & 0xFF);
  969. xlog_hex(" ep0 ", ep0[0] & 7);
  970. xlog_hex(" mps ", ep0[1] >> 16);
  971. xlog_hex(" sts ", mmio_rd(op(XHCI_OP_USBSTS)));
  972. serial_putc(SERIAL_COM1, '\n');
  973. }
  974. static int control_transfer(unsigned int ctrl, unsigned int addr,
  975. unsigned int mps, int low_speed,
  976. const unsigned char *setup, unsigned char *data,
  977. unsigned int len, int dir_in)
  978. {
  979. struct xhci_slot *s;
  980. unsigned int flags, t0, rc, st;
  981. unsigned int setup_c, data_c, status_c;
  982. unsigned int last_bell;
  983. (void)low_speed;
  984. if (!xhci_use(ctrl))
  985. return -1;
  986. /* xHCI owns SET_ADDRESS: it is an Address Device command, not a
  987. * control request on the wire. If a Default-state slot exists (new
  988. * scheme), move it to Addressed. Otherwise this is the traditional
  989. * scheme: create the slot Enabled and address it in one command,
  990. * without ever using the Default state. */
  991. if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0) {
  992. unsigned int new_addr = (unsigned int)setup[2] |
  993. ((unsigned int)setup[3] << 8);
  994. s = slot_by_addr(0);
  995. if (s)
  996. return set_address(new_addr, mps);
  997. s = new_slot_common(new_addr, mps, 0);
  998. return s ? 0 : -1;
  999. }
  1000. s = slot_by_addr(addr);
  1001. if (s && addr == 0 && (int)s->port != last_port) {
  1002. /* A failed enumeration on another port left a slot at
  1003. * address 0; it must not be reused for this device. */
  1004. free_slot(s);
  1005. s = 0;
  1006. }
  1007. if (!s) {
  1008. if (addr != 0)
  1009. return -1;
  1010. s = new_slot();
  1011. if (!s)
  1012. return -1;
  1013. }
  1014. /* The core learned the real bMaxPacketSize0 from the first 8 bytes
  1015. * of the device descriptor; reprogram EP0 before using it. Only a
  1016. * full-speed EP0 can legally differ from the speed's fixed value. */
  1017. if (s->speed == 1 && (mps == 8 || mps == 16 || mps == 32 ||
  1018. mps == 64) && mps != s->ep0_mps &&
  1019. ep0_set_mps(s, mps) != 0)
  1020. return -1;
  1021. spin_lock_irqsave(&xhci_lock, &flags);
  1022. if (ctl_busy) {
  1023. spin_unlock_irqrestore(&xhci_lock, flags);
  1024. dbg("xhci: control busy\n");
  1025. return -1;
  1026. }
  1027. ctl_busy = 1;
  1028. ctl_done = 0;
  1029. ctl_status = 0;
  1030. ctl_actual = 0;
  1031. ctl_data_seen = 0;
  1032. ctl_slot = s->slot_id;
  1033. ctl_req_len = len;
  1034. /* Encode the three stages the way EDK2's XhciDxe does: every stage
  1035. * is its own TD (no CHAIN) and carries IOC. This Braswell xHC never
  1036. * completes a chained multi-TRB control TD: with Linux's chaining
  1037. * (CHAIN on setup+data, IOC on status) the data stage hangs and no
  1038. * transfer event is ever posted, while the per-stage encoding
  1039. * completes. */
  1040. setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_IOC;
  1041. /* Transfer Type (TRT) only exists from xHCI 1.0 on; in a 0.96
  1042. * controller (Etron EJ168, early VIA/Renesas parts) bits 17:16 are
  1043. * reserved. Linux sets it only for hci_version >= 0x100 too. */
  1044. if (len && hci_version >= 0x100)
  1045. setup_c |= (dir_in ? 3u : 2u) << 16;
  1046. ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
  1047. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  1048. (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
  1049. ((unsigned int)setup[2] << 16) |
  1050. ((unsigned int)setup[3] << 24),
  1051. (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
  1052. ((unsigned int)setup[6] << 16) |
  1053. ((unsigned int)setup[7] << 24),
  1054. 8, setup_c);
  1055. ctl_data_trb = 0;
  1056. if (len) {
  1057. ctl_data_trb = s->ep0 + s->ep0_enq * 16;
  1058. data_c = TRB_TYPE(TRB_DATA) | TRB_IOC;
  1059. if (dir_in)
  1060. data_c |= TRB_DIR_IN | TRB_ISP;
  1061. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
  1062. &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
  1063. }
  1064. ctl_status_trb = s->ep0 + s->ep0_enq * 16;
  1065. status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
  1066. if (!len || !dir_in)
  1067. status_c |= TRB_DIR_IN;
  1068. ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
  1069. 0, 0, 0, status_c);
  1070. __asm__ volatile ("" ::: "memory");
  1071. mmio_wr(db_off + 4 * s->slot_id, 1);
  1072. spin_unlock_irqrestore(&xhci_lock, flags);
  1073. t0 = get_ticks_ms();
  1074. last_bell = t0;
  1075. while (get_ticks_ms() - t0 < 1000) {
  1076. spin_lock_irqsave(&xhci_lock, &flags);
  1077. xhci_poll_locked();
  1078. rc = ctl_done;
  1079. if (!rc && get_ticks_ms() - last_bell >= 100) {
  1080. /* Re-ring while the endpoint is still busy; a posted
  1081. * doorbell can be lost on some real xHCs. */
  1082. mmio_wr(db_off + 4 * s->slot_id, 1);
  1083. last_bell = get_ticks_ms();
  1084. }
  1085. spin_unlock_irqrestore(&xhci_lock, flags);
  1086. if (rc)
  1087. break;
  1088. usb_poll_relax(t0);
  1089. __asm__ volatile ("pause");
  1090. }
  1091. spin_lock_irqsave(&xhci_lock, &flags);
  1092. if (!ctl_done) {
  1093. ctl_busy = 0;
  1094. spin_unlock_irqrestore(&xhci_lock, flags);
  1095. ctl_fail_log(s, setup, 0, ctl_data_seen);
  1096. return -1;
  1097. }
  1098. rc = ctl_actual;
  1099. st = ctl_status;
  1100. ctl_busy = 0;
  1101. spin_unlock_irqrestore(&xhci_lock, flags);
  1102. if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
  1103. ctl_fail_log(s, setup, st, ctl_data_seen);
  1104. return -1;
  1105. }
  1106. return (int)rc;
  1107. }
  1108. /* ---- HID interrupt endpoint -------------------------------------------- */
  1109. static int int_in_start(unsigned int ctrl, unsigned int slot,
  1110. unsigned int addr, unsigned int ep,
  1111. unsigned int mps, unsigned int len, int low_speed)
  1112. {
  1113. struct xhci_slot *s;
  1114. struct xhci_int *in;
  1115. unsigned int ep_index, ep_id, ring, flags;
  1116. int rc;
  1117. (void)low_speed;
  1118. if (!xhci_use(ctrl))
  1119. return -1;
  1120. s = slot_by_addr(addr);
  1121. if (!s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
  1122. return -1;
  1123. int_ctrl[slot] = ctrl;
  1124. if (mps < 1)
  1125. mps = 8;
  1126. if (mps > 64)
  1127. mps = 64;
  1128. if (len < 1)
  1129. len = mps;
  1130. if (len > USB_INT_MAX)
  1131. len = USB_INT_MAX;
  1132. if (ints[slot].active)
  1133. stop_int(slot);
  1134. ep_index = (ep - 1) * 2 + 2; /* EP index 2 for EP1 IN, 4 for EP2 IN */
  1135. ep_id = 2 * ep + 1;
  1136. ring = mm_alloc_page();
  1137. if (!ring)
  1138. return -1;
  1139. zero_page(ring, 4096);
  1140. in = &ints[slot];
  1141. in->slot = s->slot_id;
  1142. in->ep_id = ep_id;
  1143. in->mps = mps;
  1144. in->ring = ring;
  1145. in->enq = 0;
  1146. in->cycle = 1;
  1147. in->trb_len = len;
  1148. in->pending = 0;
  1149. in->error = 0;
  1150. in->active = 0;
  1151. ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
  1152. rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
  1153. if (rc != 0) {
  1154. dbg_hex("xhci: configure int ep failed ", ep_index);
  1155. mm_free_page(ring);
  1156. in->ring = 0;
  1157. return -1;
  1158. }
  1159. in->active = 1;
  1160. spin_lock_irqsave(&xhci_lock, &flags);
  1161. ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  1162. (unsigned int)in->buf, 0, in->trb_len,
  1163. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1164. __asm__ volatile ("" ::: "memory");
  1165. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  1166. spin_unlock_irqrestore(&xhci_lock, flags);
  1167. return 0;
  1168. }
  1169. static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
  1170. {
  1171. struct xhci_int *in;
  1172. unsigned int flags, n, i;
  1173. if (!xhci_use_slot(slot))
  1174. return 0;
  1175. in = &ints[slot];
  1176. if (!in->active)
  1177. return 0;
  1178. spin_lock_irqsave(&xhci_lock, &flags);
  1179. xhci_poll_locked();
  1180. spin_unlock_irqrestore(&xhci_lock, flags);
  1181. if (!in->pending)
  1182. return 0;
  1183. if (in->error) {
  1184. in->pending = 0;
  1185. in->error = 0;
  1186. return -1;
  1187. }
  1188. n = in->pending_len;
  1189. if (n > len)
  1190. n = len;
  1191. for (i = 0; i < n; i++)
  1192. buf[i] = in->buf[i];
  1193. in->pending = 0;
  1194. spin_lock_irqsave(&xhci_lock, &flags);
  1195. if (in->active) {
  1196. ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
  1197. (unsigned int)in->buf, 0, in->trb_len,
  1198. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1199. __asm__ volatile ("" ::: "memory");
  1200. mmio_wr(db_off + 4 * in->slot, in->ep_id);
  1201. }
  1202. spin_unlock_irqrestore(&xhci_lock, flags);
  1203. return (int)in->pending_len;
  1204. }
  1205. static void stop_int(unsigned int slot)
  1206. {
  1207. struct xhci_int *in;
  1208. unsigned int flags;
  1209. if (slot >= USB_HCD_INT_SLOTS)
  1210. return;
  1211. in = &ints[slot];
  1212. spin_lock_irqsave(&xhci_lock, &flags);
  1213. in->active = 0;
  1214. in->pending = 0;
  1215. in->error = 0;
  1216. spin_unlock_irqrestore(&xhci_lock, flags);
  1217. }
  1218. static void int_in_stop(unsigned int slot)
  1219. {
  1220. if (!xhci_use_slot(slot))
  1221. return;
  1222. stop_int(slot);
  1223. }
  1224. /* ---- bulk transfers ---------------------------------------------------- */
  1225. static void bulk_reset(struct xhci_bulk *b)
  1226. {
  1227. if (b->ring) {
  1228. mm_free_page(b->ring);
  1229. b->ring = 0;
  1230. }
  1231. b->active = 0;
  1232. b->armed = 0;
  1233. b->slot = 0;
  1234. b->pending = 0;
  1235. b->error = 0;
  1236. b->actual = 0;
  1237. }
  1238. static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
  1239. unsigned int mps, unsigned char *buf, unsigned int len)
  1240. {
  1241. struct xhci_slot *s;
  1242. struct xhci_bulk *b;
  1243. unsigned int ep_index, ep_id, ring, flags, n, err;
  1244. int rc;
  1245. if (!xhci_use(ctrl))
  1246. return -1;
  1247. b = &bulk_in;
  1248. s = slot_by_addr(addr);
  1249. if (!s || ep < 1 || ep > 15 || !buf || !len)
  1250. return -1;
  1251. if (mps < 1)
  1252. mps = 512;
  1253. ep_index = (ep - 1) * 2 + 2;
  1254. ep_id = 2 * ep + 1;
  1255. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1256. ring = mm_alloc_page();
  1257. if (!ring)
  1258. return -1;
  1259. zero_page(ring, 4096);
  1260. bulk_reset(b);
  1261. b->slot = s->slot_id;
  1262. b->ep = ep;
  1263. b->ep_id = ep_id;
  1264. b->mps = mps;
  1265. b->ring = ring;
  1266. b->enq = 0;
  1267. b->cycle = 1;
  1268. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  1269. rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
  1270. if (rc != 0) {
  1271. bulk_reset(b);
  1272. return -1;
  1273. }
  1274. b->active = 1;
  1275. }
  1276. b->buf = (unsigned int)buf;
  1277. b->len = len;
  1278. spin_lock_irqsave(&xhci_lock, &flags);
  1279. xhci_poll_locked();
  1280. if (b->pending) {
  1281. n = b->actual;
  1282. err = (unsigned int)b->error;
  1283. b->pending = 0;
  1284. b->error = 0;
  1285. /* Do not re-arm here: the next call may use a different
  1286. * buffer/length (the mass-storage driver alternates CBW,
  1287. * data and CSW buffers). The caller's next usb_bulk_in()
  1288. * arms the new transfer with its own buffer. */
  1289. b->armed = 0;
  1290. spin_unlock_irqrestore(&xhci_lock, flags);
  1291. return err ? -1 : (int)n;
  1292. }
  1293. if (!b->armed) {
  1294. b->trb_len = len;
  1295. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1296. (unsigned int)buf, 0, len,
  1297. TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1298. b->armed = 1;
  1299. __asm__ volatile ("" ::: "memory");
  1300. mmio_wr(db_off + 4 * b->slot, b->ep_id);
  1301. }
  1302. spin_unlock_irqrestore(&xhci_lock, flags);
  1303. return 0;
  1304. }
  1305. /* Drop a bulk transfer after a timeout or a BOT reset. Stop the endpoint
  1306. * first so the xHC no longer references the old ring page, then clear the
  1307. * software state; the next bulk call configures a fresh ring. */
  1308. static void xhci_bulk_reset(unsigned int ctrl, unsigned int addr,
  1309. unsigned int ep, int dir_in)
  1310. {
  1311. struct xhci_slot *s;
  1312. struct xhci_bulk *b;
  1313. unsigned int flags, slot, ep_id;
  1314. if (!xhci_use(ctrl))
  1315. return;
  1316. s = slot_by_addr(addr);
  1317. b = dir_in ? &bulk_in : &bulk_out;
  1318. if (!s)
  1319. return;
  1320. spin_lock_irqsave(&xhci_lock, &flags);
  1321. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1322. spin_unlock_irqrestore(&xhci_lock, flags);
  1323. return;
  1324. }
  1325. slot = b->slot;
  1326. ep_id = b->ep_id;
  1327. spin_unlock_irqrestore(&xhci_lock, flags);
  1328. xhci_cmd(0, 0, TRB_TYPE(TRB_STOP_EP) | (slot << 24) | (ep_id << 16),
  1329. 0);
  1330. spin_lock_irqsave(&xhci_lock, &flags);
  1331. if (b->active && b->slot == slot && b->ep == ep)
  1332. bulk_reset(b);
  1333. spin_unlock_irqrestore(&xhci_lock, flags);
  1334. }
  1335. static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
  1336. unsigned int mps, const unsigned char *buf,
  1337. unsigned int len)
  1338. {
  1339. struct xhci_slot *s;
  1340. struct xhci_bulk *b;
  1341. unsigned int ep_index, ep_id, ring, flags, t0, done, status;
  1342. int rc;
  1343. if (!xhci_use(ctrl))
  1344. return -1;
  1345. b = &bulk_out;
  1346. s = slot_by_addr(addr);
  1347. if (!s || ep < 1 || ep > 15 || !buf || !len)
  1348. return -1;
  1349. if (mps < 1)
  1350. mps = 512;
  1351. ep_index = (ep - 1) * 2 + 1;
  1352. ep_id = 2 * ep;
  1353. if (!b->active || b->slot != s->slot_id || b->ep != ep) {
  1354. ring = mm_alloc_page();
  1355. if (!ring)
  1356. return -1;
  1357. zero_page(ring, 4096);
  1358. bulk_reset(b);
  1359. b->slot = s->slot_id;
  1360. b->ep = ep;
  1361. b->ep_id = ep_id;
  1362. b->mps = mps;
  1363. b->ring = ring;
  1364. b->enq = 0;
  1365. b->cycle = 1;
  1366. ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
  1367. rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
  1368. if (rc != 0) {
  1369. bulk_reset(b);
  1370. return -1;
  1371. }
  1372. b->active = 1;
  1373. }
  1374. spin_lock_irqsave(&xhci_lock, &flags);
  1375. if (out_busy) {
  1376. spin_unlock_irqrestore(&xhci_lock, flags);
  1377. return -1;
  1378. }
  1379. out_busy = 1;
  1380. out_done = 0;
  1381. out_status = 0;
  1382. out_slot = s->slot_id;
  1383. out_trb = b->ring + b->enq * 16;
  1384. b->trb_len = len;
  1385. ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
  1386. (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
  1387. __asm__ volatile ("" ::: "memory");
  1388. mmio_wr(db_off + 4 * s->slot_id, ep_id);
  1389. spin_unlock_irqrestore(&xhci_lock, flags);
  1390. t0 = get_ticks_ms();
  1391. while (get_ticks_ms() - t0 < 1000) {
  1392. spin_lock_irqsave(&xhci_lock, &flags);
  1393. xhci_poll_locked();
  1394. done = out_done;
  1395. spin_unlock_irqrestore(&xhci_lock, flags);
  1396. if (done)
  1397. break;
  1398. usb_poll_relax(t0);
  1399. __asm__ volatile ("pause");
  1400. }
  1401. spin_lock_irqsave(&xhci_lock, &flags);
  1402. done = out_done;
  1403. status = out_status;
  1404. out_busy = 0;
  1405. spin_unlock_irqrestore(&xhci_lock, flags);
  1406. if (!done) {
  1407. dbg("xhci: bulk out timeout\n");
  1408. return -1;
  1409. }
  1410. if (status != CC_SUCCESS) {
  1411. dbg_hex("xhci: bulk out status ", status);
  1412. return -1;
  1413. }
  1414. return 0;
  1415. }
  1416. /* ---- root hub ports ---------------------------------------------------- */
  1417. static int port_reset(unsigned int ctrl, unsigned int port)
  1418. {
  1419. unsigned int ps, t0;
  1420. if (!xhci_use(ctrl))
  1421. return -1;
  1422. if (port >= max_ports || !(port_mask & (1u << port)))
  1423. return -1;
  1424. if (port_slot[port])
  1425. free_slot(&slots[port_slot[port]]);
  1426. {
  1427. unsigned int i;
  1428. for (i = 1; i <= max_slots && i <= XHCI_MAX_SLOTS; i++)
  1429. if (slots[i].used && slots[i].route == 0 &&
  1430. slots[i].port == port)
  1431. free_slot(&slots[i]);
  1432. }
  1433. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1434. if (!(ps & PORTSC_CCS))
  1435. return -1;
  1436. /* Force the port into the Disabled state first: some xHCs ignore
  1437. * PR while the port is already enabled, leaving the device at its
  1438. * firmware-assigned address instead of address 0. USB2 ports only:
  1439. * on a SuperSpeed port PED=1 puts the link into SS.Disabled, which a
  1440. * hot reset (PR) does not leave -- software would have to write
  1441. * PLS=RxDetect -- so the BIOS-booted stick stayed unreachable. */
  1442. if ((ps & PORTSC_PED) &&
  1443. ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) < 4) {
  1444. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1445. portsc_neutral(ps) | PORTSC_PED);
  1446. t0 = get_ticks_ms();
  1447. while (get_ticks_ms() - t0 < 100) {
  1448. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1449. if (!(ps & PORTSC_PED))
  1450. break;
  1451. usb_poll_relax(t0);
  1452. }
  1453. }
  1454. /* Acknowledge stale change latches first; a leftover PRC would
  1455. * otherwise let the wait below succeed before this reset has even
  1456. * started (the firmware may have left one behind). */
  1457. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1458. portsc_neutral(ps) | PORTSC_CSC | PORTSC_PEC |
  1459. PORTSC_PRC | PORTSC_PLC);
  1460. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1461. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1462. portsc_neutral(ps) | PORTSC_PR);
  1463. t0 = get_ticks_ms();
  1464. while (get_ticks_ms() - t0 < 500) {
  1465. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1466. if (ps & PORTSC_PRC)
  1467. break;
  1468. usb_poll_relax(t0);
  1469. }
  1470. if (!(ps & PORTSC_PRC)) {
  1471. xlog_hex("xhci: port reset timeout port ", port);
  1472. xlog_hex(" portsc ", ps);
  1473. serial_putc(SERIAL_COM1, '\n');
  1474. return -1;
  1475. }
  1476. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1477. portsc_neutral(ps) | PORTSC_PRC);
  1478. t0 = get_ticks_ms();
  1479. while (get_ticks_ms() - t0 < 500) {
  1480. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1481. if (ps & PORTSC_PED)
  1482. break;
  1483. usb_poll_relax(t0);
  1484. }
  1485. if (!(ps & PORTSC_PED)) {
  1486. xlog_hex("xhci: port not enabled port ", port);
  1487. xlog_hex(" portsc ", ps);
  1488. serial_putc(SERIAL_COM1, '\n');
  1489. return -1;
  1490. }
  1491. /* Reset recovery: USB2 devices need TRSTRCY (10 ms) before the first
  1492. * control request; the delay is harmless for SuperSpeed. */
  1493. delay_ms(10);
  1494. last_port = (int)port;
  1495. return 0;
  1496. }
  1497. static int port_connected(unsigned int ctrl, unsigned int port)
  1498. {
  1499. unsigned int ps;
  1500. if (!xhci_use(ctrl))
  1501. return 0;
  1502. if (port >= max_ports || !(port_mask & (1u << port)))
  1503. return 0;
  1504. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1505. return (ps & PORTSC_CCS) ? 1 : 0;
  1506. }
  1507. static int port_low_speed(unsigned int ctrl, unsigned int port)
  1508. {
  1509. unsigned int ps;
  1510. if (!xhci_use(ctrl))
  1511. return 0;
  1512. if (port >= max_ports)
  1513. return 0;
  1514. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1515. return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
  1516. }
  1517. static int port_take_change(unsigned int ctrl, unsigned int port)
  1518. {
  1519. unsigned int ps;
  1520. if (!xhci_use(ctrl))
  1521. return 0;
  1522. if (port >= max_ports)
  1523. return 0;
  1524. ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
  1525. if (!(ps & PORTSC_CSC))
  1526. return 0;
  1527. mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
  1528. portsc_neutral(ps) | PORTSC_CSC);
  1529. return 1;
  1530. }
  1531. /* ---- controller discovery ---------------------------------------------- */
  1532. /* xHCI USB Legacy Support: take ownership from the firmware before the
  1533. * reset and stop its SMI from firing on port changes once we own the
  1534. * controller (the same idea as the PCI 0xC0 write in uhci.c/ohci.c, but
  1535. * xHCI keeps it in an extended capability).
  1536. *
  1537. * The BIOS-owned semaphore must actually clear before HCRST: while the
  1538. * firmware still owns the controller its SMI handler can interfere with
  1539. * the reset and the command ring. Linux waits for it too. */
  1540. static void legacy_handoff(unsigned int off)
  1541. {
  1542. unsigned int sup = mmio_rd(off);
  1543. unsigned int ctl, t0;
  1544. if ((sup & 0xFF) != 1)
  1545. return;
  1546. if (sup & (1u << 16)) { /* BIOS still owns it */
  1547. mmio_wr(off, sup | (1u << 24));
  1548. t0 = get_ticks_ms();
  1549. while (get_ticks_ms() - t0 < 1000) {
  1550. if (!(mmio_rd(off) & (1u << 16)))
  1551. break;
  1552. }
  1553. }
  1554. ctl = mmio_rd(off + 4);
  1555. ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
  1556. ctl |= 0x7u << 29;
  1557. mmio_wr(off + 4, ctl);
  1558. }
  1559. /* Walk the extended capabilities for the legacy handoff. Runs before the
  1560. * controller reset; scan_protocols() parses the port list afterwards. */
  1561. static void ext_caps_handoff(void)
  1562. {
  1563. unsigned int off, guard = 0;
  1564. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1565. while (off && guard++ < 32) {
  1566. unsigned int cap = mmio_rd(off);
  1567. unsigned int next = (cap >> 8) & 0xFF;
  1568. if ((cap & 0xFF) == 1)
  1569. legacy_handoff(off);
  1570. if (!next)
  1571. break;
  1572. off += next << 2;
  1573. }
  1574. }
  1575. static void scan_protocols(void)
  1576. {
  1577. unsigned int off, guard = 0;
  1578. port_mask = 0;
  1579. off = ((hcc_params >> 16) & 0xFFFF) << 2;
  1580. while (off && guard++ < 32) {
  1581. unsigned int cap = mmio_rd(off);
  1582. unsigned int id = cap & 0xFF;
  1583. unsigned int next = (cap >> 8) & 0xFF;
  1584. if (id == 2) { /* Supported Protocol */
  1585. unsigned int major = (cap >> 24) & 0xFF;
  1586. unsigned int pi = mmio_rd(off + 8);
  1587. unsigned int po = pi & 0xFF;
  1588. unsigned int pc = (pi >> 8) & 0xFF;
  1589. unsigned int i;
  1590. /* The controller may expose separate USB2 and USB3
  1591. * protocol entries for the same physical connectors;
  1592. * include both so a SuperSpeed device is not skipped. */
  1593. if ((major == 2 || major == 3) && po >= 1) {
  1594. for (i = 0; i < pc && po + i <= 32; i++)
  1595. port_mask |= 1u << (po - 1 + i);
  1596. }
  1597. }
  1598. if (!next)
  1599. break;
  1600. off += next << 2;
  1601. }
  1602. if (!port_mask)
  1603. port_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
  1604. ((1u << max_ports) - 1);
  1605. }
  1606. static int xhci_init_one(unsigned int bus, unsigned int dev,
  1607. unsigned int func, unsigned int bar)
  1608. {
  1609. unsigned int cmd;
  1610. unsigned int hcs1, hcs2, t0, i, n, scratch;
  1611. pci_bus = bus;
  1612. pci_dev = dev;
  1613. pci_func = func;
  1614. last_port = -1;
  1615. cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
  1616. pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
  1617. mmio_phys = bar;
  1618. mmio_base = (unsigned int)ioremap(bar, 0x10000, IOREMAP_MMIO);
  1619. if (!mmio_base)
  1620. return 0;
  1621. pci_id = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x00);
  1622. cap_len = mmio_rd(0) & 0xFF;
  1623. hci_version = mmio_rd(0) >> 16;
  1624. hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
  1625. hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
  1626. hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
  1627. rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
  1628. db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
  1629. max_slots = hcs1 & 0xFF;
  1630. if (max_slots > XHCI_MAX_SLOTS)
  1631. max_slots = XHCI_MAX_SLOTS;
  1632. if (!max_slots)
  1633. max_slots = 1;
  1634. max_ports = (hcs1 >> 24) & 0xFF;
  1635. if (max_ports > XHCI_MAX_PORTS)
  1636. max_ports = XHCI_MAX_PORTS;
  1637. if (!max_ports)
  1638. max_ports = 1;
  1639. ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
  1640. dbg("xhci: found\n");
  1641. dbg_hex("xhci: mmio ", mmio_phys);
  1642. dbg_dec("xhci: slots ", max_slots);
  1643. dbg_dec("xhci: ports ", max_ports);
  1644. dbg_dec("xhci: cap len ", cap_len);
  1645. dcbaa = mm_alloc_page();
  1646. input_ctx = mm_alloc_page();
  1647. cmd_ring = mm_alloc_page();
  1648. evt_ring = mm_alloc_page();
  1649. erst = mm_alloc_page();
  1650. if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
  1651. dbg("xhci: out of pages\n");
  1652. return 0;
  1653. }
  1654. zero_page(dcbaa, 4096);
  1655. zero_page(input_ctx, 4096);
  1656. zero_page(cmd_ring, 4096);
  1657. zero_page(evt_ring, 4096);
  1658. zero_page(erst, 4096);
  1659. /* Firmware handoff before the reset: while the BIOS still owns the
  1660. * controller its SMI handler can interfere with HCRST and the
  1661. * command ring. */
  1662. ext_caps_handoff();
  1663. /* Wait for the controller to become ready before touching USBCMD. */
  1664. t0 = get_ticks_ms();
  1665. while (get_ticks_ms() - t0 < 1000) {
  1666. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1667. break;
  1668. }
  1669. /* Reset. The xHC may ignore HCRST while it is running, so halt it
  1670. * and wait for HCH first (xHCI 4.2; Linux does the same). QEMU
  1671. * halts instantly, real silicon does not. */
  1672. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1673. t0 = get_ticks_ms();
  1674. while (get_ticks_ms() - t0 < 100) {
  1675. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1676. break;
  1677. }
  1678. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1679. dbg("xhci: halt timeout\n");
  1680. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
  1681. /* Intel xHCI (CherryView/Braswell included) needs 1 ms after the
  1682. * HCRST write before any other HC register is accessed; polling
  1683. * USBCMD too early can leave the controller partially reset. */
  1684. delay_ms(2);
  1685. t0 = get_ticks_ms();
  1686. while (get_ticks_ms() - t0 < 1000) {
  1687. if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
  1688. break;
  1689. }
  1690. t0 = get_ticks_ms();
  1691. while (get_ticks_ms() - t0 < 1000) {
  1692. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
  1693. break;
  1694. }
  1695. if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
  1696. dbg_hex("xhci: reset failed ",
  1697. mmio_rd(op(XHCI_OP_USBSTS)));
  1698. return 0;
  1699. }
  1700. /* After HCRST the controller must be halted before the ring
  1701. * registers are written; otherwise the writes can be dropped and
  1702. * every command times out. */
  1703. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)) {
  1704. mmio_wr(op(XHCI_OP_USBCMD), 0);
  1705. t0 = get_ticks_ms();
  1706. while (get_ticks_ms() - t0 < 100) {
  1707. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH)
  1708. break;
  1709. }
  1710. }
  1711. /* A working HCRST clears DCBAAP/CRCR/CONFIG. If firmware values
  1712. * survive, the reset did not take effect: the ports keep their old
  1713. * device addresses and every control transfer to address 0 fails. */
  1714. if (mmio_rd(op(XHCI_OP_DCBAAP)) != 0 ||
  1715. (mmio_rd(op(XHCI_OP_CRCR)) & ~0xFu) != 0 ||
  1716. (mmio_rd(op(XHCI_OP_CONFIG)) & 0xFFu) != 0)
  1717. dbg("xhci: hcreset not applied\n");
  1718. /* Clear stale status latches from the firmware's session. */
  1719. mmio_wr(op(XHCI_OP_USBSTS), (1u << 2) | (1u << 3) | (1u << 4));
  1720. mmio_wr(op(XHCI_OP_PAGESIZE), 1);
  1721. mmio_wr(op(XHCI_OP_CONFIG), max_slots);
  1722. mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
  1723. /* Scratchpad buffers, if the controller asks for any. */
  1724. n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
  1725. if (n) {
  1726. unsigned int arr;
  1727. if (n > 512)
  1728. n = 512;
  1729. arr = mm_alloc_page();
  1730. if (!arr) {
  1731. dbg("xhci: no scratchpad array\n");
  1732. return 0;
  1733. }
  1734. zero_page(arr, 4096);
  1735. for (i = 0; i < n; i++) {
  1736. scratch = mm_alloc_page();
  1737. if (!scratch) {
  1738. dbg("xhci: no scratchpad page\n");
  1739. return 0;
  1740. }
  1741. zero_page(scratch, 4096);
  1742. ram_wr64(arr + i * 8, scratch);
  1743. }
  1744. ram_wr64(dcbaa, arr);
  1745. dbg_dec("xhci: scratchpads ", n);
  1746. }
  1747. /* Command ring and event ring. */
  1748. ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
  1749. mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
  1750. evt_idx = 0;
  1751. evt_cycle = 1;
  1752. *(volatile unsigned int *)(erst + 0) = evt_ring;
  1753. *(volatile unsigned int *)(erst + 4) = 0;
  1754. *(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
  1755. *(volatile unsigned int *)(erst + 12) = 0;
  1756. mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
  1757. mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
  1758. mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
  1759. mmio_wr(rt(XHCI_RT_IMAN), 0);
  1760. mmio_wr(rt(XHCI_RT_IMOD), 0);
  1761. /* Run. */
  1762. mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
  1763. t0 = get_ticks_ms();
  1764. while (get_ticks_ms() - t0 < 1000) {
  1765. if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
  1766. break;
  1767. }
  1768. if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
  1769. dbg("xhci: controller did not start\n");
  1770. return 0;
  1771. }
  1772. /* The first command completion (or timeout) tells whether the ring is
  1773. * actually serviced. Don't gate on CRR: QEMU leaves it clear even
  1774. * while commands complete normally. */
  1775. scan_protocols();
  1776. xlog_hex("xhci: ", pci_id);
  1777. xlog_hex(" hci ", hci_version);
  1778. xlog_hex(" mmio ", mmio_phys);
  1779. xlog_hex(" ports ", port_mask);
  1780. xlog_hex(" csz ", ctx_size);
  1781. xlog_hex(" scratch ", n);
  1782. serial_putc(SERIAL_COM1, '\n');
  1783. /* Power every root port. VBus is supposed to come up after HCRST,
  1784. * but some firmware/PPC combinations leave PP clear; the USB3
  1785. * entries may share a switch with a USB2 connector, so write all of
  1786. * them, not just the USB2 range. */
  1787. for (i = 0; i < max_ports; i++) {
  1788. unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
  1789. if (!(ps & PORTSC_PP))
  1790. mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
  1791. portsc_neutral(ps) | PORTSC_PP);
  1792. }
  1793. delay_ms(20);
  1794. found = 1;
  1795. return 1;
  1796. }
  1797. static int xhci_seen(unsigned int bus, unsigned int dev, unsigned int func)
  1798. {
  1799. unsigned int i;
  1800. for (i = 0; i < xhci_nctrl; i++)
  1801. if (xhci_hcds[i].bus_no == bus &&
  1802. xhci_hcds[i].dev_no == dev &&
  1803. xhci_hcds[i].func_no == func)
  1804. return 1;
  1805. return 0;
  1806. }
  1807. static int xhci_init(void)
  1808. {
  1809. unsigned int bus, dev, func, id, class_rev, bar, bar_hi;
  1810. if (xhci_nctrl)
  1811. return (int)xhci_nctrl;
  1812. if (!xhci_hcds) {
  1813. unsigned int pages = (XHCI_MAX_HCD * sizeof(*xhci_hcds) +
  1814. 4095u) / 4096u;
  1815. xhci_hcds = (struct xhci_hcd *)mm_alloc_pages(pages);
  1816. if (!xhci_hcds)
  1817. return 0;
  1818. zero_page((unsigned int)xhci_hcds, pages * 4096);
  1819. }
  1820. for (bus = 0; bus < 256; bus++) {
  1821. for (dev = 0; dev < 32; dev++) {
  1822. for (func = 0; func < 8; func++) {
  1823. id = pci_cfg_read32(bus, dev, func, 0x00);
  1824. if (id == 0xFFFFFFFFu || id == 0)
  1825. continue;
  1826. class_rev = pci_cfg_read32(bus, dev, func, 0x08);
  1827. if ((class_rev >> 16) != 0x0C03)
  1828. continue;
  1829. if (((class_rev >> 8) & 0xFF) != 0x30)
  1830. continue;
  1831. if (xhci_seen(bus, dev, func))
  1832. continue;
  1833. bar = pci_cfg_read32(bus, dev, func, 0x10);
  1834. if (bar & 1)
  1835. continue;
  1836. bar_hi = 0;
  1837. if ((bar & 0x6) == 0x4)
  1838. bar_hi = pci_cfg_read32(bus, dev,
  1839. func, 0x14);
  1840. bar &= 0xFFFFFFF0u;
  1841. xlog_hex("xhci: pci ", id);
  1842. xlog_hex(" loc ",
  1843. (bus << 16) | (dev << 11) |
  1844. (func << 8));
  1845. xlog_hex(" bar ", bar);
  1846. xlog_hex(" hi ", bar_hi);
  1847. serial_putc(SERIAL_COM1, '\n');
  1848. if (!bar || bar_hi)
  1849. continue;
  1850. if (xhci_nctrl >= XHCI_MAX_HCD)
  1851. return (int)xhci_nctrl;
  1852. xh = &xhci_hcds[xhci_nctrl];
  1853. zero_page((unsigned int)xh, sizeof(*xh));
  1854. xh->last_port_num = -1;
  1855. if (xhci_init_one(bus, dev, func, bar))
  1856. xhci_nctrl++;
  1857. }
  1858. }
  1859. }
  1860. return (int)xhci_nctrl;
  1861. }
  1862. static unsigned int xhci_count(void)
  1863. {
  1864. return xhci_nctrl;
  1865. }
  1866. static void xhci_info(unsigned int ctrl, unsigned int *bus,
  1867. unsigned int *dev, unsigned int *func,
  1868. unsigned int *base, int *mmio)
  1869. {
  1870. if (!xhci_use(ctrl)) {
  1871. *bus = *dev = *func = 0;
  1872. *base = 0;
  1873. *mmio = 1;
  1874. return;
  1875. }
  1876. *bus = pci_bus;
  1877. *dev = pci_dev;
  1878. *func = pci_func;
  1879. *base = mmio_phys;
  1880. *mmio = 1;
  1881. }
  1882. static unsigned int xhci_port_count(unsigned int ctrl)
  1883. {
  1884. if (!xhci_use(ctrl))
  1885. return 0;
  1886. return max_ports;
  1887. }
  1888. const struct usb_hcd_ops usb_xhci_ops = {
  1889. "xHCI",
  1890. xhci_init,
  1891. xhci_count,
  1892. xhci_info,
  1893. control_transfer,
  1894. xhci_port_count,
  1895. port_connected,
  1896. port_low_speed,
  1897. port_reset,
  1898. port_take_change,
  1899. int_in_start,
  1900. int_in_poll,
  1901. int_in_stop,
  1902. xhci_bulk_out,
  1903. xhci_bulk_in,
  1904. xhci_bulk_reset,
  1905. 0,
  1906. set_route,
  1907. };