Просмотр исходного кода

Add xHCI host controller and ASIX USB Ethernet support

Johannes Zellner 2 недель назад
Родитель
Сommit
927715bdae
16 измененных файлов с 2108 добавлено и 9 удалено
  1. 25 3
      Makefile
  2. 457 0
      asix.c
  3. 12 0
      asix.h
  4. 2 0
      kernel.c
  5. 2 0
      netdev.c
  6. 2 0
      ohci.c
  7. 1 1
      tcp.c
  8. 3 1
      tools/qmp_cursor_test.py
  9. 7 2
      tools/qmp_keyboard_test.py
  10. 2 0
      uhci.c
  11. 53 1
      usb.c
  12. 22 0
      usb.h
  13. 13 0
      usb_hcd.h
  14. 1 1
      vfs.c
  15. 1495 0
      xhci.c
  16. 11 0
      xhci.h

+ 25 - 3
Makefile

@@ -24,7 +24,7 @@ EFI_LDFLAGS = -nostdlib -Wl,--subsystem,10 -Wl,-e,efi_main \
               -Wl,--image-base,0x0 -Wl,--dynamicbase \
               -Wl,--enable-reloc-section -Wl,--no-insert-timestamp
 
-.PHONY: all clean run run-smp run-floppy run-usb run-uhci run-net run-e1000 run-ahci test-usb run-serial floppy usb uefi run-uefi test-uefi flash space prog bootstrap dev net-dev cmds-dev net-cmds browser-dev netmon monitor test-crypto test-images test-images-asan
+.PHONY: all clean run run-smp run-floppy run-usb run-uhci run-xhci run-xhci-asix run-net run-e1000 run-ahci test-usb run-serial floppy usb uefi run-uefi test-uefi flash space prog bootstrap dev net-dev cmds-dev net-cmds browser-dev netmon monitor test-crypto test-images test-images-asan
 
 all: floppy.img usb.img final.bin root.img
 
@@ -154,6 +154,12 @@ uhci.o: uhci.c uhci.h usb_hcd.h pci.h serial.h timer.h
 ohci.o: ohci.c ohci.h usb_hcd.h pci.h paging.h mm.h serial.h timer.h
 	$(CC) $(CFLAGS) -Os -o $@ $<
 
+xhci.o: xhci.c xhci.h usb_hcd.h pci.h paging.h mm.h serial.h timer.h spinlock.h
+	$(CC) $(CFLAGS) -Os -o $@ $<
+
+asix.o: asix.c asix.h netdev.h usb.h io.h serial.h timer.h
+	$(CC) $(CFLAGS) -Os -o $@ $<
+
 usb.o: usb.c usb.h usb_hcd.h io.h serial.h timer.h
 	$(CC) $(CFLAGS) -Os -o $@ $<
 
@@ -319,7 +325,7 @@ sys_net.o: sys_net.c sys_net.h syscall.h dhcp.h dns.h net.h netdev.h netlog.h pr
 syscall_stub.o: syscall_stub.S irqret.inc
 	$(AS) $(ASFLAGS) -o $@ $<
 
-kernel.elf: vesa_stage.o entry.o kernel.o timer.o switch.o sched.o process.o mm.o kmalloc.o paging.o elf.o tss.o syscall.o sys_wm.o syscall_stub.o screen.o io.o tty.o rtc.o terminal.o wm.o reboot.o poweroff.o disk.o fat.o run.o serial.o serial_ring_buf.o serial_loop.o serial_recv.o bin_recv.o vga.o blit.o cpu.o thermal.o it87.o vga_copy.o gfxbench.o gpu.o ati.o cirrus.o nvidia.o nv50.o radeon.o mtrr.o font_8x8.o fbtext.o gfx.o sys_gfx.o display.o mouse.o uhci.o ohci.o usb.o usbhid.o lsusb.o panic.o bootcon.o blkdev.o ata.o ahci.o fatdetect.o fatformat.o format.o vfs.o sys_fs.o sys_info.o sys_mem.o sys_net.o idle.o kexec.o kexec_tramp.o dbg.o lspci.o peek.o ne2000.o r8168.o e1000e.o netdev.o net.o netkexec.o netlog.o dhcp.o dns.o tcp.o acpi.o lapic.o smp.o ap_tramp_embed.o prog_embed.o space_embed.o inflate.o cmds_pack.o ramfs.o share.o payload.o
+kernel.elf: vesa_stage.o entry.o kernel.o timer.o switch.o sched.o process.o mm.o kmalloc.o paging.o elf.o tss.o syscall.o sys_wm.o syscall_stub.o screen.o io.o tty.o rtc.o terminal.o wm.o reboot.o poweroff.o disk.o fat.o run.o serial.o serial_ring_buf.o serial_loop.o serial_recv.o bin_recv.o vga.o blit.o cpu.o thermal.o it87.o vga_copy.o gfxbench.o gpu.o ati.o cirrus.o nvidia.o nv50.o radeon.o mtrr.o font_8x8.o fbtext.o gfx.o sys_gfx.o display.o mouse.o uhci.o ohci.o xhci.o asix.o usb.o usbhid.o lsusb.o panic.o bootcon.o blkdev.o ata.o ahci.o fatdetect.o fatformat.o format.o vfs.o sys_fs.o sys_info.o sys_mem.o sys_net.o idle.o kexec.o kexec_tramp.o dbg.o lspci.o peek.o ne2000.o r8168.o e1000e.o netdev.o net.o netkexec.o netlog.o dhcp.o dns.o tcp.o acpi.o lapic.o smp.o ap_tramp_embed.o prog_embed.o space_embed.o inflate.o cmds_pack.o ramfs.o share.o payload.o
 	$(LD) $(KERNEL_LDFLAGS) -o $@ $^
 
 MBR_LDFLAGS = -T mbr.ld -nostdlib -m elf_i386
@@ -737,6 +743,20 @@ run-uhci: floppy.img root.img
 	qemu-system-i386 -M pc -device piix3-usb-uhci -device usb-mouse -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy 2>/dev/null || \
 	qemu-system-x86_64 -M pc -device piix3-usb-uhci -device usb-mouse -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy
 
+# xHCI variant (Braswell/Intel PCH and other USB 3.0-only machines).
+run-xhci: floppy.img root.img
+	qemu-system-i386 -M pc -device qemu-xhci -device usb-mouse -device usb-kbd -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy 2>/dev/null || \
+	qemu-system-x86_64 -M pc -device qemu-xhci -device usb-mouse -device usb-kbd -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy
+
+# Passthrough of the real ASIX AX88772 dongle (requires the host's asix
+# driver to be unbound and permission on /dev/bus/usb/...; typically run with
+# sudo).  `-nic none` keeps the QEMU e1000 out of the way so the USB NIC is
+# the only netdev.
+ASIX_HOST ?= vendorid=0x0b95,productid=0x7720
+run-xhci-asix: floppy.img root.img
+	qemu-system-i386 -M pc -nic none -device qemu-xhci -device usb-host,$(ASIX_HOST) -serial file:/tmp/opencode/serial-asix.log -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy 2>/dev/null || \
+	qemu-system-x86_64 -M pc -nic none -device qemu-xhci -device usb-host,$(ASIX_HOST) -serial file:/tmp/opencode/serial-asix.log -drive file=floppy.img,format=raw,if=floppy -drive file=root.img,format=raw,if=floppy
+
 # Network variant: a PCI NE2000 (RTL8029, the driver's QEMU match) on QEMU's
 # user-mode network (slirp).  DHCP/DNS come from slirp, the host is 10.0.2.2
 # (e.g. `python3 -m http.server` + `curl http://10.0.2.2:8000/` in a window).
@@ -784,8 +804,10 @@ test-usb: floppy.img root.img
 	python3 tools/qmp_cursor_test.py --uhci
 	python3 tools/qmp_cursor_test.py --uhci72
 	python3 tools/qmp_cursor_test.py --hotplug
+	python3 tools/qmp_cursor_test.py --xhci
 	python3 tools/qmp_keyboard_test.py
 	python3 tools/qmp_keyboard_test.py --hotplug
+	python3 tools/qmp_keyboard_test.py --xhci
 
 scp-final: final.bin
 	scp final.bin root@192.168.0.136:
@@ -871,4 +893,4 @@ net-dev: kernel.bin netkexec_send
 	./netkexec_send $(BOARD_IP) $(NETKEXEC_PORT) 0x$$entry kernel.bin
 
 clean:
-	rm -f boot.o vesa_stage.o entry.o kernel.o timer.o switch.o sched.o process.o mm.o kmalloc.o paging.o elf.o tss.o syscall.o sys_wm.o syscall_stub.o screen.o io.o tty.o rtc.o terminal.o wm.o hello.o reboot.o poweroff.o fat.o run.o disk.o serial.o serial_ring_buf.o serial_loop.o serial_recv.o bin_recv.o vga.o blit.o cpu.o thermal.o it87.o vga_copy.o gfxbench.o gpu.o ati.o cirrus.o nvidia.o nv50.o radeon.o mtrr.o font_8x8.o font_8x16.o fbtext.o gfx.o sys_gfx.o display.o mouse.o uhci.o ohci.o usb.o usbhid.o lsusb.o panic.o bootcon.o blkdev.o ata.o ahci.o fatdetect.o fatformat.o format.o vfs.o sys_fs.o sys_info.o sys_mem.o inflate.o idle.o kexec.o kexec_tramp.o dbg.o lspci.o peek.o ne2000.o r8168.o e1000e.o netdev.o net.o netkexec.o netlog.o dhcp.o dns.o tcp.o acpi.o lapic.o smp.o ap_tramp.o ap_tramp.elf ap_tramp.bin ap_tramp_embed.o mbr.o boot.elf kernel.elf mbr.elf boot.bin mbr.bin kernel.bin boot.bin.tmp boot.bin.patched image.bin floppy.img usb.img final.bin root.img prog.o prog.elf prog.bin space.o space.elf space.bin hello.elf hello_embed.o prog_embed.o space_embed.o cmds.pack cmds_pack_gz.bin cmds_pack.o ramfs.o share.o payload.o mkpack uwm_lib.o ngui.o img.o img_mem.o img_blit.o img_png.o img_bmp.o img_gif.o img_jpeg.o img_tga.o img_pnm.o img_ico.o img_pam.o img_gif_anim.o img_file.o browser.o browser_html.o uhttp.o browser.elf img_test img_test_asan *.boot.tmp $(addsuffix .o,$(UCMDS)) $(addsuffix .elf,$(UCMDS)) $(TLSLIB_OBJS) tls_test bootstrap.o bootstrap.elf bootstrap.bin send_serial kexec_send netkexec_send netmonitor BOOTX64.EFI esp.img usb-efi.img efi/main.o efi/payload.o efi/tramp.o efi/tramp.elf efi/tramp.bin efi/efi_entry.h
+	rm -f boot.o vesa_stage.o entry.o kernel.o timer.o switch.o sched.o process.o mm.o kmalloc.o paging.o elf.o tss.o syscall.o sys_wm.o syscall_stub.o screen.o io.o tty.o rtc.o terminal.o wm.o hello.o reboot.o poweroff.o fat.o run.o disk.o serial.o serial_ring_buf.o serial_loop.o serial_recv.o bin_recv.o vga.o blit.o cpu.o thermal.o it87.o vga_copy.o gfxbench.o gpu.o ati.o cirrus.o nvidia.o nv50.o radeon.o mtrr.o font_8x8.o font_8x16.o fbtext.o gfx.o sys_gfx.o display.o mouse.o uhci.o ohci.o xhci.o asix.o usb.o usbhid.o lsusb.o panic.o bootcon.o blkdev.o ata.o ahci.o fatdetect.o fatformat.o format.o vfs.o sys_fs.o sys_info.o sys_mem.o inflate.o idle.o kexec.o kexec_tramp.o dbg.o lspci.o peek.o ne2000.o r8168.o e1000e.o netdev.o net.o netkexec.o netlog.o dhcp.o dns.o tcp.o acpi.o lapic.o smp.o ap_tramp.o ap_tramp.elf ap_tramp.bin ap_tramp_embed.o mbr.o boot.elf kernel.elf mbr.elf boot.bin mbr.bin kernel.bin boot.bin.tmp boot.bin.patched image.bin floppy.img usb.img final.bin root.img prog.o prog.elf prog.bin space.o space.elf space.bin hello.elf hello_embed.o prog_embed.o space_embed.o cmds.pack cmds_pack_gz.bin cmds_pack.o ramfs.o share.o payload.o mkpack uwm_lib.o ngui.o img.o img_mem.o img_blit.o img_png.o img_bmp.o img_gif.o img_jpeg.o img_tga.o img_pnm.o img_ico.o img_pam.o img_gif_anim.o img_file.o browser.o browser_html.o uhttp.o browser.elf img_test img_test_asan *.boot.tmp $(addsuffix .o,$(UCMDS)) $(addsuffix .elf,$(UCMDS)) $(TLSLIB_OBJS) tls_test bootstrap.o bootstrap.elf bootstrap.bin send_serial kexec_send netkexec_send netmonitor BOOTX64.EFI esp.img usb-efi.img efi/main.o efi/payload.o efi/tramp.o efi/tramp.elf efi/tramp.bin efi/efi_entry.h

+ 457 - 0
asix.c

@@ -0,0 +1,457 @@
+/* ASIX AX88772 USB 2.0 Ethernet driver.
+ *
+ * The AX88772 is a vendor-specific USB device with one bulk IN and one bulk
+ * OUT endpoint.  Configuration, the embedded PHY and the MAC are reached with
+ * the ASIX vendor control requests; the receive path prepends a 4-byte header
+ * (length in the low 16 bits and its complement in the high 16 bits), the
+ * transmit path expects the same header before the Ethernet frame.
+ *
+ * The USB core's bulk API is start-or-poll: usb_bulk_in() arms on the first
+ * call and leaves the transfer disarmed after delivering a chunk, so the
+ * RX buffer stays stable while its (possibly several) frames are consumed. */
+
+#include "asix.h"
+#include "io.h"
+#include "timer.h"
+#include "usb.h"
+
+/* ASIX vendor requests. */
+#define AX_CMD_SET_SW_MII	0x06
+#define AX_CMD_READ_MII_REG	0x07
+#define AX_CMD_WRITE_MII_REG	0x08
+#define AX_CMD_SET_HW_MII	0x0A
+#define AX_CMD_READ_RX_CTL	0x0F
+#define AX_CMD_WRITE_RX_CTL	0x10
+#define AX_CMD_WRITE_IPG0	0x12
+#define AX_CMD_READ_NODE_ID	0x13
+#define AX_CMD_WRITE_NODE_ID	0x14
+#define AX_CMD_READ_PHY_ID	0x19
+#define AX_CMD_WRITE_MEDIUM_MODE 0x1B
+#define AX_CMD_WRITE_GPIOS	0x1F
+#define AX_CMD_SW_RESET		0x20
+#define AX_CMD_SW_PHY_SELECT	0x22
+
+#define AX_GPIO_RSE		0x80
+#define AX_GPIO_GPO_2		0x20
+#define AX_GPIO_GPO2EN		0x10
+
+#define AX_SWRESET_CLEAR	0x00
+#define AX_SWRESET_PRTE		0x04
+#define AX_SWRESET_PRL		0x08
+#define AX_SWRESET_IPRL		0x20
+#define AX_SWRESET_IPPD		0x40
+
+#define AX88772_IPG0_DEFAULT	0x15
+#define AX88772_IPG1_DEFAULT	0x0C
+#define AX88772_IPG2_DEFAULT	0x12
+
+#define AX_MEDIUM_FD		0x0002
+#define AX_MEDIUM_AC		0x0004
+#define AX_MEDIUM_RFC		0x0010
+#define AX_MEDIUM_TFC		0x0020
+#define AX_MEDIUM_RE		0x0100
+#define AX_MEDIUM_PS		0x0200
+#define AX88772_MEDIUM_DEFAULT	(AX_MEDIUM_FD | AX_MEDIUM_RFC | \
+				 AX_MEDIUM_TFC | AX_MEDIUM_PS | \
+				 AX_MEDIUM_AC | AX_MEDIUM_RE)
+
+#define AX_RX_CTL_SO		0x0080
+#define AX_RX_CTL_AB		0x0008
+#define AX_DEFAULT_RX_CTL	(AX_RX_CTL_SO | AX_RX_CTL_AB)
+
+#define MII_BMCR		0
+#define MII_BMSR		1
+#define MII_ADVERTISE		4
+#define BMCR_RESET		0x8000
+#define BMCR_ANENABLE		0x1000
+#define BMCR_ANRESTART		0x0200
+#define BMSR_LSTATUS		0x0004
+#define ADVERTISE_ALL		0x01E0
+#define ADVERTISE_CSMA		0x0001
+
+#define ASIX_RX_MAX		2048
+#define ASIX_TX_MAX		(1514 + 4)
+
+
+struct asix_dev {
+	int present;
+	int chip_ready;
+	int link;
+	unsigned int ctrl;
+	unsigned int addr;
+	unsigned int mps;
+	unsigned int port;
+	int low;
+	unsigned int ep_in, ep_in_mps;
+	unsigned int ep_out, ep_out_mps;
+	unsigned int phy;
+	unsigned char mac[6];
+};
+
+static struct asix_dev dev;
+static unsigned char rx_buf[ASIX_RX_MAX] __attribute__((aligned(64)));
+static unsigned char tx_buf[ASIX_TX_MAX] __attribute__((aligned(64)));
+static unsigned int rx_pos, rx_end;
+
+
+
+static void delay_ms(unsigned int ms)
+{
+	unsigned int t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < ms)
+		;
+}
+
+static int read_cmd(unsigned char cmd, unsigned short value,
+		    unsigned short index, void *buf, unsigned int len)
+{
+	int n = usb_control_in(dev.ctrl, dev.addr, dev.mps, dev.low,
+			       0xC0, cmd, value, index, (unsigned char *)buf,
+			       len);
+	return n == (int)len ? 0 : -1;
+}
+
+static int write_cmd(unsigned char cmd, unsigned short value,
+		     unsigned short index, const void *buf, unsigned int len)
+{
+	if (len) {
+		int n = usb_control_out_data(dev.ctrl, dev.addr, dev.mps,
+					     dev.low, 0x40, cmd, value, index,
+					     (const unsigned char *)buf, len);
+		return n < 0 ? -1 : 0;
+	}
+	return usb_control_out(dev.ctrl, dev.addr, dev.mps, dev.low,
+			       0x40, cmd, value, index) < 0 ? -1 : 0;
+}
+
+static int mdio_read(unsigned int loc)
+{
+	unsigned char b[2];
+
+	if (write_cmd(AX_CMD_SET_SW_MII, 0, 0, 0, 0) < 0)
+		return -1;
+	if (read_cmd(AX_CMD_READ_MII_REG, (unsigned short)dev.phy,
+		     (unsigned short)loc, b, 2) < 0) {
+		write_cmd(AX_CMD_SET_HW_MII, 0, 0, 0, 0);
+		return -1;
+	}
+	write_cmd(AX_CMD_SET_HW_MII, 0, 0, 0, 0);
+	return (int)((unsigned int)b[0] | ((unsigned int)b[1] << 8));
+}
+
+static void mdio_write(unsigned int loc, unsigned int val)
+{
+	unsigned char b[2];
+
+	b[0] = (unsigned char)(val & 0xFF);
+	b[1] = (unsigned char)((val >> 8) & 0xFF);
+	write_cmd(AX_CMD_SET_SW_MII, 0, 0, 0, 0);
+	write_cmd(AX_CMD_WRITE_MII_REG, (unsigned short)dev.phy,
+		  (unsigned short)loc, b, 2);
+	write_cmd(AX_CMD_SET_HW_MII, 0, 0, 0, 0);
+}
+
+static int chip_init(void)
+{
+	unsigned char b[2];
+	unsigned int phy, rx_ctl, i;
+	int bmcr;
+
+	dev.chip_ready = 0;
+	dev.link = 0;
+
+	if (write_cmd(AX_CMD_WRITE_GPIOS,
+		      AX_GPIO_RSE | AX_GPIO_GPO_2 | AX_GPIO_GPO2EN,
+		      0, 0, 0) < 0)
+		return -1;
+	delay_ms(5);
+
+	if (read_cmd(AX_CMD_READ_PHY_ID, 0, 0, b, 2) < 0)
+		return -1;
+	phy = b[1];
+	dev.phy = phy;
+
+	if (write_cmd(AX_CMD_SW_PHY_SELECT,
+		      (phy & 0x1F) == 0x10 ? 1 : 0, 0, 0, 0) < 0)
+		return -1;
+
+	if (write_cmd(AX_CMD_SW_RESET, AX_SWRESET_IPPD | AX_SWRESET_PRL,
+		      0, 0, 0) < 0)
+		return -1;
+	delay_ms(150);
+	if (write_cmd(AX_CMD_SW_RESET, AX_SWRESET_CLEAR, 0, 0, 0) < 0)
+		return -1;
+	delay_ms(150);
+	if ((phy & 0x1F) == 0x10)
+		write_cmd(AX_CMD_SW_RESET, AX_SWRESET_IPRL, 0, 0, 0);
+	else
+		write_cmd(AX_CMD_SW_RESET, AX_SWRESET_PRTE, 0, 0, 0);
+	delay_ms(150);
+
+	if (read_cmd(AX_CMD_READ_RX_CTL, 0, 0, b, 2) < 0)
+		return -1;
+	rx_ctl = (unsigned int)b[0] | ((unsigned int)b[1] << 8);
+	(void)rx_ctl;
+	if (write_cmd(AX_CMD_WRITE_RX_CTL, 0, 0, 0, 0) < 0)
+		return -1;
+
+	if (read_cmd(AX_CMD_READ_NODE_ID, 0, 0, dev.mac, 6) < 0)
+		return -1;
+
+	/* PHY reset and 10/100 autonegotiation. */
+	mdio_write(MII_BMCR, BMCR_RESET);
+	delay_ms(2);
+	mdio_write(MII_ADVERTISE, ADVERTISE_ALL | ADVERTISE_CSMA);
+	bmcr = mdio_read(MII_BMCR);
+	if (bmcr >= 0 && (bmcr & BMCR_ANENABLE))
+		mdio_write(MII_BMCR, (unsigned int)bmcr | BMCR_ANRESTART);
+
+	if (write_cmd(AX_CMD_WRITE_MEDIUM_MODE, AX88772_MEDIUM_DEFAULT,
+		      0, 0, 0) < 0)
+		return -1;
+	if (write_cmd(AX_CMD_WRITE_IPG0,
+		      AX88772_IPG0_DEFAULT | AX88772_IPG1_DEFAULT,
+		      AX88772_IPG2_DEFAULT, 0, 0) < 0)
+		return -1;
+	if (write_cmd(AX_CMD_WRITE_RX_CTL, AX_DEFAULT_RX_CTL,
+		      0, 0, 0) < 0)
+		return -1;
+	if (write_cmd(AX_CMD_WRITE_NODE_ID, 0, 0, dev.mac, 6) < 0)
+		return -1;
+
+	/* Wait for the PHY link, but do not fail the NIC if no cable is
+	 * plugged in yet: dhcp/netinfo can retry later. */
+	for (i = 0; i < 60; i++) {
+		int st = mdio_read(MII_BMSR);
+		if (st >= 0 && (st & BMSR_LSTATUS)) {
+			dev.link = 1;
+			break;
+		}
+		delay_ms(50);
+	}
+
+	dev.chip_ready = 1;
+	return 0;
+}
+
+static void put_mac(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < 6; i++) {
+		static const char hex[] = "0123456789ABCDEF";
+		if (i)
+			io_putchar(':');
+		io_putchar(hex[(dev.mac[i] >> 4) & 0xF]);
+		io_putchar(hex[dev.mac[i] & 0xF]);
+	}
+}
+
+static int net_init(void)
+{
+	if (!dev.present)
+		return -1;
+	if (!dev.chip_ready && chip_init() < 0)
+		return -1;
+	return 0;
+}
+
+static void net_get_mac(unsigned char mac[6])
+{
+	unsigned int i;
+
+	for (i = 0; i < 6; i++)
+		mac[i] = dev.mac[i];
+}
+
+static int net_rx(unsigned char *buf, unsigned int maxlen)
+{
+	unsigned int hdr, len, i;
+	int n;
+
+	if (!dev.present)
+		return -1;
+
+	for (;;) {
+		if (rx_pos < rx_end) {
+			if (rx_pos + 4 > rx_end) {
+				rx_pos = rx_end;
+				return -1;
+			}
+			hdr = (unsigned int)rx_buf[rx_pos] |
+			      ((unsigned int)rx_buf[rx_pos + 1] << 8) |
+			      ((unsigned int)rx_buf[rx_pos + 2] << 16) |
+			      ((unsigned int)rx_buf[rx_pos + 3] << 24);
+			len = hdr & 0x7FF;
+			if (((~hdr >> 16) & 0x7FF) != len ||
+			    len < 14 || len > maxlen ||
+			    rx_pos + 4 + len > rx_end) {
+				rx_pos = rx_end;
+				return -1;
+			}
+			for (i = 0; i < len; i++)
+				buf[i] = rx_buf[rx_pos + 4 + i];
+			rx_pos += 4 + ((len + 1) & ~1u);
+			return (int)len;
+		}
+
+		n = usb_bulk_in(dev.ctrl, dev.addr, dev.ep_in,
+				dev.ep_in_mps, rx_buf, ASIX_RX_MAX);
+		if (n <= 0)
+			return -1;
+		rx_pos = 0;
+		rx_end = (unsigned int)n;
+	}
+}
+
+static int net_tx(const unsigned char *buf, unsigned int len)
+{
+	unsigned int hdr;
+
+	if (!dev.present || len > 1514)
+		return -1;
+	hdr = len | ((~len & 0xFFFF) << 16);
+	tx_buf[0] = (unsigned char)(hdr & 0xFF);
+	tx_buf[1] = (unsigned char)((hdr >> 8) & 0xFF);
+	tx_buf[2] = (unsigned char)((hdr >> 16) & 0xFF);
+	tx_buf[3] = (unsigned char)((hdr >> 24) & 0xFF);
+	{
+		unsigned int i;
+		for (i = 0; i < len; i++)
+			tx_buf[4 + i] = buf[i];
+	}
+	return usb_bulk_out(dev.ctrl, dev.addr, dev.ep_out,
+			    dev.ep_out_mps, tx_buf, len + 4) == 0 ? 0 : -1;
+}
+
+static void net_print_info(void)
+{
+	int st;
+
+	io_putstr("ASIX AX88772 USB 2.0 Ethernet\n");
+	io_putstr("  MAC ");
+	put_mac();
+	io_putstr("\n");
+	st = dev.phy ? mdio_read(MII_BMSR) : -1;
+	if (st >= 0 && (st & BMSR_LSTATUS))
+		io_putstr("  link up, 100 Mbit capable\n");
+	else
+		io_putstr("  link down\n");
+}
+
+static void net_reset(void)
+{
+	rx_pos = 0;
+	rx_end = 0;
+	dev.chip_ready = 0;
+	if (dev.present)
+		chip_init();
+}
+
+const struct netdev asix_netdev = {
+	"asix",
+	"AX88772 USB 2.0 Ethernet",
+	net_init,
+	net_get_mac,
+	net_rx,
+	net_tx,
+	net_print_info,
+	0,
+	0,
+	net_reset,
+};
+
+/* Read and parse the configuration descriptor.  The AX88772 has one
+ * vendor-specific interface with a bulk IN and a bulk OUT endpoint. */
+static int asix_config(void)
+{
+	unsigned char cfg[128];
+	unsigned int total, i, config_value = 0;
+	unsigned int ep_in = 0, ep_out = 0, ep_in_mps = 0, ep_out_mps = 0;
+
+	if (usb_control_in(dev.ctrl, dev.addr, dev.mps, dev.low,
+			   0x80, 0x06, 0x0200, 0, cfg, 9) < 0)
+		return -1;
+	total = (unsigned int)cfg[2] | ((unsigned int)cfg[3] << 8);
+	if (total < 9)
+		return -1;
+	if (total > sizeof(cfg))
+		total = sizeof(cfg);
+	if (usb_control_in(dev.ctrl, dev.addr, dev.mps, dev.low,
+			   0x80, 0x06, 0x0200, 0, cfg, total) < 0)
+		return -1;
+
+	config_value = cfg[5];
+	i = 0;
+	while (i + 2 <= total) {
+		unsigned int blen = cfg[i];
+		unsigned int btype = cfg[i + 1];
+
+		if (blen < 2 || i + blen > total)
+			break;
+		if (btype == 0x05 && blen >= 7) {
+			unsigned int epa = cfg[i + 2];
+			unsigned int attr = cfg[i + 3] & 3u;
+			unsigned int mps = ((unsigned int)cfg[i + 4] |
+					    ((unsigned int)cfg[i + 5] << 8)) &
+					   0x7FFu;
+
+			if (attr == 2) {
+				if (epa & 0x80) {
+					ep_in = epa & 0x0F;
+					ep_in_mps = mps;
+				} else {
+					ep_out = epa & 0x0F;
+					ep_out_mps = mps;
+				}
+			}
+		}
+		i += blen;
+	}
+
+	if (!ep_in || !ep_out || !config_value)
+		return -1;
+	if (usb_control_out(dev.ctrl, dev.addr, dev.mps, dev.low,
+			    0x00, 0x09, (unsigned short)config_value, 0) < 0)
+		return -1;
+
+	dev.ep_in = ep_in;
+	dev.ep_in_mps = ep_in_mps ? ep_in_mps : 64;
+	dev.ep_out = ep_out;
+	dev.ep_out_mps = ep_out_mps ? ep_out_mps : 64;
+	return 0;
+}
+
+static int device_probe(unsigned int ctrl, unsigned int addr,
+			unsigned int mps, int low_speed, unsigned int port,
+			unsigned int vendor, unsigned int product)
+{
+	if (vendor != 0x0B95 || (product != 0x7720 && product != 0x772A &&
+				 product != 0x772B))
+		return 0;
+
+	dev.ctrl = ctrl;
+	dev.addr = addr;
+	dev.mps = mps;
+	dev.low = low_speed;
+	dev.port = port;
+	dev.present = 1;
+	dev.chip_ready = 0;
+	rx_pos = rx_end = 0;
+
+	if (asix_config() < 0) {
+		dev.present = 0;
+		return 0;
+	}
+	if (chip_init() < 0) {
+		dev.present = 0;
+		return 0;
+	}
+
+	return 1;
+}
+
+void asix_register(void)
+{
+	usb_register_device_probe(device_probe);
+}

+ 12 - 0
asix.h

@@ -0,0 +1,12 @@
+/* ASIX AX88772 USB 2.0 Ethernet driver. */
+#ifndef ASIX_H
+#define ASIX_H
+
+#include "netdev.h"
+
+extern const struct netdev asix_netdev;
+
+/* Register the USB device probe (call before the first USB scan). */
+void asix_register(void);
+
+#endif

+ 2 - 0
kernel.c

@@ -1,6 +1,7 @@
 /* Kernel: entry point at 0x7E00, timer (PIT 1 kHz), process init, terminal. */
 
 #include "acpi.h"
+#include "asix.h"
 #include "bootcon.h"
 #include "cpu.h"
 #include "idle.h"
@@ -390,6 +391,7 @@ void kernel_main(void)
 	/* USB HID backend (mouse + keyboard): controller bring-up + first
 	 * scan; devices plugged in later are picked up by the hotplug
 	 * watcher in the WM loop. */
+	asix_register();
 	usbhid_init();
 	boot_trace("usb");
 	vga_shadow_init();

+ 2 - 0
netdev.c

@@ -1,6 +1,7 @@
 /* NIC driver registry: see netdev.h. */
 
 #include "netdev.h"
+#include "asix.h"
 #include "e1000e.h"
 #include "ne2000.h"
 #include "process.h"
@@ -11,6 +12,7 @@ static const struct netdev *const netdev_drivers[] = {
 	&e1000e_netdev,
 	&r8168_netdev,
 	&ne2000_netdev,
+	&asix_netdev,
 };
 
 static const struct netdev *active;

+ 2 - 0
ohci.c

@@ -681,4 +681,6 @@ const struct usb_hcd_ops usb_ohci_ops = {
 	ohci_int_in_start,
 	ohci_int_in_poll,
 	ohci_int_in_stop,
+	0,
+	0,
 };

+ 1 - 1
tcp.c

@@ -503,7 +503,7 @@ void tcp_close(struct tcp_conn *c)
  * is absorbed instead of answered with RST (libsoup treats a reset after a
  * complete response as an error). */
 
-#define TCP_SRV_SLOTS      8          /* 1 active + 7 backlog/closing */
+#define TCP_SRV_SLOTS      4          /* 1 active + 3 backlog/closing */
 #define TCP_SRV_CLOSE_MS   2000
 #define TCP_SRV_BACKLOG_MS 30000
 

+ 3 - 1
tools/qmp_cursor_test.py

@@ -118,7 +118,7 @@ def move(s, dx, dy):
 
 def main():
     mode = "ohci"
-    for m in ("uhci", "uhci72", "multi", "hotplug"):
+    for m in ("uhci", "uhci72", "multi", "hotplug", "xhci"):
         if "--" + m in sys.argv:
             mode = m
 
@@ -141,6 +141,8 @@ def main():
                  "-device", "pci-ohci,id=o2", "-device", "usb-mouse,bus=o2.0"]
     elif mode == "hotplug":
         qemu += ["-device", "pci-ohci"]
+    elif mode == "xhci":
+        qemu += ["-device", "qemu-xhci", "-device", "usb-mouse"]
     else:
         qemu += ["-device", "pci-ohci", "-device", "usb-mouse"]
     qemu += ["-drive", "file=floppy.img,format=raw,if=floppy",

+ 7 - 2
tools/qmp_keyboard_test.py

@@ -60,6 +60,7 @@ def read_ppm(path):
 
 def main():
     hotplug = "--hotplug" in sys.argv
+    xhci = "--xhci" in sys.argv
     keys = ["h", "e", "l", "p", "ret"]
     if "--keys" in sys.argv:
         keys = sys.argv[sys.argv.index("--keys") + 1].split(",")
@@ -74,8 +75,12 @@ def main():
     ppm = os.path.join(workdir, "screen.ppm")
 
     kbddev = [] if hotplug else ["-device", "usb-kbd"]
-    qemu = ["qemu-system-i386", "-M", "pc",
-            "-device", "pci-ohci", "-device", "usb-mouse"] + kbddev + [
+    qemu = ["qemu-system-i386", "-M", "pc"]
+    if xhci:
+        qemu += ["-device", "qemu-xhci", "-device", "usb-mouse"]
+    else:
+        qemu += ["-device", "pci-ohci", "-device", "usb-mouse"]
+    qemu += kbddev + [
         "-drive", "file=floppy.img,format=raw,if=floppy",
         "-drive", "file=root.img,format=raw,if=floppy",
         "-display", "none",

+ 2 - 0
uhci.c

@@ -543,4 +543,6 @@ const struct usb_hcd_ops usb_uhci_ops = {
 	uhci_int_in_start,
 	uhci_int_in_poll,
 	uhci_int_in_stop,
+	0,
+	0,
 };

+ 53 - 1
usb.c

@@ -89,6 +89,7 @@ static const struct usb_hcd_ops *hcd;
 static unsigned int generation;
 static unsigned int next_addr[USB_HCD_MAX_CTRL];
 static usb_probe_fn current_probe;
+static usb_device_probe_fn device_probe;
 static int print_devices;
 static int scan_busy;
 
@@ -208,6 +209,28 @@ int usb_control_out(unsigned int ctrl, unsigned int addr, unsigned int mps,
 	return hcd->control_transfer(ctrl, addr, mps, low_speed, setup, 0, 0, 0);
 }
 
+int usb_control_out_data(unsigned int ctrl, unsigned int addr,
+			 unsigned int mps, int low_speed, unsigned char rt,
+			 unsigned char req, unsigned short value,
+			 unsigned short index, const unsigned char *buf,
+			 unsigned int len)
+{
+	unsigned char setup[8];
+
+	if (!hcd)
+		return -1;
+	setup[0] = rt;
+	setup[1] = req;
+	setup[2] = (unsigned char)(value & 0xFF);
+	setup[3] = (unsigned char)((value >> 8) & 0xFF);
+	setup[4] = (unsigned char)(index & 0xFF);
+	setup[5] = (unsigned char)((index >> 8) & 0xFF);
+	setup[6] = (unsigned char)(len & 0xFF);
+	setup[7] = (unsigned char)((len >> 8) & 0xFF);
+	return hcd->control_transfer(ctrl, addr, mps, low_speed, setup,
+				     (unsigned char *)buf, len, 0);
+}
+
 /* Read the configuration descriptor and hand every interface with an
    interrupt-IN endpoint to the active probe. */
 static void probe_config(unsigned int ctrl, unsigned int addr, unsigned int mps,
@@ -350,6 +373,9 @@ static void enumerate_device(unsigned int ctrl, unsigned int addr,
 	subcls  = dev_desc[5];
 	proto   = dev_desc[6];
 
+	if (device_probe)
+		device_probe(ctrl, addr, mps, low, port, vendor, product);
+
 	if (print_devices) {
 		indent(depth);
 		io_putstr("Bus ");
@@ -462,7 +488,7 @@ static void enumerate_new_device(unsigned int ctrl, unsigned int port, int low,
 		mps = 8;
 
 	/* Assign an address. */
-	if (usb_control_out(ctrl, 0, 8, low, RT_HOST2DEV_STD,
+	if (usb_control_out(ctrl, 0, mps, low, RT_HOST2DEV_STD,
 			    REQ_SET_ADDRESS, (unsigned short)addr, 0) < 0) {
 		dbg("usb:   SET_ADDRESS failed\n");
 		if (print_devices) {
@@ -490,6 +516,10 @@ int usb_hcd_init(void)
 		hcd = &usb_ohci_ops;
 		return 0;
 	}
+	if (usb_xhci_ops.init() > 0) {
+		hcd = &usb_xhci_ops;
+		return 0;
+	}
 	return -1;
 }
 
@@ -531,6 +561,28 @@ void usb_int_in_stop(unsigned int slot)
 		hcd->int_in_stop(slot);
 }
 
+void usb_register_device_probe(usb_device_probe_fn fn)
+{
+	device_probe = fn;
+}
+
+int usb_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		 unsigned int mps, const unsigned char *buf,
+		 unsigned int len)
+{
+	if (!hcd || !hcd->bulk_out)
+		return -1;
+	return hcd->bulk_out(ctrl, addr, ep, mps, buf, len);
+}
+
+int usb_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		unsigned int mps, unsigned char *buf, unsigned int len)
+{
+	if (!hcd || !hcd->bulk_in)
+		return -1;
+	return hcd->bulk_in(ctrl, addr, ep, mps, buf, len);
+}
+
 int usb_scan(usb_probe_fn probe)
 {
 	unsigned int c, p;

+ 22 - 0
usb.h

@@ -39,6 +39,28 @@ int usb_control_in(unsigned int ctrl, unsigned int addr, unsigned int mps,
 int usb_control_out(unsigned int ctrl, unsigned int addr, unsigned int mps,
 		    int low_speed, unsigned char rt, unsigned char req,
 		    unsigned short value, unsigned short index);
+int usb_control_out_data(unsigned int ctrl, unsigned int addr,
+			 unsigned int mps, int low_speed, unsigned char rt,
+			 unsigned char req, unsigned short value,
+			 unsigned short index, const unsigned char *buf,
+			 unsigned int len);
+
+/* Bulk transfers: bulk_in is start-or-poll (see usb_hcd.h). */
+int usb_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		 unsigned int mps, const unsigned char *buf,
+		 unsigned int len);
+int usb_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		unsigned int mps, unsigned char *buf, unsigned int len);
+
+/* Device-level probe for drivers that do not use HID interfaces (the
+ * USB Ethernet driver parses the whole configuration itself).  Return 1
+ * when the device was claimed.  Called for every non-hub device during a
+ * scan, before the per-interface HID probe. */
+typedef int (*usb_device_probe_fn)(unsigned int ctrl, unsigned int addr,
+				   unsigned int mps, int low_speed,
+				   unsigned int port, unsigned int vendor,
+				   unsigned int product);
+void usb_register_device_probe(usb_device_probe_fn fn);
 
 /* Interrupt-IN endpoints (slots 0..USB_HCD_INT_SLOTS-1; usbhid assigns
    one per HID interface it claims). */

+ 13 - 0
usb_hcd.h

@@ -75,9 +75,22 @@ struct usb_hcd_ops {
 	int (*int_in_poll)(unsigned int slot, unsigned char *buf,
 			   unsigned int len);
 	void (*int_in_stop)(unsigned int slot);
+
+	/* Bulk transfers (USB Ethernet / mass storage).  bulk_in is
+	 * start-or-poll: the first call arms the transfer and returns 0,
+	 * later calls return the frame length, 0 while NAKed, or -1.  A
+	 * delivered chunk stays in the caller's buffer until the next call,
+	 * which re-arms the transfer.  May be NULL. */
+	int (*bulk_out)(unsigned int ctrl, unsigned int addr,
+			unsigned int ep, unsigned int mps,
+			const unsigned char *buf, unsigned int len);
+	int (*bulk_in)(unsigned int ctrl, unsigned int addr,
+		       unsigned int ep, unsigned int mps,
+		       unsigned char *buf, unsigned int len);
 };
 
 extern const struct usb_hcd_ops usb_uhci_ops;
 extern const struct usb_hcd_ops usb_ohci_ops;
+extern const struct usb_hcd_ops usb_xhci_ops;
 
 #endif

+ 1 - 1
vfs.c

@@ -247,7 +247,7 @@ static int fat_op_rmdir(struct vfs_mount *m, const char *path)
  * the WebDAV server asks for 0..N-1 in one PROPFIND.  Cache the first 64
  * entries of the last directory so a listing is one directory walk; a write
  * invalidates it.  Larger directories fall back to the old walk past 64. */
-#define FAT_LIST_CACHE 64
+#define FAT_LIST_CACHE 32
 
 static struct vfs_dirent fat_cache[FAT_LIST_CACHE];
 static unsigned int fat_cache_count;

+ 1495 - 0
xhci.c

@@ -0,0 +1,1495 @@
+/* xHCI (USB 3.0) host controller driver.
+ *
+ * Scope: one controller, USB2 root ports and devices plugged directly into
+ * them.  The USB3 ports are left alone (devices there need SuperSpeed
+ * resource/companion-descriptor handling that the HID stack does not need
+ * yet).  Control transfers and one interrupt-IN endpoint per HID interface
+ * are implemented, which is exactly what usbhid.c uses.
+ *
+ * The schedule is polled, like the rest of the kernel.  Rings and contexts
+ * are page-allocated so the driver does not grow the kernel BSS into the
+ * 0x7F000 wall. */
+
+#include "mm.h"
+#include "paging.h"
+#include "pci.h"
+#include "serial.h"
+#include "spinlock.h"
+#include "timer.h"
+#include "usb_hcd.h"
+#include "xhci.h"
+
+#define XHCI_DEBUG 0
+
+/* ---- controller limits ------------------------------------------------- */
+#define XHCI_MAX_SLOTS		4
+#define XHCI_MAX_PORTS		32
+#define XHCI_EVT_TRBS		256
+#define XHCI_CMD_TRBS		32
+#define XHCI_EP0_TRBS		16
+#define XHCI_INT_TRBS		16
+#define XHCI_BULK_TRBS		32
+
+/* ---- capability / operational / runtime offsets (bytes) --------------- */
+#define XHCI_CAP_HCSPARAMS1	0x04
+#define XHCI_CAP_HCSPARAMS2	0x08
+#define XHCI_CAP_HCCPARAMS1	0x10
+#define XHCI_CAP_DBOFF		0x14
+#define XHCI_CAP_RTSOFF		0x18
+
+#define XHCI_OP_USBCMD		0x00
+#define XHCI_OP_USBSTS		0x04
+#define XHCI_OP_PAGESIZE	0x08
+#define XHCI_OP_CRCR		0x18
+#define XHCI_OP_DCBAAP		0x30
+#define XHCI_OP_CONFIG		0x38
+#define XHCI_OP_PORTSC		0x400
+
+#define XHCI_RT_IMAN		0x20
+#define XHCI_RT_IMOD		0x24
+#define XHCI_RT_ERSTSZ		0x28
+#define XHCI_RT_ERSTBA		0x30
+#define XHCI_RT_ERDP		0x38
+
+/* USBCMD / USBSTS */
+#define XHCI_CMD_RS		0x0001u
+#define XHCI_CMD_HCRST		0x0002u
+#define XHCI_STS_HCH		0x0001u
+#define XHCI_STS_CNR		0x0800u
+#define XHCI_STS_HCE		0x1000u
+
+/* PORTSC */
+#define PORTSC_CCS		0x00000001u
+#define PORTSC_PED		0x00000002u
+#define PORTSC_PR		0x00000010u
+#define PORTSC_PP		0x00000200u
+#define PORTSC_SPEED_SHIFT	10
+#define PORTSC_SPEED_MASK	0xFu
+#define PORTSC_CSC		0x00020000u
+#define PORTSC_PEC		0x00040000u
+#define PORTSC_WRC		0x00080000u
+#define PORTSC_OCC		0x00100000u
+#define PORTSC_PRC		0x00200000u
+#define PORTSC_PLC		0x00400000u
+#define PORTSC_CEC		0x00800000u
+#define PORTSC_RWC		(PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | \
+				 PORTSC_OCC | PORTSC_PRC | PORTSC_PLC | \
+				 PORTSC_CEC | PORTSC_PED)
+
+/* TRB types */
+#define TRB_NORMAL		1
+#define TRB_SETUP		2
+#define TRB_DATA		3
+#define TRB_STATUS		4
+#define TRB_LINK		6
+#define TRB_ENABLE_SLOT		9
+#define TRB_DISABLE_SLOT	10
+#define TRB_ADDR_DEV		11
+#define TRB_CONFIG_EP		12
+#define TRB_TRANSFER		32
+#define TRB_COMPLETION		33
+
+/* TRB control bits */
+#define TRB_CYCLE		0x00000001u
+#define TRB_TC			0x00000002u
+#define TRB_ISP			0x00000004u
+#define TRB_CHAIN		0x00000010u
+#define TRB_IOC			0x00000020u
+#define TRB_IDT			0x00000040u
+#define TRB_BSR			0x00000200u
+#define TRB_DIR_IN		0x00010000u
+#define TRB_TYPE(t)		((t) << 10)
+
+/* completion codes */
+#define CC_SUCCESS		1
+#define CC_SHORT_PACKET		13
+
+/* slot/EP context flags and endpoint types */
+#define SLOT_FLAG		0x00000001u
+#define EP0_FLAG		0x00000002u
+#define EP_CTRL			4
+#define EP_BULK_OUT		2
+#define EP_BULK_IN		6
+#define EP_INT_IN		7
+
+#define XHCI_CTRL_MPS		8
+
+struct xhci_trb {
+	unsigned int p0;
+	unsigned int p1;
+	unsigned int status;
+	unsigned int control;
+};
+
+struct xhci_slot {
+	int used;
+	unsigned int slot_id;
+	unsigned int addr;
+	unsigned int port;
+	unsigned int speed;
+	unsigned int dc;	/* device context page */
+	unsigned int ep0;	/* EP0 transfer ring page */
+	unsigned int ep0_enq;
+	unsigned int ep0_cycle;
+	unsigned int ep_mask;	/* configured endpoint flags */
+};
+
+struct xhci_int {
+	int active;
+	unsigned int slot;	/* hardware slot id */
+	unsigned int ep_id;	/* xHCI endpoint id (2n+dir) */
+	unsigned int mps;
+	unsigned int ring;
+	unsigned int enq;
+	unsigned int cycle;
+	unsigned int trb_len;
+	unsigned int pending_len;
+	int pending;
+	int error;
+	unsigned char buf[8] __attribute__((aligned(16)));
+};
+
+struct xhci_bulk {
+	int active;
+	int armed;
+	unsigned int slot;
+	unsigned int ep;	/* endpoint number */
+	unsigned int ep_id;
+	unsigned int mps;
+	unsigned int ring;
+	unsigned int enq;
+	unsigned int cycle;
+	unsigned int buf;
+	unsigned int len;
+	unsigned int trb_len;
+	unsigned int actual;
+	int pending;
+	int error;
+};
+
+static unsigned int mmio_base;
+static unsigned int cap_len;
+static unsigned int rt_off;
+static unsigned int db_off;
+static unsigned int max_slots;
+static unsigned int max_ports;
+static unsigned int ctx_size;
+static unsigned int usb2_mask;
+static unsigned int hcc_params;
+static unsigned int pci_bus, pci_dev, pci_func;
+static int found;
+static int last_port = -1;
+
+static unsigned int dcbaa;
+static unsigned int input_ctx;
+static unsigned int cmd_ring;
+static unsigned int evt_ring;
+static unsigned int erst;
+static unsigned int cmd_enq;
+static unsigned int cmd_cycle;
+static unsigned int evt_idx;
+static unsigned int evt_cycle;
+
+static unsigned int cmd_busy;
+static unsigned int cmd_done;
+static unsigned int cmd_status;
+static unsigned int cmd_slot;
+static unsigned int cmd_trb_addr;
+
+static unsigned int ctl_busy;
+static unsigned int ctl_done;
+static unsigned int ctl_slot;
+static unsigned int ctl_status;
+static unsigned int ctl_req_len;
+static unsigned int ctl_actual;
+static unsigned int ctl_data_seen;
+static unsigned int ctl_setup_trb;
+static unsigned int ctl_data_trb;
+static unsigned int ctl_status_trb;
+
+static struct xhci_bulk bulk_in;
+static struct xhci_bulk bulk_out;
+static unsigned int out_busy;
+static unsigned int out_done;
+static unsigned int out_status;
+static unsigned int out_slot;
+static unsigned int out_trb;
+
+static struct xhci_slot slots[XHCI_MAX_SLOTS + 1];
+static unsigned int addr_slot[128];
+static unsigned int port_slot[XHCI_MAX_PORTS];
+static struct xhci_int ints[USB_HCD_INT_SLOTS];
+
+static spinlock_t xhci_lock;
+
+/* ---- debug ------------------------------------------------------------- */
+#if XHCI_DEBUG
+static void dbg(const char *s)
+{
+	serial_puts(SERIAL_COM1, s);
+}
+
+static void dbg_hex(const char *s, unsigned int v)
+{
+	serial_puts(SERIAL_COM1, s);
+	serial_puthex(SERIAL_COM1, v);
+	serial_putc(SERIAL_COM1, '\n');
+}
+
+static void dbg_dec(const char *s, unsigned int v)
+{
+	serial_puts(SERIAL_COM1, s);
+	serial_putdec(SERIAL_COM1, v);
+	serial_putc(SERIAL_COM1, '\n');
+}
+#else
+static void dbg(const char *s) { (void)s; }
+static void dbg_hex(const char *s, unsigned int v) { (void)s; (void)v; }
+static void dbg_dec(const char *s, unsigned int v) { (void)s; (void)v; }
+#endif
+
+/* ---- MMIO helpers ------------------------------------------------------ */
+static unsigned int mmio_rd(unsigned int off)
+{
+	return *(volatile unsigned int *)(mmio_base + off);
+}
+
+static void mmio_wr(unsigned int off, unsigned int val)
+{
+	*(volatile unsigned int *)(mmio_base + off) = val;
+}
+
+static void mmio_wr64(unsigned int off, unsigned int lo)
+{
+	mmio_wr(off, lo);
+	mmio_wr(off + 4, 0);
+}
+
+/* Normal RAM (DCBAA, ERST, scratchpad array) is not MMIO. */
+static void ram_wr64(unsigned int addr, unsigned int lo)
+{
+	*(volatile unsigned int *)addr = lo;
+	*(volatile unsigned int *)(addr + 4) = 0;
+}
+
+static void zero_page(unsigned int addr, unsigned int len)
+{
+	unsigned int *p = (unsigned int *)addr;
+	while (len >= 4) {
+		*p++ = 0;
+		len -= 4;
+	}
+}
+
+static unsigned int op(unsigned int reg)
+{
+	return cap_len + reg;
+}
+
+static unsigned int rt(unsigned int reg)
+{
+	return rt_off + reg;
+}
+
+/* ---- rings ------------------------------------------------------------- */
+static void ring_init(unsigned int base, unsigned int size,
+		      unsigned int *enq, unsigned int *cycle)
+{
+	struct xhci_trb *link = (struct xhci_trb *)(base + (size - 1) * 16);
+
+	link->p0 = base;
+	link->p1 = 0;
+	link->status = 0;
+	link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
+	*enq = 0;
+	*cycle = 1;
+}
+
+static void ring_queue(unsigned int base, unsigned int size,
+		       unsigned int *enq, unsigned int *cycle,
+		       unsigned int p0, unsigned int p1,
+		       unsigned int status, unsigned int control)
+{
+	struct xhci_trb *trb = (struct xhci_trb *)(base + *enq * 16);
+
+	trb->p0 = p0;
+	trb->p1 = p1;
+	trb->status = status;
+	__asm__ volatile ("" ::: "memory");
+	trb->control = control | *cycle;
+	__asm__ volatile ("" ::: "memory");
+
+	(*enq)++;
+	if (*enq == size - 1) {
+		struct xhci_trb *link =
+			(struct xhci_trb *)(base + (size - 1) * 16);
+		__asm__ volatile ("" ::: "memory");
+		link->control ^= TRB_CYCLE;
+		*enq = 0;
+		*cycle ^= 1;
+	}
+}
+
+/* ---- event handling ---------------------------------------------------- */
+static void xhci_poll_locked(void);
+static void stop_int(unsigned int slot);
+static void bulk_reset(struct xhci_bulk *b);
+
+static void xhci_int_event(unsigned int slot, unsigned int ep,
+			   unsigned int code, unsigned int residual)
+{
+	unsigned int i;
+
+	for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
+		if (!ints[i].active || ints[i].slot != slot ||
+		    ints[i].ep_id != ep)
+			continue;
+		if (!ints[i].pending) {
+			if (code == CC_SUCCESS || code == CC_SHORT_PACKET) {
+				ints[i].pending_len =
+					(ints[i].trb_len >= residual) ?
+					ints[i].trb_len - residual : 0;
+				ints[i].pending = 1;
+			} else {
+				ints[i].pending = 1;
+				ints[i].error = 1;
+			}
+		}
+		break;
+	}
+}
+
+static void xhci_poll_locked(void)
+{
+	unsigned int n = 0;
+
+	while (n++ < 512) {
+		struct xhci_trb *ev =
+			(struct xhci_trb *)(evt_ring + evt_idx * 16);
+		unsigned int type;
+
+		if ((ev->control & TRB_CYCLE) != evt_cycle)
+			break;
+
+		type = (ev->control >> 10) & 0x3F;
+
+		if (type == TRB_COMPLETION) {
+			unsigned int code = (ev->status >> 24) & 0xFF;
+			unsigned int slot = (ev->control >> 24) & 0xFF;
+
+			if (cmd_busy && ev->p0 == cmd_trb_addr) {
+				cmd_status = code;
+				cmd_slot = slot;
+				cmd_done = 1;
+			}
+		} else if (type == TRB_TRANSFER) {
+			unsigned int code = (ev->status >> 24) & 0xFF;
+			unsigned int residual = ev->status & 0xFFFFFF;
+			unsigned int slot = (ev->control >> 24) & 0xFF;
+			unsigned int ep = (ev->control >> 16) & 0x1F;
+
+			if (ctl_busy && slot == ctl_slot && ep == 1) {
+				if (ev->p0 == ctl_data_trb) {
+					ctl_actual = (ctl_req_len >= residual) ?
+						ctl_req_len - residual : 0;
+					ctl_data_seen = 1;
+				} else if (ev->p0 == ctl_status_trb) {
+					if (!ctl_data_seen)
+						ctl_actual =
+							(code == CC_SUCCESS) ?
+							ctl_req_len : 0;
+					ctl_status = code;
+					ctl_done = 1;
+				}
+			} else if (out_busy && slot == out_slot &&
+				   ep == bulk_out.ep_id &&
+				   ev->p0 == out_trb) {
+				out_status = code;
+				out_done = 1;
+			} else if (bulk_in.active && slot == bulk_in.slot &&
+				   ep == bulk_in.ep_id) {
+				bulk_in.armed = 0;
+				if (!bulk_in.pending) {
+					if (code == CC_SUCCESS ||
+					    code == CC_SHORT_PACKET) {
+						bulk_in.actual =
+							(bulk_in.trb_len >=
+							 residual) ?
+							bulk_in.trb_len -
+								residual : 0;
+						bulk_in.pending = 1;
+					} else {
+						bulk_in.pending = 1;
+						bulk_in.error = 1;
+					}
+				}
+			} else {
+				xhci_int_event(slot, ep, code, residual);
+			}
+		}
+
+		evt_idx++;
+		if (evt_idx >= XHCI_EVT_TRBS) {
+			evt_idx = 0;
+			evt_cycle ^= 1;
+		}
+	}
+
+	mmio_wr64(rt(XHCI_RT_ERDP),
+		  (evt_ring + evt_idx * 16) | 8);
+}
+
+/* ---- commands ---------------------------------------------------------- */
+static int xhci_cmd(unsigned int p0, unsigned int p1, unsigned int control,
+		    unsigned int *out_slot)
+{
+	unsigned int flags, t0, rc;
+
+	if (!found)
+		return -1;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	if (cmd_busy) {
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		return -1;
+	}
+	cmd_busy = 1;
+	cmd_done = 0;
+	cmd_status = 0;
+	cmd_slot = 0;
+	cmd_trb_addr = cmd_ring + cmd_enq * 16;
+	ring_queue(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle,
+		   p0, p1, 0, control);
+	__asm__ volatile ("" ::: "memory");
+	mmio_wr(db_off, 0);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		spin_lock_irqsave(&xhci_lock, &flags);
+		xhci_poll_locked();
+		rc = cmd_done;
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		if (rc)
+			break;
+		__asm__ volatile ("pause");
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	cmd_busy = 0;
+	if (!cmd_done) {
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		dbg("xhci: command timeout\n");
+		return -1;
+	}
+	if (out_slot)
+		*out_slot = cmd_slot;
+	rc = cmd_status;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+	if (rc != CC_SUCCESS) {
+		dbg_hex("xhci: command status ", rc);
+		return -1;
+	}
+	return 0;
+}
+
+/* ---- contexts ---------------------------------------------------------- */
+static void fill_slot_ctx(unsigned int addr, unsigned int port,
+			  unsigned int speed, unsigned int last_ctx)
+{
+	unsigned int *p = (unsigned int *)(input_ctx + ctx_size);
+
+	p[0] = (speed << 20) | (last_ctx << 27);
+	p[1] = (port + 1) << 16;
+	p[2] = 0;
+	p[3] = addr;
+}
+
+static void fill_ep0_ctx(unsigned int mps, unsigned int ring,
+			 unsigned int enq, unsigned int cycle)
+{
+	unsigned int *p = (unsigned int *)(input_ctx + 2 * ctx_size);
+
+	p[0] = 0;
+	p[1] = (EP_CTRL << 3) | (3 << 1) | (mps << 16);
+	p[2] = (ring + enq * 16) | cycle;
+	p[3] = 0;
+	p[4] = 8;
+}
+
+static void fill_ep_ctx(unsigned int ep_index, unsigned int type,
+			unsigned int mps, unsigned int interval,
+			unsigned int ring)
+{
+	unsigned int *p =
+		(unsigned int *)(input_ctx + (ep_index + 2) * ctx_size);
+
+	p[0] = interval << 16;
+	p[1] = (type << 3) | (3 << 1) | (mps << 16);
+	p[2] = ring | 1;
+	p[3] = 0;
+	p[4] = mps;
+}
+
+static struct xhci_slot *slot_by_addr(unsigned int addr)
+{
+	unsigned int id;
+
+	if (addr > 127)
+		return 0;
+	id = addr_slot[addr];
+	if (id < 1 || id > max_slots || !slots[id].used)
+		return 0;
+	return &slots[id];
+}
+
+static unsigned int port_speed(unsigned int port)
+{
+	unsigned int ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+	unsigned int sp = (ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK;
+
+	return sp ? sp : 1;
+}
+
+static int xhci_config_ep(struct xhci_slot *s, unsigned int ep_index,
+			  unsigned int type, unsigned int mps,
+			  unsigned int interval, unsigned int ring)
+{
+	unsigned int ep_flag = 1u << (ep_index + 1);
+	unsigned int last = ep_index + 1;
+	unsigned int i, flags;
+	int rc;
+
+	for (i = 1; i < 32; i++)
+		if (s->ep_mask & (1u << i))
+			last = i;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	zero_page(input_ctx, 4096);
+	*(unsigned int *)input_ctx = 0;
+	*(unsigned int *)(input_ctx + 4) = SLOT_FLAG | ep_flag;
+	fill_slot_ctx(s->addr, s->port, s->speed, last);
+	fill_ep_ctx(ep_index, type, mps, interval, ring);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	rc = xhci_cmd(input_ctx, 0,
+		      TRB_TYPE(TRB_CONFIG_EP) | (s->slot_id << 24), 0);
+	if (rc == 0)
+		s->ep_mask |= ep_flag;
+	return rc;
+}
+
+/* ---- slots ------------------------------------------------------------- */
+static struct xhci_slot *new_slot(void)
+{
+	struct xhci_slot *s;
+	unsigned int slot_id, dc, ep0, flags;
+	int rc;
+
+	if (last_port < 0 || last_port >= (int)max_ports)
+		return 0;
+	if (port_slot[last_port])
+		return 0;
+
+	slot_id = 0;
+	rc = xhci_cmd(0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &slot_id);
+	if (rc != 0 || slot_id == 0 || slot_id > max_slots) {
+		dbg("xhci: enable slot failed\n");
+		return 0;
+	}
+
+	dc = mm_alloc_page();
+	ep0 = mm_alloc_page();
+	if (!dc || !ep0) {
+		if (dc)
+			mm_free_page(dc);
+		if (ep0)
+			mm_free_page(ep0);
+		return 0;
+	}
+	zero_page(dc, 4096);
+	zero_page(ep0, 4096);
+
+	s = &slots[slot_id];
+	s->used = 1;
+	s->slot_id = slot_id;
+	s->addr = 0;
+	s->port = (unsigned int)last_port;
+	s->speed = port_speed((unsigned int)last_port);
+	s->dc = dc;
+	s->ep0 = ep0;
+	s->ep0_enq = 0;
+	s->ep0_cycle = 1;
+	s->ep_mask = 0;
+	ring_init(ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle);
+
+	ram_wr64(dcbaa + slot_id * 8, dc);
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	zero_page(input_ctx, 4096);
+	*(unsigned int *)input_ctx = 0;
+	*(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
+	fill_slot_ctx(0, s->port, s->speed, 1);
+	fill_ep0_ctx(XHCI_CTRL_MPS, ep0, s->ep0_enq, s->ep0_cycle);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	rc = xhci_cmd(input_ctx, 0,
+		      TRB_TYPE(TRB_ADDR_DEV) | (slot_id << 24) | TRB_BSR, 0);
+	if (rc != 0) {
+		dbg("xhci: address default failed\n");
+		mm_free_page(dc);
+		mm_free_page(ep0);
+		s->used = 0;
+		return 0;
+	}
+
+	addr_slot[0] = slot_id;
+	port_slot[s->port] = slot_id;
+	return s;
+}
+
+static void free_slot(struct xhci_slot *s)
+{
+	unsigned int i, flags;
+
+	if (!s || !s->used)
+		return;
+
+	xhci_cmd(0, 0, TRB_TYPE(TRB_DISABLE_SLOT) | (s->slot_id << 24), 0);
+	ram_wr64(dcbaa + s->slot_id * 8, 0);
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	for (i = 0; i < USB_HCD_INT_SLOTS; i++) {
+		if (ints[i].slot == s->slot_id) {
+			ints[i].active = 0;
+			ints[i].pending = 0;
+			ints[i].error = 0;
+			if (ints[i].ring) {
+				mm_free_page(ints[i].ring);
+				ints[i].ring = 0;
+			}
+			ints[i].slot = 0;
+		}
+	}
+	if (s->dc) {
+		mm_free_page(s->dc);
+		s->dc = 0;
+	}
+	if (s->ep0) {
+		mm_free_page(s->ep0);
+		s->ep0 = 0;
+	}
+	if (s->addr < 128)
+		addr_slot[s->addr] = 0;
+	if (bulk_in.active && bulk_in.slot == s->slot_id)
+		bulk_reset(&bulk_in);
+	if (bulk_out.active && bulk_out.slot == s->slot_id)
+		bulk_reset(&bulk_out);
+	if (s->port < XHCI_MAX_PORTS)
+		port_slot[s->port] = 0;
+	s->used = 0;
+	s->addr = 0;
+	s->ep_mask = 0;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+}
+
+static int set_address(unsigned int new_addr, unsigned int mps)
+{
+	struct xhci_slot *s = slot_by_addr(0);
+	unsigned int flags;
+	int rc;
+
+	if (!s || new_addr == 0 || new_addr > 127)
+		return -1;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	zero_page(input_ctx, 4096);
+	*(unsigned int *)input_ctx = 0;
+	*(unsigned int *)(input_ctx + 4) = SLOT_FLAG | EP0_FLAG;
+	fill_slot_ctx(new_addr, s->port, s->speed, 1);
+	fill_ep0_ctx(mps, s->ep0, s->ep0_enq, s->ep0_cycle);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	rc = xhci_cmd(input_ctx, 0,
+		      TRB_TYPE(TRB_ADDR_DEV) | (s->slot_id << 24), 0);
+	if (rc != 0) {
+		dbg("xhci: set address failed\n");
+		return -1;
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	addr_slot[0] = 0;
+	s->addr = new_addr;
+	addr_slot[new_addr] = s->slot_id;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+	return 0;
+}
+
+/* ---- control transfers ------------------------------------------------- */
+static int control_transfer(unsigned int ctrl, unsigned int addr,
+			    unsigned int mps, int low_speed,
+			    const unsigned char *setup, unsigned char *data,
+			    unsigned int len, int dir_in)
+{
+	struct xhci_slot *s;
+	unsigned int flags, t0, rc, st;
+	unsigned int setup_c, data_c, status_c;
+
+	(void)ctrl;
+	(void)low_speed;
+
+	if (!found)
+		return -1;
+
+	/* xHCI owns SET_ADDRESS: it is an Address Device command, not a
+	 * control request on the wire. */
+	if (setup[0] == 0x00 && setup[1] == 0x05 && addr == 0)
+		return set_address((unsigned int)setup[2] |
+				   ((unsigned int)setup[3] << 8), mps);
+
+	s = slot_by_addr(addr);
+	if (!s) {
+		if (addr != 0)
+			return -1;
+		s = new_slot();
+		if (!s)
+			return -1;
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	if (ctl_busy) {
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		dbg("xhci: control busy\n");
+		return -1;
+	}
+	ctl_busy = 1;
+	ctl_done = 0;
+	ctl_status = 0;
+	ctl_actual = 0;
+	ctl_data_seen = 0;
+	ctl_slot = s->slot_id;
+	ctl_req_len = len;
+
+	setup_c = TRB_TYPE(TRB_SETUP) | TRB_IDT | TRB_CHAIN;
+	if (len)
+		setup_c |= (dir_in ? 3u : 2u) << 16;
+	ctl_setup_trb = s->ep0 + s->ep0_enq * 16;
+	ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
+		   (unsigned int)setup[0] | ((unsigned int)setup[1] << 8) |
+		   ((unsigned int)setup[2] << 16) |
+		   ((unsigned int)setup[3] << 24),
+		   (unsigned int)setup[4] | ((unsigned int)setup[5] << 8) |
+		   ((unsigned int)setup[6] << 16) |
+		   ((unsigned int)setup[7] << 24),
+		   8, setup_c);
+
+	ctl_data_trb = 0;
+	if (len) {
+		ctl_data_trb = s->ep0 + s->ep0_enq * 16;
+		data_c = TRB_TYPE(TRB_DATA) | TRB_CHAIN;
+		if (dir_in)
+			data_c |= TRB_DIR_IN | TRB_ISP;
+		ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq,
+			   &s->ep0_cycle, (unsigned int)data, 0, len, data_c);
+	}
+
+	ctl_status_trb = s->ep0 + s->ep0_enq * 16;
+	status_c = TRB_TYPE(TRB_STATUS) | TRB_IOC;
+	if (!len || !dir_in)
+		status_c |= TRB_DIR_IN;
+	ring_queue(s->ep0, XHCI_EP0_TRBS, &s->ep0_enq, &s->ep0_cycle,
+		   0, 0, 0, status_c);
+
+	__asm__ volatile ("" ::: "memory");
+	mmio_wr(db_off + 4 * s->slot_id, 1);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		spin_lock_irqsave(&xhci_lock, &flags);
+		xhci_poll_locked();
+		rc = ctl_done;
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		if (rc)
+			break;
+		__asm__ volatile ("pause");
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	if (!ctl_done) {
+		ctl_busy = 0;
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		dbg("xhci: control timeout\n");
+		return -1;
+	}
+	rc = ctl_actual;
+	st = ctl_status;
+	ctl_busy = 0;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	if (st != CC_SUCCESS && st != CC_SHORT_PACKET) {
+		dbg_hex("xhci: control status ", st);
+		return -1;
+	}
+	return (int)rc;
+}
+
+/* ---- HID interrupt endpoint -------------------------------------------- */
+static int int_in_start(unsigned int ctrl, unsigned int slot,
+			unsigned int addr, unsigned int ep,
+			unsigned int mps, int low_speed)
+{
+	struct xhci_slot *s = slot_by_addr(addr);
+	struct xhci_int *in;
+	unsigned int ep_index, ep_id, ring, flags;
+	int rc;
+
+	(void)ctrl;
+	(void)low_speed;
+
+	if (!found || !s || slot >= USB_HCD_INT_SLOTS || ep < 1 || ep > 15)
+		return -1;
+	if (mps < 1)
+		mps = 8;
+	if (mps > 8)
+		mps = 8;
+
+	if (ints[slot].active)
+		stop_int(slot);
+
+	ep_index = (ep - 1) * 2 + 2;	/* EP index 2 for EP1 IN, 4 for EP2 IN */
+	ep_id = 2 * ep + 1;
+	ring = mm_alloc_page();
+	if (!ring)
+		return -1;
+	zero_page(ring, 4096);
+
+	in = &ints[slot];
+	in->slot = s->slot_id;
+	in->ep_id = ep_id;
+	in->mps = mps;
+	in->ring = ring;
+	in->enq = 0;
+	in->cycle = 1;
+	in->trb_len = mps;
+	in->pending = 0;
+	in->error = 0;
+	in->active = 0;
+	ring_init(ring, XHCI_INT_TRBS, &in->enq, &in->cycle);
+
+	rc = xhci_config_ep(s, ep_index, EP_INT_IN, mps, 1, ring);
+	if (rc != 0) {
+		dbg_hex("xhci: configure int ep failed ", ep_index);
+		mm_free_page(ring);
+		in->ring = 0;
+		return -1;
+	}
+
+	in->active = 1;
+	spin_lock_irqsave(&xhci_lock, &flags);
+	ring_queue(ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
+		   (unsigned int)in->buf, 0, in->trb_len,
+		   TRB_TYPE(TRB_NORMAL) | TRB_IOC);
+	__asm__ volatile ("" ::: "memory");
+	mmio_wr(db_off + 4 * s->slot_id, ep_id);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	return 0;
+}
+
+static int int_in_poll(unsigned int slot, unsigned char *buf, unsigned int len)
+{
+	struct xhci_int *in;
+	unsigned int flags, n, i;
+
+	if (!found || slot >= USB_HCD_INT_SLOTS)
+		return 0;
+
+	in = &ints[slot];
+	if (!in->active)
+		return 0;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	xhci_poll_locked();
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	if (!in->pending)
+		return 0;
+	if (in->error) {
+		in->pending = 0;
+		in->error = 0;
+		return -1;
+	}
+
+	n = in->pending_len;
+	if (n > len)
+		n = len;
+	for (i = 0; i < n; i++)
+		buf[i] = in->buf[i];
+	in->pending = 0;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	if (in->active) {
+		ring_queue(in->ring, XHCI_INT_TRBS, &in->enq, &in->cycle,
+			   (unsigned int)in->buf, 0, in->trb_len,
+			   TRB_TYPE(TRB_NORMAL) | TRB_IOC);
+		__asm__ volatile ("" ::: "memory");
+		mmio_wr(db_off + 4 * in->slot, in->ep_id);
+	}
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	return (int)in->pending_len;
+}
+
+static void stop_int(unsigned int slot)
+{
+	struct xhci_int *in;
+	unsigned int flags;
+
+	if (slot >= USB_HCD_INT_SLOTS)
+		return;
+	in = &ints[slot];
+	spin_lock_irqsave(&xhci_lock, &flags);
+	in->active = 0;
+	in->pending = 0;
+	in->error = 0;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+}
+
+/* ---- bulk transfers ---------------------------------------------------- */
+static void bulk_reset(struct xhci_bulk *b)
+{
+	if (b->ring) {
+		mm_free_page(b->ring);
+		b->ring = 0;
+	}
+	b->active = 0;
+	b->armed = 0;
+	b->slot = 0;
+	b->pending = 0;
+	b->error = 0;
+	b->actual = 0;
+}
+
+static int xhci_bulk_in(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		   unsigned int mps, unsigned char *buf, unsigned int len)
+{
+	struct xhci_slot *s = slot_by_addr(addr);
+	struct xhci_bulk *b = &bulk_in;
+	unsigned int ep_index, ep_id, ring, flags, n, err;
+	int rc;
+
+	(void)ctrl;
+	if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
+		return -1;
+	if (mps < 1)
+		mps = 512;
+
+	ep_index = (ep - 1) * 2 + 2;
+	ep_id = 2 * ep + 1;
+
+	if (!b->active || b->slot != s->slot_id || b->ep != ep) {
+		ring = mm_alloc_page();
+		if (!ring)
+			return -1;
+		zero_page(ring, 4096);
+		bulk_reset(b);
+		b->slot = s->slot_id;
+		b->ep = ep;
+		b->ep_id = ep_id;
+		b->mps = mps;
+		b->ring = ring;
+		b->enq = 0;
+		b->cycle = 1;
+		ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
+		rc = xhci_config_ep(s, ep_index, EP_BULK_IN, mps, 0, ring);
+		if (rc != 0) {
+			bulk_reset(b);
+			return -1;
+		}
+		b->active = 1;
+	}
+
+	b->buf = (unsigned int)buf;
+	b->len = len;
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	xhci_poll_locked();
+	if (b->pending) {
+		n = b->actual;
+		err = (unsigned int)b->error;
+		b->pending = 0;
+		b->error = 0;
+		if (!err && b->active) {
+			b->trb_len = len;
+			ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq,
+				   &b->cycle, (unsigned int)buf, 0, len,
+				   TRB_TYPE(TRB_NORMAL) | TRB_IOC);
+			b->armed = 1;
+			__asm__ volatile ("" ::: "memory");
+			mmio_wr(db_off + 4 * b->slot, b->ep_id);
+		}
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		return err ? -1 : (int)n;
+	}
+	if (!b->armed) {
+		b->trb_len = len;
+		ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
+			   (unsigned int)buf, 0, len,
+			   TRB_TYPE(TRB_NORMAL) | TRB_IOC);
+		b->armed = 1;
+		__asm__ volatile ("" ::: "memory");
+		mmio_wr(db_off + 4 * b->slot, b->ep_id);
+	}
+	spin_unlock_irqrestore(&xhci_lock, flags);
+	return 0;
+}
+
+static int xhci_bulk_out(unsigned int ctrl, unsigned int addr, unsigned int ep,
+		    unsigned int mps, const unsigned char *buf,
+		    unsigned int len)
+{
+	struct xhci_slot *s = slot_by_addr(addr);
+	struct xhci_bulk *b = &bulk_out;
+	unsigned int ep_index, ep_id, ring, flags, t0, done, status;
+	int rc;
+
+	(void)ctrl;
+	if (!found || !s || ep < 1 || ep > 15 || !buf || !len)
+		return -1;
+	if (mps < 1)
+		mps = 512;
+
+	ep_index = (ep - 1) * 2 + 1;
+	ep_id = 2 * ep;
+
+	if (!b->active || b->slot != s->slot_id || b->ep != ep) {
+		ring = mm_alloc_page();
+		if (!ring)
+			return -1;
+		zero_page(ring, 4096);
+		bulk_reset(b);
+		b->slot = s->slot_id;
+		b->ep = ep;
+		b->ep_id = ep_id;
+		b->mps = mps;
+		b->ring = ring;
+		b->enq = 0;
+		b->cycle = 1;
+		ring_init(ring, XHCI_BULK_TRBS, &b->enq, &b->cycle);
+		rc = xhci_config_ep(s, ep_index, EP_BULK_OUT, mps, 0, ring);
+		if (rc != 0) {
+			bulk_reset(b);
+			return -1;
+		}
+		b->active = 1;
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	if (out_busy) {
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		return -1;
+	}
+	out_busy = 1;
+	out_done = 0;
+	out_status = 0;
+	out_slot = s->slot_id;
+	out_trb = b->ring + b->enq * 16;
+	b->trb_len = len;
+	ring_queue(b->ring, XHCI_BULK_TRBS, &b->enq, &b->cycle,
+		   (unsigned int)buf, 0, len, TRB_TYPE(TRB_NORMAL) | TRB_IOC);
+	__asm__ volatile ("" ::: "memory");
+	mmio_wr(db_off + 4 * s->slot_id, ep_id);
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		spin_lock_irqsave(&xhci_lock, &flags);
+		xhci_poll_locked();
+		done = out_done;
+		spin_unlock_irqrestore(&xhci_lock, flags);
+		if (done)
+			break;
+		__asm__ volatile ("pause");
+	}
+
+	spin_lock_irqsave(&xhci_lock, &flags);
+	done = out_done;
+	status = out_status;
+	out_busy = 0;
+	spin_unlock_irqrestore(&xhci_lock, flags);
+
+	if (!done) {
+		dbg("xhci: bulk out timeout\n");
+		return -1;
+	}
+	if (status != CC_SUCCESS) {
+		dbg_hex("xhci: bulk out status ", status);
+		return -1;
+	}
+	return 0;
+}
+
+/* ---- root hub ports ---------------------------------------------------- */
+static int port_reset(unsigned int ctrl, unsigned int port)
+{
+	unsigned int ps, t0;
+
+	(void)ctrl;
+	if (!found || port >= max_ports ||
+	    !(usb2_mask & (1u << port)))
+		return -1;
+
+	if (port_slot[port])
+		free_slot(&slots[port_slot[port]]);
+
+	ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+	if (!(ps & PORTSC_CCS))
+		return -1;
+
+	mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
+		(ps & ~PORTSC_RWC) | PORTSC_PR);
+
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 200) {
+		ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+		if (ps & PORTSC_PRC)
+			break;
+	}
+	if (!(ps & PORTSC_PRC)) {
+		dbg_hex("xhci: port reset timeout ", port);
+		return -1;
+	}
+	mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
+		(ps & ~PORTSC_RWC) | PORTSC_PRC);
+
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 100) {
+		ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+		if (ps & PORTSC_PED)
+			break;
+	}
+	if (!(ps & PORTSC_PED)) {
+		dbg_hex("xhci: port not enabled ", port);
+		return -1;
+	}
+
+	last_port = (int)port;
+	return 0;
+}
+
+static int port_connected(unsigned int ctrl, unsigned int port)
+{
+	unsigned int ps;
+
+	(void)ctrl;
+	if (!found || port >= max_ports ||
+	    !(usb2_mask & (1u << port)))
+		return 0;
+	ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+	return (ps & PORTSC_CCS) ? 1 : 0;
+}
+
+static int port_low_speed(unsigned int ctrl, unsigned int port)
+{
+	unsigned int ps;
+
+	(void)ctrl;
+	if (!found || port >= max_ports)
+		return 0;
+	ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+	return ((ps >> PORTSC_SPEED_SHIFT) & PORTSC_SPEED_MASK) == 2;
+}
+
+static int port_take_change(unsigned int ctrl, unsigned int port)
+{
+	unsigned int ps;
+
+	(void)ctrl;
+	if (!found || port >= max_ports)
+		return 0;
+	ps = mmio_rd(op(XHCI_OP_PORTSC) + port * 0x10);
+	if (!(ps & PORTSC_CSC))
+		return 0;
+	mmio_wr(op(XHCI_OP_PORTSC) + port * 0x10,
+		(ps & ~PORTSC_RWC) | PORTSC_CSC);
+	return 1;
+}
+
+/* ---- controller discovery ---------------------------------------------- */
+/* xHCI USB Legacy Support: stop the firmware's SMI from firing on port
+ * changes once we own the controller (the same idea as the PCI 0xC0 write
+ * in uhci.c/ohci.c, but xHCI keeps it in an extended capability). */
+static void legacy_smi_disable(unsigned int off)
+{
+	unsigned int sup = mmio_rd(off);
+	unsigned int ctl;
+
+	if ((sup & 0xFF) != 1)
+		return;
+	if (sup & (1u << 16))		/* BIOS still owns it */
+		mmio_wr(off, sup | (1u << 24));
+	ctl = mmio_rd(off + 4);
+	ctl &= ~((0x7u << 1) | (0xFFu << 5) | (0x7u << 17));
+	ctl |= 0x7u << 29;
+	mmio_wr(off + 4, ctl);
+}
+
+static void scan_protocols(void)
+{
+	unsigned int off, guard = 0;
+
+	usb2_mask = 0;
+	off = ((hcc_params >> 16) & 0xFFFF) << 2;
+
+	while (off && guard++ < 32) {
+		unsigned int cap = mmio_rd(off);
+		unsigned int id = cap & 0xFF;
+		unsigned int next = (cap >> 8) & 0xFF;
+
+		if (id == 1) {
+			legacy_smi_disable(off);
+		} else if (id == 2) {	/* Supported Protocol */
+			unsigned int major = (cap >> 24) & 0xFF;
+			unsigned int pi = mmio_rd(off + 8);
+			unsigned int po = pi & 0xFF;
+			unsigned int pc = (pi >> 8) & 0xFF;
+			unsigned int i;
+
+			if (major <= 2 && po >= 1) {
+				for (i = 0; i < pc && po + i <= 32; i++)
+					usb2_mask |= 1u << (po - 1 + i);
+			}
+		}
+		if (!next)
+			break;
+		off += next << 2;
+	}
+
+	if (!usb2_mask)
+		usb2_mask = (max_ports >= 32) ? 0xFFFFFFFFu :
+			    ((1u << max_ports) - 1);
+}
+
+static int xhci_init(void)
+{
+	unsigned int bus, dev, func, id, class_rev, bar, bar_hi, cmd;
+	unsigned int hcs1, hcs2, t0, i, n, scratch;
+
+	if (found)
+		return 1;
+
+	for (bus = 0; bus < 256; bus++) {
+		for (dev = 0; dev < 32; dev++) {
+			for (func = 0; func < 8; func++) {
+				id = pci_cfg_read32(bus, dev, func, 0x00);
+				if (id == 0xFFFFFFFFu || id == 0)
+					continue;
+				class_rev = pci_cfg_read32(bus, dev, func, 0x08);
+				if ((class_rev >> 16) != 0x0C03)
+					continue;
+				if (((class_rev >> 8) & 0xFF) != 0x30)
+					continue;
+				bar = pci_cfg_read32(bus, dev, func, 0x10);
+				if (bar & 1)
+					continue;
+				bar_hi = 0;
+				if ((bar & 0x6) == 0x4)
+					bar_hi = pci_cfg_read32(bus, dev,
+								func, 0x14);
+				bar &= 0xFFFFFFF0u;
+				if (!bar || bar_hi)
+					continue;
+				pci_bus = bus;
+				pci_dev = dev;
+				pci_func = func;
+				goto found_xhci;
+			}
+		}
+	}
+	return 0;
+
+found_xhci:
+	cmd = pci_cfg_read32(pci_bus, pci_dev, pci_func, 0x04);
+	pci_cfg_write32(pci_bus, pci_dev, pci_func, 0x04, cmd | 0x0006);
+
+	mmio_base = bar;
+	paging_map_mmio(mmio_base, 0x10000);
+
+	cap_len = mmio_rd(0) & 0xFF;
+	hcs1 = mmio_rd(XHCI_CAP_HCSPARAMS1);
+	hcs2 = mmio_rd(XHCI_CAP_HCSPARAMS2);
+	hcc_params = mmio_rd(XHCI_CAP_HCCPARAMS1);
+	rt_off = mmio_rd(XHCI_CAP_RTSOFF) & ~0x1Fu;
+	db_off = mmio_rd(XHCI_CAP_DBOFF) & ~0x3u;
+
+	max_slots = hcs1 & 0xFF;
+	if (max_slots > XHCI_MAX_SLOTS)
+		max_slots = XHCI_MAX_SLOTS;
+	if (!max_slots)
+		max_slots = 1;
+	max_ports = (hcs1 >> 24) & 0xFF;
+	if (max_ports > XHCI_MAX_PORTS)
+		max_ports = XHCI_MAX_PORTS;
+	if (!max_ports)
+		max_ports = 1;
+	ctx_size = (hcc_params & 0x0004u) ? 64 : 32;
+
+	dbg("xhci: found\n");
+	dbg_hex("xhci: mmio ", mmio_base);
+	dbg_dec("xhci: slots ", max_slots);
+	dbg_dec("xhci: ports ", max_ports);
+	dbg_dec("xhci: cap len ", cap_len);
+
+	dcbaa = mm_alloc_page();
+	input_ctx = mm_alloc_page();
+	cmd_ring = mm_alloc_page();
+	evt_ring = mm_alloc_page();
+	erst = mm_alloc_page();
+	if (!dcbaa || !input_ctx || !cmd_ring || !evt_ring || !erst) {
+		dbg("xhci: out of pages\n");
+		return 0;
+	}
+	zero_page(dcbaa, 4096);
+	zero_page(input_ctx, 4096);
+	zero_page(cmd_ring, 4096);
+	zero_page(evt_ring, 4096);
+	zero_page(erst, 4096);
+
+	/* Reset.  HCRST self-clears; wait for CNR to drop. */
+	mmio_wr(op(XHCI_OP_USBCMD), 0);
+	mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_HCRST);
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		if (!(mmio_rd(op(XHCI_OP_USBCMD)) & XHCI_CMD_HCRST))
+			break;
+	}
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_CNR))
+			break;
+	}
+	if (mmio_rd(op(XHCI_OP_USBSTS)) & (XHCI_STS_CNR | XHCI_STS_HCE)) {
+		dbg_hex("xhci: reset failed ",
+			mmio_rd(op(XHCI_OP_USBSTS)));
+		return 0;
+	}
+
+	mmio_wr(op(XHCI_OP_PAGESIZE), 1);
+	mmio_wr(op(XHCI_OP_CONFIG), max_slots);
+	mmio_wr64(op(XHCI_OP_DCBAAP), dcbaa);
+
+	/* Scratchpad buffers, if the controller asks for any. */
+	n = (((hcs2 >> 21) & 0x1F) << 5) | ((hcs2 >> 27) & 0x1F);
+	if (n) {
+		unsigned int arr;
+
+		if (n > 512)
+			n = 512;
+		arr = mm_alloc_page();
+		if (!arr) {
+			dbg("xhci: no scratchpad array\n");
+			return 0;
+		}
+		zero_page(arr, 4096);
+		for (i = 0; i < n; i++) {
+			scratch = mm_alloc_page();
+			if (!scratch) {
+				dbg("xhci: no scratchpad page\n");
+				return 0;
+			}
+			zero_page(scratch, 4096);
+			ram_wr64(arr + i * 8, scratch);
+		}
+		ram_wr64(dcbaa, arr);
+		dbg_dec("xhci: scratchpads ", n);
+	}
+
+	/* Command ring and event ring. */
+	ring_init(cmd_ring, XHCI_CMD_TRBS, &cmd_enq, &cmd_cycle);
+	mmio_wr64(op(XHCI_OP_CRCR), cmd_ring | 1);
+
+	evt_idx = 0;
+	evt_cycle = 1;
+	*(volatile unsigned int *)(erst + 0) = evt_ring;
+	*(volatile unsigned int *)(erst + 4) = 0;
+	*(volatile unsigned int *)(erst + 8) = XHCI_EVT_TRBS;
+	*(volatile unsigned int *)(erst + 12) = 0;
+	mmio_wr(rt(XHCI_RT_ERSTSZ), 1);
+	mmio_wr64(rt(XHCI_RT_ERSTBA), erst);
+	mmio_wr64(rt(XHCI_RT_ERDP), evt_ring);
+	mmio_wr(rt(XHCI_RT_IMAN), 0);
+	mmio_wr(rt(XHCI_RT_IMOD), 0);
+
+	/* Run. */
+	mmio_wr(op(XHCI_OP_USBCMD), XHCI_CMD_RS);
+	t0 = get_ticks_ms();
+	while (get_ticks_ms() - t0 < 1000) {
+		if (!(mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH))
+			break;
+	}
+	if (mmio_rd(op(XHCI_OP_USBSTS)) & XHCI_STS_HCH) {
+		dbg("xhci: controller did not start\n");
+		return 0;
+	}
+
+	scan_protocols();
+	dbg_hex("xhci: usb2 ports ", usb2_mask);
+
+	/* Port power, where the controller has per-port switching. */
+	if (hcc_params & 0x0008u) {
+		for (i = 0; i < max_ports; i++) {
+			unsigned int ps;
+			if (!(usb2_mask & (1u << i)))
+				continue;
+			ps = mmio_rd(op(XHCI_OP_PORTSC) + i * 0x10);
+			if (!(ps & PORTSC_PP))
+				mmio_wr(op(XHCI_OP_PORTSC) + i * 0x10,
+					(ps & ~PORTSC_RWC) | PORTSC_PP);
+		}
+	}
+
+	found = 1;
+	return 1;
+}
+
+static unsigned int xhci_count(void)
+{
+	return found ? 1u : 0u;
+}
+
+static void xhci_info(unsigned int ctrl, unsigned int *bus,
+		      unsigned int *dev, unsigned int *func,
+		      unsigned int *base, int *mmio)
+{
+	(void)ctrl;
+	*bus = pci_bus;
+	*dev = pci_dev;
+	*func = pci_func;
+	*base = mmio_base;
+	*mmio = 1;
+}
+
+static unsigned int xhci_port_count(unsigned int ctrl)
+{
+	(void)ctrl;
+	return found ? max_ports : 0;
+}
+
+const struct usb_hcd_ops usb_xhci_ops = {
+	"xHCI",
+	xhci_init,
+	xhci_count,
+	xhci_info,
+	control_transfer,
+	xhci_port_count,
+	port_connected,
+	port_low_speed,
+	port_reset,
+	port_take_change,
+	int_in_start,
+	int_in_poll,
+	stop_int,
+	xhci_bulk_out,
+	xhci_bulk_in,
+};

+ 11 - 0
xhci.h

@@ -0,0 +1,11 @@
+/* xHCI (USB 3.0) host controller driver: implements the usb_hcd_ops vtable
+   declared in usb_hcd.h.  USB2 root ports only for now (the USB3 ports are
+   left alone so the existing HID stack can be validated first). */
+#ifndef XHCI_H
+#define XHCI_H
+
+#include "usb_hcd.h"
+
+extern const struct usb_hcd_ops usb_xhci_ops;
+
+#endif