iwl_trans.c 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375
  1. /* Intel 7265D/3165 PCIe transport: see iwl_trans.h.
  2. *
  3. * Register sequences follow Linux iwlwifi (pcie/trans.c, pcie/rx.c,
  4. * pcie/tx.c) for the 7000 device family: no MSI-X, no ICT, legacy single
  5. * RX queue (256 x 4 KB RBDs), gen1 128-byte TFDs, byte-count table in
  6. * dwords. Everything is polled; CSR_INT_MASK stays zero and PCI INTx is
  7. * disabled, so the card never raises an interrupt.
  8. *
  9. * The NIC is kept awake (MAC_ACCESS_REQ held) the whole time it is up: the
  10. * firmware is configured without power save, and holding the request is
  11. * what Linux does while host commands are in flight (apmg_wake_up_wa). */
  12. #include "iwl_trans.h"
  13. #include "mm.h"
  14. #include "paging.h"
  15. #include "pci.h"
  16. #include "process.h"
  17. #include "ramfs.h"
  18. #include "serial.h"
  19. /* Per-family transport parameters (Linux: iwl_cfg_trans_params plus the
  20. * op_mode's iwl_trans_config). The 7000 family runs the MVM firmware with
  21. * DQA queues and wide command headers; the 6000 family runs the DVM
  22. * firmware with the four AC queues, the command queue on 4 and 4-byte
  23. * headers. */
  24. struct iwl_family {
  25. const char *fw_path;
  26. unsigned char dvm; /* DVM firmware (6000) vs MVM (7000) */
  27. unsigned char tlv; /* TLV .ucode vs the v2 fixed header */
  28. unsigned char cmd_q, cmd_fifo, nq;
  29. unsigned char wide_hdr; /* mvm: trans->wide_cmd_header */
  30. unsigned char bc_dword; /* byte-count table in dwords */
  31. unsigned char scd_set_active; /* mvm: trans_cfg.scd_set_active */
  32. unsigned char chain_ext_wa; /* base_params->scd_chain_ext_wa */
  33. unsigned char shadow_reg; /* base_params->shadow_reg_enable */
  34. unsigned char wake_me; /* tell ME we leave (7000 family) */
  35. unsigned char q[4], q_slots[4]; /* data/mgmt queues to allocate */
  36. };
  37. static const struct iwl_family fam_7000 = {
  38. "firmware/iwlwifi-7265D-29.ucode", 0, 1, IWL_Q_CMD, 7, 31,
  39. 1, 1, 1, 0, 1, 1,
  40. { IWL_Q_AUX, IWL_Q_BSS, 0xff, 0xff }, { 8, 32, 0, 0 },
  41. };
  42. static const struct iwl_family fam_6000 = {
  43. "firmware/iwlwifi-6000-4.ucode", 1, 0, IWL_DVM_Q_CMD, 7, 20,
  44. 0, 0, 0, 1, 0, 0,
  45. { IWL_DVM_Q_VO, IWL_DVM_Q_VI, IWL_DVM_Q_BE, IWL_DVM_Q_BK },
  46. { 16, 4, 32, 4 },
  47. };
  48. /* ---- CSR / HBUS / FH / PRPH registers (iwl-csr.h, iwl-fh.h, iwl-prph.h) */
  49. #define CSR_HW_IF_CONFIG_REG 0x000
  50. #define CSR_INT_COALESCING 0x004
  51. #define CSR_INT 0x008
  52. #define CSR_INT_MASK 0x00c
  53. #define CSR_FH_INT_STATUS 0x010
  54. #define CSR_RESET 0x020
  55. #define CSR_GP_CNTRL 0x024
  56. #define CSR_HW_REV 0x028
  57. #define CSR_GIO_REG 0x03c
  58. #define CSR_UCODE_DRV_GP1_CLR 0x05c
  59. #define CSR_MBOX_SET_REG 0x088
  60. #define CSR_MAC_SHADOW_REG_CTRL 0x0a8
  61. #define CSR_GIO_CHICKEN_BITS 0x100
  62. #define CSR_DBG_HPET_MEM_REG 0x240
  63. #define CSR_DBG_LINK_PWR_MGMT_REG 0x250
  64. #define HW_IF_NIC_READY 0x00400000u
  65. #define HW_IF_PREPARE 0x08000000u
  66. #define HW_IF_HAP_WAKE_L1A 0x00080000u
  67. #define INT_FH_RX (1u << 31)
  68. #define INT_HW_ERR (1u << 29)
  69. #define INT_FH_TX (1u << 27)
  70. #define INT_SW_ERR (1u << 25)
  71. #define INT_SW_RX (1u << 3)
  72. #define INT_ALIVE (1u << 0)
  73. #define RESET_SW_RESET 0x00000080u
  74. #define RESET_MASTER_DISABLED 0x00000100u
  75. #define RESET_STOP_MASTER 0x00000200u
  76. #define GP_MAC_CLOCK_READY 0x00000001u
  77. #define GP_INIT_DONE 0x00000004u
  78. #define GP_MAC_ACCESS_REQ 0x00000008u
  79. #define GP_GOING_TO_SLEEP 0x00000010u
  80. #define GP_HW_RF_KILL_SW 0x08000000u
  81. #define UCODE_SW_BIT_RFKILL 0x00000002u
  82. #define UCODE_CMD_BLOCKED 0x00000004u
  83. #define MBOX_OS_ALIVE (1u << 5)
  84. #define GIO_L0S_DISABLED 0x00000002u
  85. #define GIO_CHICKEN_L1A_NO_L0S_RX 0x00800000u
  86. #define GIO_CHICKEN_DIS_L0S_TIMER 0x20000000u
  87. #define LINK_PWR_MGMT_DISABLED 0x80000000u
  88. #define HBUS_TARG_MEM_WADDR 0x410
  89. #define HBUS_TARG_MEM_WDAT 0x418
  90. #define HBUS_TARG_PRPH_WADDR 0x444
  91. #define HBUS_TARG_PRPH_RADDR 0x448
  92. #define HBUS_TARG_PRPH_WDAT 0x44c
  93. #define HBUS_TARG_PRPH_RDAT 0x450
  94. #define HBUS_TARG_WRPTR 0x460
  95. #define FH_KW_MEM_ADDR_REG 0x197c
  96. #define FH_RSCSR_STTS_WPTR 0x1bc0
  97. #define FH_RSCSR_RBDCB_BASE 0x1bc4
  98. #define FH_RSCSR_RBDCB_WPTR 0x1bc8
  99. #define FH_RSCSR_RDPTR 0x1bcc
  100. #define FH_RCSR_CONFIG 0x1c00
  101. #define FH_RCSR_RBDCB_WPTR 0x1c08
  102. #define FH_RCSR_FLUSH_RB_REQ 0x1c10
  103. #define FH_TFDIB_CTRL0(c) (0x1900 + 8 * (c))
  104. #define FH_TFDIB_CTRL1(c) (0x1904 + 8 * (c))
  105. #define FH_TCSR_CONFIG(c) (0x1d00 + 0x20 * (c))
  106. #define FH_TCSR_BUF_STS(c) (0x1d08 + 0x20 * (c))
  107. #define FH_SRVC_CHNL 9
  108. #define FH_SRVC_SRAM_ADDR 0x19c8
  109. #define FH_TX_CHICKEN_BITS 0x1e98
  110. #define FH_TCSR_DMA_ENABLE 0x80000000u
  111. #define FH_TCSR_CREDIT_ENABLE 0x00000008u
  112. #define FH_TCSR_HOST_ENDTFD 0x00100000u
  113. #define FH_MEM_TB_MAX_LENGTH 0x20000u
  114. #define APMG_CLK_EN_REG 0x3004
  115. #define APMG_PS_CTRL_REG 0x300c
  116. #define APMG_PCIDEV_STT_REG 0x3010
  117. #define APMG_RTC_INT_STT_REG 0x301c
  118. #define APMG_CLK_DMA_RQT 0x00000200u
  119. #define APMG_PS_PWR_SRC_MSK 0x03000000u
  120. #define APMG_PCIDEV_L1_ACT_DIS 0x00000800u
  121. #define APMG_PCIDEV_WAKE_ME 0x00004000u
  122. #define APMG_RTC_INT_RFKILL 0x10000000u
  123. #define LMPM_CHICK 0xa01ff8
  124. #define LMPM_EXTENDED_ADDR 0x1u
  125. #define FW_MEM_EXT_START 0x40000u
  126. #define FW_MEM_EXT_END 0x57fffu
  127. #define SCD_BASE 0xa02c00
  128. #define SCD_SRAM_BASE_ADDR (SCD_BASE + 0x0)
  129. #define SCD_DRAM_BASE_ADDR (SCD_BASE + 0x8)
  130. #define SCD_TXFACT (SCD_BASE + 0x10)
  131. #define SCD_QUEUECHAIN_SEL (SCD_BASE + 0xe8)
  132. #define SCD_CHAINEXT_EN (SCD_BASE + 0x244)
  133. #define SCD_AGGR_SEL (SCD_BASE + 0x248)
  134. #define SCD_GP_CTRL (SCD_BASE + 0x1a8)
  135. #define SCD_EN_CTRL (SCD_BASE + 0x254)
  136. #define SCD_GP_AUTO_ACTIVE (1u << 18)
  137. #define SCD_GP_31_QUEUES (1u << 0)
  138. #define SCD_CONTEXT_MEM 0x600
  139. #define SCD_TRANS_TBL_MEM 0x7e0
  140. #define SCD_STTS_ACTIVE 3
  141. #define SCD_STTS_TXF 0
  142. #define SCD_STTS_WSL 4
  143. #define SCD_STTS_ACT_EN 19
  144. #define SCD_STTS_MSK 0x017f0000u
  145. static unsigned int scd_queue_rdptr(unsigned int q)
  146. {
  147. return q < 20 ? SCD_BASE + 0x68 + q * 4 : SCD_BASE + 0x2b4 + q * 4;
  148. }
  149. static unsigned int scd_queue_status(unsigned int q)
  150. {
  151. return q < 20 ? SCD_BASE + 0x10c + q * 4 : SCD_BASE + 0x334 + (q - 20) * 4;
  152. }
  153. static unsigned int fh_cbbc(unsigned int q)
  154. {
  155. if (q < 16)
  156. return 0x19d0 + 4 * q;
  157. if (q < 20)
  158. return 0x1bf0 + 4 * (q - 16);
  159. return 0x1b20 + 4 * (q - 20);
  160. }
  161. /* ---- device state ------------------------------------------------------ */
  162. #define RX_RING 256
  163. #define TFD_SIZE 128
  164. #define TFD_RING 256
  165. #define BC_ENTRIES 320 /* 256 + 64 duplicate */
  166. #define CMD_SLOTS 32
  167. #define CMD_BUF 4096 /* one page per command slot */
  168. #define DATA_SLOTS 32
  169. #define DATA_BUF 2048
  170. #define FIRST_TB 20 /* IWL_FIRST_TB_SIZE */
  171. struct txq {
  172. unsigned char *tfds; /* 256 x 128-byte TFDs */
  173. unsigned char *bufs; /* slots x bufsize */
  174. unsigned int nslots, bufsize;
  175. unsigned int wptr, rptr; /* 0..255 */
  176. int active;
  177. };
  178. struct iwl_state {
  179. volatile unsigned char *mmio;
  180. unsigned int bar, bus, dev, func, devid, hw_rev;
  181. const char *chip;
  182. unsigned int *rbd; /* RX descriptors (page) */
  183. unsigned char *rb[RX_RING]; /* RX buffers (4 KB pages) */
  184. volatile unsigned short *rb_stts;
  185. unsigned int rx_read;
  186. unsigned short *bc; /* byte-count tables, IWL_NUM_Q x 320 */
  187. unsigned char *kw; /* keep-warm page */
  188. unsigned char *fwbuf; /* 128 KB firmware chunk bounce buffer */
  189. struct txq q[IWL_NUM_Q];
  190. unsigned int scd_base;
  191. iwl_rx_handler handler;
  192. /* synchronous command bookkeeping */
  193. int waiting; /* a command is waiting for its response */
  194. unsigned short wait_seq;
  195. void *resp;
  196. unsigned int resp_max;
  197. int resp_len, resp_done;
  198. unsigned char wait_notif;
  199. int notif_seen;
  200. int fatal;
  201. struct iwl_fw_info fw;
  202. const unsigned char *fw_data;
  203. unsigned int fw_size;
  204. const struct iwl_family *fam;
  205. unsigned int sub; /* IWL_SUB_* within the family */
  206. void (*nic_config)(void); /* op-mode NIC config (DVM), may be 0 */
  207. /* v2 (pre-TLV) image: [IWL_IMG_*] instruction and data sections */
  208. const unsigned char *img_inst[2], *img_data[2];
  209. unsigned int img_inst_len[2], img_data_len[2];
  210. };
  211. static struct iwl_state *S; /* page-allocated on probe */
  212. typedef char iwl_state_fits_page[sizeof(struct iwl_state) <= 4096 ? 1 : -1];
  213. void iwl_log(const char *s)
  214. {
  215. serial_puts(SERIAL_COM1, "iwl: ");
  216. serial_puts(SERIAL_COM1, s);
  217. serial_puts(SERIAL_COM1, "\n");
  218. }
  219. void iwl_loghex(const char *s, unsigned int v)
  220. {
  221. serial_puts(SERIAL_COM1, "iwl: ");
  222. serial_puts(SERIAL_COM1, s);
  223. serial_puts(SERIAL_COM1, " ");
  224. serial_puthex(SERIAL_COM1, v);
  225. serial_puts(SERIAL_COM1, "\n");
  226. }
  227. /* Byte helpers: the kernel has no libc, and `rep` string ops keep gcc from
  228. * turning a loop back into a memcpy/memset call. */
  229. void iwl_memcpy(void *dst, const void *src, unsigned int n)
  230. {
  231. __asm__ volatile ("cld; rep movsb"
  232. : "+D"(dst), "+S"(src), "+c"(n) : : "memory");
  233. }
  234. void iwl_memzero(void *dst, unsigned int n)
  235. {
  236. __asm__ volatile ("cld; rep stosb"
  237. : "+D"(dst), "+c"(n) : "a"(0) : "memory");
  238. }
  239. /* ~1 us per port-0x80 write on every chipset; works with interrupts off
  240. * (some network syscalls run with IF=0, which freezes get_ticks_ms()). */
  241. void iwl_udelay(unsigned int us)
  242. {
  243. while (us--)
  244. port_outb(0x80, 0);
  245. }
  246. unsigned int iwl_rd(unsigned int reg)
  247. {
  248. return *(volatile unsigned int *)(S->mmio + reg);
  249. }
  250. void iwl_wr(unsigned int reg, unsigned int val)
  251. {
  252. *(volatile unsigned int *)(S->mmio + reg) = val;
  253. }
  254. static void iwl_set(unsigned int reg, unsigned int bits)
  255. {
  256. iwl_wr(reg, iwl_rd(reg) | bits);
  257. }
  258. static void iwl_clr(unsigned int reg, unsigned int bits)
  259. {
  260. iwl_wr(reg, iwl_rd(reg) & ~bits);
  261. }
  262. /* Poll until (reg & mask) == (bits & mask); timeout in microseconds. */
  263. static int iwl_poll(unsigned int reg, unsigned int bits, unsigned int mask,
  264. unsigned int us)
  265. {
  266. unsigned int t;
  267. for (t = 0; t < us; t += 10) {
  268. if ((iwl_rd(reg) & mask) == (bits & mask))
  269. return 0;
  270. iwl_udelay(10);
  271. }
  272. return -1;
  273. }
  274. unsigned int iwl_rd_prph(unsigned int addr)
  275. {
  276. iwl_wr(HBUS_TARG_PRPH_RADDR, (addr & 0x000fffffu) | (3u << 24));
  277. return iwl_rd(HBUS_TARG_PRPH_RDAT);
  278. }
  279. void iwl_wr_prph(unsigned int addr, unsigned int val)
  280. {
  281. iwl_wr(HBUS_TARG_PRPH_WADDR, (addr & 0x000fffffu) | (3u << 24));
  282. iwl_wr(HBUS_TARG_PRPH_WDAT, val);
  283. }
  284. static void prph_set(unsigned int addr, unsigned int bits)
  285. {
  286. iwl_wr_prph(addr, iwl_rd_prph(addr) | bits);
  287. }
  288. static void prph_clr(unsigned int addr, unsigned int bits)
  289. {
  290. iwl_wr_prph(addr, iwl_rd_prph(addr) & ~bits);
  291. }
  292. unsigned int iwl_mem_rd32(unsigned int addr)
  293. {
  294. iwl_wr(0x40c /* HBUS_TARG_MEM_RADDR */, addr);
  295. return iwl_rd(0x41c /* HBUS_TARG_MEM_RDAT */);
  296. }
  297. static void mem_wr32(unsigned int addr, unsigned int val)
  298. {
  299. iwl_wr(HBUS_TARG_MEM_WADDR, addr);
  300. iwl_wr(HBUS_TARG_MEM_WDAT, val);
  301. }
  302. static void mem_zero(unsigned int addr, unsigned int dwords)
  303. {
  304. iwl_wr(HBUS_TARG_MEM_WADDR, addr);
  305. while (dwords--)
  306. iwl_wr(HBUS_TARG_MEM_WDAT, 0);
  307. }
  308. /* Wake the MAC and keep it awake (MAC_ACCESS_REQ stays set). */
  309. static int grab_nic(void)
  310. {
  311. iwl_set(CSR_GP_CNTRL, GP_MAC_ACCESS_REQ);
  312. if (iwl_poll(CSR_GP_CNTRL, GP_MAC_CLOCK_READY,
  313. GP_MAC_CLOCK_READY | GP_GOING_TO_SLEEP, 15000) < 0) {
  314. iwl_loghex("nic access timeout, GP_CNTRL", iwl_rd(CSR_GP_CNTRL));
  315. return -1;
  316. }
  317. return 0;
  318. }
  319. /* ---- probe --------------------------------------------------------------- */
  320. static const char *chip_for(unsigned int devid,
  321. const struct iwl_family **fam, unsigned int *sub)
  322. {
  323. *fam = &fam_7000;
  324. *sub = IWL_SUB_7265;
  325. switch (devid) {
  326. case 0x095a: case 0x095b: return "Intel Wireless 7265";
  327. case 0x3165: case 0x3166: return "Intel Wireless 3165";
  328. }
  329. *fam = &fam_6000;
  330. switch (devid) {
  331. case 0x4239: case 0x422c:
  332. *sub = IWL_SUB_6000I;
  333. return "Intel Centrino Advanced-N 6200";
  334. case 0x4238: case 0x422b:
  335. *sub = IWL_SUB_6000;
  336. return "Intel Centrino Ultimate-N 6300";
  337. }
  338. return 0;
  339. }
  340. /* The firmware reported SW_ERR/HW_ERR: it is dead until reloaded. */
  341. int iwl_fatal(void)
  342. {
  343. return S && S->fatal;
  344. }
  345. int iwl_is_dvm(void)
  346. {
  347. return S && S->fam->dvm;
  348. }
  349. unsigned int iwl_subfamily(void)
  350. {
  351. return S ? S->sub : 0;
  352. }
  353. void iwl_set_nic_config(void (*fn)(void))
  354. {
  355. if (S)
  356. S->nic_config = fn;
  357. }
  358. int iwl_present(void)
  359. {
  360. return S != 0;
  361. }
  362. const char *iwl_chip_name(void)
  363. {
  364. return S ? S->chip : "none";
  365. }
  366. unsigned int iwl_hw_rev(void)
  367. {
  368. return S ? S->hw_rev : 0;
  369. }
  370. const struct iwl_fw_info *iwl_fw(void)
  371. {
  372. return S ? &S->fw : 0;
  373. }
  374. const unsigned char *iwl_fw_data(unsigned int *size)
  375. {
  376. if (!S)
  377. return 0;
  378. *size = S->fw_size;
  379. return S->fw_data;
  380. }
  381. /* Put the function in D0 (PCI PM capability), as pci_enable_device()
  382. * does; firmware can leave an unused WiFi card in D3hot. */
  383. static void pci_d0(unsigned int bus, unsigned int dev, unsigned int func)
  384. {
  385. unsigned int cap, n;
  386. if (!((pci_cfg_read32(bus, dev, func, 0x04) >> 16) & 0x10))
  387. return;
  388. cap = pci_cfg_read32(bus, dev, func, 0x34) & 0xfcu;
  389. for (n = 0; cap && n < 48; n++) {
  390. unsigned int hdr = pci_cfg_read32(bus, dev, func, cap);
  391. if ((hdr & 0xff) == 0x01) {
  392. unsigned int pmcsr = pci_cfg_read32(bus, dev, func, cap + 4);
  393. if (pmcsr & 3u) {
  394. pci_cfg_write32(bus, dev, func, cap + 4,
  395. pmcsr & ~3u);
  396. iwl_udelay(10000);
  397. iwl_log("PCI D3 -> D0");
  398. }
  399. return;
  400. }
  401. cap = (hdr >> 8) & 0xfcu;
  402. }
  403. }
  404. int iwl_probe(void)
  405. {
  406. unsigned int bus, dev, func;
  407. if (S)
  408. return 0;
  409. for (bus = 0; bus < 256; bus++) {
  410. for (dev = 0; dev < 32; dev++) {
  411. unsigned int id = pci_cfg_read32(bus, dev, 0, 0);
  412. unsigned int nfunc;
  413. if (id == 0xffffffffu || id == 0)
  414. continue;
  415. nfunc = (pci_cfg_read32(bus, dev, 0, 0x0c) & 0x800000u) ? 8 : 1;
  416. for (func = 0; func < nfunc; func++) {
  417. unsigned int idf = pci_cfg_read32(bus, dev, func, 0);
  418. unsigned int bar, cmd, page, sub;
  419. const struct iwl_family *fam;
  420. const char *name;
  421. if ((idf & 0xffffu) != 0x8086)
  422. continue;
  423. name = chip_for(idf >> 16, &fam, &sub);
  424. if (!name)
  425. continue;
  426. bar = pci_cfg_read32(bus, dev, func, 0x10);
  427. if (bar & 1u)
  428. continue;
  429. bar &= 0xfffffff0u;
  430. if (!bar) {
  431. iwl_log("BAR0 unassigned");
  432. continue;
  433. }
  434. page = mm_alloc_page();
  435. if (!page)
  436. return -1;
  437. S = (struct iwl_state *)page;
  438. iwl_memzero(S, 4096);
  439. S->bar = bar;
  440. S->bus = bus;
  441. S->dev = dev;
  442. S->func = func;
  443. S->devid = idf >> 16;
  444. S->chip = name;
  445. S->fam = fam;
  446. S->sub = sub;
  447. pci_d0(bus, dev, func);
  448. /* Memory + bus master, INTx off (polled). */
  449. cmd = pci_cfg_read32(bus, dev, func, 0x04);
  450. pci_cfg_write32(bus, dev, func, 0x04,
  451. (cmd & 0xffffu) | 0x0006u | 0x0400u);
  452. /* PCI_CFG_RETRY_TIMEOUT (0x41) = 0, like Linux. */
  453. pci_cfg_write8(bus, dev, func, 0x41, 0);
  454. /* The 8 KB CSR window (the ioremap window is
  455. * shared by every address space). */
  456. S->mmio = ioremap(bar, 0x2000, IOREMAP_MMIO);
  457. if (!S->mmio)
  458. return -1;
  459. iwl_wr(CSR_INT_MASK, 0);
  460. S->hw_rev = iwl_rd(CSR_HW_REV);
  461. serial_puts(SERIAL_COM1, "iwl: ");
  462. serial_puts(SERIAL_COM1, name);
  463. serial_puts(SERIAL_COM1, " dev ");
  464. serial_puthex(SERIAL_COM1, S->devid);
  465. serial_puts(SERIAL_COM1, " hw_rev ");
  466. serial_puthex(SERIAL_COM1, S->hw_rev);
  467. serial_puts(SERIAL_COM1, " bar ");
  468. serial_puthex(SERIAL_COM1, bar);
  469. serial_puts(SERIAL_COM1, "\n");
  470. if (S->hw_rev == 0xffffffffu) {
  471. iwl_log("HW_REV reads ~0: PCI issue");
  472. S = 0;
  473. return -1;
  474. }
  475. /* 0x095a/b: only rev-D silicon runs the 7265D
  476. * firmware (Linux: CSR_HW_REV_TYPE_7265D). */
  477. if (!fam->dvm &&
  478. S->devid != 0x3165 && S->devid != 0x3166 &&
  479. (S->hw_rev & 0xfff0u) != 0x210u)
  480. iwl_log("warning: not a 7265D; the 7265D firmware may fail");
  481. return 0;
  482. }
  483. }
  484. }
  485. return -1;
  486. }
  487. /* ---- firmware file ------------------------------------------------------- */
  488. #define TLV_INST 1
  489. #define TLV_FLAGS 18
  490. #define TLV_SEC_RT 19
  491. #define TLV_SEC_INIT 20
  492. #define TLV_DEF_CALIB 22
  493. #define TLV_PHY_SKU 23
  494. #define TLV_API_CHANGES 29
  495. #define TLV_ENABLED_CAPA 30
  496. static unsigned int le32(const unsigned char *p)
  497. {
  498. return p[0] | (p[1] << 8) | (p[2] << 16) | ((unsigned int)p[3] << 24);
  499. }
  500. /* Pre-TLV image (iwl-drv.c iwl_parse_v1_v2_firmware(), API >= 3 = v2):
  501. * ver, build, inst, data, init, init_data, boot sizes, then the sections
  502. * in that order. */
  503. static int fw_parse_v2(const unsigned char *d, unsigned int size)
  504. {
  505. unsigned int inst, data, init, initd, boot, o = 28;
  506. if (size < 28)
  507. return -1;
  508. S->fw.ver = le32(d);
  509. inst = le32(d + 8);
  510. data = le32(d + 12);
  511. init = le32(d + 16);
  512. initd = le32(d + 20);
  513. boot = le32(d + 24);
  514. if (((S->fw.ver >> 8) & 0xff) < 3 ||
  515. o + inst + data + init + initd + boot != size || !inst || !init) {
  516. iwl_log("firmware is not a v2 .ucode image");
  517. return -1;
  518. }
  519. S->img_inst[IWL_IMG_RT] = d + o;
  520. S->img_inst_len[IWL_IMG_RT] = inst;
  521. o += inst;
  522. S->img_data[IWL_IMG_RT] = d + o;
  523. S->img_data_len[IWL_IMG_RT] = data;
  524. o += data;
  525. S->img_inst[IWL_IMG_INIT] = d + o;
  526. S->img_inst_len[IWL_IMG_INIT] = init;
  527. o += init;
  528. S->img_data[IWL_IMG_INIT] = d + o;
  529. S->img_data_len[IWL_IMG_INIT] = initd;
  530. {
  531. static const char hx[] = "0123456789abcdef";
  532. unsigned int i, v = S->fw.ver;
  533. char *h = S->fw.human;
  534. /* "6000 v9.221.4.1", like Linux's "%u.%u.%u.%u" */
  535. h[0] = 'v';
  536. for (i = 0; i < 4; i++) {
  537. unsigned int b = (v >> (24 - 8 * i)) & 0xff;
  538. *++h = hx[b >> 4 & 0xf];
  539. *++h = hx[b & 0xf];
  540. if (i < 3)
  541. *++h = '.';
  542. }
  543. *++h = 0;
  544. }
  545. return 0;
  546. }
  547. int iwl_fw_parse(void)
  548. {
  549. const unsigned char *d;
  550. unsigned int size, off, i;
  551. if (!S)
  552. return -1;
  553. if (S->fw.present)
  554. return 0;
  555. d = ramfs_lookup(S->fam->fw_path, &size);
  556. if (!d) {
  557. iwl_log("firmware missing from /ramfs:");
  558. iwl_log(S->fam->fw_path);
  559. return -1;
  560. }
  561. if (!S->fam->tlv) {
  562. if (fw_parse_v2(d, size) < 0)
  563. return -1;
  564. S->fw_data = d;
  565. S->fw_size = size;
  566. S->fw.present = 1;
  567. iwl_log(S->fw.human);
  568. return 0;
  569. }
  570. if (size < 88 || le32(d) != 0 || le32(d + 4) != 0x0a4c5749u) {
  571. iwl_log("firmware is not a TLV image");
  572. return -1;
  573. }
  574. S->fw_data = d;
  575. S->fw_size = size;
  576. for (i = 0; i < sizeof(S->fw.human) - 1 && d[8 + i]; i++)
  577. S->fw.human[i] = (char)d[8 + i];
  578. S->fw.ver = le32(d + 72);
  579. off = 88;
  580. while (off + 8 <= size) {
  581. unsigned int t = le32(d + off), l = le32(d + off + 4);
  582. const unsigned char *p = d + off + 8;
  583. if (off + 8 + l > size)
  584. break;
  585. if (t == TLV_API_CHANGES && l >= 8 && le32(p) < 4)
  586. S->fw.api[le32(p)] = le32(p + 4);
  587. else if (t == TLV_ENABLED_CAPA && l >= 8 && le32(p) < 4)
  588. S->fw.capa[le32(p)] = le32(p + 4);
  589. else if (t == TLV_PHY_SKU && l >= 4)
  590. S->fw.phy_sku = le32(p);
  591. else if (t == TLV_DEF_CALIB && l >= 12) {
  592. unsigned int type = le32(p);
  593. /* enum iwl_ucode_type: 0 REGULAR, 1 INIT */
  594. if (type < 2) {
  595. S->fw.flow_trigger[type] = le32(p + 4);
  596. S->fw.event_trigger[type] = le32(p + 8);
  597. }
  598. }
  599. off += 8 + ((l + 3) & ~3u);
  600. }
  601. S->fw.present = 1;
  602. iwl_log(S->fw.human);
  603. return 0;
  604. }
  605. /* ---- hardware bring-up ------------------------------------------------- */
  606. int iwl_rfkill(void)
  607. {
  608. return S && !(iwl_rd(CSR_GP_CNTRL) & GP_HW_RF_KILL_SW);
  609. }
  610. static int set_hw_ready(void)
  611. {
  612. iwl_set(CSR_HW_IF_CONFIG_REG, HW_IF_NIC_READY);
  613. if (iwl_poll(CSR_HW_IF_CONFIG_REG, HW_IF_NIC_READY, HW_IF_NIC_READY, 50) < 0)
  614. return -1;
  615. iwl_set(CSR_MBOX_SET_REG, MBOX_OS_ALIVE);
  616. return 0;
  617. }
  618. static int prepare_card(void)
  619. {
  620. int iter, t;
  621. if (set_hw_ready() == 0)
  622. return 0;
  623. iwl_set(CSR_DBG_LINK_PWR_MGMT_REG, LINK_PWR_MGMT_DISABLED);
  624. iwl_udelay(1000);
  625. for (iter = 0; iter < 10; iter++) {
  626. iwl_set(CSR_HW_IF_CONFIG_REG, HW_IF_PREPARE);
  627. for (t = 0; t < 150000; t += 200) {
  628. if (set_hw_ready() == 0)
  629. return 0;
  630. iwl_udelay(200);
  631. }
  632. iwl_udelay(25000);
  633. }
  634. iwl_log("card not ready (owned by ME/AMT?)");
  635. return -1;
  636. }
  637. static int apm_init(void)
  638. {
  639. iwl_set(CSR_GIO_CHICKEN_BITS, GIO_CHICKEN_DIS_L0S_TIMER);
  640. iwl_set(CSR_GIO_CHICKEN_BITS, GIO_CHICKEN_L1A_NO_L0S_RX);
  641. iwl_set(CSR_DBG_HPET_MEM_REG, 0xffff0000u);
  642. iwl_set(CSR_HW_IF_CONFIG_REG, HW_IF_HAP_WAKE_L1A);
  643. iwl_set(CSR_GIO_REG, GIO_L0S_DISABLED);
  644. /* iwl_finish_nic_init(): INIT_DONE, then wait for the MAC clock. */
  645. iwl_set(CSR_GP_CNTRL, GP_INIT_DONE);
  646. if (iwl_poll(CSR_GP_CNTRL, GP_MAC_CLOCK_READY, GP_MAC_CLOCK_READY,
  647. 25000) < 0) {
  648. iwl_log("MAC clock not ready");
  649. return -1;
  650. }
  651. if (grab_nic() < 0)
  652. return -1;
  653. iwl_wr_prph(APMG_CLK_EN_REG, APMG_CLK_DMA_RQT);
  654. iwl_udelay(20);
  655. prph_set(APMG_PCIDEV_STT_REG, APMG_PCIDEV_L1_ACT_DIS);
  656. iwl_wr_prph(APMG_RTC_INT_STT_REG, APMG_RTC_INT_RFKILL);
  657. return 0;
  658. }
  659. static void stop_master(void)
  660. {
  661. iwl_set(CSR_RESET, RESET_STOP_MASTER);
  662. if (iwl_poll(CSR_RESET, RESET_MASTER_DISABLED, RESET_MASTER_DISABLED,
  663. 100) < 0)
  664. iwl_log("master disable timed out");
  665. }
  666. int iwl_start_hw(void)
  667. {
  668. if (!S)
  669. return -1;
  670. if (prepare_card() < 0)
  671. return -1;
  672. iwl_set(CSR_RESET, RESET_SW_RESET);
  673. iwl_udelay(6000);
  674. if (prepare_card() < 0)
  675. return -1;
  676. if (apm_init() < 0)
  677. return -1;
  678. iwl_wr(CSR_INT, 0xffffffffu);
  679. iwl_wr(CSR_FH_INT_STATUS, 0xffffffffu);
  680. return 0;
  681. }
  682. static void rx_stop(void)
  683. {
  684. iwl_wr(FH_RCSR_CONFIG, 0);
  685. iwl_poll(0x1c44 /* FH_MEM_RSSR_RX_STATUS_REG */, 0x01000000u,
  686. 0x01000000u, 1000);
  687. }
  688. void iwl_stop_device(void)
  689. {
  690. unsigned int c;
  691. if (!S || !S->mmio)
  692. return;
  693. iwl_wr(CSR_INT_MASK, 0);
  694. if (iwl_rd(CSR_GP_CNTRL) & GP_MAC_CLOCK_READY) {
  695. /* Stop the scheduler, every TX DMA channel and RX DMA. */
  696. iwl_wr_prph(SCD_TXFACT, 0);
  697. for (c = 0; c < 8; c++)
  698. iwl_wr(FH_TCSR_CONFIG(c), 0);
  699. rx_stop();
  700. iwl_wr_prph(APMG_CLK_EN_REG, APMG_CLK_DMA_RQT);
  701. /* Inform ME that we are leaving (7000 family). */
  702. if (S->fam->wake_me) {
  703. prph_set(APMG_PCIDEV_STT_REG, APMG_PCIDEV_WAKE_ME);
  704. iwl_udelay(5000);
  705. }
  706. }
  707. stop_master();
  708. iwl_set(CSR_RESET, RESET_SW_RESET);
  709. iwl_udelay(6000);
  710. iwl_clr(CSR_GP_CNTRL, GP_INIT_DONE | GP_MAC_ACCESS_REQ);
  711. iwl_wr(CSR_INT, 0xffffffffu);
  712. iwl_wr(CSR_FH_INT_STATUS, 0xffffffffu);
  713. for (c = 0; c < IWL_NUM_Q; c++)
  714. S->q[c].active = 0;
  715. }
  716. /* ---- ring allocation ---------------------------------------------------- */
  717. static void *alloc_zero(unsigned int pages)
  718. {
  719. unsigned int p = mm_alloc_pages(pages);
  720. if (p)
  721. iwl_memzero((void *)p, pages * 4096);
  722. return (void *)p;
  723. }
  724. static int alloc_txq(int qn, unsigned int slots, unsigned int bufsize)
  725. {
  726. struct txq *q = &S->q[qn];
  727. if (!q->tfds) {
  728. q->tfds = alloc_zero(TFD_RING * TFD_SIZE / 4096);
  729. q->bufs = alloc_zero((slots * bufsize + 4095) / 4096);
  730. if (!q->tfds || !q->bufs)
  731. return -1;
  732. q->nslots = slots;
  733. q->bufsize = bufsize;
  734. }
  735. iwl_memzero(q->tfds, TFD_RING * TFD_SIZE);
  736. q->wptr = q->rptr = 0;
  737. q->active = 0;
  738. return 0;
  739. }
  740. static int alloc_rings(void)
  741. {
  742. unsigned int i;
  743. if (!S->rbd) {
  744. S->rbd = alloc_zero(1);
  745. S->rb_stts = alloc_zero(1);
  746. S->bc = alloc_zero((IWL_NUM_Q * BC_ENTRIES * 2 + 4095) / 4096);
  747. S->kw = alloc_zero(1);
  748. S->fwbuf = alloc_zero(FH_MEM_TB_MAX_LENGTH / 4096);
  749. if (!S->rbd || !S->rb_stts || !S->bc || !S->kw || !S->fwbuf)
  750. return -1;
  751. for (i = 0; i < RX_RING; i++) {
  752. S->rb[i] = (unsigned char *)mm_alloc_page();
  753. if (!S->rb[i])
  754. return -1;
  755. }
  756. }
  757. if (alloc_txq(S->fam->cmd_q, CMD_SLOTS, CMD_BUF) < 0)
  758. return -1;
  759. for (i = 0; i < 4; i++)
  760. if (S->fam->q[i] != 0xff &&
  761. alloc_txq(S->fam->q[i], S->fam->q_slots[i], DATA_BUF) < 0)
  762. return -1;
  763. return 0;
  764. }
  765. static void rx_init(void)
  766. {
  767. unsigned int i;
  768. for (i = 0; i < RX_RING; i++) {
  769. *(volatile unsigned int *)S->rb[i] = 0x55550000u; /* FRAME_INVALID */
  770. S->rbd[i] = (unsigned int)S->rb[i] >> 8;
  771. }
  772. iwl_memzero((void *)S->rb_stts, 16);
  773. S->rx_read = 0;
  774. iwl_wr(FH_RCSR_CONFIG, 0);
  775. iwl_wr(FH_RCSR_RBDCB_WPTR, 0);
  776. iwl_wr(FH_RCSR_FLUSH_RB_REQ, 0);
  777. iwl_wr(FH_RSCSR_RDPTR, 0);
  778. iwl_wr(FH_RSCSR_RBDCB_WPTR, 0);
  779. iwl_wr(FH_RSCSR_RBDCB_BASE, (unsigned int)S->rbd >> 8);
  780. iwl_wr(FH_RSCSR_STTS_WPTR, (unsigned int)S->rb_stts >> 4);
  781. /* enable | ignore RXF empty | irq to host | 4K RBs | RB timeout 0x11 |
  782. * 256 RBDs (log 8). */
  783. iwl_wr(FH_RCSR_CONFIG, 0x80000000u | 0x4u | 0x1000u | 0 |
  784. (0x11u << 4) | (8u << 20));
  785. *(volatile unsigned char *)(S->mmio + CSR_INT_COALESCING) = 0x40;
  786. /* Hand every buffer but the last 8 to the hardware (write pointer
  787. * moves in multiples of 8). */
  788. iwl_wr(FH_RSCSR_RBDCB_WPTR, (RX_RING - 8) & ~7u);
  789. }
  790. static void tx_init(void)
  791. {
  792. unsigned int q;
  793. iwl_wr_prph(SCD_TXFACT, 0);
  794. iwl_wr(FH_KW_MEM_ADDR_REG, (unsigned int)S->kw >> 4);
  795. for (q = 0; q < IWL_NUM_Q; q++)
  796. if (S->q[q].tfds)
  797. iwl_wr(fh_cbbc(q), (unsigned int)S->q[q].tfds >> 8);
  798. prph_set(SCD_GP_CTRL, SCD_GP_AUTO_ACTIVE |
  799. (S->fam->nq > 20 ? SCD_GP_31_QUEUES : 0));
  800. }
  801. /* ---- firmware load ------------------------------------------------------ */
  802. static int load_chunk(unsigned int dst, const unsigned char *src,
  803. unsigned int n)
  804. {
  805. int ext = dst >= FW_MEM_EXT_START && dst <= FW_MEM_EXT_END;
  806. unsigned int t;
  807. iwl_memcpy(S->fwbuf, src, n);
  808. if (ext)
  809. prph_set(LMPM_CHICK, LMPM_EXTENDED_ADDR);
  810. iwl_wr(CSR_INT, INT_FH_TX);
  811. iwl_wr(CSR_FH_INT_STATUS, 0xffffffffu);
  812. iwl_wr(FH_TCSR_CONFIG(FH_SRVC_CHNL), 0);
  813. iwl_wr(FH_SRVC_SRAM_ADDR, dst);
  814. iwl_wr(FH_TFDIB_CTRL0(FH_SRVC_CHNL), (unsigned int)S->fwbuf);
  815. iwl_wr(FH_TFDIB_CTRL1(FH_SRVC_CHNL), n);
  816. iwl_wr(FH_TCSR_BUF_STS(FH_SRVC_CHNL), (1u << 20) | (1u << 12) | 3u);
  817. iwl_wr(FH_TCSR_CONFIG(FH_SRVC_CHNL),
  818. FH_TCSR_DMA_ENABLE | FH_TCSR_HOST_ENDTFD);
  819. for (t = 0; t < 5000000; t += 10) {
  820. if (iwl_rd(CSR_INT) & INT_FH_TX)
  821. break;
  822. iwl_udelay(10);
  823. }
  824. iwl_wr(CSR_INT, INT_FH_TX);
  825. iwl_wr(CSR_FH_INT_STATUS, 0xffffffffu);
  826. if (ext)
  827. prph_clr(LMPM_CHICK, LMPM_EXTENDED_ADDR);
  828. if (t >= 5000000) {
  829. iwl_loghex("firmware chunk timeout at", dst);
  830. return -1;
  831. }
  832. return 0;
  833. }
  834. static int load_section(unsigned int dst, const unsigned char *p,
  835. unsigned int len)
  836. {
  837. unsigned int o;
  838. for (o = 0; o < len; o += FH_MEM_TB_MAX_LENGTH) {
  839. unsigned int n = len - o;
  840. if (n > FH_MEM_TB_MAX_LENGTH)
  841. n = FH_MEM_TB_MAX_LENGTH;
  842. if (load_chunk(dst + o, p + o, n) < 0)
  843. return -1;
  844. }
  845. return 0;
  846. }
  847. static int load_sections(int img)
  848. {
  849. unsigned int want = img == IWL_IMG_INIT ? TLV_SEC_INIT : TLV_SEC_RT;
  850. const unsigned char *d = S->fw_data;
  851. unsigned int off = 88, nsec = 0;
  852. if (!S->fam->tlv) {
  853. /* IWLAGN_RTC_INST_LOWER_BOUND / IWLAGN_RTC_DATA_LOWER_BOUND */
  854. if (load_section(0x000000, S->img_inst[img],
  855. S->img_inst_len[img]) < 0 ||
  856. load_section(0x800000, S->img_data[img],
  857. S->img_data_len[img]) < 0)
  858. return -1;
  859. return 0;
  860. }
  861. while (off + 8 <= S->fw_size) {
  862. unsigned int t = le32(d + off), l = le32(d + off + 4);
  863. const unsigned char *p = d + off + 8;
  864. if (t == want && l > 4) {
  865. unsigned int dst = le32(p), len = l - 4, o;
  866. /* CPU1_CPU2 / paging separators end the CPU1 list
  867. * (single-CPU image: nothing follows that we load). */
  868. if (dst == 0xffffccccu || dst == 0xaaaabbbbu)
  869. break;
  870. for (o = 0; o < len; o += FH_MEM_TB_MAX_LENGTH) {
  871. unsigned int n = len - o;
  872. if (n > FH_MEM_TB_MAX_LENGTH)
  873. n = FH_MEM_TB_MAX_LENGTH;
  874. if (load_chunk(dst + o, p + 4 + o, n) < 0)
  875. return -1;
  876. }
  877. nsec++;
  878. }
  879. off += 8 + ((l + 3) & ~3u);
  880. }
  881. return nsec ? 0 : -1;
  882. }
  883. int iwl_load_fw(int img)
  884. {
  885. if (!S || !S->fw.present)
  886. return -1;
  887. if (prepare_card() < 0)
  888. return -1;
  889. iwl_wr(CSR_INT, 0xffffffffu);
  890. iwl_wr(CSR_UCODE_DRV_GP1_CLR, UCODE_SW_BIT_RFKILL);
  891. iwl_wr(CSR_UCODE_DRV_GP1_CLR, UCODE_CMD_BLOCKED);
  892. iwl_wr(CSR_INT, 0xffffffffu);
  893. /* iwl_pcie_nic_init(): APM, power source VMAIN, NIC config, rings. */
  894. if (apm_init() < 0)
  895. return -1;
  896. iwl_wr_prph(APMG_PS_CTRL_REG,
  897. iwl_rd_prph(APMG_PS_CTRL_REG) & ~APMG_PS_PWR_SRC_MSK);
  898. if (alloc_rings() < 0) {
  899. iwl_log("out of memory for rings");
  900. return -1;
  901. }
  902. S->fatal = 0;
  903. S->waiting = 0;
  904. /* iwl_op_mode_nic_config(): radio SKU bits (DVM) before the rings. */
  905. if (S->nic_config)
  906. S->nic_config();
  907. rx_init();
  908. tx_init();
  909. if (S->fam->shadow_reg)
  910. iwl_wr(CSR_MAC_SHADOW_REG_CTRL,
  911. iwl_rd(CSR_MAC_SHADOW_REG_CTRL) | 0x800fffffu);
  912. iwl_wr(CSR_UCODE_DRV_GP1_CLR, UCODE_SW_BIT_RFKILL);
  913. if (load_sections(img) < 0) {
  914. iwl_log("firmware section load failed");
  915. return -1;
  916. }
  917. iwl_wr(CSR_INT, 0xffffffffu);
  918. iwl_wr(CSR_RESET, 0); /* release the CPU */
  919. return 0;
  920. }
  921. /* After ALIVE: scheduler context, byte-count tables, cmd queue on FIFO 7,
  922. * TX DMA channels (iwl_pcie_tx_start()). */
  923. /* iwl_trans_pcie_txq_enable() with a cfg (the transport programs the
  924. * scheduler: FIFO, window, frame limit), as iwl_trans_ac_txq_enable()
  925. * does for the command queue and for DVM's AC queues. */
  926. static void scd_enable_queue(unsigned int qn, unsigned int fifo)
  927. {
  928. struct txq *q = &S->q[qn];
  929. int cmdq = qn == S->fam->cmd_q;
  930. if (cmdq && S->fam->scd_set_active)
  931. iwl_wr_prph(SCD_EN_CTRL, 0);
  932. iwl_wr_prph(scd_queue_status(qn), 1u << SCD_STTS_ACT_EN);
  933. if (!cmdq)
  934. prph_set(SCD_QUEUECHAIN_SEL, 1u << qn);
  935. prph_clr(SCD_AGGR_SEL, 1u << qn);
  936. q->wptr = q->rptr = 0;
  937. iwl_wr(HBUS_TARG_WRPTR, 0 | (qn << 8));
  938. iwl_wr_prph(scd_queue_rdptr(qn), 0);
  939. mem_wr32(S->scd_base + SCD_CONTEXT_MEM + qn * 8, 0);
  940. mem_wr32(S->scd_base + SCD_CONTEXT_MEM + qn * 8 + 4,
  941. (64u & 0x7f) | ((64u << 16) & 0x7f0000u));
  942. iwl_wr_prph(scd_queue_status(qn),
  943. (1u << SCD_STTS_ACTIVE) | (fifo << SCD_STTS_TXF) |
  944. (1u << SCD_STTS_WSL) | SCD_STTS_MSK);
  945. if (cmdq && S->fam->scd_set_active)
  946. iwl_wr_prph(SCD_EN_CTRL, 1u << qn);
  947. q->active = 1;
  948. }
  949. void iwl_txq_enable_fifo(int qn, unsigned int fifo)
  950. {
  951. if (S && qn >= 0 && qn < IWL_NUM_Q && S->q[qn].tfds)
  952. scd_enable_queue((unsigned int)qn, fifo);
  953. }
  954. void iwl_fw_alive(unsigned int scd_base)
  955. {
  956. unsigned int c;
  957. S->scd_base = iwl_rd_prph(SCD_SRAM_BASE_ADDR);
  958. if (scd_base && scd_base != S->scd_base)
  959. iwl_loghex("scd base mismatch, alive says", scd_base);
  960. mem_zero(S->scd_base + SCD_CONTEXT_MEM,
  961. (((SCD_TRANS_TBL_MEM + S->fam->nq * 2) & 0xfffcu) -
  962. SCD_CONTEXT_MEM) / 4);
  963. iwl_wr_prph(SCD_DRAM_BASE_ADDR, (unsigned int)S->bc >> 10);
  964. /* The SCD chain extension misbehaves on the 6000 family. */
  965. if (S->fam->chain_ext_wa)
  966. iwl_wr_prph(SCD_CHAINEXT_EN, 0);
  967. scd_enable_queue(S->fam->cmd_q, S->fam->cmd_fifo);
  968. iwl_wr_prph(SCD_TXFACT, 0xffu); /* FIFOs 0..7 */
  969. for (c = 0; c < 8; c++)
  970. iwl_wr(FH_TCSR_CONFIG(c),
  971. FH_TCSR_DMA_ENABLE | FH_TCSR_CREDIT_ENABLE);
  972. iwl_wr(FH_TX_CHICKEN_BITS, iwl_rd(FH_TX_CHICKEN_BITS) | 0x2u);
  973. prph_clr(APMG_PCIDEV_STT_REG, APMG_PCIDEV_L1_ACT_DIS);
  974. }
  975. /* DQA data/mgmt queue (iwl_trans_pcie_txq_enable() with cfg == NULL: the
  976. * firmware's SCD_QUEUE_CFG command configures the scheduler itself). */
  977. void iwl_txq_enable(int qn, unsigned short ssn)
  978. {
  979. struct txq *q = &S->q[qn];
  980. /* SCD write-pointer quirk: never move by a multiple of 0x40. */
  981. if (!((ssn - q->wptr) & 0x3f) && ssn != q->wptr)
  982. ssn++;
  983. q->rptr = q->wptr = ssn & 0xff;
  984. iwl_wr(HBUS_TARG_WRPTR, (ssn & 0xff) | ((unsigned int)qn << 8));
  985. q->active = 1;
  986. }
  987. unsigned int iwl_txq_wptr(int q)
  988. {
  989. return S->q[q].wptr;
  990. }
  991. /* ---- TX ---------------------------------------------------------------- */
  992. static void tfd_set(unsigned char *tfd, unsigned int tb, unsigned int addr,
  993. unsigned int len)
  994. {
  995. unsigned char *e = tfd + 4 + tb * 6;
  996. e[0] = addr; e[1] = addr >> 8; e[2] = addr >> 16; e[3] = addr >> 24;
  997. e[4] = (len << 4) & 0xf0;
  998. e[5] = len >> 4;
  999. tfd[3] = (unsigned char)(tb + 1);
  1000. }
  1001. static void bc_set(int qn, unsigned int idx, unsigned int bytes,
  1002. unsigned int sta)
  1003. {
  1004. unsigned short *t = S->bc + qn * BC_ENTRIES;
  1005. unsigned short v;
  1006. /* + CRC + delimiter; in dwords with bc_table_dword (7000), else
  1007. * bytes (iwl_txq_gen1_update_byte_cnt_tbl()). */
  1008. bytes += 8;
  1009. if (S->fam->bc_dword)
  1010. bytes = (bytes + 3) / 4;
  1011. v = (unsigned short)(bytes | (sta << 12));
  1012. t[idx] = v;
  1013. if (idx < BC_ENTRIES - TFD_RING)
  1014. t[TFD_RING + idx] = v;
  1015. }
  1016. static int txq_space(struct txq *q)
  1017. {
  1018. return ((q->wptr - q->rptr) & 0xff) < q->nslots - 1;
  1019. }
  1020. static unsigned char *slot_buf(struct txq *q, unsigned int idx)
  1021. {
  1022. return q->bufs + (idx & (q->nslots - 1)) * q->bufsize;
  1023. }
  1024. static void ring_doorbell(int qn)
  1025. {
  1026. struct txq *q = &S->q[qn];
  1027. q->wptr = (q->wptr + 1) & 0xff;
  1028. iwl_wr(HBUS_TARG_WRPTR, q->wptr | ((unsigned int)qn << 8));
  1029. }
  1030. static int enqueue_cmd(unsigned int id, const void *data, unsigned int len,
  1031. unsigned short *seq_out)
  1032. {
  1033. unsigned int cq = S->fam->cmd_q;
  1034. struct txq *q = &S->q[cq];
  1035. /* mvm sets wide_cmd_header: legacy (group 0) commands are sent as
  1036. * group 1 with the 8-byte header (iwl_trans_send_cmd(), DEF_ID).
  1037. * DVM uses the plain 4-byte iwl_cmd_header. */
  1038. unsigned char group = (id >> 8) & 0xff ? (id >> 8) & 0xff : 1;
  1039. unsigned char *buf, *tfd;
  1040. unsigned int hdr = S->fam->wide_hdr ? 8 : 4, total = hdr + len, idx;
  1041. unsigned short seq;
  1042. if (!q->active || total > CMD_BUF)
  1043. return -1;
  1044. if (!txq_space(q)) {
  1045. /* Command responses reclaim; every one we sent was synchronous,
  1046. * so the ring can simply be considered drained. */
  1047. q->rptr = q->wptr;
  1048. }
  1049. idx = q->wptr;
  1050. buf = slot_buf(q, idx);
  1051. seq = (unsigned short)((cq << 8) | idx);
  1052. buf[0] = id & 0xff;
  1053. buf[1] = S->fam->wide_hdr ? group : 0;
  1054. buf[2] = seq & 0xff;
  1055. buf[3] = seq >> 8;
  1056. if (S->fam->wide_hdr) {
  1057. buf[4] = len & 0xff;
  1058. buf[5] = len >> 8;
  1059. buf[6] = 0;
  1060. buf[7] = (id >> 16) & 0xff;
  1061. }
  1062. iwl_memcpy(buf + hdr, data, len);
  1063. tfd = q->tfds + idx * TFD_SIZE;
  1064. iwl_memzero(tfd, TFD_SIZE);
  1065. if (total <= FIRST_TB) {
  1066. tfd_set(tfd, 0, (unsigned int)buf, total);
  1067. } else {
  1068. tfd_set(tfd, 0, (unsigned int)buf, FIRST_TB);
  1069. tfd_set(tfd, 1, (unsigned int)buf + FIRST_TB, total - FIRST_TB);
  1070. }
  1071. *seq_out = seq;
  1072. ring_doorbell(cq);
  1073. return 0;
  1074. }
  1075. int iwl_tx(int qn, const void *txcmd, unsigned int txlen,
  1076. const void *frame, unsigned int flen, unsigned char sta_id,
  1077. unsigned int mic_len)
  1078. {
  1079. struct txq *q = &S->q[qn];
  1080. unsigned char *buf, *tfd;
  1081. unsigned int idx, total, t;
  1082. if (!q->active || 4 + txlen + flen > q->bufsize)
  1083. return -1;
  1084. for (t = 0; !txq_space(q); t++) {
  1085. if (t > 2000 || iwl_rx_poll() < 0)
  1086. return -1;
  1087. iwl_udelay(50);
  1088. }
  1089. idx = q->wptr;
  1090. buf = slot_buf(q, idx);
  1091. /* struct iwl_device_tx_cmd: 4-byte header (TX_CMD, seq) + tx_cmd. */
  1092. buf[0] = 0x1c; /* TX_CMD */
  1093. buf[1] = 0;
  1094. buf[2] = idx;
  1095. buf[3] = (unsigned char)qn;
  1096. iwl_memcpy(buf + 4, txcmd, txlen);
  1097. iwl_memcpy(buf + 4 + txlen, frame, flen);
  1098. /* dram_lsb_ptr/msb: where the firmware writes the tx_cmd scratch
  1099. * (offset 8 of struct iwl_tx_cmd), as iwl_pcie_tx() does. */
  1100. if (txlen >= 56) {
  1101. unsigned int scratch = (unsigned int)buf + 4 + 8;
  1102. buf[4 + 44] = scratch;
  1103. buf[4 + 45] = scratch >> 8;
  1104. buf[4 + 46] = scratch >> 16;
  1105. buf[4 + 47] = scratch >> 24;
  1106. buf[4 + 48] = 0;
  1107. }
  1108. total = 4 + txlen + flen;
  1109. tfd = q->tfds + idx * TFD_SIZE;
  1110. iwl_memzero(tfd, TFD_SIZE);
  1111. tfd_set(tfd, 0, (unsigned int)buf, FIRST_TB);
  1112. tfd_set(tfd, 1, (unsigned int)buf + FIRST_TB, total - FIRST_TB);
  1113. /* Byte count: the over-the-air MPDU (tx_cmd->len) plus the MIC the
  1114. * firmware appends for a hardware-encrypted frame. */
  1115. bc_set(qn, idx, flen + mic_len, sta_id);
  1116. ring_doorbell(qn);
  1117. return 0;
  1118. }
  1119. void iwl_tx_reclaim(int qn, unsigned int idx)
  1120. {
  1121. if (qn >= 0 && qn < IWL_NUM_Q)
  1122. S->q[qn].rptr = (idx + 1) & 0xff;
  1123. }
  1124. /* ---- RX ---------------------------------------------------------------- */
  1125. void iwl_set_rx_handler(iwl_rx_handler h)
  1126. {
  1127. S->handler = h;
  1128. }
  1129. static void handle_pkt(const struct iwl_pkt *pkt)
  1130. {
  1131. if (S->waiting && !(pkt->seq & 0x8000u) && pkt->seq == S->wait_seq) {
  1132. unsigned int n = iwl_pkt_payload_len(pkt);
  1133. if (n > S->resp_max)
  1134. n = S->resp_max;
  1135. if (S->resp && n)
  1136. iwl_memcpy(S->resp, pkt->data, n);
  1137. S->resp_len = (int)iwl_pkt_payload_len(pkt);
  1138. S->resp_done = 1;
  1139. return;
  1140. }
  1141. if (S->wait_notif && pkt->group == 0 && pkt->cmd == S->wait_notif)
  1142. S->notif_seen = 1;
  1143. if (S->handler)
  1144. S->handler(pkt);
  1145. }
  1146. int iwl_rx_poll(void)
  1147. {
  1148. unsigned int r, n = 0, ints;
  1149. if (!S || !S->rbd)
  1150. return 0;
  1151. ints = iwl_rd(CSR_INT);
  1152. if (ints & (INT_SW_ERR | INT_HW_ERR)) {
  1153. if (!S->fatal) {
  1154. iwl_loghex("firmware/hardware error, CSR_INT", ints);
  1155. iwl_loghex("GP_CNTRL", iwl_rd(CSR_GP_CNTRL));
  1156. }
  1157. S->fatal = 1;
  1158. iwl_wr(CSR_INT, ints);
  1159. return -1;
  1160. }
  1161. if (ints)
  1162. iwl_wr(CSR_INT, ints);
  1163. if (iwl_rd(CSR_FH_INT_STATUS))
  1164. iwl_wr(CSR_FH_INT_STATUS, 0xffffffffu);
  1165. r = S->rb_stts[0] & 0x0fff;
  1166. r &= RX_RING - 1;
  1167. while (S->rx_read != r) {
  1168. unsigned char *page = S->rb[S->rx_read];
  1169. unsigned int off = 0;
  1170. while (off + 12 < 4096) {
  1171. const struct iwl_pkt *pkt =
  1172. (const struct iwl_pkt *)(page + off);
  1173. unsigned int len;
  1174. if (pkt->len_n_flags == 0x55550000u)
  1175. break;
  1176. len = (pkt->len_n_flags & 0x3fffu) + 4;
  1177. if (len < 8 || off + len > 4096)
  1178. break;
  1179. handle_pkt(pkt);
  1180. off += (len + 63) & ~63u;
  1181. }
  1182. *(volatile unsigned int *)page = 0x55550000u;
  1183. S->rx_read = (S->rx_read + 1) & (RX_RING - 1);
  1184. n++;
  1185. /* Give the hardware the slot 8 behind us (multiple of 8). */
  1186. if ((S->rx_read & 7) == 0)
  1187. iwl_wr(FH_RSCSR_RBDCB_WPTR,
  1188. (S->rx_read + RX_RING - 8) & (RX_RING - 1));
  1189. }
  1190. return S->fatal ? -1 : (int)n;
  1191. }
  1192. int iwl_wait_notif(unsigned char cmd, unsigned int ms)
  1193. {
  1194. unsigned int t;
  1195. S->wait_notif = cmd;
  1196. S->notif_seen = 0;
  1197. for (t = 0; t < ms * 10; t++) {
  1198. if (iwl_rx_poll() < 0)
  1199. break;
  1200. if (S->notif_seen)
  1201. break;
  1202. iwl_udelay(100);
  1203. }
  1204. S->wait_notif = 0;
  1205. return S->notif_seen ? 0 : -1;
  1206. }
  1207. int iwl_send_cmd(unsigned int id, const void *data, unsigned int len,
  1208. void *resp, unsigned int resp_max)
  1209. {
  1210. unsigned short seq;
  1211. unsigned int t;
  1212. if (!S || S->fatal)
  1213. return -1;
  1214. S->resp = resp;
  1215. S->resp_max = resp_max;
  1216. S->resp_done = 0;
  1217. S->resp_len = -1;
  1218. if (enqueue_cmd(id, data, len, &seq) < 0)
  1219. return -1;
  1220. S->wait_seq = seq;
  1221. S->waiting = 1;
  1222. for (t = 0; t < 20000 && !S->resp_done; t++) { /* 2 s */
  1223. if (iwl_rx_poll() < 0)
  1224. break;
  1225. if (!S->resp_done)
  1226. iwl_udelay(100);
  1227. }
  1228. S->waiting = 0;
  1229. S->q[S->fam->cmd_q].rptr = S->q[S->fam->cmd_q].wptr;
  1230. if (!S->resp_done) {
  1231. iwl_loghex("command timeout, id", id);
  1232. return -1;
  1233. }
  1234. return S->resp_len;
  1235. }
  1236. int iwl_send_cmd_async(unsigned int id, const void *data, unsigned int len)
  1237. {
  1238. unsigned short seq;
  1239. if (!S || S->fatal)
  1240. return -1;
  1241. return enqueue_cmd(id, data, len, &seq);
  1242. }