FreeBSD kernel usb device Code
xhci.c
Go to the documentation of this file.
1/* $FreeBSD$ */
2/*-
3 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 *
5 * Copyright (c) 2010-2022 Hans Petter Selasky
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29/*
30 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
31 *
32 * The XHCI 1.0 spec can be found at
33 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
34 * and the USB 3.0 spec at
35 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
36 */
37
38/*
39 * A few words about the design implementation: This driver emulates
40 * the concept about TDs which is found in EHCI specification. This
41 * way we achieve that the USB controller drivers look similar to
42 * eachother which makes it easier to understand the code.
43 */
44
45#ifdef USB_GLOBAL_INCLUDE_FILE
46#include USB_GLOBAL_INCLUDE_FILE
47#else
48#include <sys/stdint.h>
49#include <sys/stddef.h>
50#include <sys/param.h>
51#include <sys/queue.h>
52#include <sys/types.h>
53#include <sys/systm.h>
54#include <sys/kernel.h>
55#include <sys/bus.h>
56#include <sys/module.h>
57#include <sys/lock.h>
58#include <sys/mutex.h>
59#include <sys/condvar.h>
60#include <sys/sysctl.h>
61#include <sys/sx.h>
62#include <sys/unistd.h>
63#include <sys/callout.h>
64#include <sys/malloc.h>
65#include <sys/priv.h>
66
67#include <dev/usb/usb.h>
68#include <dev/usb/usbdi.h>
69
70#define USB_DEBUG_VAR xhcidebug
71
72#include <dev/usb/usb_core.h>
73#include <dev/usb/usb_debug.h>
74#include <dev/usb/usb_busdma.h>
75#include <dev/usb/usb_process.h>
77#include <dev/usb/usb_device.h>
78#include <dev/usb/usb_hub.h>
79#include <dev/usb/usb_util.h>
80
82#include <dev/usb/usb_bus.h>
83#endif /* USB_GLOBAL_INCLUDE_FILE */
84
87
88#define XHCI_BUS2SC(bus) \
89 __containerof(bus, struct xhci_softc, sc_bus)
90
91#define XHCI_GET_CTX(sc, which, field, ptr) \
92 ((sc)->sc_ctx_is_64_byte ? \
93 &((struct which##64 *)(ptr))->field.ctx : \
94 &((struct which *)(ptr))->field)
95
96static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
97 "USB XHCI");
98
99static int xhcistreams;
100SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RWTUN,
101 &xhcistreams, 0, "Set to enable streams mode support");
102
103static int xhcictlquirk = 1;
104SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlquirk, CTLFLAG_RWTUN,
105 &xhcictlquirk, 0, "Set to enable control endpoint quirk");
106
107static int xhcidcepquirk;
108SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dcepquirk, CTLFLAG_RWTUN,
109 &xhcidcepquirk, 0, "Set to disable endpoint deconfigure command");
110
111#ifdef USB_DEBUG
112static int xhcidebug;
113static int xhciroute;
114static int xhcipolling;
115static int xhcidma32;
116static int xhcictlstep;
117
118SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RWTUN,
119 &xhcidebug, 0, "Debug level");
120SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RWTUN,
121 &xhciroute, 0, "Routing bitmap for switching EHCI ports to the XHCI controller");
122SYSCTL_INT(_hw_usb_xhci, OID_AUTO, use_polling, CTLFLAG_RWTUN,
123 &xhcipolling, 0, "Set to enable software interrupt polling for the XHCI controller");
124SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dma32, CTLFLAG_RWTUN,
125 &xhcidma32, 0, "Set to only use 32-bit DMA for the XHCI controller");
126SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlstep, CTLFLAG_RWTUN,
127 &xhcictlstep, 0, "Set to enable control endpoint status stage stepping");
128#else
129#define xhciroute 0
130#define xhcidma32 0
131#define xhcictlstep 0
132#endif
133
134#define XHCI_INTR_ENDPT 1
135
137 struct xhci_softc *sc;
139 struct xhci_td *td;
141 uint32_t len;
142 uint32_t offset;
144 uint32_t average;
145 uint32_t isoc_frame;
146 uint16_t isoc_delta;
147 uint8_t shortpkt;
148 uint8_t multishort;
149 uint8_t last_frame;
150 uint8_t trb_type;
151 uint8_t direction;
152 uint8_t tbc;
153 uint8_t tlbpc;
154 uint8_t step_td;
156};
157
158static void xhci_do_poll(struct usb_bus *);
159static void xhci_device_done(struct usb_xfer *, usb_error_t);
160static void xhci_root_intr(struct xhci_softc *);
161static void xhci_free_device_ext(struct usb_device *);
163 struct usb_endpoint_descriptor *);
168 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
169 uint8_t);
171 uint32_t, uint8_t);
173 uint64_t, uint8_t);
174static void xhci_endpoint_doorbell(struct usb_xfer *);
175
177
178#ifdef USB_DEBUG
179static void
180xhci_dump_trb(struct xhci_trb *trb)
181{
182 DPRINTFN(5, "trb = %p\n", trb);
183 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
184 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
185 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
186}
187
188static void
189xhci_dump_endpoint(struct xhci_endp_ctx *pep)
190{
191 DPRINTFN(5, "pep = %p\n", pep);
192 DPRINTFN(5, "dwEpCtx0=0x%08x\n", le32toh(pep->dwEpCtx0));
193 DPRINTFN(5, "dwEpCtx1=0x%08x\n", le32toh(pep->dwEpCtx1));
194 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)le64toh(pep->qwEpCtx2));
195 DPRINTFN(5, "dwEpCtx4=0x%08x\n", le32toh(pep->dwEpCtx4));
196 DPRINTFN(5, "dwEpCtx5=0x%08x\n", le32toh(pep->dwEpCtx5));
197 DPRINTFN(5, "dwEpCtx6=0x%08x\n", le32toh(pep->dwEpCtx6));
198 DPRINTFN(5, "dwEpCtx7=0x%08x\n", le32toh(pep->dwEpCtx7));
199}
200
201static void
202xhci_dump_device(struct xhci_slot_ctx *psl)
203{
204 DPRINTFN(5, "psl = %p\n", psl);
205 DPRINTFN(5, "dwSctx0=0x%08x\n", le32toh(psl->dwSctx0));
206 DPRINTFN(5, "dwSctx1=0x%08x\n", le32toh(psl->dwSctx1));
207 DPRINTFN(5, "dwSctx2=0x%08x\n", le32toh(psl->dwSctx2));
208 DPRINTFN(5, "dwSctx3=0x%08x\n", le32toh(psl->dwSctx3));
209}
210#endif
211
212uint8_t
214{
215#ifdef USB_DEBUG
216 return (xhcipolling != 0);
217#else
218 return (0);
219#endif
220}
221
222static void
224{
225 struct xhci_softc *sc = XHCI_BUS2SC(bus);
226 uint16_t i;
227
228 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
229 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
230
231 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
232 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
233
234 for (i = 0; i != sc->sc_noscratch; i++) {
235 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
237 }
238}
239
240static int
242{
243 struct usb_page_search buf_res;
244 struct xhci_hw_root *phwr;
245 uint64_t addr;
246 uint32_t temp;
247
248 DPRINTF("\n");
249
250 temp = XREAD4(sc, oper, XHCI_CRCR_LO);
251 if (temp & XHCI_CRCR_LO_CRR) {
252 DPRINTF("Command ring running\n");
253 temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA);
254
255 /*
256 * Try to abort the last command as per section
257 * 4.6.1.2 "Aborting a Command" of the XHCI
258 * specification:
259 */
260
261 /* stop and cancel */
262 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS);
263 XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
264
265 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA);
266 XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
267
268 /* wait 250ms */
269 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 4);
270
271 /* check if command ring is still running */
272 temp = XREAD4(sc, oper, XHCI_CRCR_LO);
273 if (temp & XHCI_CRCR_LO_CRR) {
274 DPRINTF("Comand ring still running\n");
275 return (USB_ERR_IOERROR);
276 }
277 }
278
279 /* reset command ring */
280 sc->sc_command_ccs = 1;
281 sc->sc_command_idx = 0;
282
283 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
284
285 /* set up command ring control base address */
286 addr = buf_res.physaddr;
287 phwr = buf_res.buffer;
288 addr += __offsetof(struct xhci_hw_root, hwr_commands[0]);
289
290 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
291
292 memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands));
293 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
294
296
297 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
298 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
299
300 return (0);
301}
302
305{
306 struct usb_page_search buf_res;
307 struct xhci_hw_root *phwr;
308 struct xhci_dev_ctx_addr *pdctxa;
309 usb_error_t err;
310 uint64_t addr;
311 uint32_t temp;
312 uint16_t i;
313
314 DPRINTF("\n");
315
316 sc->sc_event_ccs = 1;
317 sc->sc_event_idx = 0;
318 sc->sc_command_ccs = 1;
319 sc->sc_command_idx = 0;
320
321 err = xhci_reset_controller(sc);
322 if (err)
323 return (err);
324
325 /* set up number of device slots */
326 DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
327 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
328
329 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
330
331 temp = XREAD4(sc, oper, XHCI_USBSTS);
332
333 /* clear interrupts */
334 XWRITE4(sc, oper, XHCI_USBSTS, temp);
335 /* disable all device notifications */
336 XWRITE4(sc, oper, XHCI_DNCTRL, 0);
337
338 /* set up device context base address */
339 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
340 pdctxa = buf_res.buffer;
341 memset(pdctxa, 0, sizeof(*pdctxa));
342
343 addr = buf_res.physaddr;
344 addr += __offsetof(struct xhci_dev_ctx_addr, qwSpBufPtr[0]);
345
346 /* slot 0 points to the table of scratchpad pointers */
347 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
348
349 for (i = 0; i != sc->sc_noscratch; i++) {
350 struct usb_page_search buf_scp;
351 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
352 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
353 }
354
355 addr = buf_res.physaddr;
356
357 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
358 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
359 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
360 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
361
362 /* set up event table size */
363 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
364 XREAD4(sc, runt, XHCI_ERSTSZ(0)), sc->sc_erst_max);
365
366 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(sc->sc_erst_max));
367
368 /* set up interrupt rate */
369 XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default);
370
371 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
372
373 phwr = buf_res.buffer;
374 addr = buf_res.physaddr;
375 addr += __offsetof(struct xhci_hw_root, hwr_events[0]);
376
377 /* reset hardware root structure */
378 memset(phwr, 0, sizeof(*phwr));
379
380 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
381 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
382
383 /*
384 * PR 237666:
385 *
386 * According to the XHCI specification, the XWRITE4's to
387 * XHCI_ERSTBA_LO and _HI lead to the XHCI to copy the
388 * qwEvrsTablePtr and dwEvrsTableSize values above at that
389 * time, as the XHCI initializes its event ring support. This
390 * is before the event ring starts to pay attention to the
391 * RUN/STOP bit. Thus, make sure the values are observable to
392 * the XHCI before that point.
393 */
395
396 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
397
398 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
399 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
400
401 addr = buf_res.physaddr;
402
403 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
404
405 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
406 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
407
408 /* set up interrupter registers */
409 temp = XREAD4(sc, runt, XHCI_IMAN(0));
410 temp |= XHCI_IMAN_INTR_ENA;
411 XWRITE4(sc, runt, XHCI_IMAN(0), temp);
412
413 /* set up command ring control base address */
414 addr = buf_res.physaddr;
415 addr += __offsetof(struct xhci_hw_root, hwr_commands[0]);
416
417 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
418
419 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
420 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
421
422 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
423
425
426 /* Go! */
427 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
429
430 for (i = 0; i != 100; i++) {
431 usb_pause_mtx(NULL, hz / 100);
432 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
433 if (!temp)
434 break;
435 }
436 if (temp) {
437 XWRITE4(sc, oper, XHCI_USBCMD, 0);
438 device_printf(sc->sc_bus.parent, "Run timeout.\n");
439 return (USB_ERR_IOERROR);
440 }
441
442 /* catch any lost interrupts */
443 xhci_do_poll(&sc->sc_bus);
444
445 if (sc->sc_port_route != NULL) {
446 /* Route all ports to the XHCI by default */
449 }
450 return (0);
451}
452
455{
456 uint32_t temp;
457 uint16_t i;
458
459 DPRINTF("\n");
460
461 sc->sc_capa_off = 0;
462 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
463 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
464 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
465
466 /* Halt controller */
467 XWRITE4(sc, oper, XHCI_USBCMD, 0);
468
469 for (i = 0; i != 100; i++) {
470 usb_pause_mtx(NULL, hz / 100);
471 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
472 if (temp)
473 break;
474 }
475
476 if (!temp) {
477 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
478 return (USB_ERR_IOERROR);
479 }
480 return (0);
481}
482
485{
486 uint32_t temp = 0;
487 uint16_t i;
488
489 DPRINTF("\n");
490
491 /* Reset controller */
493
494 for (i = 0; i != 100; i++) {
495 usb_pause_mtx(NULL, hz / 100);
496 temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) |
497 (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR);
498 if (!temp)
499 break;
500 }
501
502 if (temp) {
503 device_printf(sc->sc_bus.parent, "Controller "
504 "reset timeout.\n");
505 return (USB_ERR_IOERROR);
506 }
507 return (0);
508}
509
511xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32)
512{
513 uint32_t temp;
514
515 DPRINTF("\n");
516
517 /* initialize some bus fields */
518 sc->sc_bus.parent = self;
519
520 /* set the bus revision */
522
523 /* set up the bus struct */
525
526 /* set up devices array */
527 sc->sc_bus.devices = sc->sc_devices;
529
530 /* set default cycle state in case of early interrupts */
531 sc->sc_event_ccs = 1;
532 sc->sc_command_ccs = 1;
533
534 /* set up bus space offsets */
535 sc->sc_capa_off = 0;
536 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
537 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
538 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
539
540 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
541 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
542 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
543
544 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
545
546 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
547 device_printf(sc->sc_bus.parent, "Controller does "
548 "not support 4K page size.\n");
549 return (ENXIO);
550 }
551
552 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
553
554 DPRINTF("HCS0 = 0x%08x\n", temp);
555
556 /* set up context size */
557 if (XHCI_HCS0_CSZ(temp)) {
558 sc->sc_ctx_is_64_byte = 1;
559 } else {
560 sc->sc_ctx_is_64_byte = 0;
561 }
562
563 /* get DMA bits */
564 sc->sc_bus.dma_bits = (XHCI_HCS0_AC64(temp) &&
565 xhcidma32 == 0 && dma32 == 0) ? 64 : 32;
566
567 device_printf(self, "%d bytes context size, %d-bit DMA\n",
568 sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits);
569
570 /* enable 64Kbyte control endpoint quirk */
571 sc->sc_bus.control_ep_quirk = (xhcictlquirk ? 1 : 0);
572
573 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
574
575 /* get number of device slots */
576 sc->sc_noport = XHCI_HCS1_N_PORTS(temp);
577
578 if (sc->sc_noport == 0) {
579 device_printf(sc->sc_bus.parent, "Invalid number "
580 "of ports: %u\n", sc->sc_noport);
581 return (ENXIO);
582 }
583
584 sc->sc_noport = sc->sc_noport;
586
587 DPRINTF("Max slots: %u\n", sc->sc_noslot);
588
589 if (sc->sc_noslot > XHCI_MAX_DEVICES)
591
592 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
593
594 DPRINTF("HCS2=0x%08x\n", temp);
595
596 /* get isochronous scheduling threshold */
597 sc->sc_ist = XHCI_HCS2_IST(temp);
598
599 /* get number of scratchpads */
601
603 device_printf(sc->sc_bus.parent, "XHCI request "
604 "too many scratchpads\n");
605 return (ENOMEM);
606 }
607
608 DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
609
610 /* get event table size */
611 sc->sc_erst_max = 1U << XHCI_HCS2_ERST_MAX(temp);
612 if (sc->sc_erst_max > XHCI_MAX_RSEG)
614
615 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
616
617 /* get maximum exit latency */
619 XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
620
621 /* Check if we should use the default IMOD value. */
622 if (sc->sc_imod_default == 0)
624
625 /* get all DMA memory */
628 return (ENOMEM);
629 }
630
631 /* set up command queue mutex and condition varible */
632 cv_init(&sc->sc_cmd_cv, "CMDQ");
633 sx_init(&sc->sc_cmd_sx, "CMDQ lock");
634
636 sc->sc_config_msg[0].bus = &sc->sc_bus;
638 sc->sc_config_msg[1].bus = &sc->sc_bus;
639
640 return (0);
641}
642
643void
645{
646 /*
647 * NOTE: At this point the control transfer process is gone
648 * and "xhci_configure_msg" is no longer called. Consequently
649 * waiting for the configuration messages to complete is not
650 * needed.
651 */
653
654 cv_destroy(&sc->sc_cmd_cv);
655 sx_destroy(&sc->sc_cmd_sx);
656}
657
658static void
659xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
660{
661 struct xhci_softc *sc = XHCI_BUS2SC(bus);
662
663 switch (state) {
665 DPRINTF("Stopping the XHCI\n");
668 break;
670 DPRINTF("Stopping the XHCI\n");
673 break;
675 DPRINTF("Starting the XHCI\n");
677 break;
678 default:
679 break;
680 }
681}
682
683static usb_error_t
685{
686 struct xhci_td *td;
687 struct xhci_td *td_alt_next;
688 uint32_t len;
689 uint8_t status;
690
691 td = xfer->td_transfer_cache;
692 td_alt_next = td->alt_next;
693
694 if (xfer->aframes != xfer->nframes)
695 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
696
697 while (1) {
699
700 status = td->status;
701 len = td->remainder;
702
703 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
704 xfer, (unsigned int)xfer->aframes,
705 (unsigned int)xfer->nframes,
706 (unsigned int)len, (unsigned int)td->len,
707 (unsigned int)status);
708
709 /*
710 * Verify the status length and
711 * add the length to "frlengths[]":
712 */
713 if (len > td->len) {
714 /* should not happen */
715 DPRINTF("Invalid status length, "
716 "0x%04x/0x%04x bytes\n", len, td->len);
718 } else if (xfer->aframes != xfer->nframes) {
719 xfer->frlengths[xfer->aframes] += td->len - len;
720 }
721 /* Check for last transfer */
722 if (((void *)td) == xfer->td_transfer_last) {
723 td = NULL;
724 break;
725 }
726 /* Check for transfer error */
729 /* the transfer is finished */
730 td = NULL;
731 break;
732 }
733 /* Check for short transfer */
734 if (len > 0) {
735 if (xfer->flags_int.short_frames_ok ||
737 xfer->flags_int.control_xfr) {
738 /* follow alt next */
739 td = td->alt_next;
740 } else {
741 /* the transfer is finished */
742 td = NULL;
743 }
744 break;
745 }
746 td = td->obj_next;
747
748 if (td->alt_next != td_alt_next) {
749 /* this USB frame is complete */
750 break;
751 }
752 }
753
754 /* update transfer cache */
755
756 xfer->td_transfer_cache = td;
757
762}
763
764static void
766{
767 usb_error_t err = 0;
768
769 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
770 xfer, xfer->endpoint);
771
772 /* reset scanner */
773
775
776 if (xfer->flags_int.control_xfr) {
777 if (xfer->flags_int.control_hdr)
778 err = xhci_generic_done_sub(xfer);
779
780 xfer->aframes = 1;
781
782 if (xfer->td_transfer_cache == NULL)
783 goto done;
784 }
785
786 while (xfer->aframes != xfer->nframes) {
787 err = xhci_generic_done_sub(xfer);
788 xfer->aframes++;
789
790 if (xfer->td_transfer_cache == NULL)
791 goto done;
792 }
793
794 if (xfer->flags_int.control_xfr &&
795 !xfer->flags_int.control_act)
796 err = xhci_generic_done_sub(xfer);
797done:
798 /* transfer is complete */
799 xhci_device_done(xfer, err);
800}
801
802static void
804{
805 struct xhci_td *td;
806
807 td = xfer->td_transfer_cache;
808
810
811 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
812 /* activate the transfer */
813
814 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
816
818 }
819}
820
821static void
823{
824 struct xhci_td *td;
825 struct xhci_td *td_last;
826
827 td = xfer->td_transfer_cache;
828 td_last = xfer->td_transfer_last;
829
830 td = td->alt_next;
831
833
834 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
836
837 /* copy LINK TRB to current waiting location */
838
839 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
840 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
842
843 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
845
847 }
848}
849
850/*------------------------------------------------------------------------*
851 * xhci_check_transfer
852 *------------------------------------------------------------------------*/
853static void
855{
856 struct xhci_endpoint_ext *pepext;
857 int64_t offset;
858 uint64_t td_event;
859 uint32_t temp;
860 uint32_t remainder;
861 uint16_t stream_id = 0;
862 uint16_t i;
863 uint8_t status;
864 uint8_t halted;
865 uint8_t epno;
866 uint8_t index;
867
868 /* decode TRB */
869 td_event = le64toh(trb->qwTrb0);
870 temp = le32toh(trb->dwTrb2);
871
874
875 temp = le32toh(trb->dwTrb3);
876 epno = XHCI_TRB_3_EP_GET(temp);
878
879 /* check if error means halted */
880 halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
882
883 DPRINTF("slot=%u epno=%u remainder=%u status=%u\n",
884 index, epno, remainder, status);
885
886 if (index > sc->sc_noslot) {
887 DPRINTF("Invalid slot.\n");
888 return;
889 }
890
891 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
892 DPRINTF("Invalid endpoint.\n");
893 return;
894 }
895
896 pepext = &sc->sc_hw.devs[index].endp[epno];
897
898 /* try to find the USB transfer that generated the event */
899 for (i = 0;; i++) {
900 struct usb_xfer *xfer;
901 struct xhci_td *td;
902
903 if (i == (XHCI_MAX_TRANSFERS - 1)) {
904 if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS ||
905 stream_id == (XHCI_MAX_STREAMS - 1))
906 break;
907 stream_id++;
908 i = 0;
909 DPRINTFN(5, "stream_id=%u\n", stream_id);
910 }
911
912 xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)];
913 if (xfer == NULL)
914 continue;
915
916 td = xfer->td_transfer_cache;
917
918 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
919 (long long)td_event,
920 (long long)td->td_self,
921 (long long)td->td_self + sizeof(td->td_trb));
922
923 /*
924 * NOTE: Some XHCI implementations might not trigger
925 * an event on the last LINK TRB so we need to
926 * consider both the last and second last event
927 * address as conditions for a successful transfer.
928 *
929 * NOTE: We assume that the XHCI will only trigger one
930 * event per chain of TRBs.
931 */
932
933 offset = td_event - td->td_self;
934
935 if (offset >= 0 &&
936 offset < (int64_t)sizeof(td->td_trb)) {
938
939 /* compute rest of remainder, if any */
940 for (i = (offset / 16) + 1; i < td->ntrb; i++) {
941 temp = le32toh(td->td_trb[i].dwTrb2);
943 }
944
945 DPRINTFN(5, "New remainder: %u\n", remainder);
946
947 /* clear isochronous transfer errors */
948 if (xfer->flags_int.isochronous_xfr) {
949 if (halted) {
950 halted = 0;
952 remainder = td->len;
953 }
954 }
955
956 /* "td->remainder" is verified later */
957 td->remainder = remainder;
958 td->status = status;
959
961
962 /*
963 * 1) Last transfer descriptor makes the
964 * transfer done
965 */
966 if (((void *)td) == xfer->td_transfer_last) {
967 DPRINTF("TD is last\n");
968 xhci_generic_done(xfer);
969 break;
970 }
971
972 /*
973 * 2) Any kind of error makes the transfer
974 * done
975 */
976 if (halted) {
977 DPRINTF("TD has I/O error\n");
978 xhci_generic_done(xfer);
979 break;
980 }
981
982 /*
983 * 3) If there is no alternate next transfer,
984 * a short packet also makes the transfer done
985 */
986 if (td->remainder > 0) {
987 if (td->alt_next == NULL) {
988 DPRINTF(
989 "short TD has no alternate next\n");
990 xhci_generic_done(xfer);
991 break;
992 }
993 DPRINTF("TD has short pkt\n");
994 if (xfer->flags_int.short_frames_ok ||
996 xfer->flags_int.control_xfr) {
997 /* follow the alt next */
998 xfer->td_transfer_cache = td->alt_next;
1000 break;
1001 }
1002 xhci_skip_transfer(xfer);
1003 xhci_generic_done(xfer);
1004 break;
1005 }
1006
1007 /*
1008 * 4) Transfer complete - go to next TD
1009 */
1010 DPRINTF("Following next TD\n");
1011 xfer->td_transfer_cache = td->obj_next;
1013 break; /* there should only be one match */
1014 }
1015 }
1016}
1017
1018static int
1020{
1021 if (sc->sc_cmd_addr == trb->qwTrb0) {
1022 DPRINTF("Received command event\n");
1023 sc->sc_cmd_result[0] = trb->dwTrb2;
1024 sc->sc_cmd_result[1] = trb->dwTrb3;
1025 cv_signal(&sc->sc_cmd_cv);
1026 return (1); /* command match */
1027 }
1028 return (0);
1029}
1030
1031static int
1033{
1034 struct usb_page_search buf_res;
1035 struct xhci_hw_root *phwr;
1036 uint64_t addr;
1037 uint32_t temp;
1038 int retval = 0;
1039 uint16_t i;
1040 uint8_t event;
1041 uint8_t j;
1042 uint8_t k;
1043 uint8_t t;
1044
1045 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1046
1047 phwr = buf_res.buffer;
1048
1049 /* Receive any events */
1050
1052
1053 i = sc->sc_event_idx;
1054 j = sc->sc_event_ccs;
1055 t = 2;
1056
1057 while (1) {
1058 temp = le32toh(phwr->hwr_events[i].dwTrb3);
1059
1060 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
1061
1062 if (j != k)
1063 break;
1064
1065 event = XHCI_TRB_3_TYPE_GET(temp);
1066
1067 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
1068 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
1069 (long)le32toh(phwr->hwr_events[i].dwTrb2),
1070 (long)le32toh(phwr->hwr_events[i].dwTrb3));
1071
1072 switch (event) {
1074 xhci_check_transfer(sc, &phwr->hwr_events[i]);
1075 break;
1077 retval |= xhci_check_command(sc, &phwr->hwr_events[i]);
1078 break;
1079 default:
1080 DPRINTF("Unhandled event = %u\n", event);
1081 break;
1082 }
1083
1084 i++;
1085
1086 if (i == XHCI_MAX_EVENTS) {
1087 i = 0;
1088 j ^= 1;
1089
1090 /* check for timeout */
1091 if (!--t)
1092 break;
1093 }
1094 }
1095
1096 sc->sc_event_idx = i;
1097 sc->sc_event_ccs = j;
1098
1099 /*
1100 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1101 * latched. That means to activate the register we need to
1102 * write both the low and high double word of the 64-bit
1103 * register.
1104 */
1105
1106 addr = buf_res.physaddr;
1107 addr += __offsetof(struct xhci_hw_root, hwr_events[i]);
1108
1109 /* try to clear busy bit */
1111
1112 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
1113 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
1114
1115 return (retval);
1116}
1117
1118static usb_error_t
1119xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
1120 uint16_t timeout_ms)
1121{
1122 struct usb_page_search buf_res;
1123 struct xhci_hw_root *phwr;
1124 uint64_t addr;
1125 uint32_t temp;
1126 uint8_t i;
1127 uint8_t j;
1128 uint8_t timeout = 0;
1129 int err;
1130
1132
1133 /* get hardware root structure */
1134
1135 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1136
1137 phwr = buf_res.buffer;
1138
1139 /* Queue command */
1140
1141 USB_BUS_LOCK(&sc->sc_bus);
1142retry:
1143 i = sc->sc_command_idx;
1144 j = sc->sc_command_ccs;
1145
1146 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1147 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
1148 (long long)le64toh(trb->qwTrb0),
1149 (long)le32toh(trb->dwTrb2),
1150 (long)le32toh(trb->dwTrb3));
1151
1152 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
1153 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
1154
1156
1157 temp = trb->dwTrb3;
1158
1159 if (j)
1160 temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
1161 else
1162 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1163
1164 temp &= ~htole32(XHCI_TRB_3_TC_BIT);
1165
1166 phwr->hwr_commands[i].dwTrb3 = temp;
1167
1169
1170 addr = buf_res.physaddr;
1171 addr += __offsetof(struct xhci_hw_root, hwr_commands[i]);
1172
1173 sc->sc_cmd_addr = htole64(addr);
1174
1175 i++;
1176
1177 if (i == (XHCI_MAX_COMMANDS - 1)) {
1178 if (j) {
1179 temp = htole32(XHCI_TRB_3_TC_BIT |
1182 } else {
1183 temp = htole32(XHCI_TRB_3_TC_BIT |
1185 }
1186
1187 phwr->hwr_commands[i].dwTrb3 = temp;
1188
1190
1191 i = 0;
1192 j ^= 1;
1193 }
1194
1195 sc->sc_command_idx = i;
1196 sc->sc_command_ccs = j;
1197
1198 XWRITE4(sc, door, XHCI_DOORBELL(0), 0);
1199
1200 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx,
1201 USB_MS_TO_TICKS(timeout_ms));
1202
1203 /*
1204 * In some error cases event interrupts are not generated.
1205 * Poll one time to see if the command has completed.
1206 */
1207 if (err != 0 && xhci_interrupt_poll(sc) != 0) {
1208 DPRINTF("Command was completed when polling\n");
1209 err = 0;
1210 }
1211 if (err != 0) {
1212 DPRINTF("Command timeout!\n");
1213 /*
1214 * After some weeks of continuous operation, it has
1215 * been observed that the ASMedia Technology, ASM1042
1216 * SuperSpeed USB Host Controller can suddenly stop
1217 * accepting commands via the command queue. Try to
1218 * first reset the command queue. If that fails do a
1219 * host controller reset.
1220 */
1221 if (timeout == 0 &&
1223 temp = le32toh(trb->dwTrb3);
1224
1225 /*
1226 * Avoid infinite XHCI reset loops if the set
1227 * address command fails to respond due to a
1228 * non-enumerating device:
1229 */
1231 (temp & XHCI_TRB_3_BSR_BIT) == 0) {
1232 DPRINTF("Set address timeout\n");
1233 } else {
1234 timeout = 1;
1235 goto retry;
1236 }
1237 } else {
1238 DPRINTF("Controller reset!\n");
1240 }
1241 err = USB_ERR_TIMEOUT;
1242 trb->dwTrb2 = 0;
1243 trb->dwTrb3 = 0;
1244 } else {
1245 temp = le32toh(sc->sc_cmd_result[0]);
1247 err = USB_ERR_IOERROR;
1248
1249 trb->dwTrb2 = sc->sc_cmd_result[0];
1250 trb->dwTrb3 = sc->sc_cmd_result[1];
1251 }
1252
1253 USB_BUS_UNLOCK(&sc->sc_bus);
1254
1255 return (err);
1256}
1257
1258#if 0
1259static usb_error_t
1260xhci_cmd_nop(struct xhci_softc *sc)
1261{
1262 struct xhci_trb trb;
1263 uint32_t temp;
1264
1265 DPRINTF("\n");
1266
1267 trb.qwTrb0 = 0;
1268 trb.dwTrb2 = 0;
1270
1271 trb.dwTrb3 = htole32(temp);
1272
1273 return (xhci_do_command(sc, &trb, 100 /* ms */));
1274}
1275#endif
1276
1277static usb_error_t
1278xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
1279{
1280 struct xhci_trb trb;
1281 uint32_t temp;
1282 usb_error_t err;
1283
1284 DPRINTF("\n");
1285
1286 trb.qwTrb0 = 0;
1287 trb.dwTrb2 = 0;
1289
1290 err = xhci_do_command(sc, &trb, 100 /* ms */);
1291 if (err)
1292 goto done;
1293
1294 temp = le32toh(trb.dwTrb3);
1295
1296 *pslot = XHCI_TRB_3_SLOT_GET(temp);
1297
1298done:
1299 return (err);
1300}
1301
1302static usb_error_t
1303xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
1304{
1305 struct xhci_trb trb;
1306 uint32_t temp;
1307
1308 DPRINTF("\n");
1309
1310 trb.qwTrb0 = 0;
1311 trb.dwTrb2 = 0;
1313 XHCI_TRB_3_SLOT_SET(slot_id);
1314
1315 trb.dwTrb3 = htole32(temp);
1316
1317 return (xhci_do_command(sc, &trb, 100 /* ms */));
1318}
1319
1320static usb_error_t
1321xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx,
1322 uint8_t bsr, uint8_t slot_id)
1323{
1324 struct xhci_trb trb;
1325 uint32_t temp;
1326
1327 DPRINTF("\n");
1328
1329 trb.qwTrb0 = htole64(input_ctx);
1330 trb.dwTrb2 = 0;
1332 XHCI_TRB_3_SLOT_SET(slot_id);
1333
1334 if (bsr)
1335 temp |= XHCI_TRB_3_BSR_BIT;
1336
1337 trb.dwTrb3 = htole32(temp);
1338
1339 return (xhci_do_command(sc, &trb, 500 /* ms */));
1340}
1341
1342static usb_error_t
1343xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
1344{
1345 struct usb_page_search buf_inp;
1346 struct usb_page_search buf_dev;
1347 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
1348 struct xhci_hw_dev *hdev;
1349 struct xhci_slot_ctx *slot;
1350 struct xhci_endpoint_ext *pepext;
1351 uint32_t temp;
1352 uint16_t mps;
1353 usb_error_t err;
1354 uint8_t index;
1355
1356 /* the root HUB case is not handled here */
1357 if (udev->parent_hub == NULL)
1358 return (USB_ERR_INVAL);
1359
1360 index = udev->controller_slot_id;
1361
1362 hdev = &sc->sc_hw.devs[index];
1363
1364 if (mtx != NULL)
1365 mtx_unlock(mtx);
1366
1367 XHCI_CMD_LOCK(sc);
1368
1369 switch (hdev->state) {
1370 case XHCI_ST_DEFAULT:
1371 case XHCI_ST_ENABLED:
1372
1373 hdev->state = XHCI_ST_ENABLED;
1374
1375 /* set configure mask to slot and EP0 */
1376 xhci_configure_mask(udev, 3, 0);
1377
1378 /* configure input slot context structure */
1379 err = xhci_configure_device(udev);
1380
1381 if (err != 0) {
1382 DPRINTF("Could not configure device\n");
1383 break;
1384 }
1385
1386 /* configure input endpoint context structure */
1387 switch (udev->speed) {
1388 case USB_SPEED_LOW:
1389 case USB_SPEED_FULL:
1390 mps = 8;
1391 break;
1392 case USB_SPEED_HIGH:
1393 mps = 64;
1394 break;
1395 default:
1396 mps = 512;
1397 break;
1398 }
1399
1400 pepext = xhci_get_endpoint_ext(udev,
1401 &udev->ctrl_ep_desc);
1402
1403 /* ensure the control endpoint is setup again */
1404 USB_BUS_LOCK(udev->bus);
1405 pepext->trb_halted = 1;
1406 pepext->trb_running = 0;
1407 USB_BUS_UNLOCK(udev->bus);
1408
1409 err = xhci_configure_endpoint(udev,
1410 &udev->ctrl_ep_desc, pepext,
1411 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
1412
1413 if (err != 0) {
1414 DPRINTF("Could not configure default endpoint\n");
1415 break;
1416 }
1417
1418 /* execute set address command */
1419 usbd_get_page(&hdev->input_pc, 0, &buf_inp);
1420
1421 err = xhci_cmd_set_address(sc, buf_inp.physaddr,
1422 (address == 0), index);
1423
1424 if (err != 0) {
1425 temp = le32toh(sc->sc_cmd_result[0]);
1426 if (address == 0 && sc->sc_port_route != NULL &&
1427 XHCI_TRB_2_ERROR_GET(temp) ==
1429 /* LynxPoint XHCI - ports are not switchable */
1430 /* Un-route all ports from the XHCI */
1431 sc->sc_port_route(sc->sc_bus.parent, 0, ~0);
1432 }
1433 DPRINTF("Could not set address "
1434 "for slot %u.\n", index);
1435 if (address != 0)
1436 break;
1437 }
1438
1439 /* update device address to new value */
1440
1441 usbd_get_page(&hdev->device_pc, 0, &buf_dev);
1443 buf_dev.buffer);
1445
1446 temp = le32toh(slot->dwSctx3);
1447 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp);
1448
1449 /* update device state to new value */
1450
1451 if (address != 0)
1452 hdev->state = XHCI_ST_ADDRESSED;
1453 else
1454 hdev->state = XHCI_ST_DEFAULT;
1455 break;
1456
1457 default:
1458 DPRINTF("Wrong state for set address.\n");
1459 err = USB_ERR_IOERROR;
1460 break;
1461 }
1462 XHCI_CMD_UNLOCK(sc);
1463
1464 if (mtx != NULL)
1465 mtx_lock(mtx);
1466
1467 return (err);
1468}
1469
1470static usb_error_t
1471xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
1472 uint8_t deconfigure, uint8_t slot_id)
1473{
1474 struct xhci_trb trb;
1475 uint32_t temp;
1476
1477 DPRINTF("\n");
1478
1479 trb.qwTrb0 = htole64(input_ctx);
1480 trb.dwTrb2 = 0;
1482 XHCI_TRB_3_SLOT_SET(slot_id);
1483
1484 if (deconfigure) {
1485 if (sc->sc_no_deconfigure != 0 || xhcidcepquirk != 0)
1486 return (0); /* Success */
1487 temp |= XHCI_TRB_3_DCEP_BIT;
1488 }
1489
1490 trb.dwTrb3 = htole32(temp);
1491
1492 return (xhci_do_command(sc, &trb, 100 /* ms */));
1493}
1494
1495static usb_error_t
1496xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
1497 uint8_t slot_id)
1498{
1499 struct xhci_trb trb;
1500 uint32_t temp;
1501
1502 DPRINTF("\n");
1503
1504 trb.qwTrb0 = htole64(input_ctx);
1505 trb.dwTrb2 = 0;
1507 XHCI_TRB_3_SLOT_SET(slot_id);
1508 trb.dwTrb3 = htole32(temp);
1509
1510 return (xhci_do_command(sc, &trb, 100 /* ms */));
1511}
1512
1513static usb_error_t
1514xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
1515 uint8_t ep_id, uint8_t slot_id)
1516{
1517 struct xhci_trb trb;
1518 uint32_t temp;
1519
1520 DPRINTF("\n");
1521
1522 trb.qwTrb0 = 0;
1523 trb.dwTrb2 = 0;
1525 XHCI_TRB_3_SLOT_SET(slot_id) |
1526 XHCI_TRB_3_EP_SET(ep_id);
1527
1528 if (preserve)
1529 temp |= XHCI_TRB_3_PRSV_BIT;
1530
1531 trb.dwTrb3 = htole32(temp);
1532
1533 return (xhci_do_command(sc, &trb, 100 /* ms */));
1534}
1535
1536static usb_error_t
1537xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
1538 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
1539{
1540 struct xhci_trb trb;
1541 uint32_t temp;
1542
1543 DPRINTF("\n");
1544
1545 trb.qwTrb0 = htole64(dequeue_ptr);
1546
1547 temp = XHCI_TRB_2_STREAM_SET(stream_id);
1548 trb.dwTrb2 = htole32(temp);
1549
1551 XHCI_TRB_3_SLOT_SET(slot_id) |
1552 XHCI_TRB_3_EP_SET(ep_id);
1553 trb.dwTrb3 = htole32(temp);
1554
1555 return (xhci_do_command(sc, &trb, 100 /* ms */));
1556}
1557
1558static usb_error_t
1559xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
1560 uint8_t ep_id, uint8_t slot_id)
1561{
1562 struct xhci_trb trb;
1563 uint32_t temp;
1564
1565 DPRINTF("\n");
1566
1567 trb.qwTrb0 = 0;
1568 trb.dwTrb2 = 0;
1570 XHCI_TRB_3_SLOT_SET(slot_id) |
1571 XHCI_TRB_3_EP_SET(ep_id);
1572
1573 if (suspend)
1574 temp |= XHCI_TRB_3_SUSP_EP_BIT;
1575
1576 trb.dwTrb3 = htole32(temp);
1577
1578 return (xhci_do_command(sc, &trb, 100 /* ms */));
1579}
1580
1581static usb_error_t
1582xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
1583{
1584 struct xhci_trb trb;
1585 uint32_t temp;
1586
1587 DPRINTF("\n");
1588
1589 trb.qwTrb0 = 0;
1590 trb.dwTrb2 = 0;
1592 XHCI_TRB_3_SLOT_SET(slot_id);
1593
1594 trb.dwTrb3 = htole32(temp);
1595
1596 return (xhci_do_command(sc, &trb, 100 /* ms */));
1597}
1598
1599/*------------------------------------------------------------------------*
1600 * xhci_interrupt - XHCI interrupt handler
1601 *------------------------------------------------------------------------*/
1602void
1604{
1605 uint32_t status;
1606 uint32_t temp;
1607
1608 USB_BUS_LOCK(&sc->sc_bus);
1609
1610 status = XREAD4(sc, oper, XHCI_USBSTS);
1611
1612 /* acknowledge interrupts, if any */
1613 if (status != 0) {
1614 XWRITE4(sc, oper, XHCI_USBSTS, status);
1615 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status);
1616 }
1617
1618 temp = XREAD4(sc, runt, XHCI_IMAN(0));
1619
1620 /* force clearing of pending interrupts */
1621 if (temp & XHCI_IMAN_INTR_PEND)
1622 XWRITE4(sc, runt, XHCI_IMAN(0), temp);
1623
1624 /* check for event(s) */
1626
1629 if (status & XHCI_STS_PCD) {
1630 xhci_root_intr(sc);
1631 }
1632
1633 if (status & XHCI_STS_HCH) {
1634 printf("%s: host controller halted\n",
1635 __FUNCTION__);
1636 }
1637
1638 if (status & XHCI_STS_HSE) {
1639 printf("%s: host system error\n",
1640 __FUNCTION__);
1641 }
1642
1643 if (status & XHCI_STS_HCE) {
1644 printf("%s: host controller error\n",
1645 __FUNCTION__);
1646 }
1647 }
1648 USB_BUS_UNLOCK(&sc->sc_bus);
1649}
1650
1651/*------------------------------------------------------------------------*
1652 * xhci_timeout - XHCI timeout handler
1653 *------------------------------------------------------------------------*/
1654static void
1656{
1657 struct usb_xfer *xfer = arg;
1658
1659 DPRINTF("xfer=%p\n", xfer);
1660
1661 USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1662
1663 /* transfer is transferred */
1665}
1666
1667static void
1669{
1670 struct xhci_softc *sc = XHCI_BUS2SC(bus);
1671
1672 USB_BUS_LOCK(&sc->sc_bus);
1674 USB_BUS_UNLOCK(&sc->sc_bus);
1675}
1676
1677static void
1679{
1680 struct usb_page_search buf_res;
1681 struct xhci_td *td;
1682 struct xhci_td *td_next;
1683 struct xhci_td *td_alt_next;
1684 struct xhci_td *td_first;
1685 uint32_t buf_offset;
1686 uint32_t average;
1687 uint32_t len_old;
1688 uint32_t npkt_off;
1689 uint32_t dword;
1690 uint8_t shortpkt_old;
1691 uint8_t precompute;
1692 uint8_t x;
1693
1694 td_alt_next = NULL;
1695 buf_offset = 0;
1696 shortpkt_old = temp->shortpkt;
1697 len_old = temp->len;
1698 npkt_off = 0;
1699 precompute = 1;
1700
1701restart:
1702
1703 td = temp->td;
1704 td_next = td_first = temp->td_next;
1705
1706 while (1) {
1707 if (temp->len == 0) {
1708 if (temp->shortpkt)
1709 break;
1710
1711 /* send a Zero Length Packet, ZLP, last */
1712
1713 temp->shortpkt = 1;
1714 average = 0;
1715
1716 } else {
1717 average = temp->average;
1718
1719 if (temp->len < average) {
1720 if (temp->len % temp->max_packet_size) {
1721 temp->shortpkt = 1;
1722 }
1723 average = temp->len;
1724 }
1725 }
1726
1727 if (td_next == NULL)
1728 panic("%s: out of XHCI transfer descriptors!", __FUNCTION__);
1729
1730 /* get next TD */
1731
1732 td = td_next;
1733 td_next = td->obj_next;
1734
1735 /* check if we are pre-computing */
1736
1737 if (precompute) {
1738 /* update remaining length */
1739
1740 temp->len -= average;
1741
1742 continue;
1743 }
1744 /* fill out current TD */
1745
1746 td->len = average;
1747 td->remainder = 0;
1748 td->status = 0;
1749
1750 /* update remaining length */
1751
1752 temp->len -= average;
1753
1754 /* reset TRB index */
1755
1756 x = 0;
1757
1758 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) {
1759 /* immediate data */
1760
1761 if (average > 8)
1762 average = 8;
1763
1764 td->td_trb[0].qwTrb0 = 0;
1765
1766 usbd_copy_out(temp->pc, temp->offset + buf_offset,
1767 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0,
1768 average);
1769
1770 dword = XHCI_TRB_2_BYTES_SET(8) |
1773
1774 td->td_trb[0].dwTrb2 = htole32(dword);
1775
1778
1779 /* check wLength */
1780 if (td->td_trb[0].qwTrb0 &
1781 htole64(XHCI_TRB_0_WLENGTH_MASK)) {
1782 if (td->td_trb[0].qwTrb0 &
1783 htole64(XHCI_TRB_0_DIR_IN_MASK))
1784 dword |= XHCI_TRB_3_TRT_IN;
1785 else
1786 dword |= XHCI_TRB_3_TRT_OUT;
1787 }
1788
1789 td->td_trb[0].dwTrb3 = htole32(dword);
1790#ifdef USB_DEBUG
1791 xhci_dump_trb(&td->td_trb[x]);
1792#endif
1793 x++;
1794
1795 } else do {
1796 uint32_t npkt;
1797
1798 /* fill out buffer pointers */
1799
1800 if (average == 0) {
1801 memset(&buf_res, 0, sizeof(buf_res));
1802 } else {
1803 usbd_get_page(temp->pc, temp->offset +
1804 buf_offset, &buf_res);
1805
1806 /* get length to end of page */
1807 if (buf_res.length > average)
1808 buf_res.length = average;
1809
1810 /* check for maximum length */
1811 if (buf_res.length > XHCI_TD_PAGE_SIZE)
1812 buf_res.length = XHCI_TD_PAGE_SIZE;
1813
1814 npkt_off += buf_res.length;
1815 }
1816
1817 /* set up npkt */
1818 npkt = howmany(len_old - npkt_off,
1819 temp->max_packet_size);
1820
1821 if (npkt == 0)
1822 npkt = 1;
1823 else if (npkt > 31)
1824 npkt = 31;
1825
1826 /* fill out TRB's */
1827 td->td_trb[x].qwTrb0 =
1828 htole64((uint64_t)buf_res.physaddr);
1829
1830 dword =
1831 XHCI_TRB_2_BYTES_SET(buf_res.length) |
1832 XHCI_TRB_2_TDSZ_SET(npkt) |
1834
1835 td->td_trb[x].dwTrb2 = htole32(dword);
1836
1837 switch (temp->trb_type) {
1840 XHCI_TRB_3_TBC_SET(temp->tbc) |
1842 if (td != td_first) {
1844 } else if (temp->do_isoc_sync != 0) {
1845 temp->do_isoc_sync = 0;
1846 /* wait until "isoc_frame" */
1849 } else {
1850 /* start data transfer at next interval */
1853 }
1854 if (temp->direction == UE_DIR_IN)
1855 dword |= XHCI_TRB_3_ISP_BIT;
1856 break;
1860 if (temp->direction == UE_DIR_IN)
1862 /*
1863 * Section 3.2.9 in the XHCI
1864 * specification about control
1865 * transfers says that we should use a
1866 * normal-TRB if there are more TRBs
1867 * extending the data-stage
1868 * TRB. Update the "trb_type".
1869 */
1871 break;
1875 if (temp->direction == UE_DIR_IN)
1876 dword |= XHCI_TRB_3_DIR_IN;
1877 break;
1878 default: /* XHCI_TRB_TYPE_NORMAL */
1881 if (temp->direction == UE_DIR_IN)
1882 dword |= XHCI_TRB_3_ISP_BIT;
1883 break;
1884 }
1885 td->td_trb[x].dwTrb3 = htole32(dword);
1886
1887 average -= buf_res.length;
1888 buf_offset += buf_res.length;
1889#ifdef USB_DEBUG
1890 xhci_dump_trb(&td->td_trb[x]);
1891#endif
1892 x++;
1893
1894 } while (average != 0);
1895
1896 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT);
1897
1898 /* store number of data TRB's */
1899
1900 td->ntrb = x;
1901
1902 DPRINTF("NTRB=%u\n", x);
1903
1904 /* fill out link TRB */
1905
1906 if (td_next != NULL) {
1907 /* link the current TD with the next one */
1908 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self);
1909 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self);
1910 } else {
1911 /* this field will get updated later */
1912 DPRINTF("NOLINK\n");
1913 }
1914
1915 dword = XHCI_TRB_2_IRQ_SET(0);
1916
1917 td->td_trb[x].dwTrb2 = htole32(dword);
1918
1921 /*
1922 * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint
1923 * frame only receives a single short packet event
1924 * by setting the CHAIN bit in the LINK field. In
1925 * addition some XHCI controllers have problems
1926 * sending a ZLP unless the CHAIN-BIT is set in
1927 * the LINK TRB.
1928 */
1930
1931 td->td_trb[x].dwTrb3 = htole32(dword);
1932
1933 td->alt_next = td_alt_next;
1934#ifdef USB_DEBUG
1935 xhci_dump_trb(&td->td_trb[x]);
1936#endif
1938 }
1939
1940 if (precompute) {
1941 precompute = 0;
1942
1943 /* set up alt next pointer, if any */
1944 if (temp->last_frame) {
1945 td_alt_next = NULL;
1946 } else {
1947 /* we use this field internally */
1948 td_alt_next = td_next;
1949 }
1950
1951 /* restore */
1952 temp->shortpkt = shortpkt_old;
1953 temp->len = len_old;
1954 goto restart;
1955 }
1956
1957 /*
1958 * Remove cycle bit from the first TRB if we are
1959 * stepping them:
1960 */
1961 if (temp->step_td != 0) {
1962 td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1963 usb_pc_cpu_flush(td_first->page_cache);
1964 }
1965
1966 /* clear TD SIZE to zero, hence this is the last TRB */
1967 /* remove chain bit because this is the last data TRB in the chain */
1968 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(31));
1969 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
1970 /* remove CHAIN-BIT from last LINK TRB */
1971 td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
1972
1974
1975 temp->td = td;
1976 temp->td_next = td_next;
1977}
1978
1979static void
1981{
1982 struct xhci_std_temp temp;
1983 struct xhci_td *td;
1984 uint32_t x;
1985 uint32_t y;
1986 uint8_t mult;
1987
1988 temp.do_isoc_sync = 0;
1989 temp.step_td = 0;
1990 temp.tbc = 0;
1991 temp.tlbpc = 0;
1992 temp.average = xfer->max_hc_frame_size;
1993 temp.max_packet_size = xfer->max_packet_size;
1994 temp.sc = XHCI_BUS2SC(xfer->xroot->bus);
1995 temp.pc = NULL;
1996 temp.last_frame = 0;
1997 temp.offset = 0;
1998 temp.multishort = xfer->flags_int.isochronous_xfr ||
1999 xfer->flags_int.control_xfr ||
2001
2002 /* toggle the DMA set we are using */
2003 xfer->flags_int.curr_dma_set ^= 1;
2004
2005 /* get next DMA set */
2006 td = xfer->td_start[xfer->flags_int.curr_dma_set];
2007
2008 temp.td = NULL;
2009 temp.td_next = td;
2010
2011 xfer->td_transfer_first = td;
2012 xfer->td_transfer_cache = td;
2013
2014 if (xfer->flags_int.isochronous_xfr) {
2015 uint8_t shift;
2016
2017 /* compute multiplier for ISOCHRONOUS transfers */
2018 mult = xfer->endpoint->ecomp ?
2020 : 0;
2021 /* check for USB 2.0 multiplier */
2022 if (mult == 0) {
2023 mult = (xfer->endpoint->edesc->
2024 wMaxPacketSize[1] >> 3) & 3;
2025 }
2026 /* range check */
2027 if (mult > 2)
2028 mult = 3;
2029 else
2030 mult++;
2031
2032 x = XREAD4(temp.sc, runt, XHCI_MFINDEX);
2033
2034 DPRINTF("MFINDEX=0x%08x IST=0x%x\n", x, temp.sc->sc_ist);
2035
2036 switch (usbd_get_speed(xfer->xroot->udev)) {
2037 case USB_SPEED_FULL:
2038 shift = 3;
2039 temp.isoc_delta = 8; /* 1ms */
2040 break;
2041 default:
2042 shift = usbd_xfer_get_fps_shift(xfer);
2043 temp.isoc_delta = 1U << shift;
2044 break;
2045 }
2046
2047 /* Compute isochronous scheduling threshold. */
2048 if (temp.sc->sc_ist & 8)
2049 y = (temp.sc->sc_ist & 7) << 3;
2050 else
2051 y = (temp.sc->sc_ist & 7);
2052
2053 /* Range check the IST. */
2054 if (y < 8) {
2055 y = 0;
2056 } else if (y > 15) {
2057 DPRINTFN(3, "IST(%d) is too big!\n", temp.sc->sc_ist);
2058 /*
2059 * The USB stack minimum isochronous transfer
2060 * size is typically 2x2 ms of payload. If the
2061 * IST makes is above 15 microframes, we have
2062 * an effective scheduling delay of more than
2063 * or equal to 2 milliseconds, which is too
2064 * much.
2065 */
2066 y = 7;
2067 } else {
2068 /*
2069 * Subtract one millisecond, because the
2070 * generic code adds that to the latency.
2071 */
2072 y -= 8;
2073 }
2074
2076 xfer, x, y, 8, XHCI_MFINDEX_GET(-1), &temp.isoc_frame)) {
2077 /* Start isochronous transfer at specified time. */
2078 temp.do_isoc_sync = 1;
2079
2080 DPRINTFN(3, "start next=%d\n", temp.isoc_frame);
2081 }
2082
2083 x = 0;
2085
2086 } else if (xfer->flags_int.control_xfr) {
2087 /* check if we should prepend a setup message */
2088
2089 if (xfer->flags_int.control_hdr) {
2090 temp.len = xfer->frlengths[0];
2091 temp.pc = xfer->frbuffers + 0;
2092 temp.shortpkt = temp.len ? 1 : 0;
2094 temp.direction = 0;
2095
2096 /* check for last frame */
2097 if (xfer->nframes == 1) {
2098 /* no STATUS stage yet, SETUP is last */
2099 if (xfer->flags_int.control_act)
2100 temp.last_frame = 1;
2101 }
2102
2104 }
2105 x = 1;
2106 mult = 1;
2107 temp.isoc_delta = 0;
2108 temp.isoc_frame = 0;
2109 temp.trb_type = xfer->flags_int.control_did_data ?
2111 } else {
2112 x = 0;
2113 mult = 1;
2114 temp.isoc_delta = 0;
2115 temp.isoc_frame = 0;
2117 }
2118
2119 if (x != xfer->nframes) {
2120 /* set up page_cache pointer */
2121 temp.pc = xfer->frbuffers + x;
2122 /* set endpoint direction */
2123 temp.direction = UE_GET_DIR(xfer->endpointno);
2124 }
2125
2126 while (x != xfer->nframes) {
2127 /* DATA0 / DATA1 message */
2128
2129 temp.len = xfer->frlengths[x];
2130 temp.step_td = ((xfer->endpointno & UE_DIR_IN) &&
2131 x != 0 && temp.multishort == 0);
2132
2133 x++;
2134
2135 if (x == xfer->nframes) {
2136 if (xfer->flags_int.control_xfr) {
2137 /* no STATUS stage yet, DATA is last */
2138 if (xfer->flags_int.control_act)
2139 temp.last_frame = 1;
2140 } else {
2141 temp.last_frame = 1;
2142 }
2143 }
2144 if (temp.len == 0) {
2145 /* make sure that we send an USB packet */
2146
2147 temp.shortpkt = 0;
2148
2149 temp.tbc = 0;
2150 temp.tlbpc = mult - 1;
2151
2152 } else if (xfer->flags_int.isochronous_xfr) {
2153 uint8_t tdpc;
2154
2155 /*
2156 * Isochronous transfers don't have short
2157 * packet termination:
2158 */
2159
2160 temp.shortpkt = 1;
2161
2162 /* isochronous transfers have a transfer limit */
2163
2164 if (temp.len > xfer->max_frame_size)
2165 temp.len = xfer->max_frame_size;
2166
2167 /* compute TD packet count */
2168 tdpc = howmany(temp.len, xfer->max_packet_size);
2169
2170 temp.tbc = howmany(tdpc, mult) - 1;
2171 temp.tlbpc = (tdpc % mult);
2172
2173 if (temp.tlbpc == 0)
2174 temp.tlbpc = mult - 1;
2175 else
2176 temp.tlbpc--;
2177 } else {
2178 /* regular data transfer */
2179
2180 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1;
2181 }
2182
2184
2185 if (xfer->flags_int.isochronous_xfr) {
2186 temp.offset += xfer->frlengths[x - 1];
2187 temp.isoc_frame += temp.isoc_delta;
2188 } else {
2189 /* get next Page Cache pointer */
2190 temp.pc = xfer->frbuffers + x;
2191 }
2192 }
2193
2194 /* check if we should append a status stage */
2195
2196 if (xfer->flags_int.control_xfr &&
2197 !xfer->flags_int.control_act) {
2198 /*
2199 * Send a DATA1 message and invert the current
2200 * endpoint direction.
2201 */
2202 if (xhcictlstep || temp.sc->sc_ctlstep) {
2203 /*
2204 * Some XHCI controllers will not delay the
2205 * status stage until the next SOF. Force this
2206 * behaviour to avoid failed control
2207 * transfers.
2208 */
2209 temp.step_td = (xfer->nframes != 0);
2210 } else {
2211 temp.step_td = 0;
2212 }
2213 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN;
2214 temp.len = 0;
2215 temp.pc = NULL;
2216 temp.shortpkt = 0;
2217 temp.last_frame = 1;
2219
2221 }
2222
2223 td = temp.td;
2224
2225 /* must have at least one frame! */
2226
2227 xfer->td_transfer_last = td;
2228
2229 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td);
2230}
2231
2232static void
2233xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
2234{
2235 struct usb_page_search buf_res;
2236 struct xhci_dev_ctx_addr *pdctxa;
2237
2238 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
2239
2240 pdctxa = buf_res.buffer;
2241
2242 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr);
2243
2244 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr);
2245
2247}
2248
2249static usb_error_t
2250xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop)
2251{
2252 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2253 struct usb_page_search buf_inp;
2254 struct xhci_input_ctx *input;
2255 struct xhci_slot_ctx *slot;
2256 uint32_t temp;
2257 uint8_t index;
2258 uint8_t x;
2259
2260 index = udev->controller_slot_id;
2261
2262 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2263
2264 input = XHCI_GET_CTX(sc, xhci_input_dev_ctx, ctx_input,
2265 buf_inp.buffer);
2267
2268 if (drop) {
2270 input->dwInCtx0 = htole32(mask);
2271 input->dwInCtx1 = htole32(0);
2272 } else {
2273 /*
2274 * Some hardware requires that we drop the endpoint
2275 * context before adding it again:
2276 */
2277 input->dwInCtx0 = htole32(mask & XHCI_INCTX_NON_CTRL_MASK);
2278
2279 /* Add new endpoint context */
2280 input->dwInCtx1 = htole32(mask);
2281
2282 /* find most significant set bit */
2283 for (x = 31; x != 1; x--) {
2284 if (mask & (1 << x))
2285 break;
2286 }
2287
2288 /* adjust */
2289 x--;
2290
2291 /* figure out the maximum number of contexts */
2292 if (x > sc->sc_hw.devs[index].context_num)
2293 sc->sc_hw.devs[index].context_num = x;
2294 else
2295 x = sc->sc_hw.devs[index].context_num;
2296
2297 /* update number of contexts */
2298 temp = le32toh(slot->dwSctx0);
2299 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2300 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1);
2301 slot->dwSctx0 = htole32(temp);
2302 }
2304 return (0);
2305}
2306
2307static usb_error_t
2309 struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext,
2310 uint16_t interval, uint8_t max_packet_count,
2311 uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size,
2312 uint16_t max_frame_size, uint8_t ep_mode)
2313{
2314 struct usb_page_search buf_inp;
2315 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2316 struct xhci_endp_ctx *endp;
2317 uint64_t ring_addr = pepext->physaddr;
2318 uint32_t temp;
2319 uint8_t index;
2320 uint8_t epno;
2321 uint8_t type;
2322
2323 index = udev->controller_slot_id;
2324
2325 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2326
2327 epno = edesc->bEndpointAddress;
2328 type = edesc->bmAttributes & UE_XFERTYPE;
2329
2330 if (type == UE_CONTROL)
2331 epno |= UE_DIR_IN;
2332
2333 epno = XHCI_EPNO2EPID(epno);
2334
2335 if (epno == 0)
2336 return (USB_ERR_NO_PIPE); /* invalid */
2337
2338 if (max_packet_count == 0)
2339 return (USB_ERR_BAD_BUFSIZE);
2340
2341 max_packet_count--;
2342
2343 if (mult == 0)
2344 return (USB_ERR_BAD_BUFSIZE);
2345
2346 endp = XHCI_GET_CTX(sc, xhci_input_dev_ctx, ctx_ep[epno - 1],
2347 buf_inp.buffer);
2348
2349 /* store endpoint mode */
2350 pepext->trb_ep_mode = ep_mode;
2351 /* store bMaxPacketSize for control endpoints */
2352 pepext->trb_ep_maxp = edesc->wMaxPacketSize[0];
2354
2355 if (ep_mode == USB_EP_MODE_STREAMS) {
2356 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2357 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
2359
2360 ring_addr += sizeof(struct xhci_trb) *
2361 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
2362 } else {
2363 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2366
2367 ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
2368 }
2369
2370 switch (udev->speed) {
2371 case USB_SPEED_FULL:
2372 case USB_SPEED_LOW:
2373 /* 1ms -> 125us */
2374 fps_shift += 3;
2375 break;
2376 default:
2377 break;
2378 }
2379
2380 switch (type) {
2381 case UE_INTERRUPT:
2382 if (fps_shift > 3)
2383 fps_shift--;
2384 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2385 break;
2386 case UE_ISOCHRONOUS:
2387 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2388
2389 switch (udev->speed) {
2390 case USB_SPEED_SUPER:
2391 if (mult > 3)
2392 mult = 3;
2393 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1);
2394 max_packet_count /= mult;
2395 break;
2396 default:
2397 break;
2398 }
2399 break;
2400 default:
2401 break;
2402 }
2403
2404 endp->dwEpCtx0 = htole32(temp);
2405
2406 temp =
2408 XHCI_EPCTX_1_MAXB_SET(max_packet_count) |
2409 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size);
2410
2411 /*
2412 * Always enable the "three strikes and you are gone" feature
2413 * except for ISOCHRONOUS endpoints. This is suggested by
2414 * section 4.3.3 in the XHCI specification about device slot
2415 * initialisation.
2416 */
2417 if (type != UE_ISOCHRONOUS)
2418 temp |= XHCI_EPCTX_1_CERR_SET(3);
2419
2420 switch (type) {
2421 case UE_CONTROL:
2422 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2423 break;
2424 case UE_ISOCHRONOUS:
2425 temp |= XHCI_EPCTX_1_EPTYPE_SET(1);
2426 break;
2427 case UE_BULK:
2428 temp |= XHCI_EPCTX_1_EPTYPE_SET(2);
2429 break;
2430 default:
2431 temp |= XHCI_EPCTX_1_EPTYPE_SET(3);
2432 break;
2433 }
2434
2435 /* check for IN direction */
2436 if (epno & 1)
2437 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2438
2439 endp->dwEpCtx1 = htole32(temp);
2440 endp->qwEpCtx2 = htole64(ring_addr);
2441
2442 switch (edesc->bmAttributes & UE_XFERTYPE) {
2443 case UE_INTERRUPT:
2444 case UE_ISOCHRONOUS:
2445 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) |
2447 max_frame_size));
2448 break;
2449 case UE_CONTROL:
2451 break;
2452 default:
2454 break;
2455 }
2456
2457 endp->dwEpCtx4 = htole32(temp);
2458
2459#ifdef USB_DEBUG
2460 xhci_dump_endpoint(endp);
2461#endif
2463
2464 return (0); /* success */
2465}
2466
2467static usb_error_t
2469{
2470 struct xhci_endpoint_ext *pepext;
2471 struct usb_endpoint_ss_comp_descriptor *ecomp;
2472 usb_stream_t x;
2473
2474 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2475 xfer->endpoint->edesc);
2476
2477 ecomp = xfer->endpoint->ecomp;
2478
2479 for (x = 0; x != XHCI_MAX_STREAMS; x++) {
2480 uint64_t temp;
2481
2482 /* halt any transfers */
2483 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
2484
2485 /* compute start of TRB ring for stream "x" */
2486 temp = pepext->physaddr +
2487 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
2489
2490 /* make tree structure */
2491 pepext->trb[(XHCI_MAX_TRANSFERS *
2492 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
2493
2494 /* reserved fields */
2495 pepext->trb[(XHCI_MAX_TRANSFERS *
2496 XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
2497 pepext->trb[(XHCI_MAX_TRANSFERS *
2498 XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
2499 }
2501
2502 return (xhci_configure_endpoint(xfer->xroot->udev,
2503 xfer->endpoint->edesc, pepext,
2504 xfer->interval, xfer->max_packet_count,
2505 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
2507 xfer->max_frame_size, xfer->endpoint->ep_mode));
2508}
2509
2510static usb_error_t
2512{
2513 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2514 struct usb_page_search buf_inp;
2515 struct usb_page_cache *pcinp;
2516 struct xhci_slot_ctx *slot;
2517 struct usb_device *hubdev;
2518 uint32_t temp;
2519 uint32_t route;
2520 uint32_t rh_port;
2521 uint8_t is_hub;
2522 uint8_t index;
2523 uint8_t depth;
2524
2525 index = udev->controller_slot_id;
2526
2527 DPRINTF("index=%u\n", index);
2528
2529 pcinp = &sc->sc_hw.devs[index].input_pc;
2530
2531 usbd_get_page(pcinp, 0, &buf_inp);
2532
2534
2535 rh_port = 0;
2536 route = 0;
2537
2538 /* figure out route string and root HUB port number */
2539
2540 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) {
2541 if (hubdev->parent_hub == NULL)
2542 break;
2543
2544 depth = hubdev->parent_hub->depth;
2545
2546 /*
2547 * NOTE: HS/FS/LS devices and the SS root HUB can have
2548 * more than 15 ports
2549 */
2550
2551 rh_port = hubdev->port_no;
2552
2553 if (depth == 0)
2554 break;
2555
2556 if (rh_port > 15)
2557 rh_port = 15;
2558
2559 if (depth < 6)
2560 route |= rh_port << (4 * (depth - 1));
2561 }
2562
2563 DPRINTF("Route=0x%08x\n", route);
2564
2565 temp = XHCI_SCTX_0_ROUTE_SET(route) |
2567 sc->sc_hw.devs[index].context_num + 1);
2568
2569 switch (udev->speed) {
2570 case USB_SPEED_LOW:
2571 temp |= XHCI_SCTX_0_SPEED_SET(2);
2572 if (udev->parent_hs_hub != NULL &&
2575 DPRINTF("Device inherits MTT\n");
2576 temp |= XHCI_SCTX_0_MTT_SET(1);
2577 }
2578 break;
2579 case USB_SPEED_HIGH:
2580 temp |= XHCI_SCTX_0_SPEED_SET(3);
2581 if (sc->sc_hw.devs[index].nports != 0 &&
2583 DPRINTF("HUB supports MTT\n");
2584 temp |= XHCI_SCTX_0_MTT_SET(1);
2585 }
2586 break;
2587 case USB_SPEED_FULL:
2588 temp |= XHCI_SCTX_0_SPEED_SET(1);
2589 if (udev->parent_hs_hub != NULL &&
2592 DPRINTF("Device inherits MTT\n");
2593 temp |= XHCI_SCTX_0_MTT_SET(1);
2594 }
2595 break;
2596 default:
2597 temp |= XHCI_SCTX_0_SPEED_SET(4);
2598 break;
2599 }
2600
2601 is_hub = sc->sc_hw.devs[index].nports != 0 &&
2602 (udev->speed == USB_SPEED_SUPER ||
2603 udev->speed == USB_SPEED_HIGH);
2604
2605 if (is_hub)
2606 temp |= XHCI_SCTX_0_HUB_SET(1);
2607
2608 slot->dwSctx0 = htole32(temp);
2609
2610 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port);
2611
2612 if (is_hub) {
2614 sc->sc_hw.devs[index].nports);
2615 }
2616
2617 slot->dwSctx1 = htole32(temp);
2618
2620
2621 if (is_hub) {
2623 sc->sc_hw.devs[index].tt);
2624 }
2625
2626 hubdev = udev->parent_hs_hub;
2627
2628 /* check if we should activate the transaction translator */
2629 switch (udev->speed) {
2630 case USB_SPEED_FULL:
2631 case USB_SPEED_LOW:
2632 if (hubdev != NULL) {
2634 hubdev->controller_slot_id);
2636 udev->hs_port_no);
2637 }
2638 break;
2639 default:
2640 break;
2641 }
2642
2643 slot->dwSctx2 = htole32(temp);
2644
2645 /*
2646 * These fields should be initialized to zero, according to
2647 * XHCI section 6.2.2 - slot context:
2648 */
2649 temp = XHCI_SCTX_3_DEV_ADDR_SET(0) |
2651
2652 slot->dwSctx3 = htole32(temp);
2653
2654#ifdef USB_DEBUG
2655 xhci_dump_device(slot);
2656#endif
2657 usb_pc_cpu_flush(pcinp);
2658
2659 return (0); /* success */
2660}
2661
2662static usb_error_t
2664{
2665 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2666 struct usb_page_search buf_dev;
2667 struct usb_page_search buf_ep;
2668 struct xhci_trb *trb;
2669 struct usb_page_cache *pc;
2670 struct usb_page *pg;
2671 uint64_t addr;
2672 uint8_t index;
2673 uint8_t i;
2674
2675 index = udev->controller_slot_id;
2676
2677 pc = &sc->sc_hw.devs[index].device_pc;
2678 pg = &sc->sc_hw.devs[index].device_pg;
2679
2680 /* need to initialize the page cache */
2681 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2682
2683 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2684 sizeof(struct xhci_dev_ctx64) :
2685 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE))
2686 goto error;
2687
2688 usbd_get_page(pc, 0, &buf_dev);
2689
2690 pc = &sc->sc_hw.devs[index].input_pc;
2691 pg = &sc->sc_hw.devs[index].input_pg;
2692
2693 /* need to initialize the page cache */
2694 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2695
2696 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2697 sizeof(struct xhci_input_dev_ctx64) :
2698 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) {
2699 goto error;
2700 }
2701
2702 /* initialize all endpoint LINK TRBs */
2703
2704 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) {
2705 pc = &sc->sc_hw.devs[index].endpoint_pc[i];
2706 pg = &sc->sc_hw.devs[index].endpoint_pg[i];
2707
2708 /* need to initialize the page cache */
2709 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2710
2711 if (usb_pc_alloc_mem(pc, pg,
2712 sizeof(struct xhci_dev_endpoint_trbs), XHCI_TRB_ALIGN)) {
2713 goto error;
2714 }
2715
2716 /* lookup endpoint TRB ring */
2717 usbd_get_page(pc, 0, &buf_ep);
2718
2719 /* get TRB pointer */
2720 trb = buf_ep.buffer;
2721 trb += XHCI_MAX_TRANSFERS - 1;
2722
2723 /* get TRB start address */
2724 addr = buf_ep.physaddr;
2725
2726 /* create LINK TRB */
2727 trb->qwTrb0 = htole64(addr);
2728 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2729 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2731
2732 usb_pc_cpu_flush(pc);
2733 }
2734
2735 xhci_set_slot_pointer(sc, index, buf_dev.physaddr);
2736
2737 return (0);
2738
2739error:
2741
2742 return (USB_ERR_NOMEM);
2743}
2744
2745static void
2747{
2748 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2749 uint8_t index;
2750 uint8_t i;
2751
2752 index = udev->controller_slot_id;
2754
2757 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++)
2759}
2760
2761static struct xhci_endpoint_ext *
2763{
2764 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2765 struct xhci_endpoint_ext *pepext;
2766 struct usb_page_cache *pc;
2767 struct usb_page_search buf_ep;
2768 uint8_t epno;
2769 uint8_t index;
2770
2771 epno = edesc->bEndpointAddress;
2772 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
2773 epno |= UE_DIR_IN;
2774
2775 epno = XHCI_EPNO2EPID(epno);
2776
2777 index = udev->controller_slot_id;
2778
2779 pc = &sc->sc_hw.devs[index].endpoint_pc[epno];
2780
2781 usbd_get_page(pc, 0, &buf_ep);
2782
2783 pepext = &sc->sc_hw.devs[index].endp[epno];
2784 pepext->page_cache = pc;
2785 pepext->trb = buf_ep.buffer;
2786 pepext->physaddr = buf_ep.physaddr;
2787
2788 return (pepext);
2789}
2790
2791static void
2793{
2794 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2795 uint8_t epno;
2796 uint8_t index;
2797
2798 epno = xfer->endpointno;
2799 if (xfer->flags_int.control_xfr)
2800 epno |= UE_DIR_IN;
2801
2802 epno = XHCI_EPNO2EPID(epno);
2804
2805 if (xfer->xroot->udev->flags.self_suspended == 0) {
2806 XWRITE4(sc, door, XHCI_DOORBELL(index),
2807 epno | XHCI_DB_SID_SET(xfer->stream_id));
2808 }
2809}
2810
2811static void
2813{
2814 struct xhci_endpoint_ext *pepext;
2815
2818
2820 xfer->endpoint->edesc);
2821
2822 pepext->trb_used[xfer->stream_id]--;
2823
2824 pepext->xfer[xfer->qh_pos] = NULL;
2825
2826 if (error && pepext->trb_running != 0) {
2827 pepext->trb_halted = 1;
2828 pepext->trb_running = 0;
2829 }
2830 }
2831}
2832
2833static usb_error_t
2835{
2836 struct xhci_td *td_first;
2837 struct xhci_td *td_last;
2838 struct xhci_trb *trb_link;
2839 struct xhci_endpoint_ext *pepext;
2840 uint64_t addr;
2842 uint8_t i;
2843 uint8_t inext;
2844 uint8_t trb_limit;
2845
2846 DPRINTFN(8, "\n");
2847
2848 id = xfer->stream_id;
2849
2850 /* check if already inserted */
2852 DPRINTFN(8, "Already in schedule\n");
2853 return (0);
2854 }
2855
2857 xfer->endpoint->edesc);
2858
2859 td_first = xfer->td_transfer_first;
2860 td_last = xfer->td_transfer_last;
2861 addr = pepext->physaddr;
2862
2864 case UE_CONTROL:
2865 case UE_INTERRUPT:
2866 /* single buffered */
2867 trb_limit = 1;
2868 break;
2869 default:
2870 /* multi buffered */
2871 trb_limit = (XHCI_MAX_TRANSFERS - 2);
2872 break;
2873 }
2874
2875 if (pepext->trb_used[id] >= trb_limit) {
2876 DPRINTFN(8, "Too many TDs queued.\n");
2877 return (USB_ERR_NOMEM);
2878 }
2879
2880 /* check if bMaxPacketSize changed */
2881 if (xfer->flags_int.control_xfr != 0 &&
2882 pepext->trb_ep_maxp != xfer->endpoint->edesc->wMaxPacketSize[0]) {
2883 DPRINTFN(8, "Reconfigure control endpoint\n");
2884
2885 /* force driver to reconfigure endpoint */
2886 pepext->trb_halted = 1;
2887 pepext->trb_running = 0;
2888 }
2889
2890 /* check for stopped condition, after putting transfer on interrupt queue */
2891 if (pepext->trb_running == 0) {
2892 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2893
2894 DPRINTFN(8, "Not running\n");
2895
2896 /* start configuration */
2897 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
2898 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
2899 return (0);
2900 }
2901
2902 pepext->trb_used[id]++;
2903
2904 /* get current TRB index */
2905 i = pepext->trb_index[id];
2906
2907 /* get next TRB index */
2908 inext = (i + 1);
2909
2910 /* the last entry of the ring is a hardcoded link TRB */
2911 if (inext >= (XHCI_MAX_TRANSFERS - 1))
2912 inext = 0;
2913
2914 /* store next TRB index, before stream ID offset is added */
2915 pepext->trb_index[id] = inext;
2916
2917 /* offset for stream */
2918 i += id * XHCI_MAX_TRANSFERS;
2919 inext += id * XHCI_MAX_TRANSFERS;
2920
2921 /* compute terminating return address */
2922 addr += (inext * sizeof(struct xhci_trb));
2923
2924 /* compute link TRB pointer */
2925 trb_link = td_last->td_trb + td_last->ntrb;
2926
2927 /* update next pointer of last link TRB */
2928 trb_link->qwTrb0 = htole64(addr);
2929 trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2930 trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT |
2933
2934#ifdef USB_DEBUG
2935 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]);
2936#endif
2937 usb_pc_cpu_flush(td_last->page_cache);
2938
2939 /* write ahead chain end marker */
2940
2941 pepext->trb[inext].qwTrb0 = 0;
2942 pepext->trb[inext].dwTrb2 = 0;
2943 pepext->trb[inext].dwTrb3 = 0;
2944
2945 /* update next pointer of link TRB */
2946
2947 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self);
2948 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2949
2950#ifdef USB_DEBUG
2951 xhci_dump_trb(&pepext->trb[i]);
2952#endif
2954
2955 /* toggle cycle bit which activates the transfer chain */
2956
2957 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2959
2961
2962 DPRINTF("qh_pos = %u\n", i);
2963
2964 pepext->xfer[i] = xfer;
2965
2966 xfer->qh_pos = i;
2967
2969
2971
2972 return (0);
2973}
2974
2975static void
2977{
2978 uint16_t i;
2979
2980 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2981
2982 /* clear any old interrupt data */
2983 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2984
2985 for (i = 1; i <= sc->sc_noport; i++) {
2986 /* pick out CHANGE bits from the status register */
2987 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & (
2991 XHCI_PS_CEC)) {
2992 sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2993 DPRINTF("port %d changed\n", i);
2994 }
2995 }
2997 sizeof(sc->sc_hub_idata));
2998}
2999
3000/*------------------------------------------------------------------------*
3001 * xhci_device_done - XHCI done handler
3002 *
3003 * NOTE: This function can be called two times in a row on
3004 * the same USB transfer. From close and from interrupt.
3005 *------------------------------------------------------------------------*/
3006static void
3008{
3009 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
3010 xfer, xfer->endpoint, error);
3011
3012 /* remove transfer from HW queue */
3014
3015 /* dequeue transfer and start next transfer */
3017}
3018
3019/*------------------------------------------------------------------------*
3020 * XHCI data transfer support (generic type)
3021 *------------------------------------------------------------------------*/
3022static void
3024{
3025 DPRINTF("\n");
3026}
3027
3028static void
3030{
3031 DPRINTF("\n");
3032
3034}
3035
3036static void
3038 usb_stream_t stream_id, struct usb_xfer *enter_xfer)
3039{
3040 struct usb_xfer *xfer;
3041
3042 /* check if there is a current transfer */
3043 xfer = ep->endpoint_q[stream_id].curr;
3044 if (xfer == NULL)
3045 return;
3046
3047 /*
3048 * Check if the current transfer is started and then pickup
3049 * the next one, if any. Else wait for next start event due to
3050 * block on failure feature.
3051 */
3052 if (!xfer->flags_int.bandwidth_reclaimed)
3053 return;
3054
3055 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
3056 if (xfer == NULL) {
3057 /*
3058 * In case of enter we have to consider that the
3059 * transfer is queued by the USB core after the enter
3060 * method is called.
3061 */
3062 xfer = enter_xfer;
3063
3064 if (xfer == NULL)
3065 return;
3066 }
3067
3068 /* try to multi buffer */
3070}
3071
3072static void
3074{
3075 DPRINTF("\n");
3076
3077 /* set up TD's and QH */
3079
3081 xfer->stream_id, xfer);
3082}
3083
3084static void
3086{
3087 DPRINTF("\n");
3088
3089 /* try to insert xfer on HW queue */
3091
3092 /* try to multi buffer */
3094 xfer->stream_id, NULL);
3095
3096 /* add transfer last on interrupt queue */
3097 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3098
3099 /* start timeout, if any */
3100 if (xfer->timeout != 0)
3102}
3103
3105{
3110};
3111
3112/*------------------------------------------------------------------------*
3113 * xhci root HUB support
3114 *------------------------------------------------------------------------*
3115 * Simulate a hardware HUB by handling all the necessary requests.
3116 *------------------------------------------------------------------------*/
3117#define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
3118
3119static const
3121{
3122 .bLength = sizeof(xhci_devd),
3123 .bDescriptorType = UDESC_DEVICE, /* type */
3124 HSETW(.bcdUSB, 0x0300), /* USB version */
3125 .bDeviceClass = UDCLASS_HUB, /* class */
3126 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */
3127 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */
3128 .bMaxPacketSize = 9, /* max packet size */
3129 HSETW(.idVendor, 0x0000), /* vendor */
3130 HSETW(.idProduct, 0x0000), /* product */
3131 HSETW(.bcdDevice, 0x0100), /* device version */
3132 .iManufacturer = 1,
3133 .iProduct = 2,
3134 .iSerialNumber = 0,
3135 .bNumConfigurations = 1, /* # of configurations */
3136};
3137
3138static const
3140 .bosd = {
3141 .bLength = sizeof(xhci_bosd.bosd),
3142 .bDescriptorType = UDESC_BOS,
3143 HSETW(.wTotalLength, sizeof(xhci_bosd)),
3144 .bNumDeviceCaps = 3,
3145 },
3146 .usb2extd = {
3147 .bLength = sizeof(xhci_bosd.usb2extd),
3148 .bDescriptorType = 1,
3149 .bDevCapabilityType = 2,
3150 .bmAttributes[0] = 2,
3151 },
3152 .usbdcd = {
3153 .bLength = sizeof(xhci_bosd.usbdcd),
3154 .bDescriptorType = UDESC_DEVICE_CAPABILITY,
3155 .bDevCapabilityType = 3,
3156 .bmAttributes = 0, /* XXX */
3157 HSETW(.wSpeedsSupported, 0x000C),
3158 .bFunctionalitySupport = 8,
3159 .bU1DevExitLat = 255, /* dummy - not used */
3160 .wU2DevExitLat = { 0x00, 0x08 },
3161 },
3162 .cidd = {
3163 .bLength = sizeof(xhci_bosd.cidd),
3164 .bDescriptorType = 1,
3165 .bDevCapabilityType = 4,
3166 .bReserved = 0,
3167 .bContainerID = 0, /* XXX */
3168 },
3169};
3170
3171static const
3173 .confd = {
3174 .bLength = sizeof(xhci_confd.confd),
3175 .bDescriptorType = UDESC_CONFIG,
3176 .wTotalLength[0] = sizeof(xhci_confd),
3177 .bNumInterface = 1,
3178 .bConfigurationValue = 1,
3179 .iConfiguration = 0,
3180 .bmAttributes = UC_SELF_POWERED,
3181 .bMaxPower = 0 /* max power */
3182 },
3183 .ifcd = {
3184 .bLength = sizeof(xhci_confd.ifcd),
3185 .bDescriptorType = UDESC_INTERFACE,
3186 .bNumEndpoints = 1,
3189 .bInterfaceProtocol = 0,
3190 },
3191 .endpd = {
3192 .bLength = sizeof(xhci_confd.endpd),
3193 .bDescriptorType = UDESC_ENDPOINT,
3194 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT,
3195 .bmAttributes = UE_INTERRUPT,
3196 .wMaxPacketSize[0] = 2, /* max 15 ports */
3197 .bInterval = 255,
3198 },
3199 .endpcd = {
3200 .bLength = sizeof(xhci_confd.endpcd),
3201 .bDescriptorType = UDESC_ENDPOINT_SS_COMP,
3202 .bMaxBurst = 0,
3203 .bmAttributes = 0,
3204 },
3205};
3206
3207static const
3209 .bLength = sizeof(xhci_hubd),
3211};
3212
3213static usb_error_t
3215 struct usb_device_request *req, const void **pptr, uint16_t *plength)
3216{
3217 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3218 const char *str_ptr;
3219 const void *ptr;
3220 uint32_t port;
3221 uint32_t v;
3222 uint16_t len;
3223 uint16_t i;
3224 uint16_t value;
3225 uint16_t index;
3226 uint8_t j;
3227 usb_error_t err;
3228
3229 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3230
3231 /* buffer reset */
3232 ptr = (const void *)&sc->sc_hub_desc;
3233 len = 0;
3234 err = 0;
3235
3236 value = UGETW(req->wValue);
3237 index = UGETW(req->wIndex);
3238
3239 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3240 "wValue=0x%04x wIndex=0x%04x\n",
3241 req->bmRequestType, req->bRequest,
3242 UGETW(req->wLength), value, index);
3243
3244#define C(x,y) ((x) | ((y) << 8))
3245 switch (C(req->bRequest, req->bmRequestType)) {
3249 /*
3250 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3251 * for the integrated root hub.
3252 */
3253 break;
3255 len = 1;
3256 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3257 break;
3259 switch (value >> 8) {
3260 case UDESC_DEVICE:
3261 if ((value & 0xff) != 0) {
3262 err = USB_ERR_IOERROR;
3263 goto done;
3264 }
3265 len = sizeof(xhci_devd);
3266 ptr = (const void *)&xhci_devd;
3267 break;
3268
3269 case UDESC_BOS:
3270 if ((value & 0xff) != 0) {
3271 err = USB_ERR_IOERROR;
3272 goto done;
3273 }
3274 len = sizeof(xhci_bosd);
3275 ptr = (const void *)&xhci_bosd;
3276 break;
3277
3278 case UDESC_CONFIG:
3279 if ((value & 0xff) != 0) {
3280 err = USB_ERR_IOERROR;
3281 goto done;
3282 }
3283 len = sizeof(xhci_confd);
3284 ptr = (const void *)&xhci_confd;
3285 break;
3286
3287 case UDESC_STRING:
3288 switch (value & 0xff) {
3289 case 0: /* Language table */
3290 str_ptr = "\001";
3291 break;
3292
3293 case 1: /* Vendor */
3294 str_ptr = sc->sc_vendor;
3295 break;
3296
3297 case 2: /* Product */
3298 str_ptr = "XHCI root HUB";
3299 break;
3300
3301 default:
3302 str_ptr = "";
3303 break;
3304 }
3305
3307 sc->sc_hub_desc.temp,
3308 sizeof(sc->sc_hub_desc.temp),
3309 str_ptr);
3310 break;
3311
3312 default:
3313 err = USB_ERR_IOERROR;
3314 goto done;
3315 }
3316 break;
3318 len = 1;
3319 sc->sc_hub_desc.temp[0] = 0;
3320 break;
3322 len = 2;
3324 break;
3327 len = 2;
3328 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3329 break;
3331 if (value >= XHCI_MAX_DEVICES) {
3332 err = USB_ERR_IOERROR;
3333 goto done;
3334 }
3335 break;
3337 if (value != 0 && value != 1) {
3338 err = USB_ERR_IOERROR;
3339 goto done;
3340 }
3341 sc->sc_conf = value;
3342 break;
3344 break;
3348 err = USB_ERR_IOERROR;
3349 goto done;
3351 break;
3353 break;
3354 /* Hub requests */
3356 break;
3358 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3359
3360 if ((index < 1) ||
3361 (index > sc->sc_noport)) {
3362 err = USB_ERR_IOERROR;
3363 goto done;
3364 }
3365 port = XHCI_PORTSC(index);
3366
3367 v = XREAD4(sc, oper, port);
3368 i = XHCI_PS_PLS_GET(v);
3369 v &= ~XHCI_PS_CLEAR;
3370
3371 switch (value) {
3373 XWRITE4(sc, oper, port, v | XHCI_PS_WRC);
3374 break;
3376 XWRITE4(sc, oper, port, v | XHCI_PS_CEC);
3377 break;
3378 case UHF_C_PORT_SUSPEND:
3380 XWRITE4(sc, oper, port, v | XHCI_PS_PLC);
3381 break;
3383 XWRITE4(sc, oper, port, v | XHCI_PS_CSC);
3384 break;
3385 case UHF_C_PORT_ENABLE:
3386 XWRITE4(sc, oper, port, v | XHCI_PS_PEC);
3387 break;
3389 XWRITE4(sc, oper, port, v | XHCI_PS_OCC);
3390 break;
3391 case UHF_C_PORT_RESET:
3392 XWRITE4(sc, oper, port, v | XHCI_PS_PRC);
3393 break;
3394 case UHF_PORT_ENABLE:
3395 XWRITE4(sc, oper, port, v | XHCI_PS_PED);
3396 break;
3397 case UHF_PORT_POWER:
3398 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP);
3399 break;
3400 case UHF_PORT_INDICATOR:
3401 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
3402 break;
3403 case UHF_PORT_SUSPEND:
3404
3405 /* U3 -> U15 */
3406 if (i == 3) {
3407 XWRITE4(sc, oper, port, v |
3409 }
3410
3411 /* wait 20ms for resume sequence to complete */
3412 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3413
3414 /* U0 */
3415 XWRITE4(sc, oper, port, v |
3417 break;
3418 default:
3419 err = USB_ERR_IOERROR;
3420 goto done;
3421 }
3422 break;
3423
3425 if ((value & 0xff) != 0) {
3426 err = USB_ERR_IOERROR;
3427 goto done;
3428 }
3429
3430 v = XREAD4(sc, capa, XHCI_HCSPARAMS0);
3431
3433
3435
3436 if (XHCI_HCS0_PPC(v))
3438 else
3439 i = UHD_PWR_GANGED;
3440
3441 if (XHCI_HCS0_PIND(v))
3442 i |= UHD_PORT_IND;
3443
3444 i |= UHD_OC_INDIVIDUAL;
3445
3447
3448 /* see XHCI section 5.4.9: */
3450
3451 for (j = 1; j <= sc->sc_noport; j++) {
3452 v = XREAD4(sc, oper, XHCI_PORTSC(j));
3453 if (v & XHCI_PS_DR) {
3454 sc->sc_hub_desc.hubd.
3455 DeviceRemovable[j / 8] |= 1U << (j % 8);
3456 }
3457 }
3459 break;
3460
3462 len = 16;
3463 memset(sc->sc_hub_desc.temp, 0, 16);
3464 break;
3465
3467 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index);
3468
3469 if ((index < 1) ||
3470 (index > sc->sc_noport)) {
3471 err = USB_ERR_IOERROR;
3472 goto done;
3473 }
3474
3475 v = XREAD4(sc, oper, XHCI_PORTSC(index));
3476
3477 DPRINTFN(9, "port status=0x%08x\n", v);
3478
3480
3481 switch (XHCI_PS_SPEED_GET(v)) {
3482 case 3:
3483 i |= UPS_HIGH_SPEED;
3484 break;
3485 case 2:
3486 i |= UPS_LOW_SPEED;
3487 break;
3488 case 1:
3489 /* FULL speed */
3490 break;
3491 default:
3492 i |= UPS_OTHER_SPEED;
3493 break;
3494 }
3495
3496 if (v & XHCI_PS_CCS)
3498 if (v & XHCI_PS_PED)
3499 i |= UPS_PORT_ENABLED;
3500 if (v & XHCI_PS_OCA)
3502 if (v & XHCI_PS_PR)
3503 i |= UPS_RESET;
3504#if 0
3505 if (v & XHCI_PS_PP)
3506 /* XXX undefined */
3507#endif
3509
3510 i = 0;
3511 if (v & XHCI_PS_CSC)
3513 if (v & XHCI_PS_PEC)
3514 i |= UPS_C_PORT_ENABLED;
3515 if (v & XHCI_PS_OCC)
3517 if (v & XHCI_PS_WRC)
3519 if (v & XHCI_PS_PRC)
3520 i |= UPS_C_PORT_RESET;
3521 if (v & XHCI_PS_PLC)
3523 if (v & XHCI_PS_CEC)
3525
3527 len = sizeof(sc->sc_hub_desc.ps);
3528 break;
3529
3531 err = USB_ERR_IOERROR;
3532 goto done;
3533
3535 break;
3536
3538
3539 i = index >> 8;
3540 index &= 0x00FF;
3541
3542 if ((index < 1) ||
3543 (index > sc->sc_noport)) {
3544 err = USB_ERR_IOERROR;
3545 goto done;
3546 }
3547
3548 port = XHCI_PORTSC(index);
3549 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
3550
3551 switch (value) {
3553 if (XHCI_PS_SPEED_GET(v) != 4) {
3554 err = USB_ERR_IOERROR;
3555 goto done;
3556 }
3557 port = XHCI_PORTPMSC(index);
3558 v = XREAD4(sc, oper, port);
3559 v &= ~XHCI_PM3_U1TO_SET(0xFF);
3560 v |= XHCI_PM3_U1TO_SET(i);
3561 XWRITE4(sc, oper, port, v);
3562 break;
3564 if (XHCI_PS_SPEED_GET(v) != 4) {
3565 err = USB_ERR_IOERROR;
3566 goto done;
3567 }
3568 port = XHCI_PORTPMSC(index);
3569 v = XREAD4(sc, oper, port);
3570 v &= ~XHCI_PM3_U2TO_SET(0xFF);
3571 v |= XHCI_PM3_U2TO_SET(i);
3572 XWRITE4(sc, oper, port, v);
3573 break;
3574 case UHF_BH_PORT_RESET:
3575 XWRITE4(sc, oper, port, v | XHCI_PS_WPR);
3576 break;
3578 XWRITE4(sc, oper, port, v |
3580 /* 4ms settle time */
3581 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3582 break;
3583 case UHF_PORT_ENABLE:
3584 DPRINTFN(3, "set port enable %d\n", index);
3585 break;
3586 case UHF_PORT_SUSPEND:
3587 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i);
3588 j = XHCI_PS_SPEED_GET(v);
3589 if ((j < 1) || (j > 3)) {
3590 /* non-supported speed */
3591 err = USB_ERR_IOERROR;
3592 goto done;
3593 }
3594 XWRITE4(sc, oper, port, v |
3595 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS);
3596 break;
3597 case UHF_PORT_RESET:
3598 DPRINTFN(6, "reset port %d\n", index);
3599 XWRITE4(sc, oper, port, v | XHCI_PS_PR);
3600 break;
3601 case UHF_PORT_POWER:
3602 DPRINTFN(3, "set port power %d\n", index);
3603 XWRITE4(sc, oper, port, v | XHCI_PS_PP);
3604 break;
3605 case UHF_PORT_TEST:
3606 DPRINTFN(3, "set port test %d\n", index);
3607 break;
3608 case UHF_PORT_INDICATOR:
3609 DPRINTFN(3, "set port indicator %d\n", index);
3610
3611 v &= ~XHCI_PS_PIC_SET(3);
3612 v |= XHCI_PS_PIC_SET(1);
3613
3614 XWRITE4(sc, oper, port, v);
3615 break;
3616 default:
3617 err = USB_ERR_IOERROR;
3618 goto done;
3619 }
3620 break;
3621
3626 break;
3627 default:
3628 err = USB_ERR_IOERROR;
3629 goto done;
3630 }
3631done:
3632 *plength = len;
3633 *pptr = ptr;
3634 return (err);
3635}
3636
3637static void
3639{
3640 struct usb_page_search page_info;
3641 struct usb_page_cache *pc;
3642 struct usb_xfer *xfer;
3643 void *last_obj;
3644 uint32_t ntd;
3645 uint32_t n;
3646
3647 xfer = parm->curr_xfer;
3648
3649 /*
3650 * The proof for the "ntd" formula is illustrated like this:
3651 *
3652 * +------------------------------------+
3653 * | |
3654 * | |remainder -> |
3655 * | +-----+---+ |
3656 * | | xxx | x | frm 0 |
3657 * | +-----+---++ |
3658 * | | xxx | xx | frm 1 |
3659 * | +-----+----+ |
3660 * | ... |
3661 * +------------------------------------+
3662 *
3663 * "xxx" means a completely full USB transfer descriptor
3664 *
3665 * "x" and "xx" means a short USB packet
3666 *
3667 * For the remainder of an USB transfer modulo
3668 * "max_data_length" we need two USB transfer descriptors.
3669 * One to transfer the remaining data and one to finalise with
3670 * a zero length packet in case the "force_short_xfer" flag is
3671 * set. We only need two USB transfer descriptors in the case
3672 * where the transfer length of the first one is a factor of
3673 * "max_frame_size". The rest of the needed USB transfer
3674 * descriptors is given by the buffer size divided by the
3675 * maximum data payload.
3676 */
3677 parm->hc_max_packet_size = 0x400;
3678 parm->hc_max_packet_count = 16 * 3;
3680
3681 xfer->flags_int.bdma_enable = 1;
3682
3684
3685 if (xfer->flags_int.isochronous_xfr) {
3686 ntd = ((1 * xfer->nframes)
3687 + (xfer->max_data_length / xfer->max_hc_frame_size));
3688 } else if (xfer->flags_int.control_xfr) {
3689 ntd = ((2 * xfer->nframes) + 1 /* STATUS */
3690 + (xfer->max_data_length / xfer->max_hc_frame_size));
3691 } else {
3692 ntd = ((2 * xfer->nframes)
3693 + (xfer->max_data_length / xfer->max_hc_frame_size));
3694 }
3695
3696alloc_dma_set:
3697
3698 if (parm->err)
3699 return;
3700
3701 /*
3702 * Allocate queue heads and transfer descriptors
3703 */
3704 last_obj = NULL;
3705
3707 parm, &pc, sizeof(struct xhci_td),
3708 XHCI_TD_ALIGN, ntd)) {
3709 parm->err = USB_ERR_NOMEM;
3710 return;
3711 }
3712 if (parm->buf) {
3713 for (n = 0; n != ntd; n++) {
3714 struct xhci_td *td;
3715
3716 usbd_get_page(pc + n, 0, &page_info);
3717
3718 td = page_info.buffer;
3719
3720 /* init TD */
3721 td->td_self = page_info.physaddr;
3722 td->obj_next = last_obj;
3723 td->page_cache = pc + n;
3724
3725 last_obj = td;
3726
3727 usb_pc_cpu_flush(pc + n);
3728 }
3729 }
3730 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3731
3732 if (!xfer->flags_int.curr_dma_set) {
3733 xfer->flags_int.curr_dma_set = 1;
3734 goto alloc_dma_set;
3735 }
3736}
3737
3738static uint8_t
3739xhci_get_endpoint_state(struct usb_device *udev, uint8_t epno)
3740{
3741 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3742 struct usb_page_search buf_dev;
3743 struct xhci_hw_dev *hdev;
3744 struct xhci_endp_ctx *endp;
3745 uint32_t temp;
3746
3747 MPASS(epno != 0);
3748
3749 hdev = &sc->sc_hw.devs[udev->controller_slot_id];
3750
3751 usbd_get_page(&hdev->device_pc, 0, &buf_dev);
3752 endp = XHCI_GET_CTX(sc, xhci_dev_ctx, ctx_ep[epno - 1],
3753 buf_dev.buffer);
3755
3756 temp = le32toh(endp->dwEpCtx0);
3757
3758 return (XHCI_EPCTX_0_EPSTATE_GET(temp));
3759}
3760
3761static usb_error_t
3763{
3764 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3765 struct usb_page_search buf_inp;
3766 struct usb_device *udev;
3767 struct xhci_endpoint_ext *pepext;
3768 struct usb_endpoint_descriptor *edesc;
3769 struct usb_page_cache *pcinp;
3770 usb_error_t err;
3771 usb_stream_t stream_id;
3772 uint32_t mask;
3773 uint8_t index;
3774 uint8_t epno;
3775
3776 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3777 xfer->endpoint->edesc);
3778
3779 udev = xfer->xroot->udev;
3780 index = udev->controller_slot_id;
3781
3782 pcinp = &sc->sc_hw.devs[index].input_pc;
3783
3784 usbd_get_page(pcinp, 0, &buf_inp);
3785
3786 edesc = xfer->endpoint->edesc;
3787
3788 epno = edesc->bEndpointAddress;
3789 stream_id = xfer->stream_id;
3790
3791 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
3792 epno |= UE_DIR_IN;
3793
3794 epno = XHCI_EPNO2EPID(epno);
3795
3796 if (epno == 0)
3797 return (USB_ERR_NO_PIPE); /* invalid */
3798
3799 XHCI_CMD_LOCK(sc);
3800
3801 /* configure endpoint */
3802
3804
3805 if (err != 0) {
3806 XHCI_CMD_UNLOCK(sc);
3807 return (err);
3808 }
3809
3810 /*
3811 * Get the endpoint into the stopped state according to the
3812 * endpoint context state diagram in the XHCI specification:
3813 */
3814 switch (xhci_get_endpoint_state(udev, epno)) {
3816 break;
3818 break;
3820 err = xhci_cmd_reset_ep(sc, 0, epno, index);
3821 if (err != 0)
3822 DPRINTF("Could not reset endpoint %u\n", epno);
3823 break;
3824 default:
3825 err = xhci_cmd_stop_ep(sc, 0, epno, index);
3826 if (err != 0)
3827 DPRINTF("Could not stop endpoint %u\n", epno);
3828 break;
3829 }
3830
3832 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
3834 stream_id, epno, index);
3835
3836 if (err != 0)
3837 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
3838
3839 /*
3840 * Get the endpoint into the running state according to the
3841 * endpoint context state diagram in the XHCI specification:
3842 */
3843
3844 mask = (1U << epno);
3845 xhci_configure_mask(udev, mask | 1U, 0);
3846
3847 if (!(sc->sc_hw.devs[index].ep_configured & mask)) {
3848 sc->sc_hw.devs[index].ep_configured |= mask;
3849 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index);
3850 } else {
3851 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
3852 }
3853
3854 if (err != 0) {
3855 DPRINTF("Could not configure "
3856 "endpoint %u at slot %u.\n", epno, index);
3857 }
3858 XHCI_CMD_UNLOCK(sc);
3859
3860 return (0);
3861}
3862
3863static void
3865{
3866 return;
3867}
3868
3869static void
3871{
3872 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3873
3874 /* put transfer on interrupt queue (again) */
3876
3877 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
3878 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
3879}
3880
3881static void
3883{
3884 struct xhci_softc *sc;
3885 struct xhci_endpoint_ext *pepext;
3886 struct usb_xfer *xfer;
3887
3888 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus);
3889
3890restart:
3891 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3892 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3893 xfer->endpoint->edesc);
3894
3895 if ((pepext->trb_halted != 0) ||
3896 (pepext->trb_running == 0)) {
3897 uint16_t i;
3898
3899 /* clear halted and running */
3900 pepext->trb_halted = 0;
3901 pepext->trb_running = 0;
3902
3903 /* nuke remaining buffered transfers */
3904
3905 for (i = 0; i != (XHCI_MAX_TRANSFERS *
3906 XHCI_MAX_STREAMS); i++) {
3907 /*
3908 * NOTE: We need to use the timeout
3909 * error code here else existing
3910 * isochronous clients can get
3911 * confused:
3912 */
3913 if (pepext->xfer[i] != NULL) {
3914 xhci_device_done(pepext->xfer[i],
3916 }
3917 }
3918
3919 /*
3920 * NOTE: The USB transfer cannot vanish in
3921 * this state!
3922 */
3923
3924 USB_BUS_UNLOCK(&sc->sc_bus);
3925
3927
3928 USB_BUS_LOCK(&sc->sc_bus);
3929
3930 /* check if halted is still cleared */
3931 if (pepext->trb_halted == 0) {
3932 pepext->trb_running = 1;
3933 memset(pepext->trb_index, 0,
3934 sizeof(pepext->trb_index));
3935 }
3936 goto restart;
3937 }
3938
3939 if (xfer->flags_int.did_dma_delay) {
3940 /* remove transfer from interrupt queue (again) */
3942
3943 /* we are finally done */
3945
3946 /* queue changed - restart */
3947 goto restart;
3948 }
3949 }
3950
3951 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3952 /* try to insert xfer on HW queue */
3954
3955 /* try to multi buffer */
3957 xfer->stream_id, NULL);
3958 }
3959}
3960
3961static void
3963 struct usb_endpoint *ep)
3964{
3965 struct xhci_endpoint_ext *pepext;
3966 struct xhci_softc *sc;
3967 uint8_t index;
3968 uint8_t epno;
3969
3970 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
3971 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode);
3972
3973 if (udev->parent_hub == NULL) {
3974 /* root HUB has special endpoint handling */
3975 return;
3976 }
3977
3979
3980 pepext = xhci_get_endpoint_ext(udev, edesc);
3981
3982 USB_BUS_LOCK(udev->bus);
3983 pepext->trb_halted = 1;
3984 pepext->trb_running = 0;
3985
3986 /*
3987 * When doing an alternate setting, except for control
3988 * endpoints, we need to re-configure the XHCI endpoint
3989 * context:
3990 */
3991 if ((edesc->bEndpointAddress & UE_ADDR) != 0) {
3992 sc = XHCI_BUS2SC(udev->bus);
3993 index = udev->controller_slot_id;
3994 epno = XHCI_EPNO2EPID(edesc->bEndpointAddress);
3995 sc->sc_hw.devs[index].ep_configured &= ~(1U << epno);
3996 }
3997 USB_BUS_UNLOCK(udev->bus);
3998}
3999
4000static void
4001xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
4002{
4003
4004}
4005
4006static void
4008{
4009 struct xhci_endpoint_ext *pepext;
4010
4011 DPRINTF("\n");
4012
4013 if (udev->flags.usb_mode != USB_MODE_HOST) {
4014 /* not supported */
4015 return;
4016 }
4017 if (udev->parent_hub == NULL) {
4018 /* root HUB has special endpoint handling */
4019 return;
4020 }
4021
4022 pepext = xhci_get_endpoint_ext(udev, ep->edesc);
4023
4024 USB_BUS_LOCK(udev->bus);
4025 pepext->trb_halted = 1;
4026 pepext->trb_running = 0;
4027 USB_BUS_UNLOCK(udev->bus);
4028}
4029
4030static usb_error_t
4032{
4033 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4034 usb_error_t err;
4035 uint8_t temp;
4036
4037 /* no init for root HUB */
4038 if (udev->parent_hub == NULL)
4039 return (0);
4040
4041 XHCI_CMD_LOCK(sc);
4042
4043 /* set invalid default */
4044
4045 udev->controller_slot_id = sc->sc_noslot + 1;
4046
4047 /* try to get a new slot ID from the XHCI */
4048
4049 err = xhci_cmd_enable_slot(sc, &temp);
4050
4051 if (err) {
4052 XHCI_CMD_UNLOCK(sc);
4053 return (err);
4054 }
4055
4056 if (temp > sc->sc_noslot) {
4057 XHCI_CMD_UNLOCK(sc);
4058 return (USB_ERR_BAD_ADDRESS);
4059 }
4060
4061 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) {
4062 DPRINTF("slot %u already allocated.\n", temp);
4063 XHCI_CMD_UNLOCK(sc);
4064 return (USB_ERR_BAD_ADDRESS);
4065 }
4066
4067 /* store slot ID for later reference */
4068
4069 udev->controller_slot_id = temp;
4070
4071 /* reset data structure */
4072
4073 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0]));
4074
4075 /* set mark slot allocated */
4076
4077 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED;
4078
4079 err = xhci_alloc_device_ext(udev);
4080
4081 XHCI_CMD_UNLOCK(sc);
4082
4083 /* get device into default state */
4084
4085 if (err == 0)
4086 err = xhci_set_address(udev, NULL, 0);
4087
4088 return (err);
4089}
4090
4091static void
4093{
4094 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4095 uint8_t index;
4096
4097 /* no init for root HUB */
4098 if (udev->parent_hub == NULL)
4099 return;
4100
4101 XHCI_CMD_LOCK(sc);
4102
4103 index = udev->controller_slot_id;
4104
4105 if (index <= sc->sc_noslot) {
4108
4109 /* free device extension */
4111 }
4112
4113 XHCI_CMD_UNLOCK(sc);
4114}
4115
4116static void
4117xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4118{
4119 /*
4120 * Wait until the hardware has finished any possible use of
4121 * the transfer descriptor(s)
4122 */
4123 *pus = 2048; /* microseconds */
4124}
4125
4126static void
4128{
4129 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4130 uint8_t index;
4131 uint8_t n;
4132 uint8_t p;
4133
4134 DPRINTF("\n");
4135
4136 /* check for root HUB */
4137 if (udev->parent_hub == NULL)
4138 return;
4139
4140 index = udev->controller_slot_id;
4141
4142 XHCI_CMD_LOCK(sc);
4143
4144 /* blindly resume all endpoints */
4145
4146 USB_BUS_LOCK(udev->bus);
4147
4148 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4149 for (p = 0; p != XHCI_MAX_STREAMS; p++) {
4150 XWRITE4(sc, door, XHCI_DOORBELL(index),
4151 n | XHCI_DB_SID_SET(p));
4152 }
4153 }
4154
4155 USB_BUS_UNLOCK(udev->bus);
4156
4157 XHCI_CMD_UNLOCK(sc);
4158}
4159
4160static void
4162{
4163 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4164 uint8_t index;
4165 uint8_t n;
4166 usb_error_t err;
4167
4168 DPRINTF("\n");
4169
4170 /* check for root HUB */
4171 if (udev->parent_hub == NULL)
4172 return;
4173
4174 index = udev->controller_slot_id;
4175
4176 XHCI_CMD_LOCK(sc);
4177
4178 /* blindly suspend all endpoints */
4179
4180 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4181 err = xhci_cmd_stop_ep(sc, 1, n, index);
4182 if (err != 0) {
4183 DPRINTF("Failed to suspend endpoint "
4184 "%u on slot %u (ignored).\n", n, index);
4185 }
4186 }
4187
4188 XHCI_CMD_UNLOCK(sc);
4189}
4190
4191static void
4193{
4194 DPRINTF("\n");
4195}
4196
4197static void
4199{
4200 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4201 struct usb_page_search buf_inp;
4202 usb_error_t err;
4203 uint8_t index;
4204
4205 /* check for root HUB */
4206 if (udev->parent_hub == NULL)
4207 return;
4208
4209 index = udev->controller_slot_id;
4210
4211 DPRINTF("\n");
4212
4214 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports,
4215 &sc->sc_hw.devs[index].tt);
4216 if (err != 0)
4217 sc->sc_hw.devs[index].nports = 0;
4218 }
4219
4220 XHCI_CMD_LOCK(sc);
4221
4222 switch (usb_get_device_state(udev)) {
4223 case USB_STATE_POWERED:
4224 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT)
4225 break;
4226
4227 /* set default state */
4229 sc->sc_hw.devs[index].ep_configured = 3U;
4230
4231 /* reset number of contexts */
4232 sc->sc_hw.devs[index].context_num = 0;
4233
4234 err = xhci_cmd_reset_dev(sc, index);
4235
4236 if (err != 0) {
4237 DPRINTF("Device reset failed "
4238 "for slot %u.\n", index);
4239 }
4240 break;
4241
4244 break;
4245
4247 sc->sc_hw.devs[index].ep_configured = 3U;
4248
4249 /* set configure mask to slot only */
4250 xhci_configure_mask(udev, 1, 0);
4251
4252 /* deconfigure all endpoints, except EP0 */
4253 err = xhci_cmd_configure_ep(sc, 0, 1, index);
4254
4255 if (err) {
4256 DPRINTF("Failed to deconfigure "
4257 "slot %u.\n", index);
4258 }
4259 break;
4260
4262 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) {
4263 /* deconfigure all endpoints, except EP0 */
4264 err = xhci_cmd_configure_ep(sc, 0, 1, index);
4265
4266 if (err) {
4267 DPRINTF("Failed to deconfigure "
4268 "slot %u.\n", index);
4269 }
4270 }
4271
4272 /* set configured state */
4274 sc->sc_hw.devs[index].ep_configured = 3U;
4275
4276 /* reset number of contexts */
4277 sc->sc_hw.devs[index].context_num = 0;
4278
4279 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
4280
4281 xhci_configure_mask(udev, 3, 0);
4282
4283 err = xhci_configure_device(udev);
4284 if (err != 0) {
4285 DPRINTF("Could not configure device "
4286 "at slot %u.\n", index);
4287 }
4288
4289 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
4290 if (err != 0) {
4291 DPRINTF("Could not evaluate device "
4292 "context at slot %u.\n", index);
4293 }
4294 break;
4295
4296 default:
4297 break;
4298 }
4299 XHCI_CMD_UNLOCK(sc);
4300}
4301
4302static usb_error_t
4304 uint8_t ep_mode)
4305{
4306 switch (ep_mode) {
4308 return (0);
4310 if (xhcistreams == 0 ||
4311 (ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
4312 udev->speed != USB_SPEED_SUPER)
4313 return (USB_ERR_INVAL);
4314 return (0);
4315 default:
4316 return (USB_ERR_INVAL);
4317 }
4318}
4319
4320static const struct usb_bus_methods xhci_bus_methods = {
4322 .endpoint_uninit = xhci_ep_uninit,
4323 .xfer_setup = xhci_xfer_setup,
4324 .xfer_unsetup = xhci_xfer_unsetup,
4325 .get_dma_delay = xhci_get_dma_delay,
4326 .device_init = xhci_device_init,
4327 .device_uninit = xhci_device_uninit,
4328 .device_resume = xhci_device_resume,
4329 .device_suspend = xhci_device_suspend,
4330 .set_hw_power = xhci_set_hw_power,
4331 .roothub_exec = xhci_roothub_exec,
4332 .xfer_poll = xhci_do_poll,
4333 .start_dma_delay = xhci_start_dma_delay,
4334 .set_address = xhci_set_address,
4335 .clear_stall = xhci_ep_clear_stall,
4336 .device_state_change = xhci_device_state_change,
4337 .set_hw_power_sleep = xhci_set_hw_power_sleep,
4338 .set_endpoint_mode = xhci_set_endpoint_mode,
4339};
static int debug
Definition: cfumass.c:73
uint16_t len
Definition: ehci.h:41
uint8_t n
Definition: if_run.c:612
uint8_t k
Definition: if_run.c:612
uint16_t retry
Definition: if_runreg.h:7
struct @109 error
uint8_t id
Definition: if_usievar.h:4
volatile uint32_t td_next
Definition: ohci.h:19
uint32_t value
u_int index
enum pci_id_type type
int state
uint64_t address
u_int bus
uint64_t * addr
u_int slot
uByte bLength
Definition: usb.h:313
void(* endpoint_init)(struct usb_device *, struct usb_endpoint_descriptor *, struct usb_endpoint *)
struct usb_bus * bus
Definition: usb_bus.h:41
struct usb_proc_msg hdr
Definition: usb_bus.h:40
enum usb_revision usbrev
Definition: usb_bus.h:119
struct mtx bus_mtx
Definition: usb_bus.h:95
device_t parent
Definition: usb_bus.h:100
const struct usb_bus_methods * methods
Definition: usb_bus.h:107
struct usb_device ** devices
Definition: usb_bus.h:108
uint8_t control_ep_quirk
Definition: usb_bus.h:125
uint8_t devices_max
Definition: usb_bus.h:121
uint8_t dma_bits
Definition: usb_bus.h:124
struct usb_xfer_queue intr_q
Definition: usb_bus.h:97
uByte bDescriptorType
Definition: usb.h:290
uByte bDeviceProtocol
Definition: usb.h:298
uint8_t self_suspended
Definition: usb_device.h:107
enum usb_hc_mode usb_mode
Definition: usb_device.h:94
struct usb_device * parent_hs_hub
Definition: usb_device.h:219
enum usb_dev_speed speed
Definition: usb_device.h:235
uint8_t hs_port_no
Definition: usb_device.h:253
uint8_t depth
Definition: usb_device.h:249
struct usb_bus * bus
Definition: usb_device.h:216
struct usb_device_descriptor ddesc
Definition: usb_device.h:270
uint8_t controller_slot_id
Definition: usb_device.h:245
struct usb_device * parent_hub
Definition: usb_device.h:218
uint8_t port_no
Definition: usb_device.h:251
uint8_t address
Definition: usb_device.h:243
struct usb_device_flags flags
Definition: usb_device.h:266
struct usb_endpoint_descriptor ctrl_ep_desc
Definition: usb_device.h:268
uByte bEndpointAddress
Definition: usb.h:528
uWord wMaxPacketSize
Definition: usb.h:558
struct usb_endpoint_ss_comp_descriptor * ecomp
Definition: usbdi.h:145
struct usb_xfer_queue endpoint_q[USB_MAX_EP_STREAMS]
Definition: usbdi.h:142
uint8_t ep_mode
Definition: usbdi.h:168
const struct usb_pipe_methods * methods
Definition: usbdi.h:146
struct usb_endpoint_descriptor * edesc
Definition: usbdi.h:144
uByte bDescriptorType
Definition: usb.h:634
uWord wHubCharacteristics
Definition: usb.h:636
uByte bPwrOn2PwrGood
Definition: usb.h:637
struct usb_dma_parent_tag * tag_parent
Definition: usb_busdma.h:95
usb_size_t length
Definition: usb_busdma.h:82
void(* open)(struct usb_xfer *)
uWord wPortStatus
Definition: usb.h:704
uWord wPortChange
Definition: usb.h:735
usb_proc_callback_t * pm_callback
Definition: usbdi.h:522
uint32_t hc_max_frame_size
Definition: usb_transfer.h:115
uint8_t hc_max_packet_count
Definition: usb_transfer.h:117
struct usb_xfer * curr_xfer
Definition: usb_transfer.h:105
uint16_t hc_max_packet_size
Definition: usb_transfer.h:116
usb_error_t err
Definition: usb_transfer.h:120
uWord wStatus
Definition: usb.h:686
uint8_t control_hdr
Definition: usb_core.h:104
uint8_t control_act
Definition: usb_core.h:106
uint8_t short_frames_ok
Definition: usb_core.h:110
uint8_t control_did_data
Definition: usb_core.h:108
uint8_t isochronous_xfr
Definition: usb_core.h:120
uint8_t curr_dma_set
Definition: usb_core.h:121
uint8_t control_xfr
Definition: usb_core.h:103
uint8_t did_dma_delay
Definition: usb_core.h:97
uint8_t bandwidth_reclaimed
Definition: usb_core.h:102
uint8_t force_short_xfer
Definition: usbdi.h:196
struct usb_xfer * curr
Definition: usbdi.h:128
struct usb_bus * bus
Definition: usb_transfer.h:78
struct usb_device * udev
Definition: usb_transfer.h:79
usb_stream_t stream_id
Definition: usb_core.h:165
usb_frlength_t * frlengths
Definition: usb_core.h:150
usb_frcount_t nframes
Definition: usb_core.h:162
usb_frlength_t max_data_length
Definition: usb_core.h:155
uint16_t qh_pos
Definition: usb_core.h:169
struct usb_page_cache * frbuffers
Definition: usb_core.h:151
uint8_t max_packet_count
Definition: usb_core.h:175
void * td_start[2]
Definition: usb_core.h:143
usb_timeout_t timeout
Definition: usb_core.h:158
usb_frcount_t aframes
Definition: usb_core.h:163
struct usb_endpoint * endpoint
Definition: usb_core.h:140
struct usb_xfer_flags_int flags_int
Definition: usb_core.h:182
struct usb_xfer_flags flags
Definition: usb_core.h:181
void * td_transfer_cache
Definition: usb_core.h:146
struct usb_xfer_root * xroot
Definition: usb_core.h:141
usb_frlength_t max_hc_frame_size
Definition: usb_core.h:154
usb_timeout_t interval
Definition: usb_core.h:171
uint16_t max_frame_size
Definition: usb_core.h:168
void * td_transfer_last
Definition: usb_core.h:145
void * td_transfer_first
Definition: usb_core.h:144
uint16_t max_packet_size
Definition: usb_core.h:167
uint8_t endpointno
Definition: usb_core.h:174
struct usb_bos_descriptor bosd
Definition: xhci.h:475
struct usb_devcap_container_id_descriptor cidd
Definition: xhci.h:478
struct usb_devcap_ss_descriptor usbdcd
Definition: xhci.h:477
struct usb_devcap_usb2ext_descriptor usb2extd
Definition: xhci.h:476
struct usb_endpoint_descriptor endpd
Definition: xhci.h:470
struct usb_interface_descriptor ifcd
Definition: xhci.h:469
struct usb_endpoint_ss_comp_descriptor endpcd
Definition: xhci.h:471
struct usb_config_descriptor confd
Definition: xhci.h:468
volatile uint64_t qwBaaDevCtxAddr[USB_MAX_DEVICES+1]
Definition: xhci.h:65
volatile uint64_t qwSpBufPtr[XHCI_MAX_SCRATCHPADS]
Definition: xhci.h:69
volatile uint32_t dwEpCtx6
Definition: xhci.h:160
volatile uint32_t dwEpCtx0
Definition: xhci.h:120
volatile uint32_t dwEpCtx7
Definition: xhci.h:161
volatile uint64_t qwEpCtx2
Definition: xhci.h:150
volatile uint32_t dwEpCtx4
Definition: xhci.h:154
volatile uint32_t dwEpCtx1
Definition: xhci.h:139
volatile uint32_t dwEpCtx5
Definition: xhci.h:159
struct xhci_trb * trb
Definition: xhci.h:415
uint8_t trb_used[XHCI_MAX_STREAMS]
Definition: xhci.h:419
uint64_t physaddr
Definition: xhci.h:418
uint8_t trb_halted
Definition: xhci.h:421
struct usb_page_cache * page_cache
Definition: xhci.h:417
struct usb_xfer * xfer[XHCI_MAX_TRANSFERS *XHCI_MAX_STREAMS]
Definition: xhci.h:416
uint8_t trb_index[XHCI_MAX_STREAMS]
Definition: xhci.h:420
uint8_t trb_running
Definition: xhci.h:422
uint8_t trb_ep_maxp
Definition: xhci.h:424
uint8_t trb_ep_mode
Definition: xhci.h:423
volatile uint32_t dwEvrsTableSize
Definition: xhci.h:399
volatile uint64_t qwEvrsTablePtr
Definition: xhci.h:398
uint32_t ep_configured
Definition: xhci.h:447
struct usb_page_cache device_pc
Definition: xhci.h:437
struct usb_page device_pg
Definition: xhci.h:441
struct usb_page_cache endpoint_pc[XHCI_MAX_ENDPOINTS]
Definition: xhci.h:439
uint8_t state
Definition: xhci.h:449
uint8_t tt
Definition: xhci.h:451
struct usb_page endpoint_pg[XHCI_MAX_ENDPOINTS]
Definition: xhci.h:443
struct usb_page_cache input_pc
Definition: xhci.h:438
uint8_t context_num
Definition: xhci.h:452
struct xhci_endpoint_ext endp[XHCI_MAX_ENDPOINTS]
Definition: xhci.h:445
uint8_t nports
Definition: xhci.h:450
struct usb_page input_pg
Definition: xhci.h:442
struct xhci_trb hwr_events[XHCI_MAX_EVENTS]
Definition: xhci.h:408
struct xhci_event_ring_seg hwr_ring_seg[XHCI_MAX_RSEG]
Definition: xhci.h:404
struct xhci_trb hwr_commands[XHCI_MAX_COMMANDS]
Definition: xhci.h:409
struct usb_page_cache ctx_pc
Definition: xhci.h:457
struct usb_page_cache scratch_pc[XHCI_MAX_SCRATCHPADS]
Definition: xhci.h:458
struct xhci_hw_dev devs[XHCI_MAX_DEVICES+1]
Definition: xhci.h:464
struct usb_page ctx_pg
Definition: xhci.h:461
struct usb_page root_pg
Definition: xhci.h:460
struct usb_page_cache root_pc
Definition: xhci.h:456
struct usb_page scratch_pg[XHCI_MAX_SCRATCHPADS]
Definition: xhci.h:462
volatile uint32_t dwInCtx0
Definition: xhci.h:171
volatile uint32_t dwInCtx1
Definition: xhci.h:173
volatile uint32_t dwSctx3
Definition: xhci.h:103
volatile uint32_t dwSctx2
Definition: xhci.h:94
volatile uint32_t dwSctx1
Definition: xhci.h:87
volatile uint32_t dwSctx0
Definition: xhci.h:76
char sc_vendor[16]
Definition: xhci.h:565
uint8_t sc_conf
Definition: xhci.h:549
uint32_t sc_door_off
Definition: xhci.h:531
uint8_t sc_ist
Definition: xhci.h:562
xhci_port_route_t * sc_port_route
Definition: xhci.h:499
uint8_t sc_ctlstep
Definition: xhci.h:551
struct usb_bus sc_bus
Definition: xhci.h:493
uint16_t sc_event_idx
Definition: xhci.h:535
uint8_t sc_noslot
Definition: xhci.h:545
uint64_t sc_cmd_addr
Definition: xhci.h:516
struct cv sc_cmd_cv
Definition: xhci.h:503
uint32_t sc_exit_lat_max
Definition: xhci.h:522
uint16_t sc_command_idx
Definition: xhci.h:536
uint32_t sc_runt_off
Definition: xhci.h:529
union xhci_hub_desc sc_hub_desc
Definition: xhci.h:501
struct sx sc_cmd_sx
Definition: xhci.h:504
struct usb_bus_msg sc_config_msg[2]
Definition: xhci.h:495
struct usb_device * sc_devices[XHCI_MAX_DEVICES]
Definition: xhci.h:506
uint8_t sc_no_deconfigure
Definition: xhci.h:559
uint32_t sc_capa_off
Definition: xhci.h:527
uint32_t sc_oper_off
Definition: xhci.h:525
uint16_t sc_erst_max
Definition: xhci.h:534
uint32_t sc_cmd_result[2]
Definition: xhci.h:518
uint16_t sc_imod_default
Definition: xhci.h:537
uint16_t sc_noscratch
Definition: xhci.h:540
struct xhci_hw_softc sc_hw
Definition: xhci.h:491
uint8_t sc_ctx_is_64_byte
Definition: xhci.h:556
uint8_t sc_hub_idata[32]
Definition: xhci.h:553
uint8_t sc_event_ccs
Definition: xhci.h:542
uint8_t sc_noport
Definition: xhci.h:547
uint8_t sc_command_ccs
Definition: xhci.h:543
uint16_t isoc_delta
Definition: xhci.c:146
uint8_t last_frame
Definition: xhci.c:149
struct xhci_softc * sc
Definition: xhci.c:137
uint32_t max_packet_size
Definition: xhci.c:143
uint32_t average
Definition: xhci.c:144
uint8_t trb_type
Definition: xhci.c:150
uint32_t offset
Definition: xhci.c:142
struct usb_page_cache * pc
Definition: xhci.c:138
uint8_t tlbpc
Definition: xhci.c:153
uint8_t direction
Definition: xhci.c:151
uint8_t multishort
Definition: xhci.c:148
uint8_t step_td
Definition: xhci.c:154
uint8_t tbc
Definition: xhci.c:152
struct xhci_td * td
Definition: xhci.c:139
struct xhci_td * td_next
Definition: xhci.c:140
uint32_t isoc_frame
Definition: xhci.c:145
uint8_t do_isoc_sync
Definition: xhci.c:155
uint32_t len
Definition: xhci.c:141
uint8_t shortpkt
Definition: xhci.c:147
Definition: xhci.h:374
struct xhci_td * alt_next
Definition: xhci.h:383
uint32_t len
Definition: xhci.h:386
uint8_t status
Definition: xhci.h:389
uint64_t td_self
Definition: xhci.h:381
uint8_t ntrb
Definition: xhci.h:388
struct usb_page_cache * page_cache
Definition: xhci.h:385
struct xhci_td * obj_next
Definition: xhci.h:384
uint32_t remainder
Definition: xhci.h:387
struct xhci_trb td_trb[XHCI_TD_PAGE_NBUF+1]
Definition: xhci.h:376
Definition: xhci.h:229
volatile uint32_t dwTrb2
Definition: xhci.h:233
volatile uint64_t qwTrb0
Definition: xhci.h:230
volatile uint32_t dwTrb3
Definition: xhci.h:247
#define DPRINTF(...)
Definition: umass.c:179
struct usb_hub_ss_descriptor hubd
Definition: xhci.h:484
struct usb_port_status ps
Definition: xhci.h:483
struct usb_status stat
Definition: xhci.h:482
uint8_t temp[128]
Definition: xhci.h:485
#define UE_INTERRUPT
Definition: usb.h:544
#define UR_SET_CONFIG
Definition: usb.h:219
#define UPS_C_PORT_RESET
Definition: usb.h:740
#define UHD_PWR_INDIVIDUAL
Definition: usb.h:609
#define UDSUBCLASS_HUB
Definition: usb.h:374
#define UC_SELF_POWERED
Definition: usb.h:395
#define UHD_PWR_GANGED
Definition: usb.h:608
#define UPS_C_BH_PORT_RESET
Definition: usb.h:742
#define UT_WRITE_INTERFACE
Definition: usb.h:170
#define UR_SET_DESCRIPTOR
Definition: usb.h:217
#define UDPROTO_HSHUBMTT
Definition: usb.h:377
#define UE_ADDR
Definition: usb.h:536
#define UE_BULK
Definition: usb.h:543
#define UHF_PORT_POWER
Definition: usb.h:254
#define UR_GET_CONFIG
Definition: usb.h:218
#define UT_WRITE_CLASS_OTHER
Definition: usb.h:178
#define UISUBCLASS_HUB
Definition: usb.h:480
#define UT_WRITE_CLASS_DEVICE
Definition: usb.h:176
#define UR_SET_ADDRESS
Definition: usb.h:193
#define UPS_C_PORT_ENABLED
Definition: usb.h:737
#define UT_WRITE_DEVICE
Definition: usb.h:169
#define UDESC_CONFIG
Definition: usb.h:196
#define UR_GET_INTERFACE
Definition: usb.h:220
#define UDESC_DEVICE_CAPABILITY
Definition: usb.h:208
#define UPS_C_OVERCURRENT_INDICATOR
Definition: usb.h:739
#define UDCLASS_HUB
Definition: usb.h:373
#define UDPROTO_SSHUB
Definition: usb.h:378
#define UHF_PORT_LINK_STATE
Definition: usb.h:253
#define UR_CLEAR_TT_BUFFER
Definition: usb.h:228
#define UHF_BH_PORT_RESET
Definition: usb.h:272
@ USB_STATE_POWERED
Definition: usb.h:790
@ USB_STATE_CONFIGURED
Definition: usb.h:792
@ USB_STATE_ADDRESSED
Definition: usb.h:791
#define UR_GET_STATUS
Definition: usb.h:190
#define UHF_C_PORT_SUSPEND
Definition: usb.h:259
#define UR_CLEAR_FEATURE
Definition: usb.h:191
#define UDESC_ENDPOINT
Definition: usb.h:200
#define UPS_LOW_SPEED
Definition: usb.h:729
#define UDESC_ENDPOINT_SS_COMP
Definition: usb.h:216
#define UHF_PORT_U1_TIMEOUT
Definition: usb.h:267
#define UPS_PORT_LINK_STATE_SET(x)
Definition: usb.h:713
#define UDS_SELF_POWERED
Definition: usb.h:688
#define UHF_PORT_TEST
Definition: usb.h:262
#define UDESC_STRING
Definition: usb.h:197
#define UHF_C_PORT_OVER_CURRENT
Definition: usb.h:260
#define UHF_PORT_SUSPEND
Definition: usb.h:250
#define UR_STOP_TT
Definition: usb.h:231
#define UE_DIR_IN
Definition: usb.h:531
#define UR_SYNCH_FRAME
Definition: usb.h:222
#define UT_READ_DEVICE
Definition: usb.h:166
#define UDESC_BOS
Definition: usb.h:207
#define UPS_C_PORT_CONFIG_ERROR
Definition: usb.h:744
#define UR_GET_TT_STATE
Definition: usb.h:230
#define UE_XFERTYPE
Definition: usb.h:540
#define UHF_PORT_ENABLE
Definition: usb.h:249
#define UICLASS_HUB
Definition: usb.h:479
#define UDESC_DEVICE
Definition: usb.h:195
#define UDESC_INTERFACE
Definition: usb.h:199
#define UPS_C_PORT_LINK_STATE
Definition: usb.h:743
#define UPS_HIGH_SPEED
Definition: usb.h:730
@ USB_SPEED_LOW
Definition: usb.h:753
@ USB_SPEED_FULL
Definition: usb.h:754
@ USB_SPEED_HIGH
Definition: usb.h:755
@ USB_SPEED_SUPER
Definition: usb.h:756
#define UR_RESET_TT
Definition: usb.h:229
@ USB_MODE_HOST
Definition: usb.h:778
@ USB_EP_MODE_DEFAULT
Definition: usb.h:801
@ USB_EP_MODE_STREAMS
Definition: usb.h:802
#define UE_GET_DIR(a)
Definition: usb.h:529
#define UHD_OC_INDIVIDUAL
Definition: usb.h:614
#define UPS_PORT_ENABLED
Definition: usb.h:706
#define UT_WRITE_ENDPOINT
Definition: usb.h:171
#define UT_READ_CLASS_DEVICE
Definition: usb.h:172
#define UHF_PORT_RESET
Definition: usb.h:252
#define UHF_C_PORT_CONFIG_ERROR
Definition: usb.h:270
#define UPS_OTHER_SPEED
Definition: usb.h:731
#define UR_SET_INTERFACE
Definition: usb.h:221
#define UT_READ_CLASS_OTHER
Definition: usb.h:174
#define UDESC_SS_HUB
Definition: usb.h:215
#define UE_CONTROL
Definition: usb.h:541
#define UHF_PORT_INDICATOR
Definition: usb.h:263
#define UT_READ_ENDPOINT
Definition: usb.h:168
#define UHF_C_PORT_RESET
Definition: usb.h:261
@ USB_REV_3_0
Definition: usb.h:770
#define UHF_C_PORT_LINK_STATE
Definition: usb.h:269
#define UHD_PORT_IND
Definition: usb.h:621
#define UE_GET_SS_ISO_MULT(x)
Definition: usb.h:570
#define UPS_C_CONNECT_STATUS
Definition: usb.h:736
#define UPS_CURRENT_CONNECT_STATUS
Definition: usb.h:705
#define UE_ISOCHRONOUS
Definition: usb.h:542
#define UHF_C_PORT_ENABLE
Definition: usb.h:258
#define UHF_C_BH_PORT_RESET
Definition: usb.h:273
#define UR_SET_FEATURE
Definition: usb.h:192
#define UPS_OVERCURRENT_INDICATOR
Definition: usb.h:708
#define UT_READ_INTERFACE
Definition: usb.h:167
#define UR_GET_DESCRIPTOR
Definition: usb.h:194
#define UHF_C_PORT_CONNECTION
Definition: usb.h:257
#define UPS_RESET
Definition: usb.h:709
#define UHF_PORT_U2_TIMEOUT
Definition: usb.h:268
void usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, struct usb_page_search *res)
Definition: usb_busdma.c:86
void usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, void *ptr, usb_frlength_t len)
Definition: usb_busdma.c:283
void usb_pc_free_mem(struct usb_page_cache *pc)
void usb_pc_cpu_invalidate(struct usb_page_cache *pc)
void usb_pc_cpu_flush(struct usb_page_cache *pc)
#define USB_GET_DMA_TAG(dev)
Definition: usb_busdma.h:46
uint8_t usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, usb_size_t size, usb_size_t align)
void usb_bus_mem_free_all(struct usb_bus *bus, usb_bus_mem_cb_t *cb)
uint8_t usb_bus_mem_alloc_all(struct usb_bus *bus, bus_dma_tag_t dmat, usb_bus_mem_cb_t *cb)
void usb_bus_reset_async_locked(struct usb_bus *bus)
#define USB_HW_POWER_RESUME
void usb_bus_mem_flush_all(struct usb_bus *bus, usb_bus_mem_cb_t *cb)
#define USB_HW_POWER_SUSPEND
#define USB_HW_POWER_SHUTDOWN
void() usb_bus_mem_sub_cb_t(struct usb_bus *bus, struct usb_page_cache *pc, struct usb_page *pg, usb_size_t size, usb_size_t align)
#define USB_BUS_LOCK(_b)
Definition: usb_core.h:45
#define USB_BUS_UNLOCK(_b)
Definition: usb_core.h:46
#define USB_BUS_LOCK_ASSERT(_b, _t)
Definition: usb_core.h:47
enum usb_dev_speed usbd_get_speed(struct usb_device *udev)
Definition: usb_device.c:2589
enum usb_dev_state usb_get_device_state(struct usb_device *udev)
Definition: usb_device.c:2877
#define USETW(w, v)
Definition: usb_endian.h:77
#define UGETW(w)
Definition: usb_endian.h:53
uint16_t usb_stream_t
Definition: usb_freebsd.h:105
void uhub_root_intr(struct usb_bus *bus, const uint8_t *ptr, uint8_t len)
Definition: usb_hub.c:960
usb_error_t uhub_query_info(struct usb_device *udev, uint8_t *pnports, uint8_t *ptt)
Definition: usb_hub.c:1130
void ** pptr
Definition: usb_if.m:52
uint16_t offset
Definition: usb_if.m:54
const void * req
Definition: usb_if.m:51
void * usb_proc_msignal(struct usb_process *up, void *_pm0, void *_pm1)
Definition: usb_process.c:288
void usbd_transfer_setup_sub(struct usb_setup_params *parm)
Definition: usb_transfer.c:457
void usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
void usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex, usb_frlength_t len)
uint8_t usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
void usb_dma_delay_done_cb(struct usb_xfer *xfer)
void usbd_transfer_dequeue(struct usb_xfer *xfer)
void usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
uint8_t usbd_xfer_get_isochronous_start_frame(struct usb_xfer *xfer, uint32_t frame_curr, uint32_t frame_min, uint32_t frame_ms, uint32_t frame_mask, uint32_t *p_frame_start)
void usbd_transfer_timeout_ms(struct usb_xfer *xfer, void(*cb)(void *arg), usb_timeout_t ms)
uint8_t usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm, struct usb_page_cache **ppc, usb_size_t size, usb_size_t align, usb_size_t count)
void usb_pause_mtx(struct mtx *mtx, int timo)
Definition: usb_util.c:135
uint8_t usb_make_str_desc(void *ptr, uint16_t max_len, const char *s)
Definition: usb_util.c:193
void() usb_proc_callback_t(struct usb_proc_msg *)
Definition: usbdi.h:95
uint16_t idProduct
Definition: usbdi.h:35
uint8_t bInterfaceSubClass
Definition: usbdi.h:46
usb_error_t
Definition: usbdi.h:45
@ USB_ERR_NORMAL_COMPLETION
Definition: usbdi.h:46
@ USB_ERR_BAD_ADDRESS
Definition: usbdi.h:52
@ USB_ERR_STALLED
Definition: usbdi.h:68
@ USB_ERR_IOERROR
Definition: usbdi.h:64
@ USB_ERR_NO_PIPE
Definition: usbdi.h:58
@ USB_ERR_NOMEM
Definition: usbdi.h:50
@ USB_ERR_BAD_BUFSIZE
Definition: usbdi.h:53
@ USB_ERR_CANCELLED
Definition: usbdi.h:51
@ USB_ERR_INVAL
Definition: usbdi.h:49
@ USB_ERR_TIMEOUT
Definition: usbdi.h:66
uint8_t bInterfaceProtocol
Definition: usbdi.h:47
#define USB_MS_TO_TICKS(ms)
Definition: usbdi.h:120
uint16_t idVendor
Definition: usbdi.h:34
uint8_t bInterfaceClass
Definition: usbdi.h:45
static void xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
Definition: xhci.c:2812
usb_error_t xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32)
Definition: xhci.c:511
static void xhci_device_suspend(struct usb_device *udev)
Definition: xhci.c:4161
static usb_error_t xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx, uint8_t bsr, uint8_t slot_id)
Definition: xhci.c:1321
static void xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
Definition: xhci.c:223
static usb_error_t xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
Definition: xhci.c:2468
static usb_error_t xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx, uint8_t deconfigure, uint8_t slot_id)
Definition: xhci.c:1471
#define xhciroute
Definition: xhci.c:129
static void xhci_generic_done(struct usb_xfer *xfer)
Definition: xhci.c:765
static usb_error_t xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr, uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
Definition: xhci.c:1537
static const struct usb_hub_ss_descriptor xhci_hubd
Definition: xhci.c:3208
static usb_error_t xhci_configure_endpoint(struct usb_device *, struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *, uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t, uint8_t)
Definition: xhci.c:2308
usb_error_t xhci_reset_controller(struct xhci_softc *sc)
Definition: xhci.c:484
static usb_error_t xhci_configure_mask(struct usb_device *, uint32_t, uint8_t)
Definition: xhci.c:2250
static void xhci_root_intr(struct xhci_softc *)
Definition: xhci.c:2976
static void xhci_timeout(void *arg)
Definition: xhci.c:1655
static uint8_t xhci_get_endpoint_state(struct usb_device *udev, uint8_t epno)
Definition: xhci.c:3739
static void xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
Definition: xhci.c:854
static usb_error_t xhci_device_init(struct usb_device *udev)
Definition: xhci.c:4031
static usb_error_t xhci_configure_device(struct usb_device *)
Definition: xhci.c:2511
static const struct xhci_config_desc xhci_confd
Definition: xhci.c:3172
SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RWTUN, &xhcistreams, 0, "Set to enable streams mode support")
static void xhci_skip_transfer(struct usb_xfer *xfer)
Definition: xhci.c:822
static void xhci_set_hw_power(struct usb_bus *bus)
Definition: xhci.c:4192
static void xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
Definition: xhci.c:4117
static void xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
Definition: xhci.c:659
usb_error_t xhci_halt_controller(struct xhci_softc *sc)
Definition: xhci.c:454
static usb_error_t xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
Definition: xhci.c:1278
static void xhci_free_device_ext(struct usb_device *)
Definition: xhci.c:2746
static const struct usb_bus_methods xhci_bus_methods
Definition: xhci.c:176
static void xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
Definition: xhci.c:4007
static void xhci_device_done(struct usb_xfer *, usb_error_t)
Definition: xhci.c:3007
#define C(x, y)
static void xhci_device_generic_close(struct usb_xfer *xfer)
Definition: xhci.c:3029
static usb_error_t xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve, uint8_t ep_id, uint8_t slot_id)
Definition: xhci.c:1514
static int xhci_reset_command_queue_locked(struct xhci_softc *sc)
Definition: xhci.c:241
static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "USB XHCI")
static usb_error_t xhci_configure_reset_endpoint(struct usb_xfer *xfer)
Definition: xhci.c:3762
#define XHCI_GET_CTX(sc, which, field, ptr)
Definition: xhci.c:91
static usb_error_t xhci_roothub_exec(struct usb_device *udev, struct usb_device_request *req, const void **pptr, uint16_t *plength)
Definition: xhci.c:3214
static void xhci_xfer_unsetup(struct usb_xfer *xfer)
Definition: xhci.c:3864
static usb_proc_callback_t xhci_configure_msg
Definition: xhci.c:164
uint8_t xhci_use_polling(void)
Definition: xhci.c:213
static usb_error_t xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend, uint8_t ep_id, uint8_t slot_id)
Definition: xhci.c:1559
static usb_error_t xhci_transfer_insert(struct usb_xfer *xfer)
Definition: xhci.c:2834
void xhci_uninit(struct xhci_softc *sc)
Definition: xhci.c:644
static void xhci_device_state_change(struct usb_device *udev)
Definition: xhci.c:4198
static void xhci_setup_generic_chain_sub(struct xhci_std_temp *temp)
Definition: xhci.c:1678
static void xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
Definition: xhci.c:2233
static usb_error_t xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, uint8_t ep_mode)
Definition: xhci.c:4303
static int xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
Definition: xhci.c:1019
#define xhcidma32
Definition: xhci.c:130
static void xhci_setup_generic_chain(struct usb_xfer *xfer)
Definition: xhci.c:1980
static int xhcictlquirk
Definition: xhci.c:103
static void xhci_do_poll(struct usb_bus *)
Definition: xhci.c:1668
static void xhci_xfer_setup(struct usb_setup_params *parm)
Definition: xhci.c:3638
static void xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
Definition: xhci.c:4001
#define xhcictlstep
Definition: xhci.c:131
static usb_error_t xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
Definition: xhci.c:1303
static void xhci_endpoint_doorbell(struct usb_xfer *)
Definition: xhci.c:2792
static void xhci_device_generic_enter(struct usb_xfer *xfer)
Definition: xhci.c:3073
#define XHCI_INTR_ENDPT
Definition: xhci.c:134
static int xhcidcepquirk
Definition: xhci.c:107
usb_error_t xhci_start_controller(struct xhci_softc *sc)
Definition: xhci.c:304
static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *, uint64_t, uint8_t)
Definition: xhci.c:1496
static usb_error_t xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
Definition: xhci.c:1343
static void xhci_activate_transfer(struct usb_xfer *xfer)
Definition: xhci.c:803
static void xhci_device_uninit(struct usb_device *udev)
Definition: xhci.c:4092
static usb_error_t xhci_generic_done_sub(struct usb_xfer *xfer)
Definition: xhci.c:684
#define HSETW(ptr, val)
Definition: xhci.c:3117
#define XHCI_BUS2SC(bus)
Definition: xhci.c:88
static const struct usb_device_descriptor xhci_devd
Definition: xhci.c:3120
static void xhci_device_generic_open(struct usb_xfer *xfer)
Definition: xhci.c:3023
static int xhci_interrupt_poll(struct xhci_softc *sc)
Definition: xhci.c:1032
static void xhci_device_generic_start(struct usb_xfer *xfer)
Definition: xhci.c:3085
static usb_error_t xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
Definition: xhci.c:1582
static const struct xhci_bos_desc xhci_bosd
Definition: xhci.c:3139
static void xhci_device_generic_multi_enter(struct usb_endpoint *ep, usb_stream_t stream_id, struct usb_xfer *enter_xfer)
Definition: xhci.c:3037
static const struct usb_pipe_methods xhci_device_generic_methods
Definition: xhci.c:3104
void xhci_interrupt(struct xhci_softc *sc)
Definition: xhci.c:1603
static usb_error_t xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb, uint16_t timeout_ms)
Definition: xhci.c:1119
static struct xhci_endpoint_ext * xhci_get_endpoint_ext(struct usb_device *, struct usb_endpoint_descriptor *)
Definition: xhci.c:2762
static int xhcistreams
Definition: xhci.c:99
static void xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep)
Definition: xhci.c:3962
static void xhci_start_dma_delay(struct usb_xfer *xfer)
Definition: xhci.c:3870
static usb_error_t xhci_alloc_device_ext(struct usb_device *udev)
Definition: xhci.c:2663
static void xhci_device_resume(struct usb_device *udev)
Definition: xhci.c:4127
#define XHCI_EPCTX_0_LSA_SET(x)
Definition: xhci.h:135
#define XHCI_TRB_3_DIR_IN
Definition: xhci.h:38
#define XHCI_TRB_2_BYTES_SET(x)
Definition: xhci.h:11
#define XHCI_TRB_0_WLENGTH_MASK
Definition: xhci.h:2
volatile uint32_t dwTrb2
Definition: xhci.h:3
@ XHCI_ST_DISABLED
Definition: xhci.h:428
@ XHCI_ST_ADDRESSED
Definition: xhci.h:431
@ XHCI_ST_ENABLED
Definition: xhci.h:429
@ XHCI_ST_CONFIGURED
Definition: xhci.h:432
@ XHCI_ST_DEFAULT
Definition: xhci.h:430
#define XHCI_SCTX_1_NUM_PORTS_SET(x)
Definition: xhci.h:92
#define XHCI_TRB_TYPE_ENABLE_SLOT
Definition: xhci.h:60
#define XHCI_SCTX_0_MTT_SET(x)
Definition: xhci.h:81
#define XHCI_SCTX_0_SPEED_SET(x)
Definition: xhci.h:79
#define XHCI_TRB_TYPE_SET_TR_DEQUEUE
Definition: xhci.h:67
volatile uint32_t dwTrb3
Definition: xhci.h:17
#define XHCI_TRB_3_IOC_BIT
Definition: xhci.h:26
#define XHCI_TRB_TYPE_DISABLE_SLOT
Definition: xhci.h:61
struct xhci_slot_ctx ctx_slot
Definition: xhci.h:0
#define XHCI_TRB_TYPE_ISOCH
Definition: xhci.h:56
#define XHCI_TRB_3_TLBPC_SET(x)
Definition: xhci.h:40
#define XHCI_EPCTX_1_EPTYPE_SET(x)
Definition: xhci.h:142
#define XHCI_TRB_TYPE_NOOP
Definition: xhci.h:59
#define XHCI_MAX_ENDPOINTS
Definition: xhci.h:34
#define XHCI_SCTX_0_HUB_SET(x)
Definition: xhci.h:83
#define XHCI_SCTX_0_CTX_NUM_SET(x)
Definition: xhci.h:85
#define XHCI_SCTX_0_SCT_SEC_TR_RING
Definition: xhci.h:216
#define XHCI_TRB_TYPE_CONFIGURE_EP
Definition: xhci.h:63
#define XHCI_TRB_TYPE_ADDRESS_DEVICE
Definition: xhci.h:62
#define XHCI_MAX_DEVICES
Definition: xhci.h:33
#define XHCI_SCTX_2_TT_THINK_TIME_SET(x)
Definition: xhci.h:99
#define XHCI_TRB_2_ERROR_GET(x)
Definition: xhci.h:4
#define XHCI_EPCTX_4_AVG_TRB_LEN_SET(x)
Definition: xhci.h:155
#define XHCI_SCTX_3_SLOT_STATE_SET(x)
Definition: xhci.h:106
#define XHCI_CMD_UNLOCK(sc)
Definition: xhci.h:569
#define XHCI_TRB_EVENT_TRANSFER
Definition: xhci.h:77
struct xhci_endp_ctx ctx_ep[XHCI_MAX_ENDPOINTS - 1]
Definition: xhci.h:1
#define XHCI_TRB_3_SLOT_GET(x)
Definition: xhci.h:47
#define XHCI_TRB_TYPE_STOP_EP
Definition: xhci.h:66
#define XHCI_TRB_3_BSR_BIT
Definition: xhci.h:33
#define XHCI_TRB_3_TYPE_SET(x)
Definition: xhci.h:19
#define XHCI_TRB_2_STREAM_SET(x)
Definition: xhci.h:15
#define XHCI_SCTX_0_ROUTE_SET(x)
Definition: xhci.h:77
#define XHCI_EPCTX_0_MULT_SET(x)
Definition: xhci.h:131
#define XHCI_MAX_EVENTS
Definition: xhci.h:36
#define XHCI_TRB_3_FRID_SET(x)
Definition: xhci.h:44
#define XHCI_MAX_SCRATCHPADS
Definition: xhci.h:35
#define XHCI_TRB_ALIGN
Definition: xhci.h:60
#define XHCI_TRB_ERROR_SUCCESS
Definition: xhci.h:88
#define XHCI_EPCTX_0_MAXP_STREAMS_SET(x)
Definition: xhci.h:133
#define XHCI_TRB_TYPE_RESET_EP
Definition: xhci.h:65
#define XHCI_TRB_TYPE_NORMAL
Definition: xhci.h:52
uint32_t remainder
Definition: xhci.h:12
#define XHCI_TRB_2_IRQ_SET(x)
Definition: xhci.h:13
#define XHCI_TRB_3_IDT_BIT
Definition: xhci.h:27
#define XHCI_CMD_ASSERT_LOCKED(sc)
Definition: xhci.h:570
#define XHCI_TRB_3_CHAIN_BIT
Definition: xhci.h:25
#define XHCI_EPCTX_0_EPSTATE_SET(x)
Definition: xhci.h:121
#define XHCI_SCTX_3_DEV_ADDR_SET(x)
Definition: xhci.h:104
#define XHCI_MAX_RSEG
Definition: xhci.h:38
#define XHCI_TRB_0_DIR_IN_MASK
Definition: xhci.h:1
#define XHCI_TRB_3_TRT_IN
Definition: xhci.h:37
#define XHCI_TRB_3_EP_GET(x)
Definition: xhci.h:41
#define XHCI_TD_PAGE_SIZE
Definition: xhci.h:367
#define XHCI_EPNO2EPID(x)
Definition: xhci.h:72
#define XHCI_TRB_EVENT_CMD_COMPLETE
Definition: xhci.h:78
#define XHCI_SCTX_3_DEV_ADDR_GET(x)
Definition: xhci.h:105
#define XHCI_TRB_3_TC_BIT
Definition: xhci.h:21
#define XHCI_TRB_ERROR_STALL
Definition: xhci.h:93
#define XHCI_TRB_3_DCEP_BIT
Definition: xhci.h:31
#define XHCI_CMD_LOCK(sc)
Definition: xhci.h:568
#define XHCI_TRB_3_SLOT_SET(x)
Definition: xhci.h:48
#define XHCI_EPCTX_0_EPSTATE_HALTED
Definition: xhci.h:125
#define XHCI_EPCTX_0_EPSTATE_GET(x)
Definition: xhci.h:122
#define XHCI_TRB_ERROR_SHORT_PKT
Definition: xhci.h:100
#define XHCI_TRB_TYPE_STATUS_STAGE
Definition: xhci.h:55
#define XHCI_TRB_3_ISP_BIT
Definition: xhci.h:23
#define XHCI_EPCTX_0_EPSTATE_STOPPED
Definition: xhci.h:126
#define XHCI_TRB_3_CYCLE_BIT
Definition: xhci.h:20
#define XHCI_TD_ALIGN
Definition: xhci.h:61
#define XHCI_SCTX_1_RH_PORT_SET(x)
Definition: xhci.h:90
#define XHCI_EPCTX_0_IVAL_SET(x)
Definition: xhci.h:137
#define XHCI_TRB_2_REM_GET(x)
Definition: xhci.h:8
#define XHCI_TRB_ERROR_LENGTH
Definition: xhci.h:114
#define XHCI_TRB_3_PRSV_BIT
Definition: xhci.h:32
volatile uint64_t qwTrb0
Definition: xhci.h:0
#define XHCI_SCTX_2_TT_HUB_SID_SET(x)
Definition: xhci.h:95
#define XHCI_TRB_3_ISO_SIA_BIT
Definition: xhci.h:45
#define XHCI_TRB_3_SUSP_EP_BIT
Definition: xhci.h:46
#define XHCI_EPCTX_1_CERR_SET(x)
Definition: xhci.h:140
#define XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x)
Definition: xhci.h:157
#define XHCI_MAX_COMMANDS
Definition: xhci.h:37
#define XHCI_TRB_2_BYTES_GET(x)
Definition: xhci.h:10
#define XHCI_SCTX_2_TT_PORT_NUM_SET(x)
Definition: xhci.h:97
#define XHCI_PAGE_SIZE
Definition: xhci.h:62
#define XHCI_EPCTX_2_DCS_SET(x)
Definition: xhci.h:151
#define XHCI_TRB_TYPE_LINK
Definition: xhci.h:57
#define XHCI_MAX_TRANSFERS
Definition: xhci.h:39
#define XHCI_TRB_ERROR_PARAMETER
Definition: xhci.h:104
#define XHCI_TRB_TYPE_SETUP_STAGE
Definition: xhci.h:53
#define XHCI_INCTX_NON_CTRL_MASK
Definition: xhci.h:170
uint8_t status
Definition: xhci.h:14
#define XHCI_TD_PAYLOAD_MAX
Definition: xhci.h:364
#define XHCI_EPCTX_1_HID_SET(x)
Definition: xhci.h:144
#define XHCI_TRB_3_TRT_OUT
Definition: xhci.h:36
#define XHCI_EPCTX_1_MAXB_SET(x)
Definition: xhci.h:146
#define XHCI_TRB_2_TDSZ_SET(x)
Definition: xhci.h:7
#define XHCI_EPCTX_0_EPSTATE_DISABLED
Definition: xhci.h:123
#define XHCI_EPCTX_1_MAXP_SIZE_SET(x)
Definition: xhci.h:148
#define XHCI_TRB_3_TYPE_GET(x)
Definition: xhci.h:18
#define XHCI_SCTX_2_IRQ_TARGET_SET(x)
Definition: xhci.h:101
#define XHCI_TRB_TYPE_RESET_DEVICE
Definition: xhci.h:68
#define XHCI_TRB_TYPE_EVALUATE_CTX
Definition: xhci.h:64
#define XHCI_TRB_3_EP_SET(x)
Definition: xhci.h:42
#define XHCI_TRB_TYPE_DATA_STAGE
Definition: xhci.h:54
#define XHCI_TRB_3_TBC_SET(x)
Definition: xhci.h:29
#define XHCI_HCS0_PIND(x)
Definition: xhcireg.h:67
#define XHCI_DNCTRL
Definition: xhcireg.h:103
#define XHCI_STS_HCH
Definition: xhcireg.h:88
#define XHCI_PM3_U1TO_SET(x)
Definition: xhcireg.h:147
#define XHCI_IMAN_INTR_PEND
Definition: xhcireg.h:166
#define XREAD1(sc, what, a)
Definition: xhcireg.h:207
#define XHCI_HCS2_ERST_MAX(x)
Definition: xhcireg.h:56
#define XHCI_STS_PCD
Definition: xhcireg.h:91
#define XHCI_PS_CSC
Definition: xhcireg.h:130
#define XHCI_CRCR_LO_CRR
Definition: xhcireg.h:109
#define XHCI_USBCMD
Definition: xhcireg.h:77
#define XHCI_IMOD(n)
Definition: xhcireg.h:168
#define XHCI_CMD_RS
Definition: xhcireg.h:78
#define XHCI_IMOD_DEFAULT
Definition: xhcireg.h:173
#define XWRITE4(sc, what, a, x)
Definition: xhcireg.h:222
#define XHCI_RTSOFF
Definition: xhcireg.h:74
#define XHCI_CRCR_LO_CS
Definition: xhcireg.h:107
#define XHCI_CONFIG
Definition: xhcireg.h:114
#define XHCI_STS_HCE
Definition: xhcireg.h:96
#define XHCI_HCS3_U2_DEL(x)
Definition: xhcireg.h:61
#define XHCI_HCS0_CSZ(x)
Definition: xhcireg.h:65
#define XHCI_PS_PRC
Definition: xhcireg.h:134
#define XHCI_ERSTSZ(n)
Definition: xhcireg.h:175
#define XHCI_CRCR_LO_CA
Definition: xhcireg.h:108
#define XREAD2(sc, what, a)
Definition: xhcireg.h:210
#define XHCI_HCSPARAMS3
Definition: xhcireg.h:59
#define XHCI_CRCR_LO_RCS
Definition: xhcireg.h:106
#define XHCI_HCSPARAMS0
Definition: xhcireg.h:62
#define XHCI_CMD_HSEE
Definition: xhcireg.h:81
#define XHCI_STS_HSE
Definition: xhcireg.h:89
#define XHCI_HCS0_AC64(x)
Definition: xhcireg.h:63
#define XHCI_PM3_U2TO_SET(x)
Definition: xhcireg.h:149
#define XHCI_IMAN_INTR_ENA
Definition: xhcireg.h:167
#define XHCI_PAGESIZE
Definition: xhcireg.h:97
#define XHCI_PORTSC(n)
Definition: xhcireg.h:118
#define XHCI_PS_WPR
Definition: xhcireg.h:142
#define XHCI_HCSPARAMS2
Definition: xhcireg.h:54
#define XHCI_DOORBELL(n)
Definition: xhcireg.h:186
#define XHCI_IMAN(n)
Definition: xhcireg.h:165
#define XHCI_MFINDEX_GET(x)
Definition: xhcireg.h:164
#define XHCI_PS_PP
Definition: xhcireg.h:125
#define XHCI_PS_PLS_GET(x)
Definition: xhcireg.h:123
#define XHCI_PS_SPEED_GET(x)
Definition: xhcireg.h:126
#define XHCI_PS_PLS_SET(x)
Definition: xhcireg.h:124
#define XHCI_PS_LWS
Definition: xhcireg.h:129
#define XHCI_PS_OCA
Definition: xhcireg.h:121
#define XHCI_PORTPMSC(n)
Definition: xhcireg.h:145
#define XHCI_ERDP_LO(n)
Definition: xhcireg.h:180
#define XHCI_ERSTS_SET(x)
Definition: xhcireg.h:177
#define XHCI_CAPLENGTH
Definition: xhcireg.h:45
#define XHCI_DBOFF
Definition: xhcireg.h:73
#define XHCI_DCBAAP_LO
Definition: xhcireg.h:112
#define XHCI_HCS3_U1_DEL(x)
Definition: xhcireg.h:60
#define XHCI_USBSTS
Definition: xhcireg.h:87
#define XREAD4(sc, what, a)
Definition: xhcireg.h:213
#define XHCI_ERDP_HI(n)
Definition: xhcireg.h:183
#define XHCI_PS_WRC
Definition: xhcireg.h:132
#define XHCI_PS_PIC_SET(x)
Definition: xhcireg.h:128
#define XHCI_PS_PED
Definition: xhcireg.h:120
#define XHCI_HCSPARAMS1
Definition: xhcireg.h:50
#define XHCI_HCIVERSION
Definition: xhcireg.h:47
#define XHCI_PS_PLC
Definition: xhcireg.h:135
#define XHCI_CRCR_LO
Definition: xhcireg.h:105
#define XHCI_DB_SID_SET(x)
Definition: xhcireg.h:190
#define XHCI_ERSTBA_LO(n)
Definition: xhcireg.h:178
#define XHCI_CRCR_HI
Definition: xhcireg.h:111
#define XHCI_ERDP_LO_BUSY
Definition: xhcireg.h:182
#define XHCI_CMD_HCRST
Definition: xhcireg.h:79
#define XHCI_PS_DR
Definition: xhcireg.h:141
#define XHCI_HCS1_N_PORTS(x)
Definition: xhcireg.h:53
#define XHCI_DCBAAP_HI
Definition: xhcireg.h:113
#define XHCI_HCS1_DEVSLOT_MAX(x)
Definition: xhcireg.h:51
#define XHCI_ERSTBA_HI(n)
Definition: xhcireg.h:179
#define XHCI_PS_PR
Definition: xhcireg.h:122
#define XHCI_HCS2_SPB_MAX(x)
Definition: xhcireg.h:58
#define XHCI_HCS0_PPC(x)
Definition: xhcireg.h:66
#define XHCI_MFINDEX
Definition: xhcireg.h:163
#define XHCI_PS_OCC
Definition: xhcireg.h:133
#define XHCI_STS_CNR
Definition: xhcireg.h:95
#define XHCI_PAGESIZE_4K
Definition: xhcireg.h:98
#define XHCI_PS_CCS
Definition: xhcireg.h:119
#define XHCI_PS_CEC
Definition: xhcireg.h:136
#define XHCI_PS_PEC
Definition: xhcireg.h:131
#define XHCI_HCS2_IST(x)
Definition: xhcireg.h:55
#define XHCI_CMD_INTE
Definition: xhcireg.h:80