29#ifdef USB_GLOBAL_INCLUDE_FILE
30#include USB_GLOBAL_INCLUDE_FILE
32#include <sys/stdint.h>
33#include <sys/stddef.h>
38#include <sys/kernel.h>
40#include <sys/module.h>
43#include <sys/condvar.h>
44#include <sys/sysctl.h>
46#include <sys/unistd.h>
47#include <sys/callout.h>
48#include <sys/malloc.h>
55#define USB_DEBUG_VAR usb_debug
71static void usb_dma_tag_destroy(
struct usb_dma_tag *);
72static void usb_dma_lock_cb(
void *, bus_dma_lock_op_t);
73static void usb_pc_alloc_mem_cb(
void *, bus_dma_segment_t *,
int,
int);
74static void usb_pc_load_mem_cb(
void *, bus_dma_segment_t *,
int,
int);
75static void usb_pc_common_mem_cb(
void *, bus_dma_segment_t *,
int,
int,
100 offset += pc->page_offset_buf;
104 page = pc->page_start;
106 if (pc->ismultiseg) {
112 res->physaddr = page->physaddr +
offset;
115 res->physaddr = page->physaddr +
offset;
153 if (buf_res.
length & mask)
203 if (buf_res.length >
len) {
204 buf_res.length =
len;
206 error = copyin(ptr, buf_res.buffer, buf_res.length);
211 len -= buf_res.length;
222struct usb_m_copy_in_arg {
228usbd_m_copy_in_cb(
void *arg,
void *src, uint32_t count)
230 struct usb_m_copy_in_arg *ua = arg;
233 ua->dst_offset +=
count;
241 struct usb_m_copy_in_arg arg = {cache, dst_offset};
242 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
260 if (res.length >
len) {
267 error = uiomove(res.buffer, res.length, uio);
272 pc_offset += res.length;
320 if (res.length >
len) {
323 error = copyout(res.buffer, ptr, res.length);
363usb_dma_lock_cb(
void *arg, bus_dma_lock_op_t op)
380 if (bus_dma_tag_create
381 ( udt->tag_parent->tag,
384 (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
389 (align == 1 &&
size > 1) ?
393 BUS_DMA_KEEP_PG_OFFSET,
408 bus_dma_tag_destroy(udt->tag);
415usb_pc_alloc_mem_cb(
void *arg, bus_dma_segment_t *segs,
418 usb_pc_common_mem_cb(arg, segs, nseg,
error, 0);
425usb_pc_load_mem_cb(
void *arg, bus_dma_segment_t *segs,
428 usb_pc_common_mem_cb(arg, segs, nseg,
error, 1);
435usb_pc_common_mem_cb(
void *arg, bus_dma_segment_t *segs,
436 int nseg,
int error, uint8_t isload)
463 pc->page_offset_buf = rem;
464 pc->page_offset_end += rem;
469 for (x = 0; x != nseg - 1; x++) {
470 if (((segs[x].ds_addr + segs[x].ds_len) & (
USB_PAGE_SIZE - 1)) ==
478 DPRINTFN(0,
"Page offset was not preserved\n");
484 while (pc->ismultiseg) {
486 if (off >= (segs->ds_len + rem)) {
496 pg->physaddr = rounddown2(segs->ds_addr + off,
USB_PAGE_SIZE);
500 owned = mtx_owned(uptag->mtx);
504 uptag->dma_error = (
error ? 1 : 0);
506 (uptag->func) (uptag);
508 cv_broadcast(uptag->cv);
539 while (align <
size) {
573 if (bus_dmamem_alloc(
574 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
580 pc->page_offset_buf = 0;
581 pc->page_offset_end =
size;
584 pc->ismultiseg = (align == 1);
589 err = bus_dmamap_load(
590 utag->tag, map, ptr,
size, &usb_pc_alloc_mem_cb,
591 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
593 if (err == EINPROGRESS) {
594 cv_wait(uptag->cv, uptag->mtx);
599 if (err || uptag->dma_error) {
600 bus_dmamem_free(utag->tag, ptr, map);
604 memset(ptr, 0,
size);
613 pc->page_start = NULL;
614 pc->page_offset_buf = 0;
615 pc->page_offset_end = 0;
632 bus_dmamap_unload(pc->tag, pc->map);
634 bus_dmamem_free(pc->tag, pc->
buffer, pc->map);
652 pc->page_offset_buf = 0;
653 pc->page_offset_end =
size;
670 bus_dmamap_unload(pc->tag, pc->map);
675 err = bus_dmamap_load(
677 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
678 if (err == EINPROGRESS) {
679 cv_wait(uptag->cv, uptag->mtx);
682 if (err || uptag->dma_error) {
692 bus_dmamap_unload(pc->tag, pc->map);
700 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
723 if (pc->page_offset_end == pc->page_offset_buf) {
733 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
734 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
743 if (pc->page_offset_end == pc->page_offset_buf) {
747 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
775 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
797 bus_dmamap_unload(pc->tag, pc->map);
798 bus_dmamap_destroy(pc->tag, pc->map);
814 USB_ASSERT(align > 0, (
"Invalid parameter align = 0\n"));
817 udt = udpt->utag_first;
818 nudt = udpt->utag_max;
821 if (udt->align == 0) {
822 usb_dma_tag_create(udt,
size, align);
823 if (udt->tag == NULL) {
830 if ((udt->align == align) && (udt->size ==
size)) {
845 uint8_t ndmabits, uint8_t nudt)
847 memset(udpt, 0,
sizeof(*udpt));
857 cv_init(udpt->cv,
"USB DMA CV");
863 udpt->utag_first = udt;
864 udpt->utag_max = nudt;
865 udpt->dma_bits = ndmabits;
868 memset(udt, 0,
sizeof(*udt));
869 udt->tag_parent = udpt;
883 udt = udpt->utag_first;
884 nudt = udpt->utag_max;
889 usb_dma_tag_destroy(udt);
895 if (udpt->utag_max) {
897 cv_destroy(udpt->cv);
940 frlength_0 = xfer->
sumlen;
978 info->dma_nframes = nframes;
979 info->dma_currframe = 0;
980 info->dma_frlength_0 = frlength_0;
985 while (--nframes > 0) {
986 xfer->
frbuffers[nframes].isread = isread;
987 xfer->
frbuffers[nframes].page_start = pg;
993 if (info->dma_error) {
999 if (info->dma_currframe != info->dma_nframes) {
1000 if (info->dma_currframe == 0) {
1003 info->dma_frlength_0, 0);
1006 nframes = info->dma_currframe;
1012 info->dma_currframe++;
1042 info->dma_error = udpt->dma_error;
struct usb_dma_parent_tag * tag_parent
enum usb_hc_mode usb_mode
usb_frlength_t * frlengths
struct usb_page * dma_page_ptr
struct usb_page_cache * frbuffers
struct usb_xfer_flags_int flags_int
struct usb_xfer_root * xroot
void usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, usb_frlength_t len)
void usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, struct usb_page_search *res)
void usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, const void *ptr, usb_frlength_t len)
uint8_t usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset, usb_frlength_t len, usb_frlength_t mask)
void usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, void *ptr, usb_frlength_t len)
void usb_pc_dmamap_destroy(struct usb_page_cache *pc)
void usb_bdma_work_loop(struct usb_xfer_queue *pq)
void usb_bdma_pre_sync(struct usb_xfer *xfer)
void usb_pc_free_mem(struct usb_page_cache *pc)
uint8_t usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
uint8_t usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
void usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, struct usb_dma_tag *udt, bus_dma_tag_t dmat, struct mtx *mtx, usb_dma_callback_t *func, uint8_t ndmabits, uint8_t nudt)
void usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
void usb_pc_cpu_invalidate(struct usb_page_cache *pc)
struct usb_dma_tag * usb_dma_tag_find(struct usb_dma_parent_tag *udpt, usb_size_t size, usb_size_t align)
void usb_pc_cpu_flush(struct usb_page_cache *pc)
void usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
void usb_bdma_post_sync(struct usb_xfer *xfer)
void() usb_dma_callback_t(struct usb_dma_parent_tag *udpt)
int usb_uiomove(struct usb_page_cache *pc, struct uio *uio, usb_frlength_t pc_offset, usb_frlength_t len)
uint8_t usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, usb_size_t size, usb_size_t align)
#define USB_GET_DATA_ISREAD(xfer)
#define USB_ADD_BYTES(ptr, size)
#define USB_BUS_UNLOCK(_b)
void usbd_pipe_enter(struct usb_xfer *xfer)
void usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
void usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
#define USB_DMATAG_TO_XROOT(dpt)
#define USB_MTX_UNLOCK(_m)
#define USB_MTX_ASSERT(_m, _t)
int usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, const void *ptr, usb_frlength_t len)
int usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, void *ptr, usb_frlength_t len)
@ USB_ERR_DMA_LOAD_FAILED
void usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)