52#if defined(__amd64__) || defined(__i386__)
79 struct sysctl_ctx_list sysctl_tree;
80 struct sysctl_oid *sysctl_tree_top;
88static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
89static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
90static void *busdma_ih;
92static MALLOC_DEFINE(M_BOUNCE,
"bounce",
"busdma bounce pages");
95 "Total bounce pages");
101_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
int flags)
106 if (
flags & BUS_DMA_NOWAIT) {
108 map->pagesneeded = 0;
115 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
117 return (EINPROGRESS);
130 STAILQ_INIT(&bounce_zone_list);
131 STAILQ_INIT(&bounce_map_waitinglist);
132 STAILQ_INIT(&bounce_map_callbacklist);
133 mtx_init(&
bounce_lock,
"bounce pages lock", NULL, MTX_DEF);
137static struct sysctl_ctx_list *
141 return (&bz->sysctl_tree);
144static struct sysctl_oid *
148 return (bz->sysctl_tree_top);
157 STAILQ_FOREACH(bz, &bounce_zone_list, links) {
158 if ((dmat_alignment(dmat) <= bz->alignment) &&
160 dmat_domain(dmat) == bz->domain &&
162 (dmat_lowaddr(dmat) >= bz->lowaddr)) {
163 dmat->bounce_zone = bz;
169 M_NOWAIT | M_ZERO)) == NULL)
172 STAILQ_INIT(&bz->bounce_page_list);
174 bz->reserved_bpages = 0;
175 bz->active_bpages = 0;
176 bz->lowaddr = dmat_lowaddr(dmat);
177 bz->alignment = MAX(dmat_alignment(dmat), PAGE_SIZE);
180 bz->domain = dmat_domain(dmat);
184 snprintf(bz->lowaddrid,
sizeof(bz->lowaddrid),
"%#jx",
185 (uintmax_t)bz->lowaddr);
186 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
187 dmat->bounce_zone = bz;
190 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
191 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
192 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"");
193 if (bz->sysctl_tree_top == NULL) {
200 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
201 "Total bounce pages");
204 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
205 "Free bounce pages");
208 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
209 "Reserved bounce pages");
212 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
213 "Active bounce pages");
216 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
217 "Total bounce requests (pages bounced)");
220 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
221 "Total bounce requests that were deferred");
224 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0,
"");
227 "alignment", CTLFLAG_RD, &bz->alignment,
"");
231 "domain", CTLFLAG_RD, &bz->domain, 0,
244 bz = dmat->bounce_zone;
246 while (numpages > 0) {
251 DOMAINSET_PREF(bz->domain), M_NOWAIT | M_ZERO);
253 bpage =
malloc(
sizeof(*bpage), M_BUSDMA, M_NOWAIT | M_ZERO);
260 M_BOUNCE, DOMAINSET_PREF(bz->domain), M_NOWAIT,
261 0ul, bz->lowaddr, PAGE_SIZE, 0);
264 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0);
266 if (bpage->
vaddr == 0) {
267 free(bpage, M_BUSDMA);
272 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
290 bz = dmat->bounce_zone;
291 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
292 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
293 return (map->pagesneeded - (map->pagesreserved + pages));
294 bz->free_bpages -= pages;
295 bz->reserved_bpages += pages;
296 map->pagesreserved += pages;
297 pages = map->pagesneeded - map->pagesreserved;
302#if defined(__amd64__) || defined(__i386__)
304add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
305 vm_paddr_t addr1, vm_paddr_t addr2, bus_size_t size)
309 bus_addr_t
addr, bus_size_t size)
315 KASSERT(dmat->bounce_zone != NULL, (
"no bounce zone in dma tag"));
316 KASSERT(map != NULL, (
"add_bounce_page: bad map %p", map));
317#if defined(__amd64__) || defined(__i386__)
318 KASSERT(map != &nobounce_dmamap, (
"add_bounce_page: bad map %p", map));
321 KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0,
322 (
"add_bounce_page: bad map %p", map));
325 bz = dmat->bounce_zone;
326 if (map->pagesneeded == 0)
327 panic(
"add_bounce_page: map doesn't need any pages");
330 if (map->pagesreserved == 0)
331 panic(
"add_bounce_page: map doesn't need any pages");
332 map->pagesreserved--;
335 bpage = STAILQ_FIRST(&bz->bounce_page_list);
337 panic(
"add_bounce_page: free page list is empty");
339 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
340 bz->reserved_bpages--;
344 if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
346#if defined(__amd64__) || defined(__i386__)
347 bpage->
vaddr |= addr1 & PAGE_MASK;
348 bpage->
busaddr |= addr1 & PAGE_MASK;
350 (
"Trying to bounce multiple pages with BUS_DMA_KEEP_PG_OFFSET"));
357#if defined(__amd64__) || defined(__i386__)
358 bpage->
datapage[0] = PHYS_TO_VM_PAGE(addr1);
359 KASSERT((addr2 & PAGE_MASK) == 0, (
"Second page is not aligned"));
360 bpage->
datapage[1] = PHYS_TO_VM_PAGE(addr2);
361 bpage->
dataoffs = addr1 & PAGE_MASK;
367 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
374 struct bus_dmamap *map;
378 bz = dmat->bounce_zone;
381 if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
387 bpage->
vaddr &= ~PAGE_MASK;
391 schedule_swi =
false;
393 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
396 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
398 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
399 STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
401 bz->total_deferred++;
414 struct bus_dmamap *map;
417 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
418 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
421 dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat), BUS_DMA_LOCK);
423 map->callback_arg, BUS_DMA_WAITOK);
424 dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat), BUS_DMA_UNLOCK);
435 panic(
"died while creating busdma swi ithread");
MALLOC_DEFINE(M_BINMISC, KMOD_NAME, "misc binary image activator")
SYSCTL_INT(ASLR_NODE_OID, OID_AUTO, enable, CTLFLAG_RWTUN, &__elfN(aslr_enabled), 0, ": enable address map randomization")
int swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, void *arg, int pri, enum intr_type flags, void **cookiep)
void swi_sched(void *cookie, int flags)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void * contigmalloc(unsigned long size, struct malloc_type *type, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary)
void * contigmalloc_domainset(unsigned long size, struct malloc_type *type, struct domainset *ds, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary)
void * malloc_domainset(size_t size, struct malloc_type *mtp, struct domainset *ds, int flags)
void free(void *addr, struct malloc_type *mtp)
void panic(const char *fmt,...)
int sysctl_ctx_free(struct sysctl_ctx_list *clist)
int sysctl_ctx_init(struct sysctl_ctx_list *c)
struct iommu_domain ** domain
int bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg, int flags)
static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, bus_addr_t addr, bus_size_t size)
static void start_busdma_swi(void *dummy __unused)
static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
static struct sysctl_oid * busdma_sysctl_tree_top(struct bounce_zone *bz)
static struct sysctl_ctx_list * busdma_sysctl_tree(struct bounce_zone *bz)
SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL)
static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
static int busdma_zonecount
static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
static void init_bounce_pages(void *dummy __unused)
static struct mtx bounce_lock
static STAILQ_HEAD(bounce_zone)
static void busdma_swi(void *dummy __unused)
static int alloc_bounce_zone(bus_dma_tag_t dmat)
int snprintf(char *str, size_t size, const char *format,...)