diff --git a/sys/arch/arm64/arm64/autoconf.c b/sys/arch/arm64/arm64/autoconf.c index 5d3d477a5e2..6a955c61fe9 100644 --- a/sys/arch/arm64/arm64/autoconf.c +++ b/sys/arch/arm64/arm64/autoconf.c @@ -1,4 +1,4 @@ -/* $OpenBSD: autoconf.c,v 1.16 2024/11/10 06:51:59 jsg Exp $ */ +/* $OpenBSD: autoconf.c,v 1.17 2026/05/19 13:05:47 kettenis Exp $ */ /* * Copyright (c) 2009 Miodrag Vallat. * @@ -23,6 +23,8 @@ #include #include +#include + #if defined(NFSCLIENT) #include #include @@ -57,6 +59,8 @@ cpu_configure(void) splhigh(); softintr_init(); + bus_dma_init(); + config_rootfound("mainbus", NULL); unmap_startup(); diff --git a/sys/arch/arm64/arm64/bus_dma.c b/sys/arch/arm64/arm64/bus_dma.c index d260174c0fa..f064b336197 100644 --- a/sys/arch/arm64/arm64/bus_dma.c +++ b/sys/arch/arm64/arm64/bus_dma.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bus_dma.c,v 1.14 2020/11/22 15:18:35 patrick Exp $ */ +/* $OpenBSD: bus_dma.c,v 1.15 2026/05/19 13:05:47 kettenis Exp $ */ /* * Copyright (c) 2003-2004 Opsycon AB (www.opsycon.se / www.opsycon.com) @@ -54,19 +54,79 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ + +#include "kstat.h" + #include #include -#include #include #include #include +#if NKSTAT > 0 +#include +#endif + #include #include #include #include +u_long bus_dma_high_pages; +u_long bus_dma_bounce_pages; +u_long bus_dma_bounces; + +#if NKSTAT > 0 +struct bus_dma_kstat_data { + struct kstat_kv kd_bounce_pages; + struct kstat_kv kd_bounces; +}; + +static const struct bus_dma_kstat_data bus_dma_kstat_tpl = { + KSTAT_KV_INITIALIZER("bounce-pages", KSTAT_KV_T_COUNTER64), + KSTAT_KV_INITIALIZER("bounces", KSTAT_KV_T_COUNTER64), +}; + +int +bus_dma_kstat_copy(struct kstat *ks, void *dst) +{ + struct bus_dma_kstat_data *kd = dst; + + *kd = bus_dma_kstat_tpl; + kstat_kv_u64(&kd->kd_bounce_pages) = bus_dma_bounce_pages; + kstat_kv_u64(&kd->kd_bounces) = bus_dma_bounces; + + return 0; +} +#endif + +void +bus_dma_init(void) +{ + struct uvm_constraint_range high_constraint; +#if NKSTAT > 0 + struct kstat *ks; +#endif + + high_constraint.ucr_low = dma_constraint.ucr_high; + high_constraint.ucr_high = no_constraint.ucr_high; + if (high_constraint.ucr_low != high_constraint.ucr_high) + high_constraint.ucr_low++; + + bus_dma_high_pages = uvm_pagecount(&high_constraint); + +#if NKSTAT > 0 + ks = kstat_create("mainbus0", 0, "dma", 0, KSTAT_T_KV, 0); + if (ks == NULL) + return; + + ks->ks_datalen = sizeof(bus_dma_kstat_tpl); + ks->ks_copy = bus_dma_kstat_copy; + kstat_install(ks); +#endif +} + /* * Common function for DMA map creation. May be called by bus-specific * DMA map creation functions. @@ -75,9 +135,21 @@ int _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) { + struct uvm_constraint_range *constraint = &no_constraint; + int use_bounce_buffer = 0; struct machine_bus_dmamap *map; + struct pglist mlist; + struct vm_page **pg, *pgnext; + size_t mapsize, sz, ssize; + vaddr_t va, sva; void *mapstore; - size_t mapsize; + int npages, error; + const struct kmem_dyn_mode *kd; + + if (bus_dma_high_pages > 0) { + use_bounce_buffer = 1; + constraint = &dma_constraint; + } /* * Allocate and initialize the DMA map. The end of the map @@ -93,6 +165,16 @@ _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, */ mapsize = sizeof(struct machine_bus_dmamap) + (sizeof(bus_dma_segment_t) * (nsegments - 1)); + + if (use_bounce_buffer) { + /* this many pages plus one in case we get split */ + npages = round_page(size) / PAGE_SIZE + 1; + if (npages < nsegments) + npages = nsegments; + mapsize += sizeof(struct vm_page *) * npages; + atomic_add_long(&bus_dma_bounce_pages, npages); + } + if ((mapstore = malloc(mapsize, M_DEVBUF, (flags & BUS_DMA_NOWAIT) ? (M_NOWAIT | M_ZERO) : (M_WAITOK | M_ZERO))) == NULL) return (ENOMEM); @@ -103,6 +185,49 @@ _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, map->_dm_maxsegsz = maxsegsz; map->_dm_boundary = boundary; map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT); + map->_dm_flags &= ~BUS_DMA_64BIT; /* XXX */ + if (use_bounce_buffer) { + map->_dm_pages = (void *)&map->dm_segs[nsegments]; + map->_dm_npages = npages; + } + + if (!use_bounce_buffer) { + *dmamp = map; + return (0); + } + + sz = npages << PGSHIFT; + kd = flags & BUS_DMA_NOWAIT ? &kd_trylock : &kd_waitok; + va = (vaddr_t)km_alloc(sz, &kv_any, &kp_none, kd); + if (va == 0) { + map->_dm_npages = 0; + free(map, M_DEVBUF, mapsize); + return (ENOMEM); + } + + TAILQ_INIT(&mlist); + error = uvm_pglistalloc(sz, constraint->ucr_low, + constraint->ucr_high, PAGE_SIZE, 0, &mlist, nsegments, + (flags & BUS_DMA_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK); + if (error) { + map->_dm_npages = 0; + km_free((void *)va, sz, &kv_any, &kp_none); + free(map, M_DEVBUF, mapsize); + return (ENOMEM); + } + + sva = va; + ssize = sz; + pgnext = TAILQ_FIRST(&mlist); + for (pg = map->_dm_pages; npages--; va += PAGE_SIZE, pg++) { + *pg = pgnext; + pmap_kenter_pa(va, VM_PAGE_TO_PHYS(*pg), + PROT_READ | PROT_WRITE); + pgnext = TAILQ_NEXT(*pg, pageq); + memset((void *)va, 0, PAGE_SIZE); + } + pmap_update(pmap_kernel()); + map->_dm_pgva = sva; *dmamp = map; return (0); @@ -116,9 +241,26 @@ void _dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) { size_t mapsize; + struct vm_page **pg; + struct pglist mlist; + + if (map->_dm_pgva) { + km_free((void *)map->_dm_pgva, map->_dm_npages << PGSHIFT, + &kv_any, &kp_none); + } mapsize = sizeof(struct machine_bus_dmamap) + (sizeof(bus_dma_segment_t) * (map->_dm_segcnt - 1)); + mapsize += sizeof(struct vm_page *) * map->_dm_npages; + + if (map->_dm_pages) { + TAILQ_INIT(&mlist); + for (pg = map->_dm_pages; map->_dm_npages--; pg++) { + TAILQ_INSERT_TAIL(&mlist, *pg, pageq); + } + uvm_pglistfree(&mlist); + } + free(map, M_DEVBUF, mapsize); } @@ -131,7 +273,8 @@ _dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags) { paddr_t lastaddr; - int seg, error; + int seg, used, error; + int lastbounce; /* * Make sure that on error condition we return "no valid mappings". @@ -143,11 +286,14 @@ _dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, return (EINVAL); seg = 0; + used = 0; + lastbounce = 0; error = (*t->_dmamap_load_buffer)(t, map, buf, buflen, p, flags, - &lastaddr, &seg, 1); + &lastaddr, &seg, &used, &lastbounce, 1); if (error == 0) { map->dm_nsegs = seg + 1; map->dm_mapsize = buflen; + map->_dm_nused = used; } return (error); @@ -160,7 +306,8 @@ int _dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags) { paddr_t lastaddr; - int seg, error, first; + int seg, used, error, first; + int lastbounce; struct mbuf *m; /* @@ -179,17 +326,20 @@ _dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags) first = 1; seg = 0; + used = 0; + lastbounce = 0; error = 0; for (m = m0; m != NULL && error == 0; m = m->m_next) { if (m->m_len == 0) continue; error = (*t->_dmamap_load_buffer)(t, map, m->m_data, m->m_len, - NULL, flags, &lastaddr, &seg, first); + NULL, flags, &lastaddr, &seg, &used, &lastbounce, first); first = 0; } if (error == 0) { - map->dm_nsegs = seg + 1; map->dm_mapsize = m0->m_pkthdr.len; + map->dm_nsegs = seg + 1; + map->_dm_nused = used; } return (error); @@ -202,7 +352,8 @@ int _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags) { paddr_t lastaddr; - int seg, i, error, first; + int seg, used, i, error, first; + int lastbounce; bus_size_t minlen, resid; struct proc *p = NULL; struct iovec *iov; @@ -227,6 +378,8 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags) first = 1; seg = 0; + used = 0; + lastbounce = 0; error = 0; for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { /* @@ -237,7 +390,7 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags) addr = (void *)iov[i].iov_base; error = (*t->_dmamap_load_buffer)(t, map, addr, minlen, - p, flags, &lastaddr, &seg, first); + p, flags, &lastaddr, &seg, &used, &lastbounce, first); first = 0; resid -= minlen; @@ -245,6 +398,7 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags) if (error == 0) { map->dm_nsegs = seg + 1; map->dm_mapsize = uio->uio_resid; + map->_dm_nused = used; } return (error); @@ -260,9 +414,11 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, { bus_addr_t paddr, baddr, bmask, lastaddr = 0; bus_size_t plen, sgsize, mapsize; - vaddr_t vaddr; + int bounce, lastbounce = 0; int first = 1; int i, seg = 0; + int off, page = 0; + vaddr_t pgva, vaddr; /* * Make sure that on error condition we return "no valid mappings". @@ -281,7 +437,22 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, vaddr = segs[i]._ds_vaddr; plen = MIN(segs[i].ds_len, size); + bounce = 0; + if (paddr + plen - 1 > dma_constraint.ucr_high) + bounce = 1; + while (plen > 0) { + if (bounce) { + if (page >= map->_dm_npages) + return (EFBIG); + + off = paddr & PAGE_MASK; + pgva = map->_dm_pgva + (page << PGSHIFT) + off; + page++; + } else { + pgva = -1; + } + /* * Compute the segment size, and adjust counts. */ @@ -307,15 +478,17 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, map->dm_segs[seg].ds_len = sgsize; map->dm_segs[seg]._ds_paddr = paddr; map->dm_segs[seg]._ds_vaddr = vaddr; + map->dm_segs[seg]._ds_bounce_va = pgva; first = 0; } else { if (paddr == lastaddr && + bounce == lastbounce && (map->dm_segs[seg].ds_len + sgsize) <= map->_dm_maxsegsz && (map->_dm_boundary == 0 || (map->dm_segs[seg].ds_addr & bmask) == (paddr & bmask)) && - (t->_flags & BUS_DMA_COHERENT || + (t->_flags & BUS_DMA_COHERENT || !bounce || (map->dm_segs[seg]._ds_vaddr + map->dm_segs[seg].ds_len == vaddr))) map->dm_segs[seg].ds_len += sgsize; @@ -326,6 +499,7 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, map->dm_segs[seg].ds_len = sgsize; map->dm_segs[seg]._ds_paddr = paddr; map->dm_segs[seg]._ds_vaddr = vaddr; + map->dm_segs[seg]._ds_bounce_va = pgva; } } @@ -335,11 +509,13 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, size -= sgsize; lastaddr = paddr; + lastbounce = bounce; } } map->dm_mapsize = mapsize; map->dm_nsegs = seg + 1; + map->_dm_nused = page; return (0); } @@ -356,6 +532,7 @@ _dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) */ map->dm_nsegs = 0; map->dm_mapsize = 0; + map->_dm_nused = 0; } static void @@ -393,15 +570,22 @@ void _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, bus_size_t size, int op) { + int coherent = 0; + int bounce = 0; int nsegs; int curseg; + if (t->_flags & BUS_DMA_COHERENT) + coherent = 1; + + if (map->_dm_nused > 0) + bounce = 1; + /* - * If our tag tells us that the device we are doing DMA - * with is coherent, make sure the write buffer is synced - * and return. + * If we're fully coherent, just make sure the write buffer is + * synced and return. */ - if (t->_flags & BUS_DMA_COHERENT) { + if (coherent && !bounce) { membar_sync(); return; } @@ -410,11 +594,14 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, curseg = 0; while (size && nsegs) { + vaddr_t bounce_va; + vaddr_t flush_va; vaddr_t vaddr; bus_size_t ssize; ssize = map->dm_segs[curseg].ds_len; vaddr = map->dm_segs[curseg]._ds_vaddr; + bounce_va = map->dm_segs[curseg]._ds_bounce_va; if (addr != 0) { if (addr >= ssize) { @@ -422,6 +609,8 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, ssize = 0; } else { vaddr += addr; + if (bounce_va != -1) + bounce_va += addr; ssize -= addr; addr = 0; } @@ -430,7 +619,26 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr, ssize = size; if (ssize != 0) { - _dmamap_sync_segment(vaddr, ssize, op); + if (bounce_va != -1) + flush_va = bounce_va; + else + flush_va = vaddr; + + if (bounce_va != -1 && (op & BUS_DMASYNC_PREWRITE)) { + memcpy((void *)bounce_va, (void *)vaddr, + ssize); + atomic_inc_long(&bus_dma_bounces); + } + + if ((t->_flags & BUS_DMA_COHERENT) == 0) + _dmamap_sync_segment(flush_va, ssize, op); + + if (bounce_va != -1 && (op & BUS_DMASYNC_POSTREAD)) { + memcpy((void *)vaddr, (void *)bounce_va, + ssize); + atomic_inc_long(&bus_dma_bounces); + } + size -= ssize; } curseg++; @@ -588,21 +796,25 @@ _dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp, - int *segp, int first) + int *segp, int *usedp, int *lastbouncep, int first) { bus_size_t sgsize; bus_addr_t lastaddr, baddr, bmask; paddr_t curaddr; - vaddr_t vaddr = (vaddr_t)buf; - int seg; + vaddr_t pgva, vaddr = (vaddr_t)buf; + int bounce, lastbounce; + int seg, page, off; pmap_t pmap; + struct vm_page *pg; if (p != NULL) pmap = p->p_vmspace->vm_map.pmap; else pmap = pmap_kernel(); + page = *usedp; lastaddr = *lastaddrp; + lastbounce = *lastbouncep; bmask = ~(map->_dm_boundary - 1); if (t->_dma_mask != 0) bmask &= t->_dma_mask; @@ -615,6 +827,23 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, panic("_dmapmap_load_buffer: pmap_extract(%p, %lx) failed!", pmap, vaddr); + bounce = 0; + if (curaddr > dma_constraint.ucr_high && + (map->_dm_flags & BUS_DMA_64BIT) == 0) + bounce = 1; + + if (bounce) { + if (page >= map->_dm_npages) + return (EFBIG); + + off = vaddr & PAGE_MASK; + pg = map->_dm_pages[page]; + curaddr = VM_PAGE_TO_PHYS(pg) + off; + pgva = map->_dm_pgva + (page << PGSHIFT) + off; + page++; + } else + pgva = -1; + /* * Compute the segment size, and adjust counts. */ @@ -641,15 +870,17 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, map->dm_segs[seg].ds_len = sgsize; map->dm_segs[seg]._ds_paddr = curaddr; map->dm_segs[seg]._ds_vaddr = vaddr; + map->dm_segs[seg]._ds_bounce_va = pgva; first = 0; } else { if ((bus_addr_t)curaddr == lastaddr && + bounce == lastbounce && (map->dm_segs[seg].ds_len + sgsize) <= map->_dm_maxsegsz && (map->_dm_boundary == 0 || (map->dm_segs[seg].ds_addr & bmask) == ((bus_addr_t)curaddr & bmask)) && - (t->_flags & BUS_DMA_COHERENT || + (t->_flags & BUS_DMA_COHERENT || !bounce || (map->dm_segs[seg]._ds_vaddr + map->dm_segs[seg].ds_len == vaddr))) map->dm_segs[seg].ds_len += sgsize; @@ -660,16 +891,20 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, map->dm_segs[seg].ds_len = sgsize; map->dm_segs[seg]._ds_paddr = curaddr; map->dm_segs[seg]._ds_vaddr = vaddr; + map->dm_segs[seg]._ds_bounce_va = pgva; } } lastaddr = (bus_addr_t)curaddr + sgsize; + lastbounce = bounce; vaddr += sgsize; buflen -= sgsize; } *segp = seg; + *usedp = page; *lastaddrp = lastaddr; + *lastbouncep = lastbounce; /* * Did we fit? diff --git a/sys/dev/fdt/bcm2711_pcie.c b/sys/dev/fdt/bcm2711_pcie.c index a2103b649fd..5d30b85b877 100644 --- a/sys/dev/fdt/bcm2711_pcie.c +++ b/sys/dev/fdt/bcm2711_pcie.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bcm2711_pcie.c,v 1.18 2025/08/29 11:50:43 kettenis Exp $ */ +/* $OpenBSD: bcm2711_pcie.c,v 1.19 2026/05/19 13:05:47 kettenis Exp $ */ /* * Copyright (c) 2020, 2025 Mark Kettenis * @@ -254,7 +254,8 @@ int bcmpcie_bs_iomap(bus_space_tag_t, bus_addr_t, bus_size_t, int, int bcmpcie_bs_memmap(bus_space_tag_t, bus_addr_t, bus_size_t, int, bus_space_handle_t *); int bcmpcie_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, - bus_size_t, struct proc *, int, paddr_t *, int *, int); + bus_size_t, struct proc *, int, paddr_t *, int *, int *, + int *, int); int bcmpcie_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t, bus_dma_segment_t *, int, bus_size_t, int); @@ -1113,7 +1114,7 @@ bcmpcie_bs_memmap(bus_space_tag_t t, bus_addr_t addr, bus_size_t size, int bcmpcie_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp, - int *segp, int first) + int *segp, int *usedp, int *lastbouncep, int first) { struct bcmpcie_softc *sc = t->_cookie; paddr_t lastaddr = *lastaddrp; @@ -1123,7 +1124,7 @@ bcmpcie_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, lastlen = map->dm_segs[firstseg].ds_len; error = sc->sc_dmat->_dmamap_load_buffer(sc->sc_dmat, map, buf, buflen, - p, flags, lastaddrp, segp, first); + p, flags, lastaddrp, segp, usedp, lastbouncep, first); if (error) return error;