Implement bounce buffers for arm64. Almost identical to the riscv64

version, but for now this strips the BUS_DMA_64BIT flag since the DMA
constraints on arm64 also include bus constraints.  This will be fixed
in a future diff.

sys/arch/arm64/arm64/autoconf.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.16 2024/11/10 06:51:59 jsg Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.17 2026/05/19 13:05:47 kettenis Exp $	*/
 /*
  * Copyright (c) 2009 Miodrag Vallat.
  *
@@ -23,6 +23,8 @@
 #include <sys/systm.h>
 #include <uvm/uvm_extern.h>
 
+#include <machine/bus.h>
+
 #if defined(NFSCLIENT)
 #include <net/if.h>
 #include <net/if_types.h>
@@ -57,6 +59,8 @@ cpu_configure(void)
 	splhigh();
 
 	softintr_init();
+	bus_dma_init();
+
 	config_rootfound("mainbus", NULL);
 
 	unmap_startup();

sys/arch/arm64/arm64/bus_dma.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.14 2020/11/22 15:18:35 patrick Exp $ */
+/*	$OpenBSD: bus_dma.c,v 1.15 2026/05/19 13:05:47 kettenis Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -54,19 +54,79 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
+
+#include "kstat.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
-#include <sys/kernel.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 
+#if NKSTAT > 0
+#include <sys/kstat.h>
+#endif
+
 #include <uvm/uvm_extern.h>
 
 #include <machine/bus.h>
 #include <machine/cpu.h>
 #include <machine/cpufunc.h>
 
+u_long bus_dma_high_pages;
+u_long bus_dma_bounce_pages;
+u_long bus_dma_bounces;
+
+#if NKSTAT > 0
+struct bus_dma_kstat_data {
+	struct kstat_kv kd_bounce_pages;
+	struct kstat_kv kd_bounces;
+};
+
+static const struct bus_dma_kstat_data bus_dma_kstat_tpl = {
+	KSTAT_KV_INITIALIZER("bounce-pages", KSTAT_KV_T_COUNTER64),
+	KSTAT_KV_INITIALIZER("bounces", KSTAT_KV_T_COUNTER64),
+};
+
+int
+bus_dma_kstat_copy(struct kstat *ks, void *dst)
+{
+	struct bus_dma_kstat_data *kd = dst;
+
+	*kd = bus_dma_kstat_tpl;
+	kstat_kv_u64(&kd->kd_bounce_pages) = bus_dma_bounce_pages;
+	kstat_kv_u64(&kd->kd_bounces) = bus_dma_bounces;
+
+	return 0;
+}
+#endif
+
+void
+bus_dma_init(void)
+{
+	struct uvm_constraint_range high_constraint;
+#if NKSTAT > 0
+	struct kstat *ks;
+#endif
+
+	high_constraint.ucr_low = dma_constraint.ucr_high;
+	high_constraint.ucr_high = no_constraint.ucr_high;
+	if (high_constraint.ucr_low != high_constraint.ucr_high)
+		high_constraint.ucr_low++;
+
+	bus_dma_high_pages = uvm_pagecount(&high_constraint);
+
+#if NKSTAT > 0
+	ks = kstat_create("mainbus0", 0, "dma", 0, KSTAT_T_KV, 0);
+	if (ks == NULL)
+		return;
+
+	ks->ks_datalen = sizeof(bus_dma_kstat_tpl);
+	ks->ks_copy = bus_dma_kstat_copy;
+	kstat_install(ks);
+#endif
+}
+
 /*
  * Common function for DMA map creation.  May be called by bus-specific
  * DMA map creation functions.
@@ -75,9 +135,21 @@ int
 _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
     bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
 {
+	struct uvm_constraint_range *constraint = &no_constraint;
+	int use_bounce_buffer = 0;
 	struct machine_bus_dmamap *map;
+	struct pglist mlist;
+	struct vm_page **pg, *pgnext;
+	size_t mapsize, sz, ssize;
+	vaddr_t va, sva;
 	void *mapstore;
-	size_t mapsize;
+	int npages, error;
+	const struct kmem_dyn_mode *kd;
+
+	if (bus_dma_high_pages > 0) {
+		use_bounce_buffer = 1;
+		constraint = &dma_constraint;
+	}
 
 	/*
 	 * Allocate and initialize the DMA map.  The end of the map
@@ -93,6 +165,16 @@ _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
 	 */
 	mapsize = sizeof(struct machine_bus_dmamap) +
 	    (sizeof(bus_dma_segment_t) * (nsegments - 1));
+
+	if (use_bounce_buffer) {
+		/* this many pages plus one in case we get split */
+		npages = round_page(size) / PAGE_SIZE + 1;
+		if (npages < nsegments)
+			npages = nsegments;
+		mapsize += sizeof(struct vm_page *) * npages;
+		atomic_add_long(&bus_dma_bounce_pages, npages);
+	}
+
 	if ((mapstore = malloc(mapsize, M_DEVBUF, (flags & BUS_DMA_NOWAIT) ?
 	    (M_NOWAIT | M_ZERO) : (M_WAITOK | M_ZERO))) == NULL)
 		return (ENOMEM);
@@ -103,6 +185,49 @@ _dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
 	map->_dm_maxsegsz = maxsegsz;
 	map->_dm_boundary = boundary;
 	map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
+	map->_dm_flags &= ~BUS_DMA_64BIT; /* XXX */
+	if (use_bounce_buffer) {
+		map->_dm_pages = (void *)&map->dm_segs[nsegments];
+		map->_dm_npages = npages;
+	}
+
+	if (!use_bounce_buffer) {
+		*dmamp = map;
+		return (0);
+	}
+
+	sz = npages << PGSHIFT;
+	kd = flags & BUS_DMA_NOWAIT ? &kd_trylock : &kd_waitok;
+	va = (vaddr_t)km_alloc(sz, &kv_any, &kp_none, kd);
+	if (va == 0) {
+		map->_dm_npages = 0;
+		free(map, M_DEVBUF, mapsize);
+		return (ENOMEM);
+	}
+
+	TAILQ_INIT(&mlist);
+	error = uvm_pglistalloc(sz, constraint->ucr_low,
+	    constraint->ucr_high, PAGE_SIZE, 0, &mlist, nsegments,
+	    (flags & BUS_DMA_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK);
+	if (error) {
+		map->_dm_npages = 0;
+		km_free((void *)va, sz, &kv_any, &kp_none);
+		free(map, M_DEVBUF, mapsize);
+		return (ENOMEM);
+	}
+
+	sva = va;
+	ssize = sz;
+	pgnext = TAILQ_FIRST(&mlist);
+	for (pg = map->_dm_pages; npages--; va += PAGE_SIZE, pg++) {
+		*pg = pgnext;
+		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(*pg),
+		    PROT_READ | PROT_WRITE);
+		pgnext = TAILQ_NEXT(*pg, pageq);
+		memset((void *)va, 0, PAGE_SIZE);
+	}
+	pmap_update(pmap_kernel());
+	map->_dm_pgva = sva;
 
 	*dmamp = map;
 	return (0);
@@ -116,9 +241,26 @@ void
 _dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
 {
 	size_t mapsize;
+	struct vm_page **pg;
+	struct pglist mlist;
+
+	if (map->_dm_pgva) {
+		km_free((void *)map->_dm_pgva, map->_dm_npages << PGSHIFT,
+		    &kv_any, &kp_none);
+	}
 
 	mapsize = sizeof(struct machine_bus_dmamap) +
 	    (sizeof(bus_dma_segment_t) * (map->_dm_segcnt - 1));
+	mapsize += sizeof(struct vm_page *) * map->_dm_npages;
+
+	if (map->_dm_pages) {
+		TAILQ_INIT(&mlist);
+		for (pg = map->_dm_pages; map->_dm_npages--; pg++) {
+			TAILQ_INSERT_TAIL(&mlist, *pg, pageq);
+		}
+		uvm_pglistfree(&mlist);
+	}
+
 	free(map, M_DEVBUF, mapsize);
 }
 
@@ -131,7 +273,8 @@ _dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen,
     struct proc *p, int flags)
 {
 	paddr_t lastaddr;
-	int seg, error;
+	int seg, used, error;
+	int lastbounce;
 
 	/*
 	 * Make sure that on error condition we return "no valid mappings".
@@ -143,11 +286,14 @@ _dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen,
 		return (EINVAL);
 
 	seg = 0;
+	used = 0;
+	lastbounce = 0;
 	error = (*t->_dmamap_load_buffer)(t, map, buf, buflen, p, flags,
-	    &lastaddr, &seg, 1);
+	    &lastaddr, &seg, &used, &lastbounce, 1);
 	if (error == 0) {
 		map->dm_nsegs = seg + 1;
 		map->dm_mapsize = buflen;
+		map->_dm_nused = used;
 	}
 
 	return (error);
@@ -160,7 +306,8 @@ int
 _dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
 {
 	paddr_t lastaddr;
-	int seg, error, first;
+	int seg, used, error, first;
+	int lastbounce;
 	struct mbuf *m;
 
 	/*
@@ -179,17 +326,20 @@ _dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
 
 	first = 1;
 	seg = 0;
+	used = 0;
+	lastbounce = 0;
 	error = 0;
 	for (m = m0; m != NULL && error == 0; m = m->m_next) {
 		if (m->m_len == 0)
 			continue;
 		error = (*t->_dmamap_load_buffer)(t, map, m->m_data, m->m_len,
-		    NULL, flags, &lastaddr, &seg, first);
+		    NULL, flags, &lastaddr, &seg, &used, &lastbounce, first);
 		first = 0;
 	}
 	if (error == 0) {
-		map->dm_nsegs = seg + 1;
 		map->dm_mapsize = m0->m_pkthdr.len;
+		map->dm_nsegs = seg + 1;
+		map->_dm_nused = used;
 	}
 
 	return (error);
@@ -202,7 +352,8 @@ int
 _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
 {
 	paddr_t lastaddr;
-	int seg, i, error, first;
+	int seg, used, i, error, first;
+	int lastbounce;
 	bus_size_t minlen, resid;
 	struct proc *p = NULL;
 	struct iovec *iov;
@@ -227,6 +378,8 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
 
 	first = 1;
 	seg = 0;
+	used = 0;
+	lastbounce = 0;
 	error = 0;
 	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
 		/*
@@ -237,7 +390,7 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
 		addr = (void *)iov[i].iov_base;
 
 		error = (*t->_dmamap_load_buffer)(t, map, addr, minlen,
-		    p, flags, &lastaddr, &seg, first);
+		    p, flags, &lastaddr, &seg, &used, &lastbounce, first);
 		first = 0;
 
 		resid -= minlen;
@@ -245,6 +398,7 @@ _dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
 	if (error == 0) {
 		map->dm_nsegs = seg + 1;
 		map->dm_mapsize = uio->uio_resid;
+		map->_dm_nused = used;
 	}
 
 	return (error);
@@ -260,9 +414,11 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
 {
 	bus_addr_t paddr, baddr, bmask, lastaddr = 0;
 	bus_size_t plen, sgsize, mapsize;
-	vaddr_t vaddr;
+	int bounce, lastbounce = 0;
 	int first = 1;
 	int i, seg = 0;
+	int off, page = 0;
+	vaddr_t pgva, vaddr;
 
 	/*
 	 * Make sure that on error condition we return "no valid mappings".
@@ -281,7 +437,22 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
 		vaddr = segs[i]._ds_vaddr;
 		plen = MIN(segs[i].ds_len, size);
 
+		bounce = 0;
+		if (paddr + plen - 1 > dma_constraint.ucr_high)
+			bounce = 1;
+
 		while (plen > 0) {
+			if (bounce) {
+				if (page >= map->_dm_npages)
+					return (EFBIG);
+
+				off = paddr & PAGE_MASK;
+				pgva = map->_dm_pgva + (page << PGSHIFT) + off;
+				page++;
+			} else {
+				pgva = -1;
+			}
+
 			/*
 			 * Compute the segment size, and adjust counts.
 			 */
@@ -307,15 +478,17 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
 				map->dm_segs[seg].ds_len = sgsize;
 				map->dm_segs[seg]._ds_paddr = paddr;
 				map->dm_segs[seg]._ds_vaddr = vaddr;
+				map->dm_segs[seg]._ds_bounce_va = pgva;
 				first = 0;
 			} else {
 				if (paddr == lastaddr &&
+				    bounce == lastbounce &&
 				    (map->dm_segs[seg].ds_len + sgsize) <=
 				     map->_dm_maxsegsz &&
 				    (map->_dm_boundary == 0 ||
 				     (map->dm_segs[seg].ds_addr & bmask) ==
 				     (paddr & bmask)) &&
-				    (t->_flags & BUS_DMA_COHERENT ||
+				    (t->_flags & BUS_DMA_COHERENT || !bounce ||
 				     (map->dm_segs[seg]._ds_vaddr +
 				     map->dm_segs[seg].ds_len == vaddr)))
 					map->dm_segs[seg].ds_len += sgsize;
@@ -326,6 +499,7 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
 					map->dm_segs[seg].ds_len = sgsize;
 					map->dm_segs[seg]._ds_paddr = paddr;
 					map->dm_segs[seg]._ds_vaddr = vaddr;
+					map->dm_segs[seg]._ds_bounce_va = pgva;
 				}
 			}
 
@@ -335,11 +509,13 @@ _dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs,
 			size -= sgsize;
 
 			lastaddr = paddr;
+			lastbounce = bounce;
 		}
 	}
 
 	map->dm_mapsize = mapsize;
 	map->dm_nsegs = seg + 1;
+	map->_dm_nused = page;
 	return (0);
 }
 
@@ -356,6 +532,7 @@ _dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
 	 */
 	map->dm_nsegs = 0;
 	map->dm_mapsize = 0;
+	map->_dm_nused = 0;
 }
 
 static void
@@ -393,15 +570,22 @@ void
 _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
     bus_size_t size, int op)
 {
+	int coherent = 0;
+	int bounce = 0;
 	int nsegs;
 	int curseg;
 
+	if (t->_flags & BUS_DMA_COHERENT)
+		coherent = 1;
+
+	if (map->_dm_nused > 0)
+		bounce = 1;
+
 	/*
-	 * If our tag tells us that the device we are doing DMA
-	 * with is coherent, make sure the write buffer is synced
-	 * and return.
+	 * If we're fully coherent, just make sure the write buffer is
+	 * synced and return.
 	 */
-	if (t->_flags & BUS_DMA_COHERENT) {
+	if (coherent && !bounce) {
 		membar_sync();
 		return;
 	}
@@ -410,11 +594,14 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 	curseg = 0;
 
 	while (size && nsegs) {
+		vaddr_t bounce_va;
+		vaddr_t flush_va;
 		vaddr_t vaddr;
 		bus_size_t ssize;
 
 		ssize = map->dm_segs[curseg].ds_len;
 		vaddr = map->dm_segs[curseg]._ds_vaddr;
+		bounce_va = map->dm_segs[curseg]._ds_bounce_va;
 
 		if (addr != 0) {
 			if (addr >= ssize) {
@@ -422,6 +609,8 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 				ssize = 0;
 			} else {
 				vaddr += addr;
+				if (bounce_va != -1)
+					bounce_va += addr;
 				ssize -= addr;
 				addr = 0;
 			}
@@ -430,7 +619,26 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 			ssize = size;
 
 		if (ssize != 0) {
-			_dmamap_sync_segment(vaddr, ssize, op);
+			if (bounce_va != -1)
+				flush_va = bounce_va;
+			else
+				flush_va = vaddr;
+
+			if (bounce_va != -1 && (op & BUS_DMASYNC_PREWRITE)) {
+				memcpy((void *)bounce_va, (void *)vaddr,
+				    ssize);
+				atomic_inc_long(&bus_dma_bounces);
+			}
+
+			if ((t->_flags & BUS_DMA_COHERENT) == 0)
+				_dmamap_sync_segment(flush_va, ssize, op);
+
+			if (bounce_va != -1 && (op & BUS_DMASYNC_POSTREAD)) {
+				memcpy((void *)vaddr, (void *)bounce_va,
+				    ssize);
+				atomic_inc_long(&bus_dma_bounces);
+			}
+
 			size -= ssize;
 		}
 		curseg++;
@@ -588,21 +796,25 @@ _dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off,
 int
 _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
     bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp,
-    int *segp, int first)
+    int *segp, int *usedp, int *lastbouncep, int first)
 {
 	bus_size_t sgsize;
 	bus_addr_t lastaddr, baddr, bmask;
 	paddr_t curaddr;
-	vaddr_t vaddr = (vaddr_t)buf;
-	int seg;
+	vaddr_t pgva, vaddr = (vaddr_t)buf;
+	int bounce, lastbounce;
+	int seg, page, off;
 	pmap_t pmap;
+	struct vm_page *pg;
 
 	if (p != NULL)
 		pmap = p->p_vmspace->vm_map.pmap;
 	else
 		pmap = pmap_kernel();
 
+	page = *usedp;
 	lastaddr = *lastaddrp;
+	lastbounce = *lastbouncep;
 	bmask  = ~(map->_dm_boundary - 1);
 	if (t->_dma_mask != 0)
 		bmask &= t->_dma_mask;
@@ -615,6 +827,23 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 			panic("_dmapmap_load_buffer: pmap_extract(%p, %lx) failed!",
 			    pmap, vaddr);
 
+		bounce = 0;
+		if (curaddr > dma_constraint.ucr_high &&
+		    (map->_dm_flags & BUS_DMA_64BIT) == 0)
+			bounce = 1;
+
+		if (bounce) {
+			if (page >= map->_dm_npages)
+				return (EFBIG);
+
+			off = vaddr & PAGE_MASK;
+			pg = map->_dm_pages[page];
+			curaddr = VM_PAGE_TO_PHYS(pg) + off;
+			pgva = map->_dm_pgva + (page << PGSHIFT) + off;
+			page++;
+		} else
+			pgva = -1;
+
 		/*
 		 * Compute the segment size, and adjust counts.
 		 */
@@ -641,15 +870,17 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 			map->dm_segs[seg].ds_len = sgsize;
 			map->dm_segs[seg]._ds_paddr = curaddr;
 			map->dm_segs[seg]._ds_vaddr = vaddr;
+			map->dm_segs[seg]._ds_bounce_va = pgva;
 			first = 0;
 		} else {
 			if ((bus_addr_t)curaddr == lastaddr &&
+			    bounce == lastbounce &&
 			    (map->dm_segs[seg].ds_len + sgsize) <=
 			     map->_dm_maxsegsz &&
 			    (map->_dm_boundary == 0 ||
 			     (map->dm_segs[seg].ds_addr & bmask) ==
 			     ((bus_addr_t)curaddr & bmask)) &&
-			    (t->_flags & BUS_DMA_COHERENT ||
+			    (t->_flags & BUS_DMA_COHERENT || !bounce ||
 			     (map->dm_segs[seg]._ds_vaddr +
 			     map->dm_segs[seg].ds_len == vaddr)))
 				map->dm_segs[seg].ds_len += sgsize;
@@ -660,16 +891,20 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 				map->dm_segs[seg].ds_len = sgsize;
 				map->dm_segs[seg]._ds_paddr = curaddr;
 				map->dm_segs[seg]._ds_vaddr = vaddr;
+				map->dm_segs[seg]._ds_bounce_va = pgva;
 			}
 		}
 
 		lastaddr = (bus_addr_t)curaddr + sgsize;
+		lastbounce = bounce;
 		vaddr += sgsize;
 		buflen -= sgsize;
 	}
 
 	*segp = seg;
+	*usedp = page;
 	*lastaddrp = lastaddr;
+	*lastbouncep = lastbounce;
 
 	/*
 	 * Did we fit?

sys/dev/fdt/bcm2711_pcie.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: bcm2711_pcie.c,v 1.18 2025/08/29 11:50:43 kettenis Exp $	*/
+/*	$OpenBSD: bcm2711_pcie.c,v 1.19 2026/05/19 13:05:47 kettenis Exp $	*/
 /*
  * Copyright (c) 2020, 2025 Mark Kettenis <kettenis@openbsd.org>
  *
@@ -254,7 +254,8 @@ int	bcmpcie_bs_iomap(bus_space_tag_t, bus_addr_t, bus_size_t, int,
 int	bcmpcie_bs_memmap(bus_space_tag_t, bus_addr_t, bus_size_t, int,
 	    bus_space_handle_t *);
 int	bcmpcie_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *,
-	    bus_size_t, struct proc *, int, paddr_t *, int *, int);
+	    bus_size_t, struct proc *, int, paddr_t *, int *, int *,
+	    int *, int);
 int	bcmpcie_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
 	    bus_dma_segment_t *, int, bus_size_t, int);
 
@@ -1113,7 +1114,7 @@ bcmpcie_bs_memmap(bus_space_tag_t t, bus_addr_t addr, bus_size_t size,
 int
 bcmpcie_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
     bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp,
-    int *segp, int first)
+    int *segp, int *usedp, int *lastbouncep, int first)
 {
 	struct bcmpcie_softc *sc = t->_cookie;
 	paddr_t lastaddr = *lastaddrp;
@@ -1123,7 +1124,7 @@ bcmpcie_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 
 	lastlen = map->dm_segs[firstseg].ds_len;
 	error = sc->sc_dmat->_dmamap_load_buffer(sc->sc_dmat, map, buf, buflen,
-	    p, flags, lastaddrp, segp, first);
+	    p, flags, lastaddrp, segp, usedp, lastbouncep, first);
 	if (error)
 		return error;