svn commit: r244000 - in projects/physbio/sys: arm/arm mips/mips
Jeff Roberson
jeff at FreeBSD.org
Fri Dec 7 23:18:32 UTC 2012
Author: jeff
Date: Fri Dec 7 23:18:30 2012
New Revision: 244000
URL: http://svnweb.freebsd.org/changeset/base/244000
Log:
- Remember the list of virtual addresses we have mapped in the bus_dma_map
on architectures that need to invalidate virtual caches. This makes the
sync operation type agnostic. Rather than mallocing a small structure
for each virtual address as busdma_machdep-v6.c on arm did, I opted to
allocate an array sized by the maximum number of physical segments
supported based on the idea that there should always be more physical
than virtual segments.
Sponsored by: EMC / Isilon Storage Division
Modified:
projects/physbio/sys/arm/arm/busdma_machdep-v6.c
projects/physbio/sys/arm/arm/busdma_machdep.c
projects/physbio/sys/mips/mips/busdma_machdep.c
Modified: projects/physbio/sys/arm/arm/busdma_machdep-v6.c
==============================================================================
--- projects/physbio/sys/arm/arm/busdma_machdep-v6.c Fri Dec 7 22:30:30 2012 (r243999)
+++ projects/physbio/sys/arm/arm/busdma_machdep-v6.c Fri Dec 7 23:18:30 2012 (r244000)
@@ -108,7 +108,6 @@ struct sync_list {
vm_offset_t vaddr; /* kva of bounce buffer */
bus_addr_t busaddr; /* Physical address */
bus_size_t datacount; /* client data count */
- STAILQ_ENTRY(sync_list) slinks;
};
int busdma_swi_pending;
@@ -151,7 +150,8 @@ struct bus_dmamap {
bus_dmamap_callback_t *callback;
void *callback_arg;
STAILQ_ENTRY(bus_dmamap) links;
- STAILQ_HEAD(,sync_list) slist;
+ int sync_count;
+ struct sync_list slist[];
};
static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
@@ -436,17 +436,18 @@ out:
int
bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
{
+ int mapsize;
int error;
error = 0;
- *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
- M_NOWAIT | M_ZERO);
+ mapsize = sizeof(**mapp) + (sizeof(struct sync_list) * dmat->nsegments);
+ *mapp = (bus_dmamap_t)malloc(mapsize, M_DEVBUF, M_NOWAIT | M_ZERO);
if (*mapp == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
return (ENOMEM);
}
- STAILQ_INIT(&((*mapp)->slist));
+ (*mapp)->sync_count = 0;
if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc(
@@ -521,8 +522,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
int
bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
{
- if (STAILQ_FIRST(&map->bpages) != NULL ||
- STAILQ_FIRST(&map->slist) != NULL) {
+ if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d",
__func__, dmat, EBUSY);
return (EBUSY);
@@ -546,6 +546,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
bus_dmamap_t *mapp)
{
int mflags, len;
+ int mapsize;
if (flags & BUS_DMA_NOWAIT)
mflags = M_NOWAIT;
@@ -554,15 +555,15 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
/* ARM non-snooping caches need a map for the VA cache sync structure */
- *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
- M_NOWAIT | M_ZERO);
+ mapsize = sizeof(**mapp) + (sizeof(struct sync_list) * dmat->nsegments);
+ *mapp = (bus_dmamap_t)malloc(mapsize, M_DEVBUF, M_NOWAIT | M_ZERO);
if (*mapp == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->flags, ENOMEM);
return (ENOMEM);
}
- STAILQ_INIT(&((*mapp)->slist));
+ (*mapp)->sync_count = 0;
if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc(
@@ -774,18 +775,14 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
} else {
- /* add_sync_list(dmat, map, vaddr, sgsize, cflag); */
- sl = (struct sync_list *)malloc(sizeof(struct sync_list),
- M_DEVBUF, M_NOWAIT | M_ZERO);
- if (sl == NULL)
+ sl = &map->slist[map->sync_count];
+ if (++map->sync_count > dmat->nsegments)
goto cleanup;
- STAILQ_INSERT_TAIL(&(map->slist), sl, slinks);
sl->vaddr = vaddr;
sl->datacount = sgsize;
sl->busaddr = curaddr;
}
-
if (dmat->ranges) {
struct arm32_dma_range *dr;
@@ -1012,12 +1009,6 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
{
struct bounce_page *bpage;
struct bounce_zone *bz;
- struct sync_list *sl;
-
- while ((sl = STAILQ_FIRST(&map->slist)) != NULL) {
- STAILQ_REMOVE_HEAD(&map->slist, slinks);
- free(sl, M_DEVBUF);
- }
if ((bz = dmat->bounce_zone) != NULL) {
while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
@@ -1031,6 +1022,7 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
map->pagesreserved = 0;
map->pagesneeded = 0;
}
+ map->sync_count = 0;
}
#ifdef notyetbounceuser
@@ -1090,15 +1082,13 @@ void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{
struct bounce_page *bpage;
- struct sync_list *sl;
+ struct sync_list *sl, *end;
bus_size_t len, unalign;
vm_offset_t buf, ebuf;
#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD
vm_offset_t bbuf;
char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
#endif
- int listcount = 0;
-
/* if buffer was from user space, it it possible that this
* is not the same vm map. The fix is to map each page in
* the buffer into the current address space (KVM) and then
@@ -1166,29 +1156,26 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
}
}
- sl = STAILQ_FIRST(&map->slist);
- while (sl) {
- listcount++;
- sl = STAILQ_NEXT(sl, slinks);
- }
- if ((sl = STAILQ_FIRST(&map->slist)) != NULL) {
+ if (map->sync_count != 0) {
/* ARM caches are not self-snooping for dma */
+ sl = &map->slist[0];
+ end = &map->slist[map->sync_count];
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
"performing sync", __func__, dmat, dmat->flags, op);
switch (op) {
case BUS_DMASYNC_PREWRITE:
- while (sl != NULL) {
+ while (sl != end) {
cpu_dcache_wb_range(sl->vaddr, sl->datacount);
l2cache_wb_range(sl->vaddr, sl->busaddr,
sl->datacount);
- sl = STAILQ_NEXT(sl, slinks);
+ sl++;
}
break;
case BUS_DMASYNC_PREREAD:
- while (sl != NULL) {
+ while (sl != end) {
/* write back the unaligned portions */
vm_paddr_t physaddr = sl->busaddr, ephysaddr;
buf = sl->vaddr;
@@ -1228,16 +1215,16 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
cpu_dcache_inv_range(buf, len);
l2cache_inv_range(buf, physaddr, len);
}
- sl = STAILQ_NEXT(sl, slinks);
+ sl++;
}
break;
case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
- while (sl != NULL) {
+ while (sl != end) {
cpu_dcache_wbinv_range(sl->vaddr, sl->datacount);
l2cache_wbinv_range(sl->vaddr,
sl->busaddr, sl->datacount);
- sl = STAILQ_NEXT(sl, slinks);
+ sl++;
}
break;
@@ -1245,7 +1232,7 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
case BUS_DMASYNC_POSTREAD:
if (!pmap_dmap_iscurrent(map->pmap))
panic("_bus_dmamap_sync: wrong user map. apply fix");
- while (sl != NULL) {
+ while (sl != end) {
/* write back the unaligned portions */
vm_paddr_t physaddr;
buf = sl->vaddr;
@@ -1278,7 +1265,7 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
unalign = arm_dcache_align - unalign;
memcpy((void *)ebuf, _tmp_clend, unalign);
}
- sl = STAILQ_NEXT(sl, slinks);
+ sl++;
}
break;
#endif /* FIX_DMAP_BUS_DMASYNC_POSTREAD */
Modified: projects/physbio/sys/arm/arm/busdma_machdep.c
==============================================================================
--- projects/physbio/sys/arm/arm/busdma_machdep.c Fri Dec 7 22:30:30 2012 (r243999)
+++ projects/physbio/sys/arm/arm/busdma_machdep.c Fri Dec 7 23:18:30 2012 (r244000)
@@ -102,6 +102,12 @@ struct bounce_page {
STAILQ_ENTRY(bounce_page) links;
};
+struct sync_list {
+ vm_offset_t vaddr; /* kva of bounce buffer */
+ bus_addr_t busaddr; /* Physical address */
+ bus_size_t datacount; /* client data count */
+};
+
int busdma_swi_pending;
struct bounce_zone {
@@ -131,11 +137,7 @@ static SYSCTL_NODE(_hw, OID_AUTO, busdma
SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
"Total bounce pages");
-#define DMAMAP_LINEAR 0x1
-#define DMAMAP_MBUF 0x2
-#define DMAMAP_UIO 0x4
#define DMAMAP_ALLOCATED 0x10
-#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
#define DMAMAP_COHERENT 0x8
struct bus_dmamap {
struct bp_list bpages;
@@ -151,7 +153,8 @@ struct bus_dmamap {
STAILQ_ENTRY(bus_dmamap) links;
bus_dmamap_callback_t *callback;
void *callback_arg;
-
+ int sync_count;
+ struct sync_list *slist;
};
static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
@@ -290,10 +293,14 @@ dflt_lock(void *arg, bus_dma_lock_op_t o
}
static __inline bus_dmamap_t
-_busdma_alloc_dmamap(void)
+_busdma_alloc_dmamap(bus_dma_tag_t dmat)
{
+ struct sync_list *slist;
bus_dmamap_t map;
+ slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
+ if (slist == NULL)
+ return (NULL);
mtx_lock(&busdma_mtx);
map = TAILQ_FIRST(&dmamap_freelist);
if (map)
@@ -305,13 +312,17 @@ _busdma_alloc_dmamap(void)
map->flags = DMAMAP_ALLOCATED;
} else
map->flags = 0;
- STAILQ_INIT(&map->bpages);
+ if (map != NULL)
+ STAILQ_INIT(&map->bpages);
+ else
+ free(slist, M_DEVBUF);
return (map);
}
static __inline void
_busdma_free_dmamap(bus_dmamap_t map)
{
+ free(map->slist, M_DEVBUF);
if (map->flags & DMAMAP_ALLOCATED)
free(map, M_DEVBUF);
else {
@@ -494,7 +505,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
}
}
- newmap = _busdma_alloc_dmamap();
+ newmap = _busdma_alloc_dmamap(dmat);
if (newmap == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
return (ENOMEM);
@@ -502,6 +513,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
*mapp = newmap;
newmap->dmat = dmat;
newmap->allocbuffer = NULL;
+ newmap->sync_count = 0;
dmat->map_count++;
/*
@@ -565,7 +577,7 @@ int
bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
{
- if (STAILQ_FIRST(&map->bpages) != NULL) {
+ if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d",
__func__, dmat, EBUSY);
return (EBUSY);
@@ -608,7 +620,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
if (flags & BUS_DMA_ZERO)
mflags |= M_ZERO;
- newmap = _busdma_alloc_dmamap();
+ newmap = _busdma_alloc_dmamap(dmat);
if (newmap == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->flags, ENOMEM);
@@ -617,6 +629,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
dmat->map_count++;
*mapp = newmap;
newmap->dmat = dmat;
+ newmap->sync_count = 0;
if (dmat->maxsize <= PAGE_SIZE &&
(dmat->alignment < dmat->maxsize) &&
@@ -728,6 +741,8 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm
} else {
if (reserve_bounce_pages(dmat, map, 1) != 0) {
/* Queue us for resources */
+ map->buffer = buf;
+ map->len = buflen;
STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
map, links);
mtx_unlock(&bounce_lock);
@@ -751,6 +766,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
{
bus_size_t sgsize;
bus_addr_t curaddr, baddr, bmask;
+ struct sync_list *sl;
vm_offset_t vaddr = (vm_offset_t)buf;
int seg;
int error = 0;
@@ -839,8 +855,16 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
sgsize = (baddr - curaddr);
}
if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
- map->pagesneeded != 0 && run_filter(dmat, curaddr))
+ map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
+ } else {
+ sl = &map->slist[map->sync_count];
+ if (++map->sync_count > dmat->nsegments)
+ goto cleanup;
+ sl->vaddr = vaddr;
+ sl->datacount = sgsize;
+ sl->busaddr = curaddr;
+ }
if (dmat->ranges) {
struct arm32_dma_range *dr;
@@ -883,7 +907,7 @@ segdone:
}
*segp = seg;
-
+cleanup:
/*
* Did we fit?
*/
@@ -906,10 +930,6 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_
KASSERT(map != NULL, ("dmamap is NULL"));
map->callback = callback;
map->callback_arg = callback_arg;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_LINEAR|DMAMAP_COHERENT;
- map->buffer = buf;
- map->len = buflen;
error = _bus_dmamap_load_buffer(dmat,
map, buf, buflen, kernel_pmap,
flags, NULL, &nsegs);
@@ -938,10 +958,6 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
M_ASSERTPKTHDR(m0);
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT;
- map->buffer = m0;
- map->len = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
struct mbuf *m;
@@ -950,7 +966,6 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
error = _bus_dmamap_load_buffer(dmat,
map, m->m_data, m->m_len,
kernel_pmap, flags, NULL, &nsegs);
- map->len += m->m_len;
}
}
} else {
@@ -982,10 +997,6 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm
flags |= BUS_DMA_NOWAIT;
*nsegs = -1;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT;
- map->buffer = m0;
- map->len = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
struct mbuf *m;
@@ -995,7 +1006,6 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm
m->m_data, m->m_len,
kernel_pmap, flags,
segs, nsegs);
- map->len += m->m_len;
}
}
} else {
@@ -1024,10 +1034,6 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat,
resid = uio->uio_resid;
iov = uio->uio_iov;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_UIO|DMAMAP_COHERENT;
- map->buffer = uio;
- map->len = 0;
if (uio->uio_segflg == UIO_USERSPACE) {
KASSERT(uio->uio_td != NULL,
@@ -1051,7 +1057,6 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat,
error = _bus_dmamap_load_buffer(dmat,
map, addr, minlen, pmap, flags, NULL, &nsegs);
- map->len += minlen;
resid -= minlen;
}
}
@@ -1079,11 +1084,11 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
{
struct bounce_page *bpage;
- map->flags &= ~DMAMAP_TYPE_MASK;
while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
STAILQ_REMOVE_HEAD(&map->bpages, links);
free_bounce_page(dmat, bpage);
}
+ map->sync_count = 0;
return;
}
@@ -1172,28 +1177,10 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat,
}
}
-static __inline int
-_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
-{
- struct bounce_page *bpage;
-
- STAILQ_FOREACH(bpage, &map->bpages, links) {
- if ((vm_offset_t)buf >= bpage->datavaddr &&
- (vm_offset_t)buf + len <= bpage->datavaddr +
- bpage->datacount)
- return (1);
- }
- return (0);
-
-}
-
void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{
- struct mbuf *m;
- struct uio *uio;
- int resid;
- struct iovec *iov;
+ struct sync_list *sl, *end;
if (op == BUS_DMASYNC_POSTWRITE)
return;
@@ -1202,38 +1189,10 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
if (map->flags & DMAMAP_COHERENT)
return;
CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
- switch(map->flags & DMAMAP_TYPE_MASK) {
- case DMAMAP_LINEAR:
- if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
- bus_dmamap_sync_buf(map->buffer, map->len, op);
- break;
- case DMAMAP_MBUF:
- m = map->buffer;
- while (m) {
- if (m->m_len > 0 &&
- !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
- bus_dmamap_sync_buf(m->m_data, m->m_len, op);
- m = m->m_next;
- }
- break;
- case DMAMAP_UIO:
- uio = map->buffer;
- iov = uio->uio_iov;
- resid = uio->uio_resid;
- for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
- bus_size_t minlen = resid < iov[i].iov_len ? resid :
- iov[i].iov_len;
- if (minlen > 0) {
- if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
- minlen))
- bus_dmamap_sync_buf(iov[i].iov_base,
- minlen, op);
- resid -= minlen;
- }
- }
- break;
- default:
- break;
+ if (map->sync_count) {
+ end = &map->slist[map->sync_count];
+ for (sl = &map->slist[0]; sl != end; sl++)
+ bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op);
}
cpu_drain_writebuf();
}
Modified: projects/physbio/sys/mips/mips/busdma_machdep.c
==============================================================================
--- projects/physbio/sys/mips/mips/busdma_machdep.c Fri Dec 7 22:30:30 2012 (r243999)
+++ projects/physbio/sys/mips/mips/busdma_machdep.c Fri Dec 7 23:18:30 2012 (r244000)
@@ -93,6 +93,12 @@ struct bounce_page {
STAILQ_ENTRY(bounce_page) links;
};
+struct sync_list {
+ vm_offset_t vaddr; /* kva of bounce buffer */
+ bus_addr_t busaddr; /* Physical address */
+ bus_size_t datacount; /* client data count */
+};
+
int busdma_swi_pending;
struct bounce_zone {
@@ -122,10 +128,6 @@ static SYSCTL_NODE(_hw, OID_AUTO, busdma
SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
"Total bounce pages");
-#define DMAMAP_LINEAR 0x1
-#define DMAMAP_MBUF 0x2
-#define DMAMAP_UIO 0x4
-#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
#define DMAMAP_UNCACHEABLE 0x8
#define DMAMAP_ALLOCATED 0x10
#define DMAMAP_MALLOCUSED 0x20
@@ -144,7 +146,8 @@ struct bus_dmamap {
STAILQ_ENTRY(bus_dmamap) links;
bus_dmamap_callback_t *callback;
void *callback_arg;
-
+ int sync_count;
+ struct sync_list *slist;
};
static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
@@ -268,10 +271,14 @@ dflt_lock(void *arg, bus_dma_lock_op_t o
}
static __inline bus_dmamap_t
-_busdma_alloc_dmamap(void)
+_busdma_alloc_dmamap(bus_dma_tag_t dmat)
{
+ struct sync_list *slist;
bus_dmamap_t map;
+ slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
+ if (slist == NULL)
+ return (NULL);
mtx_lock(&busdma_mtx);
map = TAILQ_FIRST(&dmamap_freelist);
if (map)
@@ -283,13 +290,17 @@ _busdma_alloc_dmamap(void)
map->flags = DMAMAP_ALLOCATED;
} else
map->flags = 0;
- STAILQ_INIT(&map->bpages);
+ if (map != NULL)
+ STAILQ_INIT(&map->bpages);
+ else
+ free(slist, M_DEVBUF);
return (map);
}
static __inline void
_busdma_free_dmamap(bus_dmamap_t map)
{
+ free(map->slist, M_DEVBUF);
if (map->flags & DMAMAP_ALLOCATED)
free(map, M_DEVBUF);
else {
@@ -472,7 +483,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
}
}
- newmap = _busdma_alloc_dmamap();
+ newmap = _busdma_alloc_dmamap(dmat);
if (newmap == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM);
return (ENOMEM);
@@ -480,6 +491,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
*mapp = newmap;
newmap->dmat = dmat;
newmap->allocbuffer = NULL;
+ newmap->sync_count = 0;
dmat->map_count++;
/*
@@ -544,7 +556,7 @@ int
bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
{
- if (STAILQ_FIRST(&map->bpages) != NULL) {
+ if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d",
__func__, dmat, EBUSY);
return (EBUSY);
@@ -587,7 +599,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
if (flags & BUS_DMA_ZERO)
mflags |= M_ZERO;
- newmap = _busdma_alloc_dmamap();
+ newmap = _busdma_alloc_dmamap(dmat);
if (newmap == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->flags, ENOMEM);
@@ -596,6 +608,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
dmat->map_count++;
*mapp = newmap;
newmap->dmat = dmat;
+ nwemap->sync_count = 0;
/*
* If all the memory is coherent with DMA then we don't need to
@@ -726,6 +739,8 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm
} else {
if (reserve_bounce_pages(dmat, map, 1) != 0) {
/* Queue us for resources */
+ map->buffer = buf;
+ map->len = buflen;
STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
map, links);
mtx_unlock(&bounce_lock);
@@ -750,6 +765,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
{
bus_size_t sgsize;
bus_addr_t curaddr, baddr, bmask;
+ struct sync_list *sl;
vm_offset_t vaddr = (vm_offset_t)buf;
int seg;
int error = 0;
@@ -798,6 +814,13 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
+ } else {
+ sl = &map->slist[map->sync_count];
+ if (++map->sync_count > dmat->nsegments)
+ goto cleanup;
+ sl->vaddr = vaddr;
+ sl->datacount = sgsize;
+ sl->busaddr = curaddr;
}
/*
@@ -826,7 +849,7 @@ segdone:
}
*segp = seg;
-
+cleanup:
/*
* Did we fit?
*/
@@ -849,10 +872,6 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_
KASSERT(map != NULL, ("dmamap is NULL"));
map->callback = callback;
map->callback_arg = callback_arg;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_LINEAR;
- map->buffer = buf;
- map->len = buflen;
error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap,
flags, NULL, &nsegs);
if (error == EINPROGRESS)
@@ -880,10 +899,6 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
M_ASSERTPKTHDR(m0);
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_MBUF;
- map->buffer = m0;
- map->len = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
struct mbuf *m;
@@ -892,7 +907,6 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat,
error = _bus_dmamap_load_buffer(dmat,
map, m->m_data, m->m_len,
kernel_pmap, flags, NULL, &nsegs);
- map->len += m->m_len;
}
}
} else {
@@ -924,10 +938,6 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm
flags |= BUS_DMA_NOWAIT;
*nsegs = -1;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_MBUF;
- map->buffer = m0;
- map->len = 0;
if (m0->m_pkthdr.len <= dmat->maxsize) {
struct mbuf *m;
@@ -937,7 +947,6 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm
m->m_data, m->m_len,
kernel_pmap, flags,
segs, nsegs);
- map->len += m->m_len;
}
}
} else {
@@ -966,10 +975,6 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat,
resid = uio->uio_resid;
iov = uio->uio_iov;
- map->flags &= ~DMAMAP_TYPE_MASK;
- map->flags |= DMAMAP_UIO;
- map->buffer = uio;
- map->len = 0;
if (uio->uio_segflg == UIO_USERSPACE) {
KASSERT(uio->uio_td != NULL,
@@ -994,7 +999,6 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat,
error = _bus_dmamap_load_buffer(dmat, map, addr,
minlen, pmap, flags, NULL, &nsegs);
- map->len += minlen;
resid -= minlen;
}
}
@@ -1022,11 +1026,11 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
{
struct bounce_page *bpage;
- map->flags &= ~DMAMAP_TYPE_MASK;
while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
STAILQ_REMOVE_HEAD(&map->bpages, links);
free_bounce_page(dmat, bpage);
}
+ map->sync_count = 0;
return;
}
@@ -1154,28 +1158,10 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat,
}
}
-static __inline int
-_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
-{
- struct bounce_page *bpage;
-
- STAILQ_FOREACH(bpage, &map->bpages, links) {
- if ((vm_offset_t)buf >= bpage->datavaddr &&
- (vm_offset_t)buf + len <= bpage->datavaddr +
- bpage->datacount)
- return (1);
- }
- return (0);
-
-}
-
void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{
- struct mbuf *m;
- struct uio *uio;
- int resid;
- struct iovec *iov;
+ struct sync_list *sl, *end;
if (op == BUS_DMASYNC_POSTWRITE)
return;
@@ -1189,38 +1175,10 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
return;
CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
- switch(map->flags & DMAMAP_TYPE_MASK) {
- case DMAMAP_LINEAR:
- if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
- bus_dmamap_sync_buf(map->buffer, map->len, op);
- break;
- case DMAMAP_MBUF:
- m = map->buffer;
- while (m) {
- if (m->m_len > 0 &&
- !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
- bus_dmamap_sync_buf(m->m_data, m->m_len, op);
- m = m->m_next;
- }
- break;
- case DMAMAP_UIO:
- uio = map->buffer;
- iov = uio->uio_iov;
- resid = uio->uio_resid;
- for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
- bus_size_t minlen = resid < iov[i].iov_len ? resid :
- iov[i].iov_len;
- if (minlen > 0) {
- if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
- minlen))
- bus_dmamap_sync_buf(iov[i].iov_base,
- minlen, op);
- resid -= minlen;
- }
- }
- break;
- default:
- break;
+ if (map->sync_count) {
+ end = &map->slist[map->sync_count];
+ for (sl = &map->slist[0]; sl != end; sl++)
+ bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op);
}
}
More information about the svn-src-projects
mailing list