PERFORCE change 115507 for review
Matt Jacob
mjacob at FreeBSD.org
Thu Mar 8 04:34:24 UTC 2007
http://perforce.freebsd.org/chv.cgi?CH=115507
Change 115507 by mjacob at mjexp_6 on 2007/03/08 04:33:37
Integrate from vendor branch.
Affected files ...
.. //depot/projects/mjexp_6/sys/amd64/amd64/pmap.c#2 integrate
.. //depot/projects/mjexp_6/sys/amd64/conf/SMP#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/busdma_machdep.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/cpufunc.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/elf_trampoline.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/locore.S#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/mem.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/nexus_io.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/trap.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/arm/vm_machdep.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/at91/at91.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/conf/AVILA#1 branch
.. //depot/projects/mjexp_6/sys/arm/conf/IQ31244#2 integrate
.. //depot/projects/mjexp_6/sys/arm/include/bus.h#2 integrate
.. //depot/projects/mjexp_6/sys/arm/include/db_machdep.h#2 integrate
.. //depot/projects/mjexp_6/sys/arm/include/elf.h#2 integrate
.. //depot/projects/mjexp_6/sys/arm/include/md_var.h#2 integrate
.. //depot/projects/mjexp_6/sys/arm/sa11x0/sa11x0_io.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/sa11x0/sa11x0_irqhandler.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/xscale/i80321/i80321_pci.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/xscale/i80321/i80321_space.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/xscale/i80321/i80321_wdog.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/xscale/i80321/obio_space.c#2 integrate
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/avila_ata.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/avila_led.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/avila_machdep.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/files.avila#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/files.ixp425#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/if_npe.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/if_npereg.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixdp425_pci.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixdp425reg.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_a4x_io.S#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_a4x_space.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_iic.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_intr.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_mem.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_npe.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_npereg.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_npevar.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_pci.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_pci_asm.S#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_pci_space.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_qmgr.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_qmgr.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_space.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_timer.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425_wdog.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425reg.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/ixp425var.h#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/std.avila#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/std.ixp425#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/uart_bus_ixp425.c#1 branch
.. //depot/projects/mjexp_6/sys/arm/xscale/ixp425/uart_cpu_ixp425.c#1 branch
.. //depot/projects/mjexp_6/sys/boot/i386/cdboot/cdboot.s#2 integrate
.. //depot/projects/mjexp_6/sys/cam/scsi/scsi_da.c#3 integrate
.. //depot/projects/mjexp_6/sys/conf/Makefile.arm#2 integrate
.. //depot/projects/mjexp_6/sys/conf/files#5 integrate
.. //depot/projects/mjexp_6/sys/conf/files.arm#3 integrate
.. //depot/projects/mjexp_6/sys/conf/kern.pre.mk#2 integrate
.. //depot/projects/mjexp_6/sys/conf/options.arm#2 integrate
.. //depot/projects/mjexp_6/sys/dev/ata/atapi-cam.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/bce/if_bce.c#4 integrate
.. //depot/projects/mjexp_6/sys/dev/bge/if_bge.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/em/if_em.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/fdc/fdc.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/hme/if_hme.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/ichwd/ichwd.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/ad7418.c#1 branch
.. //depot/projects/mjexp_6/sys/dev/iicbus/ds1672.c#1 branch
.. //depot/projects/mjexp_6/sys/dev/iicbus/iic.h#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/iicbb.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/iicbus.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/iicbus_if.m#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/iiconf.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iicbus/iiconf.h#2 integrate
.. //depot/projects/mjexp_6/sys/dev/ipmi/ipmi.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/isp/isp_freebsd.h#5 integrate
.. //depot/projects/mjexp_6/sys/dev/iwi/if_iwi.c#4 integrate
.. //depot/projects/mjexp_6/sys/dev/iwi/if_iwireg.h#2 integrate
.. //depot/projects/mjexp_6/sys/dev/iwi/if_iwivar.h#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/acphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/acphyreg.h#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/amphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/bmtphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/brgphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/ciphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/exphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/gentbi.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/inphy.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/ip1000phy.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/lxtphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/mii_physubr.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/miidevs#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/miivar.h#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/mlphy.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/nsgphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/nsphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/pnaphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/qsphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/rgephy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/rlphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/ruephy.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/tdkphy.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/tlphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mii/xmphy.c#3 integrate
.. //depot/projects/mjexp_6/sys/dev/mk48txx/mk48txx.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/syscons/fire/fire_saver.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/syscons/logo/logo_saver.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/syscons/rain/rain_saver.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/syscons/warp/warp_saver.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/uart/uart_dev_ns8250.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/vge/if_vge.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/watchdog/watchdog.c#2 integrate
.. //depot/projects/mjexp_6/sys/dev/wi/if_wi_pccard.c#2 integrate
.. //depot/projects/mjexp_6/sys/fs/msdosfs/denode.h#2 integrate
.. //depot/projects/mjexp_6/sys/fs/msdosfs/msdosfs_fat.c#2 integrate
.. //depot/projects/mjexp_6/sys/geom/geom_disk.c#2 integrate
.. //depot/projects/mjexp_6/sys/i386/i386/elan-mmcr.c#2 integrate
.. //depot/projects/mjexp_6/sys/kern/kern_clock.c#2 integrate
.. //depot/projects/mjexp_6/sys/kern/kern_conf.c#2 integrate
.. //depot/projects/mjexp_6/sys/kern/subr_firmware.c#3 integrate
.. //depot/projects/mjexp_6/sys/kern/sys_generic.c#2 integrate
.. //depot/projects/mjexp_6/sys/kern/uipc_usrreq.c#3 integrate
.. //depot/projects/mjexp_6/sys/kern/vfs_lookup.c#3 integrate
.. //depot/projects/mjexp_6/sys/kern/vfs_syscalls.c#2 integrate
.. //depot/projects/mjexp_6/sys/net/bpf.c#3 integrate
.. //depot/projects/mjexp_6/sys/net/ethernet.h#3 integrate
.. //depot/projects/mjexp_6/sys/net/if_bridge.c#4 integrate
.. //depot/projects/mjexp_6/sys/net/if_ethersubr.c#2 integrate
.. //depot/projects/mjexp_6/sys/net/if_loop.c#3 integrate
.. //depot/projects/mjexp_6/sys/net/if_tap.c#3 integrate
.. //depot/projects/mjexp_6/sys/net/if_tun.c#3 integrate
.. //depot/projects/mjexp_6/sys/net/route.c#2 integrate
.. //depot/projects/mjexp_6/sys/netgraph/ng_ksocket.c#2 integrate
.. //depot/projects/mjexp_6/sys/netinet/ip_mroute.c#2 integrate
.. //depot/projects/mjexp_6/sys/netinet/tcp.h#2 integrate
.. //depot/projects/mjexp_6/sys/pci/if_pcn.c#3 integrate
.. //depot/projects/mjexp_6/sys/pci/if_pcnreg.h#3 integrate
.. //depot/projects/mjexp_6/sys/sys/conf.h#2 integrate
.. //depot/projects/mjexp_6/sys/sys/firmware.h#2 integrate
.. //depot/projects/mjexp_6/sys/sys/param.h#5 integrate
.. //depot/projects/mjexp_6/sys/sys/watchdog.h#2 integrate
.. //depot/projects/mjexp_6/sys/tools/fw_stub.awk#2 integrate
.. //depot/projects/mjexp_6/sys/ufs/ufs/ufs_vnops.c#2 integrate
.. //depot/projects/mjexp_6/sys/vm/swap_pager.c#3 integrate
.. //depot/projects/mjexp_6/sys/vm/swap_pager.h#2 integrate
Differences ...
==== //depot/projects/mjexp_6/sys/amd64/amd64/pmap.c#2 (text+ko) ====
@@ -77,7 +77,7 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.516.2.8 2006/09/27 18:10:15 alc Exp $");
+__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.516.2.9 2007/02/28 09:27:45 kib Exp $");
/*
* Manages physical address maps.
@@ -1388,9 +1388,15 @@
while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
nkpt++;
+ if (kernel_vm_end - 1 >= kernel_map->max_offset) {
+ kernel_vm_end = kernel_map->max_offset;
+ break;
+ }
}
}
addr = roundup2(addr, PAGE_SIZE * NPTEPG);
+ if (addr - 1 >= kernel_map->max_offset)
+ addr = kernel_map->max_offset;
while (kernel_vm_end < addr) {
pde = pmap_pde(kernel_pmap, kernel_vm_end);
if (pde == NULL) {
@@ -1408,6 +1414,10 @@
}
if ((*pde & PG_V) != 0) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
+ if (kernel_vm_end - 1 >= kernel_map->max_offset) {
+ kernel_vm_end = kernel_map->max_offset;
+ break;
+ }
continue;
}
@@ -1427,6 +1437,10 @@
*pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
+ if (kernel_vm_end - 1 >= kernel_map->max_offset) {
+ kernel_vm_end = kernel_map->max_offset;
+ break;
+ }
}
}
==== //depot/projects/mjexp_6/sys/amd64/conf/SMP#2 (text+ko) ====
@@ -2,8 +2,10 @@
# SMP -- Generic kernel configuration file for FreeBSD/amd64 SMP
# Use this for multi-processor machines
#
-# $FreeBSD: src/sys/amd64/conf/SMP,v 1.1.6.1 2005/09/18 03:37:58 scottl Exp $
+# $FreeBSD: src/sys/amd64/conf/SMP,v 1.1.6.2 2007/03/03 23:19:46 cperciva Exp $
include GENERIC
+ident SMP-GENERIC
+
options SMP
==== //depot/projects/mjexp_6/sys/arm/arm/busdma_machdep.c#2 (text+ko) ====
@@ -29,10 +29,10 @@
*/
#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: src/sys/arm/arm/busdma_machdep.c,v 1.22.2.2 2006/06/23 17:42:33 cognet Exp $");
+__FBSDID("$FreeBSD: src/sys/arm/arm/busdma_machdep.c,v 1.22.2.3 2007/02/26 23:23:31 cognet Exp $");
/*
- * MacPPC bus dma support routines
+ * ARM bus dma support routines
*/
#define _ARM32_BUS_DMA_PRIVATE
@@ -48,6 +48,7 @@
#include <sys/uio.h>
#include <sys/ktr.h>
#include <sys/kernel.h>
+#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
@@ -56,7 +57,14 @@
#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#define MAX_BPAGES 64
+#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
+#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
+
+struct bounce_zone;
+
struct bus_dma_tag {
bus_dma_tag_t parent;
bus_size_t alignment;
@@ -81,8 +89,47 @@
*/
struct arm32_dma_range *ranges;
int _nranges;
+ struct bounce_zone *bounce_zone;
+};
+
+struct bounce_page {
+ vm_offset_t vaddr; /* kva of bounce buffer */
+ vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */
+ bus_addr_t busaddr; /* Physical address */
+ vm_offset_t datavaddr; /* kva of client data */
+ bus_size_t datacount; /* client data count */
+ STAILQ_ENTRY(bounce_page) links;
+};
+
+int busdma_swi_pending;
+
+struct bounce_zone {
+ STAILQ_ENTRY(bounce_zone) links;
+ STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
+ int total_bpages;
+ int free_bpages;
+ int reserved_bpages;
+ int active_bpages;
+ int total_bounced;
+ int total_deferred;
+ bus_size_t alignment;
+ bus_size_t boundary;
+ bus_addr_t lowaddr;
+ char zoneid[8];
+ char lowaddrid[20];
+ struct sysctl_ctx_list sysctl_tree;
+ struct sysctl_oid *sysctl_tree_top;
};
+static struct mtx bounce_lock;
+static int total_bpages;
+static int busdma_zonecount;
+static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
+
+SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
+SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
+ "Total bounce pages");
+
#define DMAMAP_LINEAR 0x1
#define DMAMAP_MBUF 0x2
#define DMAMAP_UIO 0x4
@@ -90,6 +137,9 @@
#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
#define DMAMAP_COHERENT 0x8
struct bus_dmamap {
+ struct bp_list bpages;
+ int pagesneeded;
+ int pagesreserved;
bus_dma_tag_t dmat;
int flags;
void *buffer;
@@ -97,8 +147,15 @@
void *allocbuffer;
TAILQ_ENTRY(bus_dmamap) freelist;
int len;
+ STAILQ_ENTRY(bus_dmamap) links;
+ bus_dmamap_callback_t *callback;
+ void *callback_arg;
+
};
+static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
+static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
+
static TAILQ_HEAD(,bus_dmamap) dmamap_freelist =
TAILQ_HEAD_INITIALIZER(dmamap_freelist);
@@ -109,6 +166,45 @@
MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
+static void init_bounce_pages(void *dummy);
+static int alloc_bounce_zone(bus_dma_tag_t dmat);
+static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
+static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
+ int commit);
+static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
+ vm_offset_t vaddr, bus_size_t size);
+static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
+
+/* Default tag, as most drivers provide no parent tag. */
+bus_dma_tag_t arm_root_dma_tag;
+
+/*
+ * Return true if a match is made.
+ *
+ * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
+ *
+ * If paddr is within the bounds of the dma tag then call the filter callback
+ * to check for a match, if there is no filter callback then assume a match.
+ */
+static int
+run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
+{
+ int retval;
+
+ retval = 0;
+
+ do {
+ if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
+ || ((paddr & (dmat->alignment - 1)) != 0))
+ && (dmat->filter == NULL
+ || (*dmat->filter)(dmat->filterarg, paddr) != 0))
+ retval = 1;
+
+ dmat = dmat->parent;
+ } while (retval == 0 && dmat != NULL);
+ return (retval);
+}
+
static void
arm_dmamap_freelist_init(void *dummy)
{
@@ -129,6 +225,19 @@
bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap,
int flags, vm_offset_t *lastaddrp, int *segp);
+static __inline int
+_bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr)
+{
+ int i;
+ for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) {
+ if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1])
+ || (lowaddr < phys_avail[i] &&
+ highaddr > phys_avail[i]))
+ return (1);
+ }
+ return (0);
+}
+
static __inline struct arm32_dma_range *
_bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
bus_addr_t curaddr)
@@ -195,11 +304,12 @@
TAILQ_REMOVE(&dmamap_freelist, map, freelist);
mtx_unlock(&busdma_mtx);
if (!map) {
- map = malloc(sizeof(*map), M_DEVBUF, M_NOWAIT);
+ map = malloc(sizeof(*map), M_DEVBUF, M_NOWAIT | M_ZERO);
if (map)
map->flags = DMAMAP_ALLOCATED;
} else
map->flags = 0;
+ STAILQ_INIT(&map->bpages);
return (map);
}
@@ -232,6 +342,8 @@
int error = 0;
/* Return a NULL tag on failure */
*dmat = NULL;
+ if (!parent)
+ parent = arm_root_dma_tag;
newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
if (newtag == NULL) {
@@ -273,6 +385,9 @@
else if (parent->boundary != 0)
newtag->boundary = min(parent->boundary,
newtag->boundary);
+ if ((newtag->filter != NULL) ||
+ ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
+ newtag->flags |= BUS_DMA_COULD_BOUNCE;
if (newtag->filter == NULL) {
/*
* Short circuit looking at our parent directly
@@ -285,8 +400,38 @@
if (newtag->parent != NULL)
atomic_add_int(&parent->ref_count, 1);
}
+ if (_bus_dma_can_bounce(newtag->lowaddr, newtag->highaddr)
+ || newtag->alignment > 1)
+ newtag->flags |= BUS_DMA_COULD_BOUNCE;
- *dmat = newtag;
+ if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ (flags & BUS_DMA_ALLOCNOW) != 0) {
+ struct bounce_zone *bz;
+
+ /* Must bounce */
+
+ if ((error = alloc_bounce_zone(newtag)) != 0) {
+ free(newtag, M_DEVBUF);
+ return (error);
+ }
+ bz = newtag->bounce_zone;
+
+ if (ptoa(bz->total_bpages) < maxsize) {
+ int pages;
+
+ pages = atop(maxsize) - bz->total_bpages;
+
+ /* Add pages to our bounce pool */
+ if (alloc_bounce_pages(newtag, pages) < pages)
+ error = ENOMEM;
+ }
+ /* Performed initial allocation */
+ newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
+ }
+ if (error != 0)
+ free(newtag, M_DEVBUF);
+ else
+ *dmat = newtag;
CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
__func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
@@ -327,6 +472,7 @@
return (0);
}
+#include <sys/kdb.h>
/*
* Allocate a handle for mapping from kva/uva/physical
* address space into bus device space.
@@ -335,9 +481,7 @@
bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
{
bus_dmamap_t newmap;
-#ifdef KTR
int error = 0;
-#endif
newmap = _busdma_alloc_dmamap();
if (newmap == NULL) {
@@ -346,8 +490,55 @@
}
*mapp = newmap;
newmap->dmat = dmat;
+ newmap->allocbuffer = NULL;
dmat->map_count++;
+ /*
+ * Bouncing might be required if the driver asks for an active
+ * exclusion region, a data alignment that is stricter than 1, and/or
+ * an active address boundary.
+ */
+ if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
+
+ /* Must bounce */
+ struct bounce_zone *bz;
+ int maxpages;
+
+ if (dmat->bounce_zone == NULL) {
+ if ((error = alloc_bounce_zone(dmat)) != 0) {
+ _busdma_free_dmamap(newmap);
+ *mapp = NULL;
+ return (error);
+ }
+ }
+ bz = dmat->bounce_zone;
+
+ /* Initialize the new map */
+ STAILQ_INIT(&((*mapp)->bpages));
+
+ /*
+ * Attempt to add pages to our pool on a per-instance
+ * basis up to a sane limit.
+ */
+ maxpages = MAX_BPAGES;
+ if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
+ || (dmat->map_count > 0 && bz->total_bpages < maxpages)) {
+ int pages;
+
+ pages = MAX(atop(dmat->maxsize), 1);
+ pages = MIN(maxpages - bz->total_bpages, pages);
+ pages = MAX(pages, 1);
+ if (alloc_bounce_pages(dmat, pages) < pages)
+ error = ENOMEM;
+
+ if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
+ if (error == 0)
+ dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
+ } else {
+ error = 0;
+ }
+ }
+ }
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
__func__, dmat, dmat->flags, error);
@@ -363,6 +554,11 @@
{
_busdma_free_dmamap(map);
+ if (STAILQ_FIRST(&map->bpages) != NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, EBUSY);
+ return (EBUSY);
+ }
dmat->map_count--;
CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
return (0);
@@ -398,7 +594,9 @@
*mapp = newmap;
newmap->dmat = dmat;
- if (dmat->maxsize <= PAGE_SIZE) {
+ if (dmat->maxsize <= PAGE_SIZE &&
+ (dmat->alignment < dmat->maxsize) &&
+ !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
*vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
} else {
/*
@@ -451,7 +649,9 @@
vaddr = map->origbuffer;
arm_unmap_nocache(map->allocbuffer, dmat->maxsize);
}
- if (dmat->maxsize <= PAGE_SIZE)
+ if (dmat->maxsize <= PAGE_SIZE &&
+ dmat->alignment < dmat->maxsize &&
+ !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
free(vaddr, M_DEVBUF);
else {
contigfree(vaddr, dmat->maxsize, M_DEVBUF);
@@ -461,6 +661,64 @@
CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
}
+static int
+_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+ bus_size_t buflen, int flags, int *nb)
+{
+ vm_offset_t vaddr;
+ vm_offset_t vendaddr;
+ bus_addr_t paddr;
+ int needbounce = *nb;
+
+ if ((map->pagesneeded == 0)) {
+ CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
+ "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
+ dmat->boundary, dmat->alignment);
+ CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
+ map, &nobounce_dmamap, map->pagesneeded);
+ /*
+ * Count the number of bounce pages
+ * needed in order to complete this transfer
+ */
+ vaddr = trunc_page((vm_offset_t)buf);
+ vendaddr = (vm_offset_t)buf + buflen;
+
+ while (vaddr < vendaddr) {
+ paddr = pmap_kextract(vaddr);
+ if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ run_filter(dmat, paddr) != 0) {
+ needbounce = 1;
+ map->pagesneeded++;
+ }
+ vaddr += PAGE_SIZE;
+ }
+ CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+ }
+
+ /* Reserve Necessary Bounce Pages */
+ if (map->pagesneeded != 0) {
+ mtx_lock(&bounce_lock);
+ if (flags & BUS_DMA_NOWAIT) {
+ if (reserve_bounce_pages(dmat, map, 0) != 0) {
+ mtx_unlock(&bounce_lock);
+ return (ENOMEM);
+ }
+ } else {
+ if (reserve_bounce_pages(dmat, map, 1) != 0) {
+ /* Queue us for resources */
+ STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
+ map, links);
+ mtx_unlock(&bounce_lock);
+ return (EINPROGRESS);
+ }
+ }
+ mtx_unlock(&bounce_lock);
+ }
+
+ *nb = needbounce;
+ return (0);
+}
+
/*
* Utility function to load a linear buffer. lastaddrp holds state
* between invocations (for multiple-buffer loads). segp contains
@@ -480,10 +738,17 @@
pd_entry_t *pde;
pt_entry_t pte;
pt_entry_t *ptep;
+ int needbounce = 0;
lastaddr = *lastaddrp;
bmask = ~(dmat->boundary - 1);
+ if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
+ error = _bus_dmamap_count_pages(dmat, map, buf, buflen, flags,
+ &needbounce);
+ if (error)
+ return (error);
+ }
CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, "
"alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment);
@@ -530,20 +795,6 @@
map->flags &= ~DMAMAP_COHERENT;
}
- if (dmat->ranges) {
- struct arm32_dma_range *dr;
-
- dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
- curaddr);
- if (dr == NULL)
- return (EINVAL);
- /*
- * In a valid DMA range. Translate the physical
- * memory address to an address in the DMA window.
- */
- curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
-
- }
/*
* Compute the segment size, and adjust counts.
*/
@@ -559,12 +810,30 @@
if (sgsize > (baddr - curaddr))
sgsize = (baddr - curaddr);
}
+ if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ map->pagesneeded != 0 && run_filter(dmat, curaddr))
+ curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
+ if (dmat->ranges) {
+ struct arm32_dma_range *dr;
+
+ dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges,
+ curaddr);
+ if (dr == NULL)
+ return (EINVAL);
+ /*
+ * In a valid DMA range. Translate the physical
+ * memory address to an address in the DMA window.
+ */
+ curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
+
+ }
+
/*
* Insert chunk into a segment, coalescing with
* the previous segment if possible.
*/
- if (seg >= 0 && curaddr == lastaddr &&
+ if (needbounce == 0 && seg >= 0 && curaddr == lastaddr &&
(segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
(dmat->boundary == 0 ||
(segs[seg].ds_addr & bmask) ==
@@ -614,6 +883,8 @@
KASSERT(dmat != NULL, ("dmatag is NULL"));
KASSERT(map != NULL, ("dmamap is NULL"));
+ map->callback = callback;
+ map->callback_arg = callback_arg;
map->flags &= ~DMAMAP_TYPE_MASK;
map->flags |= DMAMAP_LINEAR|DMAMAP_COHERENT;
map->buffer = buf;
@@ -621,6 +892,8 @@
error = bus_dmamap_load_buffer(dmat,
dm_segments, map, buf, buflen, kernel_pmap,
flags, &lastaddr, &nsegs);
+ if (error == EINPROGRESS)
+ return (error);
if (error)
(*callback)(callback_arg, NULL, 0, error);
else
@@ -796,26 +1069,94 @@
void
_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
{
+ struct bounce_page *bpage;
+
map->flags &= ~DMAMAP_TYPE_MASK;
+ while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
+ STAILQ_REMOVE_HEAD(&map->bpages, links);
+ free_bounce_page(dmat, bpage);
+ }
return;
}
+
static __inline void
bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op)
{
+ char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
if (op & BUS_DMASYNC_PREWRITE)
cpu_dcache_wb_range((vm_offset_t)buf, len);
- if (op & BUS_DMASYNC_PREREAD) {
+ if (op & BUS_DMASYNC_POSTREAD) {
+ if ((vm_offset_t)buf & arm_dcache_align_mask)
+ memcpy(_tmp_cl, (void *)((vm_offset_t)buf & ~
+ arm_dcache_align_mask),
+ (vm_offset_t)buf - ((vm_offset_t)buf &~
+ arm_dcache_align_mask));
+ if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
+ memcpy(_tmp_cl, (void *)((vm_offset_t)buf & ~
+ arm_dcache_align_mask),
+ (vm_offset_t)buf - ((vm_offset_t)buf &~
+ arm_dcache_align_mask));
+ if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
+ memcpy(_tmp_clend, (void *)(((vm_offset_t)buf + len) & ~
+ arm_dcache_align_mask),
+ (vm_offset_t)buf +len - (((vm_offset_t)buf + len) &~
+ arm_dcache_align_mask));
+ cpu_dcache_inv_range((vm_offset_t)buf, len);
if ((vm_offset_t)buf & arm_dcache_align_mask)
- cpu_dcache_wbinv_range((vm_offset_t)buf &
- ~arm_dcache_align_mask, arm_dcache_align);
+ memcpy((void *)((vm_offset_t)buf &
+ ~arm_dcache_align_mask),
+ _tmp_cl,
+ (vm_offset_t)buf - ((vm_offset_t)buf &~
+ arm_dcache_align_mask));
if (((vm_offset_t)buf + len) & arm_dcache_align_mask)
- cpu_dcache_wbinv_range(((vm_offset_t)buf + len) &
- ~arm_dcache_align_mask, arm_dcache_align);
+ memcpy((void *)(((vm_offset_t)buf + len) & ~
+ arm_dcache_align_mask), _tmp_clend,
+ (vm_offset_t)buf +len - (((vm_offset_t)buf + len) &~
+ arm_dcache_align_mask));
+ }
+}
+
+static void
+_bus_dmamap_sync_bp(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+ struct bounce_page *bpage;
+
+ STAILQ_FOREACH(bpage, &map->bpages, links) {
+ if (op & BUS_DMASYNC_PREWRITE) {
+ bcopy((void *)bpage->datavaddr,
+ (void *)(bpage->vaddr_nocache != 0 ?
+ bpage->vaddr_nocache : bpage->vaddr),
+ bpage->datacount);
+ if (bpage->vaddr_nocache == 0)
+ cpu_dcache_wb_range(bpage->vaddr,
+ bpage->datacount);
+ }
+ if (op & BUS_DMASYNC_POSTREAD) {
+ if (bpage->vaddr_nocache == 0)
+ cpu_dcache_inv_range(bpage->vaddr,
+ bpage->datacount);
+ bcopy((void *)(bpage->vaddr_nocache != 0 ?
+ bpage->vaddr_nocache : bpage->vaddr),
+ (void *)bpage->datavaddr, bpage->datacount);
+ }
+ }
+}
+
+static __inline int
+_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len)
+{
+ struct bounce_page *bpage;
+
+ STAILQ_FOREACH(bpage, &map->bpages, links) {
+ if ((vm_offset_t)buf >= bpage->datavaddr &&
+ (vm_offset_t)buf + len < bpage->datavaddr +
+ bpage->datacount)
+ return (1);
}
- if (op & BUS_DMASYNC_POSTREAD)
- cpu_dcache_inv_range((vm_offset_t)buf, len);
+ return (0);
+
}
void
@@ -828,6 +1169,8 @@
if (op == BUS_DMASYNC_POSTWRITE)
return;
+ if (STAILQ_FIRST(&map->bpages))
+ _bus_dmamap_sync_bp(dmat, map, op);
if (map->flags & DMAMAP_COHERENT)
return;
if ((op && BUS_DMASYNC_POSTREAD) && (map->len > PAGE_SIZE)) {
@@ -837,12 +1180,14 @@
CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
switch(map->flags & DMAMAP_TYPE_MASK) {
case DMAMAP_LINEAR:
- bus_dmamap_sync_buf(map->buffer, map->len, op);
+ if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len)))
+ bus_dmamap_sync_buf(map->buffer, map->len, op);
break;
case DMAMAP_MBUF:
m = map->buffer;
while (m) {
- if (m->m_len > 0)
+ if (m->m_len > 0 &&
+ !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len)))
bus_dmamap_sync_buf(m->m_data, m->m_len, op);
m = m->m_next;
}
@@ -855,8 +1200,10 @@
bus_size_t minlen = resid < iov[i].iov_len ? resid :
iov[i].iov_len;
if (minlen > 0) {
- bus_dmamap_sync_buf(iov[i].iov_base, minlen,
- op);
+ if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base,
+ minlen))
+ bus_dmamap_sync_buf(iov[i].iov_base,
+ minlen, op);
resid -= minlen;
}
}
@@ -866,3 +1213,247 @@
}
cpu_drain_writebuf();
}
+
+static void
+init_bounce_pages(void *dummy __unused)
+{
+
+ total_bpages = 0;
+ STAILQ_INIT(&bounce_zone_list);
+ STAILQ_INIT(&bounce_map_waitinglist);
+ STAILQ_INIT(&bounce_map_callbacklist);
+ mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
+}
+SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
+
+static struct sysctl_ctx_list *
+busdma_sysctl_tree(struct bounce_zone *bz)
+{
+ return (&bz->sysctl_tree);
+}
+
+static struct sysctl_oid *
+busdma_sysctl_tree_top(struct bounce_zone *bz)
+{
+ return (bz->sysctl_tree_top);
+}
+
+static int
+alloc_bounce_zone(bus_dma_tag_t dmat)
+{
+ struct bounce_zone *bz;
+
+ /* Check to see if we already have a suitable zone */
+ STAILQ_FOREACH(bz, &bounce_zone_list, links) {
+ if ((dmat->alignment <= bz->alignment)
+ && (dmat->boundary <= bz->boundary)
+ && (dmat->lowaddr >= bz->lowaddr)) {
+ dmat->bounce_zone = bz;
+ return (0);
+ }
+ }
+
+ if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
+ M_NOWAIT | M_ZERO)) == NULL)
+ return (ENOMEM);
+
+ STAILQ_INIT(&bz->bounce_page_list);
+ bz->free_bpages = 0;
+ bz->reserved_bpages = 0;
+ bz->active_bpages = 0;
+ bz->lowaddr = dmat->lowaddr;
+ bz->alignment = dmat->alignment;
+ bz->boundary = dmat->boundary;
+ snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
+ busdma_zonecount++;
+ snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
+ STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
+ dmat->bounce_zone = bz;
+
+ sysctl_ctx_init(&bz->sysctl_tree);
+ bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
+ SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
+ CTLFLAG_RD, 0, "");
+ if (bz->sysctl_tree_top == NULL) {
+ sysctl_ctx_free(&bz->sysctl_tree);
+ return (0); /* XXX error code? */
+ }
+
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
+ "Total bounce pages");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
+ "Free bounce pages");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
+ "Reserved bounce pages");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
+ "Active bounce pages");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
+ "Total bounce requests");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
+ "Total bounce requests that were deferred");
+ SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
+ SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
+ SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
+ "boundary", CTLFLAG_RD, &bz->boundary, 0, "");
+
+ return (0);
+}
+
+static int
+alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
+{
+ struct bounce_zone *bz;
+ int count;
+
+ bz = dmat->bounce_zone;
+ count = 0;
+ while (numpages > 0) {
+ struct bounce_page *bpage;
+
+ bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+
+ if (bpage == NULL)
+ break;
+ bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
>>> TRUNCATED FOR MAIL (1000 lines) <<<
More information about the p4-projects
mailing list