svn commit: r199670 - head/sys/dev/bge

[John Baldwin]

On Sunday 22 November 2009 6:48:18 pm Scott Long wrote:
> By definition, PCIe can't transfer across 4GB boundaries.  It's not a  
> bug specific to Broadcom.  If you're loading dynamic buffers (i.e.  
> mbufs), setting an appropriate boundary value in the tag will allow  
> busdma to take care of this.  If you're allocating static buffers,  
> busdma won't honor this.  But what you've done here is best anyways;  
> control buffers that are going to be frequently transferred are best  
> kept in the lower 4GB of the address space.  It simplifies PCIe  
> handling, and it's significantly faster on PCI/PCI-X.  So I'd suggest  
> making this the rule rather than the exception in the driver.

Should we enforce an implicit 4GB boundary in bus_dma then?  Perhaps Host-PCI 
bridge drivers should create a tag with a 4GB boundary that devices inherit 
via bus_get_dma_tag().  For i386/PAE we might should always enforce a 4GB 
boundary as well?

> Scott
> 
> On Nov 22, 2009, at 1:50 PM, Pyun YongHyeon wrote:
> 
> > Author: yongari
> > Date: Sun Nov 22 20:50:27 2009
> > New Revision: 199670
> > URL: http://svn.freebsd.org/changeset/base/199670
> >
> > Log:
> >  Fix two long standing bugs on bge(4). Most pre BCM5755 controllers
> >  have a DMA bug when buffer address crosses a multiple of the 4GB
> >  boundary(e.g. 4GB, 8GB, 12GB etc). Limit DMA address to be within
> >  4GB address for these controllers. The second DMA bug limits DMA
> >  address to be within 40bit address space. This bug applies to
> >  BCM5714 and BCM5715 and 5708(bce(4) controller). This is not
> >  actually a MAC controller bug but an issue with the embedded PCIe
> >  to PCI-X bridge in the device. So for BCM5714/BCM5715 controllers
> >  also limit the DMA address to be within 40bit address space.
> >  Special thanks to davidch@ who gave me detailed errata information.
> >  I think this change will fix long standing bge(4) instability
> >  issues on systems with more than 4GB memory.
> >
> >  Reviewed by:	davidch
> >
> > Modified:
> >  head/sys/dev/bge/if_bge.c
> >  head/sys/dev/bge/if_bgereg.h
> >
> > Modified: head/sys/dev/bge/if_bge.c
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > ======================================================================
> > --- head/sys/dev/bge/if_bge.c	Sun Nov 22 20:45:15 2009	(r199669)
> > +++ head/sys/dev/bge/if_bge.c	Sun Nov 22 20:50:27 2009	(r199670)
> > @@ -2104,15 +2104,21 @@ bge_dma_alloc(device_t dev)
> > {
> > 	struct bge_dmamap_arg ctx;
> > 	struct bge_softc *sc;
> > +	bus_addr_t lowaddr;
> > 	int i, error;
> >
> > 	sc = device_get_softc(dev);
> >
> > +	lowaddr = BUS_SPACE_MAXADDR;
> > +	if ((sc->bge_flags & BGE_FLAG_40BIT_BUG) != 0)
> > +		lowaddr = BGE_DMA_MAXADDR;
> > +	if ((sc->bge_flags & BGE_FLAG_4G_BNDRY_BUG) != 0)
> > +		lowaddr = BUS_SPACE_MAXADDR_32BIT;
> > 	/*
> > 	 * Allocate the parent bus DMA tag appropriate for PCI.
> > 	 */
> > 	error = bus_dma_tag_create(bus_get_dma_tag(sc->bge_dev),
> > -	    1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,	NULL,
> > +	    1, 0, lowaddr, BUS_SPACE_MAXADDR, NULL,
> > 	    NULL, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
> > 	    0, NULL, NULL, &sc->bge_cdata.bge_parent_tag);
> >
> > @@ -2566,6 +2572,16 @@ bge_attach(device_t dev)
> > 			sc->bge_flags |= BGE_FLAG_BER_BUG;
> > 	}
> >
> > +	/*
> > +	 * All controllers that are not 5755 or higher have 4GB
> > +	 * boundary DMA bug.
> > +	 * Whenever an address crosses a multiple of the 4GB boundary
> > +	 * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
> > +	 * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
> > +	 * state machine will lockup and cause the device to hang.
> > +	 */
> > +	if (BGE_IS_5755_PLUS(sc) == 0)
> > +		sc->bge_flags |= BGE_FLAG_4G_BNDRY_BUG;
> >
> > 	/*
> > 	 * We could possibly check for BCOM_DEVICEID_BCM5788 in bge_probe()
> > @@ -2729,6 +2745,13 @@ bge_attach(device_t dev)
> > #ifdef DEVICE_POLLING
> > 	ifp->if_capabilities |= IFCAP_POLLING;
> > #endif
> > +	/*
> > +	 * The 40bit DMA bug applies to the 5714/5715 controllers and is
> > +	 * not actually a MAC controller bug but an issue with the embedded
> > +	 * PCIe to PCI-X bridge in the device. Use 40bit DMA workaround.
> > +	 */
> > +	if (BGE_IS_5714_FAMILY(sc) && (sc->bge_flags & BGE_FLAG_PCIX))
> > +		sc->bge_flags |= BGE_FLAG_40BIT_BUG;
> >
> > 	/*
> > 	 * 5700 B0 chips do not support checksumming correctly due
> >
> > Modified: head/sys/dev/bge/if_bgereg.h
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > = 
> > ======================================================================
> > --- head/sys/dev/bge/if_bgereg.h	Sun Nov 22 20:45:15 2009	(r199669)
> > +++ head/sys/dev/bge/if_bgereg.h	Sun Nov 22 20:50:27 2009	(r199670)
> > @@ -2484,6 +2484,13 @@ struct bge_gib {
> > #define	BGE_NSEG_JUMBO	4
> > #define	BGE_NSEG_NEW 32
> >
> > +/* Maximum DMA address for controllers that have 40bit DMA address  
> > bug. */
> > +#if (BUS_SPACE_MAXADDR < 0xFFFFFFFFFF)
> > +#define	BGE_DMA_MAXADDR		BUS_SPACE_MAXADDR
> > +#else
> > +#define	BGE_DMA_MAXADDR		0xFFFFFFFFFF
> > +#endif
> > +
> > /*
> >  * Ring structures. Most of these reside in host memory and we tell
> >  * the NIC where they are via the ring control blocks. The exceptions
> > @@ -2600,6 +2607,8 @@ struct bge_softc {
> > #define	BGE_FLAG_5714_FAMILY	0x00004000
> > #define	BGE_FLAG_575X_PLUS	0x00008000
> > #define	BGE_FLAG_5755_PLUS	0x00010000
> > +#define	BGE_FLAG_40BIT_BUG	0x00020000
> > +#define	BGE_FLAG_4G_BNDRY_BUG	0x00040000
> > #define	BGE_FLAG_RX_ALIGNBUG	0x00100000
> > #define	BGE_FLAG_NO_3LED	0x00200000
> > #define	BGE_FLAG_ADC_BUG	0x00400000
> 
> 



-- 
John Baldwin