git: ddcb65c52640 - stable/13 - LinuxKPI: implement dma_sync_single_for_*, apply to (un)map single/sg

The branch stable/13 has been updated by bz:

URL: https://cgit.FreeBSD.org/src/commit/?id=ddcb65c5264015eb57e4ab8a742a49a92d877653

commit ddcb65c5264015eb57e4ab8a742a49a92d877653
Author:     Bjoern A. Zeeb <bz@FreeBSD.org>
AuthorDate: 2021-10-01 10:51:50 +0000
Commit:     Bjoern A. Zeeb <bz@FreeBSD.org>
CommitDate: 2022-02-20 16:26:01 +0000

    LinuxKPI: implement dma_sync_single_for_*, apply to (un)map single/sg
    
    Implement dma_sync_single_for_{cpu,device} translating the Linux
    DMA_ flags to BUS_DMASYNC_ combinations.  Make map_single/unmap_single*
    functions call the respective sync function.   Apply the same logic to
    the scatter-gather list map/unmap functions.
    
    Sponsored by:   The FreeBSD Foundation
    Reviewed by:    hselasky
    Differential Revision: https://reviews.freebsd.org/D32255
    
    (cherry picked from commit 95edb10b47fc1a919cd1687aaf16be9e14456c89)
---
 .../linuxkpi/common/include/linux/dma-mapping.h    | 75 +++++++++++++++++++---
 sys/compat/linuxkpi/common/src/linux_pci.c         | 60 ++++++++++++++++-
 2 files changed, 124 insertions(+), 11 deletions(-)

diff --git a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h
index f80258892574..fcb50f4b5217 100644
--- a/sys/compat/linuxkpi/common/include/linux/dma-mapping.h
+++ b/sys/compat/linuxkpi/common/include/linux/dma-mapping.h
@@ -103,6 +103,7 @@ int linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
 void linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
     int nents __unused, enum dma_data_direction dir __unused,
     unsigned long attrs __unused);
+void linuxkpi_dma_sync(struct device *, dma_addr_t, size_t, bus_dmasync_op_t);
 
 static inline int
 dma_supported(struct device *dev, u64 dma_mask)
@@ -167,12 +168,6 @@ dma_free_coherent(struct device *dev, size_t size, void *cpu_addr,
 	kmem_free((vm_offset_t)cpu_addr, size);
 }
 
-#define	dma_map_single_attrs(dev, ptr, size, dir, attrs)	\
-	linux_dma_map_phys(dev, vtophys(ptr), size)
-
-#define	dma_unmap_single_attrs(dev, dma_addr, size, dir, attrs)	\
-	linux_dma_unmap(dev, dma_addr, size)
-
 static inline dma_addr_t
 dma_map_page_attrs(struct device *dev, struct page *page, size_t offset,
     size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -205,9 +200,28 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
 }
 
 static inline void
-dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
+dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma, size_t size,
     enum dma_data_direction direction)
 {
+	bus_dmasync_op_t op;
+
+	switch (direction) {
+	case DMA_BIDIRECTIONAL:
+		op = BUS_DMASYNC_POSTREAD;
+		linuxkpi_dma_sync(dev, dma, size, op);
+		op = BUS_DMASYNC_PREREAD;
+		break;
+	case DMA_TO_DEVICE:
+		op = BUS_DMASYNC_POSTWRITE;
+		break;
+	case DMA_FROM_DEVICE:
+		op = BUS_DMASYNC_POSTREAD;
+		break;
+	default:
+		return;
+	}
+
+	linuxkpi_dma_sync(dev, dma, size, op);
 }
 
 static inline void
@@ -218,9 +232,26 @@ dma_sync_single(struct device *dev, dma_addr_t addr, size_t size,
 }
 
 static inline void
-dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
+dma_sync_single_for_device(struct device *dev, dma_addr_t dma,
     size_t size, enum dma_data_direction direction)
 {
+	bus_dmasync_op_t op;
+
+	switch (direction) {
+	case DMA_BIDIRECTIONAL:
+		op = BUS_DMASYNC_PREWRITE;
+		break;
+	case DMA_TO_DEVICE:
+		op = BUS_DMASYNC_PREREAD;
+		break;
+	case DMA_FROM_DEVICE:
+		op = BUS_DMASYNC_PREWRITE;
+		break;
+	default:
+		return;
+	}
+
+	linuxkpi_dma_sync(dev, dma, size, op);
 }
 
 static inline void
@@ -260,6 +291,34 @@ static inline unsigned int dma_set_max_seg_size(struct device *dev,
 	return (0);
 }
 
+static inline dma_addr_t
+_dma_map_single_attrs(struct device *dev, void *ptr, size_t size,
+    enum dma_data_direction direction, unsigned long attrs __unused)
+{
+	dma_addr_t dma;
+
+	dma = linux_dma_map_phys(dev, vtophys(ptr), size);
+	if (!dma_mapping_error(dev, dma))
+		dma_sync_single_for_device(dev, dma, size, direction);
+
+	return (dma);
+}
+
+static inline void
+_dma_unmap_single_attrs(struct device *dev, dma_addr_t dma, size_t size,
+    enum dma_data_direction direction, unsigned long attrs __unused)
+{
+
+	dma_sync_single_for_cpu(dev, dma, size, direction);
+	linux_dma_unmap(dev, dma, size);
+}
+
+#define	dma_map_single_attrs(dev, ptr, size, dir, attrs)	\
+	_dma_map_single_attrs(dev, ptr, size, dir, 0)
+
+#define	dma_unmap_single_attrs(dev, dma_addr, size, dir, attrs)	\
+	_dma_unmap_single_attrs(dev, dma_addr, size, dir, 0)
+
 #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c
index 1c562b74c95a..af79ef1d8654 100644
--- a/sys/compat/linuxkpi/common/src/linux_pci.c
+++ b/sys/compat/linuxkpi/common/src/linux_pci.c
@@ -1,7 +1,7 @@
 /*-
  * Copyright (c) 2015-2016 Mellanox Technologies, Ltd.
  * All rights reserved.
- * Copyright (c) 2020-2021 The FreeBSD Foundation
+ * Copyright (c) 2020-2022 The FreeBSD Foundation
  *
  * Portions of this software were developed by Björn Zeeb
  * under sponsorship from the FreeBSD Foundation.
@@ -954,9 +954,32 @@ linux_dma_alloc_coherent(struct device *dev, size_t size,
 	return (mem);
 }
 
+void
+linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size,
+    bus_dmasync_op_t op)
+{
+	struct linux_dma_priv *priv;
+	struct linux_dma_obj *obj;
+
+	priv = dev->dma_priv;
+
+	if (pctrie_is_empty(&priv->ptree))
+		return;
+
+	DMA_PRIV_LOCK(priv);
+	obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
+	if (obj == NULL) {
+		DMA_PRIV_UNLOCK(priv);
+		return;
+	}
+
+	bus_dmamap_sync(obj->dmat, obj->dmamap, op);
+	DMA_PRIV_UNLOCK(priv);
+}
+
 int
 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
-    enum dma_data_direction dir __unused, unsigned long attrs __unused)
+    enum dma_data_direction direction, unsigned long attrs __unused)
 {
 	struct linux_dma_priv *priv;
 	struct scatterlist *sg;
@@ -989,6 +1012,21 @@ linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
 
 		sg_dma_address(sg) = seg.ds_addr;
 	}
+
+	switch (direction) {
+	case DMA_BIDIRECTIONAL:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE);
+		break;
+	case DMA_TO_DEVICE:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD);
+		break;
+	case DMA_FROM_DEVICE:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE);
+		break;
+	default:
+		break;
+	}
+
 	DMA_PRIV_UNLOCK(priv);
 
 	return (nents);
@@ -996,7 +1034,7 @@ linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
 
 void
 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
-    int nents __unused, enum dma_data_direction dir __unused,
+    int nents __unused, enum dma_data_direction direction,
     unsigned long attrs __unused)
 {
 	struct linux_dma_priv *priv;
@@ -1004,6 +1042,22 @@ linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
 	priv = dev->dma_priv;
 
 	DMA_PRIV_LOCK(priv);
+
+	switch (direction) {
+	case DMA_BIDIRECTIONAL:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD);
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD);
+		break;
+	case DMA_TO_DEVICE:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE);
+		break;
+	case DMA_FROM_DEVICE:
+		bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD);
+		break;
+	default:
+		break;
+	}
+
 	bus_dmamap_unload(priv->dmat, sgl->dma_map);
 	bus_dmamap_destroy(priv->dmat, sgl->dma_map);
 	DMA_PRIV_UNLOCK(priv);