git: de2437f19950 - main - mlx5en: Configure relaxed PCI read and write ordering for ethernet.

[Hans Petter Selasky]

The branch main has been updated by hselasky:

URL: https://cgit.FreeBSD.org/src/commit/?id=de2437f19950f6758159abbde93200468d1327fa

commit de2437f19950f6758159abbde93200468d1327fa
Author:     Hans Petter Selasky <hselasky at FreeBSD.org>
AuthorDate: 2021-06-16 13:01:57 +0000
Commit:     Hans Petter Selasky <hselasky at FreeBSD.org>
CommitDate: 2021-07-12 12:22:34 +0000

    mlx5en: Configure relaxed PCI read and write ordering for ethernet.
    
    This may improve performance in some configurations.
    
    MFC after:      1 week
    Reviewed by:    kib
    Sponsored by:   Mellanox Technologies // NVIDIA Networking
---
 sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 14 +++++++++++++-
 sys/dev/mlx5/mlx5_ifc.h             | 12 ++++++++----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 97438de3bc2a..02417f5f931c 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -3707,6 +3707,18 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
 	return (0);
 }
 
+static void
+mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
+{
+	bool ro_pci_enable =
+	    pci_get_relaxed_ordering_enabled(mdev->pdev->dev.bsddev);
+	bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
+	bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
+
+	MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
+	MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
+}
+
 static int
 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
 		  struct mlx5_core_mr *mkey)
@@ -3729,7 +3741,7 @@ mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
 	MLX5_SET(mkc, mkc, umr_en, 1);	/* used by HW TLS */
 	MLX5_SET(mkc, mkc, lw, 1);
 	MLX5_SET(mkc, mkc, lr, 1);
-
+	mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 	MLX5_SET(mkc, mkc, pd, pdn);
 	MLX5_SET(mkc, mkc, length64, 1);
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
index ce8074fba3f1..d44927236f74 100644
--- a/sys/dev/mlx5/mlx5_ifc.h
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -1069,12 +1069,14 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 
 	u8         reserved_4[0x8];
 	u8         log_max_cq_sz[0x8];
-	u8         reserved_5[0xb];
+	u8         relaxed_ordering_write_umr[0x1];
+	u8         relaxed_ordering_read_umr[0x1];
+	u8         reserved_5[0x9];
 	u8         log_max_cq[0x5];
 
 	u8         log_max_eq_sz[0x8];
-	u8         relaxed_ordering_write[1];
-	u8         reserved_6[0x1];
+	u8         relaxed_ordering_write[0x1];
+	u8         relaxed_ordering_read[0x1];
 	u8         log_max_mkey[0x6];
 	u8         reserved_7[0xb];
 	u8         fast_teardown[0x1];
@@ -2685,7 +2687,9 @@ struct mlx5_ifc_mkc_bits {
 
 	u8         translations_octword_size[0x20];
 
-	u8         reserved_7[0x1b];
+	u8         reserved_at_1c0[0x19];
+	u8         relaxed_ordering_read[0x1];
+	u8         reserved_at_1d9[0x1];
 	u8         log_page_size[0x5];
 
 	u8         reserved_8[0x20];