svn commit: r322145 - in stable/10/sys/dev/mlx5: . mlx5_core

[Hans Petter Selasky]

Author: hselasky
Date: Mon Aug  7 12:38:04 2017
New Revision: 322145
URL: https://svnweb.freebsd.org/changeset/base/322145

Log:
  MFC r312880:
  Wait for all VFs pages to be reclaimed before closing EQ pages.
  
  Sponsored by:	Mellanox Technologies

Modified:
  stable/10/sys/dev/mlx5/driver.h
  stable/10/sys/dev/mlx5/mlx5_core/mlx5_main.c
  stable/10/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c
Directory Properties:
  stable/10/   (props changed)

Modified: stable/10/sys/dev/mlx5/driver.h
==============================================================================
--- stable/10/sys/dev/mlx5/driver.h	Mon Aug  7 12:36:48 2017	(r322144)
+++ stable/10/sys/dev/mlx5/driver.h	Mon Aug  7 12:38:04 2017	(r322145)
@@ -43,6 +43,7 @@
 #include <dev/mlx5/doorbell.h>
 
 #define MLX5_QCOUNTER_SETS_NETDEV 64
+#define MLX5_MAX_NUMBER_OF_VFS 128
 
 enum {
 	MLX5_BOARD_ID_LEN = 64,
@@ -521,7 +522,7 @@ struct mlx5_priv {
 	s64			fw_pages;
 	atomic_t		reg_pages;
 	struct list_head	free_list;
-
+	s64			pages_per_func[MLX5_MAX_NUMBER_OF_VFS];
 	struct mlx5_core_health health;
 
 	struct mlx5_srq_table	srq_table;
@@ -850,6 +851,7 @@ void mlx5_core_req_pages_handler(struct mlx5_core_dev 
 				 s32 npages);
 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot);
 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev);
+s64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev);
 void mlx5_register_debugfs(void);
 void mlx5_unregister_debugfs(void);
 int mlx5_eq_init(struct mlx5_core_dev *dev);

Modified: stable/10/sys/dev/mlx5/mlx5_core/mlx5_main.c
==============================================================================
--- stable/10/sys/dev/mlx5/mlx5_core/mlx5_main.c	Mon Aug  7 12:36:48 2017	(r322144)
+++ stable/10/sys/dev/mlx5/mlx5_core/mlx5_main.c	Mon Aug  7 12:38:04 2017	(r322145)
@@ -851,6 +851,7 @@ static void mlx5_dev_cleanup(struct mlx5_core_dev *dev
 	mlx5_cleanup_qp_table(dev);
 	mlx5_cleanup_cq_table(dev);
 	unmap_bf_area(dev);
+	mlx5_wait_for_reclaim_vfs_pages(dev);
 	free_comp_eqs(dev);
 	mlx5_stop_eqs(dev);
 	mlx5_free_uuars(dev, &priv->uuari);

Modified: stable/10/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c
==============================================================================
--- stable/10/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c	Mon Aug  7 12:36:48 2017	(r322144)
+++ stable/10/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c	Mon Aug  7 12:38:04 2017	(r322145)
@@ -27,6 +27,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/delay.h>
 #include <dev/mlx5/driver.h>
 #include "mlx5_core.h"
 
@@ -282,6 +283,7 @@ retry:
 		goto out_alloc;
 	}
 	dev->priv.fw_pages += npages;
+	dev->priv.pages_per_func[func_id] += npages;
 
 	if (out.hdr.status) {
 		err = mlx5_cmd_status_to_err(&out.hdr);
@@ -355,7 +357,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u3
 		*nclaimed = num_claimed;
 
 	dev->priv.fw_pages -= num_claimed;
-
+	dev->priv.pages_per_func[func_id] -= num_claimed;
 	for (i = 0; i < num_claimed; i++) {
 		addr = be64_to_cpu(out->pas[i]);
 		free_4k(dev, addr);
@@ -422,6 +424,31 @@ int mlx5_satisfy_startup_pages(struct mlx5_core_dev *d
 enum {
 	MLX5_BLKS_FOR_RECLAIM_PAGES = 12
 };
+
+s64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev)
+{
+	int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
+	s64 prevpages = 0;
+	s64 npages = 0;
+
+	while (!time_after(jiffies, end)) {
+		/* exclude own function, VFs only */
+		npages = dev->priv.fw_pages - dev->priv.pages_per_func[0];
+		if (!npages)
+			break;
+
+		if (npages != prevpages)
+			end = end + msecs_to_jiffies(100);
+
+		prevpages = npages;
+		msleep(1);
+	}
+
+	if (npages)
+		mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n");
+
+	return -npages;
+}
 
 static int optimal_reclaimed_pages(void)
 {