svn commit: r347374 - stable/11/sys/amd64/vmm/io

[Konstantin Belousov]

On Wed, May 08, 2019 at 11:24:47PM +0000, John Baldwin wrote:
> Author: jhb
> Date: Wed May  8 23:24:47 2019
> New Revision: 347374
> URL: https://svnweb.freebsd.org/changeset/base/347374
> 
> Log:
>   MFC 333174: Use PCI power-mgmt to reset a device if FLR fails.
>   
>   A large number of devices don't support PCIe FLR, in particular
>   graphics adapters. Use PCI power management to perform the
>   reset if FLR fails or isn't available, by cycling the device
>   through the D3 state.
>   
>   This has been tested by a number of users with Nvidia and AMD GPUs.
Now stable/11 ppt_pci_reset() can be made identical to the HEAD version,
I merged all reset changes some time ago.

> 
> Modified:
>   stable/11/sys/amd64/vmm/io/ppt.c
> Directory Properties:
>   stable/11/   (props changed)
> 
> Modified: stable/11/sys/amd64/vmm/io/ppt.c
> ==============================================================================
> --- stable/11/sys/amd64/vmm/io/ppt.c	Wed May  8 23:13:27 2019	(r347373)
> +++ stable/11/sys/amd64/vmm/io/ppt.c	Wed May  8 23:24:47 2019	(r347374)
> @@ -351,6 +351,30 @@ ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
>  	return (FALSE);
>  }
>  
> +static void
> +ppt_pci_reset(device_t dev)
> +{
> +	int ps;
> +
> +	if (pcie_flr(dev,
> +		     max(pcie_get_max_completion_timeout(dev) / 1000, 10),
> +		     true))
> +		return;
> +
> +	/*
> +	 * If FLR fails, attempt a power-management reset by cycling
> +	 * the device in/out of D3 state.
> +	 * PCI spec says we can only go into D3 state from D0 state.
> +	 * Transition from D[12] into D0 before going to D3 state.
> +	 */
> +	ps = pci_get_powerstate(dev);
> +	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
> +		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
> +	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
> +		pci_set_powerstate(dev, PCI_POWERSTATE_D3);
> +	pci_set_powerstate(dev, ps);
> +}
> +
>  int
>  ppt_assign_device(struct vm *vm, int bus, int slot, int func)
>  {
> @@ -366,9 +390,7 @@ ppt_assign_device(struct vm *vm, int bus, int slot, in
>  			return (EBUSY);
>  
>  		pci_save_state(ppt->dev);
> -		pcie_flr(ppt->dev,
> -		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
> -		    true);
> +		ppt_pci_reset(ppt->dev);
>  		pci_restore_state(ppt->dev);
>  		ppt->vm = vm;
>  		iommu_add_device(vm_iommu_domain(vm), pci_get_rid(ppt->dev));
> @@ -391,9 +413,7 @@ ppt_unassign_device(struct vm *vm, int bus, int slot, 
>  			return (EBUSY);
>  
>  		pci_save_state(ppt->dev);
> -		pcie_flr(ppt->dev,
> -		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
> -		    true);
> +		ppt_pci_reset(ppt->dev);
>  		pci_restore_state(ppt->dev);
>  		ppt_unmap_mmio(vm, ppt);
>  		ppt_teardown_msi(ppt);