diff mbox series

[v3,3/3] vfio: Avoid disabling and enabling vectors repeatedly in VFIO migration

Message ID 20210223022225.50-4-lushenming@huawei.com
State New
Headers show
Series vfio: Some fixes and optimizations for VFIO migration | expand

Commit Message

Shenming Lu Feb. 23, 2021, 2:22 a.m. UTC
In VFIO migration resume phase and some guest startups, there are
already unmasked vectors in the vector table when calling
vfio_msix_enable(). So in order to avoid inefficiently disabling
and enabling vectors repeatedly, let's allocate all needed vectors
first and then enable these unmasked vectors one by one without
disabling.

Signed-off-by: Shenming Lu <lushenming@huawei.com>
---
 hw/pci/msix.c         |  2 +-
 hw/vfio/pci.c         | 20 +++++++++++++++++---
 include/hw/pci/msix.h |  1 +
 3 files changed, 19 insertions(+), 4 deletions(-)

Comments

Alex Williamson March 2, 2021, 9:26 p.m. UTC | #1
MST/Marcel,

Do you have an Ack or objection to exporting msix_masked() as below?
Thanks,

Alex

On Tue, 23 Feb 2021 10:22:25 +0800
Shenming Lu <lushenming@huawei.com> wrote:

> In VFIO migration resume phase and some guest startups, there are
> already unmasked vectors in the vector table when calling
> vfio_msix_enable(). So in order to avoid inefficiently disabling
> and enabling vectors repeatedly, let's allocate all needed vectors
> first and then enable these unmasked vectors one by one without
> disabling.
> 
> Signed-off-by: Shenming Lu <lushenming@huawei.com>
> ---
>  hw/pci/msix.c         |  2 +-
>  hw/vfio/pci.c         | 20 +++++++++++++++++---
>  include/hw/pci/msix.h |  1 +
>  3 files changed, 19 insertions(+), 4 deletions(-)
> 
> diff --git a/hw/pci/msix.c b/hw/pci/msix.c
> index ae9331cd0b..e057958fcd 100644
> --- a/hw/pci/msix.c
> +++ b/hw/pci/msix.c
> @@ -131,7 +131,7 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
>      }
>  }
>  
> -static bool msix_masked(PCIDevice *dev)
> +bool msix_masked(PCIDevice *dev)
>  {
>      return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
>  }
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index f74be78209..088fd41926 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -569,6 +569,9 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
>  
>  static void vfio_msix_enable(VFIOPCIDevice *vdev)
>  {
> +    PCIDevice *pdev = &vdev->pdev;
> +    unsigned int nr, max_vec = 0;
> +
>      vfio_disable_interrupts(vdev);
>  
>      vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
> @@ -587,11 +590,22 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
>       * triggering to userspace, then immediately release the vector, leaving
>       * the physical device with no vectors enabled, but MSI-X enabled, just
>       * like the guest view.
> +     * If there are already unmasked vectors (in migration resume phase and
> +     * some guest startups) which will be enabled soon, we can allocate all
> +     * of them here to avoid inefficiently disabling and enabling vectors
> +     * repeatedly later.
>       */
> -    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
> -    vfio_msix_vector_release(&vdev->pdev, 0);
> +    if (!msix_masked(pdev)) {
> +        for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
> +            if (!msix_is_masked(pdev, nr)) {
> +                max_vec = nr;
> +            }
> +        }
> +    }
> +    vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
> +    vfio_msix_vector_release(pdev, max_vec);
>  
> -    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
> +    if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
>                                    vfio_msix_vector_release, NULL)) {
>          error_report("vfio: msix_set_vector_notifiers failed");
>      }
> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
> index 4c4a60c739..b3cd88e262 100644
> --- a/include/hw/pci/msix.h
> +++ b/include/hw/pci/msix.h
> @@ -28,6 +28,7 @@ void msix_load(PCIDevice *dev, QEMUFile *f);
>  
>  int msix_enabled(PCIDevice *dev);
>  int msix_present(PCIDevice *dev);
> +bool msix_masked(PCIDevice *dev);
>  
>  bool msix_is_masked(PCIDevice *dev, unsigned vector);
>  void msix_set_pending(PCIDevice *dev, unsigned vector);
diff mbox series

Patch

diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index ae9331cd0b..e057958fcd 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -131,7 +131,7 @@  static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
     }
 }
 
-static bool msix_masked(PCIDevice *dev)
+bool msix_masked(PCIDevice *dev)
 {
     return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
 }
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index f74be78209..088fd41926 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -569,6 +569,9 @@  static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
 
 static void vfio_msix_enable(VFIOPCIDevice *vdev)
 {
+    PCIDevice *pdev = &vdev->pdev;
+    unsigned int nr, max_vec = 0;
+
     vfio_disable_interrupts(vdev);
 
     vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
@@ -587,11 +590,22 @@  static void vfio_msix_enable(VFIOPCIDevice *vdev)
      * triggering to userspace, then immediately release the vector, leaving
      * the physical device with no vectors enabled, but MSI-X enabled, just
      * like the guest view.
+     * If there are already unmasked vectors (in migration resume phase and
+     * some guest startups) which will be enabled soon, we can allocate all
+     * of them here to avoid inefficiently disabling and enabling vectors
+     * repeatedly later.
      */
-    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
-    vfio_msix_vector_release(&vdev->pdev, 0);
+    if (!msix_masked(pdev)) {
+        for (nr = 0; nr < msix_nr_vectors_allocated(pdev); nr++) {
+            if (!msix_is_masked(pdev, nr)) {
+                max_vec = nr;
+            }
+        }
+    }
+    vfio_msix_vector_do_use(pdev, max_vec, NULL, NULL);
+    vfio_msix_vector_release(pdev, max_vec);
 
-    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
+    if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
                                   vfio_msix_vector_release, NULL)) {
         error_report("vfio: msix_set_vector_notifiers failed");
     }
diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
index 4c4a60c739..b3cd88e262 100644
--- a/include/hw/pci/msix.h
+++ b/include/hw/pci/msix.h
@@ -28,6 +28,7 @@  void msix_load(PCIDevice *dev, QEMUFile *f);
 
 int msix_enabled(PCIDevice *dev);
 int msix_present(PCIDevice *dev);
+bool msix_masked(PCIDevice *dev);
 
 bool msix_is_masked(PCIDevice *dev, unsigned vector);
 void msix_set_pending(PCIDevice *dev, unsigned vector);