| Message ID | 1589999088-31477-19-git-send-email-kwankhede@nvidia.com |
|---|---|
| State | New |
| Headers | show |
| Series | Add migration support for VFIO devices | expand |
* Kirti Wankhede (kwankhede@nvidia.com) wrote: > Added amount of bytes transferred to the target VM by all VFIO devices > > Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> > --- > hw/vfio/common.c | 20 ++++++++++++++++++++ > hw/vfio/migration.c | 10 +++++++++- > include/qemu/vfio-helpers.h | 3 +++ > migration/migration.c | 18 ++++++++++++++++++ > monitor/hmp-cmds.c | 6 ++++++ > qapi/migration.json | 19 ++++++++++++++++++- > 6 files changed, 74 insertions(+), 2 deletions(-) > > diff --git a/hw/vfio/common.c b/hw/vfio/common.c > index 2dd9e8b24788..e1ad426a6d78 100644 > --- a/hw/vfio/common.c > +++ b/hw/vfio/common.c > @@ -40,6 +40,7 @@ > #include "trace.h" > #include "qapi/error.h" > #include "migration/migration.h" > +#include "qemu/vfio-helpers.h" > > VFIOGroupList vfio_group_list = > QLIST_HEAD_INITIALIZER(vfio_group_list); > @@ -293,6 +294,25 @@ const MemoryRegionOps vfio_region_ops = { > * Device state interfaces > */ > > +bool vfio_mig_active(void) > +{ > + VFIOGroup *group; > + VFIODevice *vbasedev; > + > + if (QLIST_EMPTY(&vfio_group_list)) { > + return false; > + } > + > + QLIST_FOREACH(group, &vfio_group_list, next) { > + QLIST_FOREACH(vbasedev, &group->device_list, next) { > + if (vbasedev->migration_blocker) { > + return false; > + } > + } > + } > + return true; > +} > + > static bool vfio_devices_are_stopped_and_saving(void) > { > VFIOGroup *group; > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > index 7729c90782bd..2eef38fe5c65 100644 > --- a/hw/vfio/migration.c > +++ b/hw/vfio/migration.c > @@ -26,7 +26,7 @@ > #include "exec/ram_addr.h" > #include "pci.h" > #include "trace.h" > - > +#include "qemu/vfio-helpers.h" > /* > * Flags used as delimiter: > * 0xffffffff => MSB 32-bit all 1s > @@ -38,6 +38,8 @@ > #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) > #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) > > +static int64_t bytes_transferred; > + eww global; but if there's no where else OK. I think you need to initialise this somewhere at the start of each migration; you can get a migration that fails or is cancelled and then start a second one, if you don't zero it then the second time will be wrong. > static void vfio_migration_region_exit(VFIODevice *vbasedev) > { > VFIOMigration *migration = vbasedev->migration; > @@ -229,6 +231,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev) > return ret; > } > > + bytes_transferred += data_size; > return data_size; > } > > @@ -744,6 +747,11 @@ static int vfio_migration_init(VFIODevice *vbasedev, > > /* ---------------------------------------------------------------------- */ > > +int64_t vfio_mig_bytes_transferred(void) > +{ > + return bytes_transferred; > +} > + > int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) > { > struct vfio_region_info *info; > diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h > index 1f057c2b9e40..26a7df0767b1 100644 > --- a/include/qemu/vfio-helpers.h > +++ b/include/qemu/vfio-helpers.h > @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, > int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, > int irq_type, Error **errp); > > +bool vfio_mig_active(void); > +int64_t vfio_mig_bytes_transferred(void); > + > #endif > diff --git a/migration/migration.c b/migration/migration.c > index 0bb042a0f78b..bcdc880e9a76 100644 > --- a/migration/migration.c > +++ b/migration/migration.c > @@ -54,6 +54,7 @@ > #include "net/announce.h" > #include "qemu/queue.h" > #include "multifd.h" > +#include "qemu/vfio-helpers.h" > > #define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */ > > @@ -970,6 +971,17 @@ static void populate_disk_info(MigrationInfo *info) > } > } > > +#ifdef CONFIG_LINUX If you put the ifdef inside this function then you can avoid all the others below. > +static void populate_vfio_info(MigrationInfo *info) > +{ > + if (vfio_mig_active()) { > + info->has_vfio = true; > + info->vfio = g_malloc0(sizeof(*info->vfio)); > + info->vfio->bytes = vfio_mig_bytes_transferred(); > + } > +} > +#endif > + > static void fill_source_migration_info(MigrationInfo *info) > { > MigrationState *s = migrate_get_current(); > @@ -995,6 +1007,9 @@ static void fill_source_migration_info(MigrationInfo *info) > populate_time_info(info, s); > populate_ram_info(info, s); > populate_disk_info(info); > +#ifdef CONFIG_LINUX > + populate_vfio_info(info); > +#endif > break; > case MIGRATION_STATUS_COLO: > info->has_status = true; > @@ -1003,6 +1018,9 @@ static void fill_source_migration_info(MigrationInfo *info) > case MIGRATION_STATUS_COMPLETED: > populate_time_info(info, s); > populate_ram_info(info, s); > +#ifdef CONFIG_LINUX > + populate_vfio_info(info); > +#endif > break; > case MIGRATION_STATUS_FAILED: > info->has_status = true; > diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c > index 9c61e769ca9a..396c195263c2 100644 > --- a/monitor/hmp-cmds.c > +++ b/monitor/hmp-cmds.c > @@ -355,6 +355,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) > } > monitor_printf(mon, "]\n"); > } > + > + if (info->has_vfio) { > + monitor_printf(mon, "vfio device bytes: %" PRIu64 " kbytes\n", > + info->vfio->bytes >> 10); > + } > + > qapi_free_MigrationInfo(info); > } > > diff --git a/qapi/migration.json b/qapi/migration.json > index d5000558c6c9..acbc42a1efe8 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -146,6 +146,18 @@ > 'active', 'postcopy-active', 'postcopy-paused', > 'postcopy-recover', 'completed', 'failed', 'colo', > 'pre-switchover', 'device', 'wait-unplug' ] } > +## > +# @VfioStats: > +# > +# Detailed VFIO devices migration statistics > +# > +# @bytes: amount of bytes transferred to the target VM by VFIO devices > +# > +# Since: 5.1 > +# > +## > +{ 'struct': 'VfioStats', > + 'data': {'bytes': 'int' } } > > ## > # @MigrationInfo: > @@ -207,6 +219,10 @@ > # > # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) > # > +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, > +# only returned if VFIO device is present, migration is supported by all > +# VFIO devices and status is 'active' or 'completed' (since 5.1) > +# > # Since: 0.14.0 > ## > { 'struct': 'MigrationInfo', > @@ -222,7 +238,8 @@ > '*postcopy-blocktime' : 'uint32', > '*postcopy-vcpu-blocktime': ['uint32'], > '*compression': 'CompressionStats', > - '*socket-address': ['SocketAddress'] } } > + '*socket-address': ['SocketAddress'], > + '*vfio': 'VfioStats' } } Is there anyway you can provide stats on the amount of main memory that's been marked as dirty because it's mapped by the device? Dave > ## > # @query-migrate: > -- > 2.7.0 > -- Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
Kirti Wankhede <kwankhede@nvidia.com> writes: > Added amount of bytes transferred to the target VM by all VFIO devices > > Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> QAPI review only. > diff --git a/qapi/migration.json b/qapi/migration.json > index d5000558c6c9..acbc42a1efe8 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -146,6 +146,18 @@ > 'active', 'postcopy-active', 'postcopy-paused', > 'postcopy-recover', 'completed', 'failed', 'colo', > 'pre-switchover', 'device', 'wait-unplug' ] } > +## > +# @VfioStats: > +# > +# Detailed VFIO devices migration statistics > +# > +# @bytes: amount of bytes transferred to the target VM by VFIO devices > +# > +# Since: 5.1 > +# > +## > +{ 'struct': 'VfioStats', > + 'data': {'bytes': 'int' } } Pardon my ignorance... What exactly do VFIO devices transfer to the target VM? How is that related to MigrationInfo member @ram? For what it's worth, MigrationStats has # @transferred: amount of bytes already transferred to the target VM Name yours @transferred as well, for consistency? MigrationStats has much more information, and some of it is pretty useful to track how migration is doing, in particular whether it converges, and how fast. Absent in VfioStats due to "not implemented", or due to "can't be done"? Byte counts should use QAPI type 'size'. Many existing ones don't. Since MigrationStats uses 'int', I'll let the migration maintainers decide whether they want 'int' or 'size' here. > > ## > # @MigrationInfo: > @@ -207,6 +219,10 @@ > # > # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) > # > +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, > +# only returned if VFIO device is present, migration is supported by all > +# VFIO devices and status is 'active' or 'completed' (since 5.1) > +# > # Since: 0.14.0 > ## > { 'struct': 'MigrationInfo', > @@ -222,7 +238,8 @@ 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats', '*disk': 'MigrationStats', '*xbzrle-cache': 'XBZRLECacheStats', '*total-time': 'int', '*expected-downtime': 'int', '*downtime': 'int', '*setup-time': 'int', '*cpu-throttle-percentage': 'int', '*error-desc': 'str', > '*postcopy-blocktime' : 'uint32', > '*postcopy-vcpu-blocktime': ['uint32'], > '*compression': 'CompressionStats', > - '*socket-address': ['SocketAddress'] } } > + '*socket-address': ['SocketAddress'], > + '*vfio': 'VfioStats' } } > > ## > # @query-migrate: If @vfio and @ram are related, add @vfio next to @ram and @disk, please.
diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 2dd9e8b24788..e1ad426a6d78 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -40,6 +40,7 @@ #include "trace.h" #include "qapi/error.h" #include "migration/migration.h" +#include "qemu/vfio-helpers.h" VFIOGroupList vfio_group_list = QLIST_HEAD_INITIALIZER(vfio_group_list); @@ -293,6 +294,25 @@ const MemoryRegionOps vfio_region_ops = { * Device state interfaces */ +bool vfio_mig_active(void) +{ + VFIOGroup *group; + VFIODevice *vbasedev; + + if (QLIST_EMPTY(&vfio_group_list)) { + return false; + } + + QLIST_FOREACH(group, &vfio_group_list, next) { + QLIST_FOREACH(vbasedev, &group->device_list, next) { + if (vbasedev->migration_blocker) { + return false; + } + } + } + return true; +} + static bool vfio_devices_are_stopped_and_saving(void) { VFIOGroup *group; diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 7729c90782bd..2eef38fe5c65 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -26,7 +26,7 @@ #include "exec/ram_addr.h" #include "pci.h" #include "trace.h" - +#include "qemu/vfio-helpers.h" /* * Flags used as delimiter: * 0xffffffff => MSB 32-bit all 1s @@ -38,6 +38,8 @@ #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) +static int64_t bytes_transferred; + static void vfio_migration_region_exit(VFIODevice *vbasedev) { VFIOMigration *migration = vbasedev->migration; @@ -229,6 +231,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev) return ret; } + bytes_transferred += data_size; return data_size; } @@ -744,6 +747,11 @@ static int vfio_migration_init(VFIODevice *vbasedev, /* ---------------------------------------------------------------------- */ +int64_t vfio_mig_bytes_transferred(void) +{ + return bytes_transferred; +} + int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) { struct vfio_region_info *info; diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h index 1f057c2b9e40..26a7df0767b1 100644 --- a/include/qemu/vfio-helpers.h +++ b/include/qemu/vfio-helpers.h @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, int irq_type, Error **errp); +bool vfio_mig_active(void); +int64_t vfio_mig_bytes_transferred(void); + #endif diff --git a/migration/migration.c b/migration/migration.c index 0bb042a0f78b..bcdc880e9a76 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -54,6 +54,7 @@ #include "net/announce.h" #include "qemu/queue.h" #include "multifd.h" +#include "qemu/vfio-helpers.h" #define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */ @@ -970,6 +971,17 @@ static void populate_disk_info(MigrationInfo *info) } } +#ifdef CONFIG_LINUX +static void populate_vfio_info(MigrationInfo *info) +{ + if (vfio_mig_active()) { + info->has_vfio = true; + info->vfio = g_malloc0(sizeof(*info->vfio)); + info->vfio->bytes = vfio_mig_bytes_transferred(); + } +} +#endif + static void fill_source_migration_info(MigrationInfo *info) { MigrationState *s = migrate_get_current(); @@ -995,6 +1007,9 @@ static void fill_source_migration_info(MigrationInfo *info) populate_time_info(info, s); populate_ram_info(info, s); populate_disk_info(info); +#ifdef CONFIG_LINUX + populate_vfio_info(info); +#endif break; case MIGRATION_STATUS_COLO: info->has_status = true; @@ -1003,6 +1018,9 @@ static void fill_source_migration_info(MigrationInfo *info) case MIGRATION_STATUS_COMPLETED: populate_time_info(info, s); populate_ram_info(info, s); +#ifdef CONFIG_LINUX + populate_vfio_info(info); +#endif break; case MIGRATION_STATUS_FAILED: info->has_status = true; diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 9c61e769ca9a..396c195263c2 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -355,6 +355,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) } monitor_printf(mon, "]\n"); } + + if (info->has_vfio) { + monitor_printf(mon, "vfio device bytes: %" PRIu64 " kbytes\n", + info->vfio->bytes >> 10); + } + qapi_free_MigrationInfo(info); } diff --git a/qapi/migration.json b/qapi/migration.json index d5000558c6c9..acbc42a1efe8 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -146,6 +146,18 @@ 'active', 'postcopy-active', 'postcopy-paused', 'postcopy-recover', 'completed', 'failed', 'colo', 'pre-switchover', 'device', 'wait-unplug' ] } +## +# @VfioStats: +# +# Detailed VFIO devices migration statistics +# +# @bytes: amount of bytes transferred to the target VM by VFIO devices +# +# Since: 5.1 +# +## +{ 'struct': 'VfioStats', + 'data': {'bytes': 'int' } } ## # @MigrationInfo: @@ -207,6 +219,10 @@ # # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) # +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, +# only returned if VFIO device is present, migration is supported by all +# VFIO devices and status is 'active' or 'completed' (since 5.1) +# # Since: 0.14.0 ## { 'struct': 'MigrationInfo', @@ -222,7 +238,8 @@ '*postcopy-blocktime' : 'uint32', '*postcopy-vcpu-blocktime': ['uint32'], '*compression': 'CompressionStats', - '*socket-address': ['SocketAddress'] } } + '*socket-address': ['SocketAddress'], + '*vfio': 'VfioStats' } } ## # @query-migrate:
Added amount of bytes transferred to the target VM by all VFIO devices Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> --- hw/vfio/common.c | 20 ++++++++++++++++++++ hw/vfio/migration.c | 10 +++++++++- include/qemu/vfio-helpers.h | 3 +++ migration/migration.c | 18 ++++++++++++++++++ monitor/hmp-cmds.c | 6 ++++++ qapi/migration.json | 19 ++++++++++++++++++- 6 files changed, 74 insertions(+), 2 deletions(-)