@@ -229,6 +229,8 @@ int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp);
void vfio_unblock_multiple_devices_migration(void);
int vfio_block_postcopy_migration(VFIODevice *vbasedev, Error **errp);
void vfio_unblock_postcopy_migration(void);
+int vfio_block_background_snapshot(VFIODevice *vbasedev, Error **errp);
+void vfio_unblock_background_snapshot(void);
bool vfio_viommu_preset(VFIODevice *vbasedev);
int64_t vfio_mig_bytes_transferred(void);
void vfio_reset_bytes_transferred(void);
@@ -516,6 +516,7 @@ void migration_populate_vfio_info(MigrationInfo *info);
void migration_reset_vfio_bytes_transferred(void);
bool migration_vfio_mig_active(void);
void migration_vfio_unblock_postcopy_migration(void);
+void migration_vfio_unblock_background_snapshot(void);
void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
#endif
@@ -345,6 +345,7 @@ static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
static Error *multiple_devices_migration_blocker;
static Error *postcopy_migration_blocker;
+static Error *background_snapshot_blocker;
static unsigned int vfio_migratable_devices_num(void)
{
@@ -470,6 +471,47 @@ void vfio_unblock_postcopy_migration(void)
postcopy_migration_blocker = NULL;
}
+int vfio_block_background_snapshot(VFIODevice *vbasedev, Error **errp)
+{
+ int ret;
+
+ if (!migrate_background_snapshot()) {
+ return 0;
+ }
+
+ if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
+ error_setg(errp,
+ "VFIO migration is not compatible with background snapshot");
+ return -EINVAL;
+ }
+
+ if (background_snapshot_blocker) {
+ return 0;
+ }
+
+ error_setg(&background_snapshot_blocker,
+ "VFIO migration is not compatible with background snapshot");
+ ret = migrate_add_blocker(background_snapshot_blocker, errp);
+ if (ret < 0) {
+ error_free(background_snapshot_blocker);
+ background_snapshot_blocker = NULL;
+ }
+
+ return ret;
+}
+
+void vfio_unblock_background_snapshot(void)
+{
+ if (!background_snapshot_blocker ||
+ (vfio_migratable_devices_num() && migrate_background_snapshot())) {
+ return;
+ }
+
+ migrate_del_blocker(background_snapshot_blocker);
+ error_free(background_snapshot_blocker);
+ background_snapshot_blocker = NULL;
+}
+
bool vfio_mig_active(void)
{
return vfio_migratable_devices_num();
@@ -857,6 +857,7 @@ static void vfio_migration_deinit(VFIODevice *vbasedev)
vfio_migration_free(vbasedev);
vfio_unblock_multiple_devices_migration();
vfio_unblock_postcopy_migration();
+ vfio_unblock_background_snapshot();
}
static int vfio_block_migration(VFIODevice *vbasedev, Error *err, Error **errp)
@@ -945,6 +946,11 @@ bool vfio_migration_realize(VFIODevice *vbasedev, Error **errp)
goto out_deinit;
}
+ ret = vfio_block_background_snapshot(vbasedev, errp);
+ if (ret) {
+ goto out_deinit;
+ }
+
if (vfio_viommu_preset(vbasedev)) {
error_setg(&err, "%s: Migration is currently not supported "
"with vIOMMU enabled", vbasedev->name);
@@ -537,6 +537,12 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
return false;
}
}
+
+ if (migration_vfio_mig_active()) {
+ error_setg(errp, "Background-snapshot is not compatible with VFIO "
+ "migration");
+ return false;
+ }
}
#ifdef CONFIG_LINUX
@@ -625,6 +631,7 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
static void migration_caps_remove_blockers(void)
{
migration_vfio_unblock_postcopy_migration();
+ migration_vfio_unblock_background_snapshot();
}
bool migrate_cap_set(int cap, bool value, Error **errp)
@@ -37,6 +37,11 @@ void migration_vfio_unblock_postcopy_migration(void)
{
vfio_unblock_postcopy_migration();
}
+
+void migration_vfio_unblock_background_snapshot(void)
+{
+ vfio_unblock_background_snapshot();
+}
#else
void migration_populate_vfio_info(MigrationInfo *info)
{
@@ -54,4 +59,8 @@ bool migration_vfio_mig_active(void)
void migration_vfio_unblock_postcopy_migration()
{
}
+
+void migration_vfio_unblock_background_snapshot(void)
+{
+}
#endif
Background snapshot allows creating a snapshot of the VM while it's running and keeping it small by not including dirty RAM pages. The way it works is by first stopping the VM, saving the non-iterable devices' state and then starting the VM and saving the RAM while write protecting it with UFFD. The resulting snapshot represents the VM state at snapshot start. VFIO migration is not compatible with background snapshot. First of all, VFIO device state is not even saved in background snapshot because only non-iterable device state is saved. But even if it was saved, after starting the VM, a VFIO device could dirty pages without it being detected by UFFD write protection. This would corrupt the snapshot, as the RAM in it would not represent the RAM at snapshot start. To prevent this and to be explicit about supported features, block VFIO migration with background snapshot: Fail setting background snapshot capability if a VFIO device is present, and add a migration blocker if a VFIO device is added when background snapshot capability is on. Signed-off-by: Avihai Horon <avihaih@nvidia.com> --- include/hw/vfio/vfio-common.h | 2 ++ migration/migration.h | 1 + hw/vfio/common.c | 42 +++++++++++++++++++++++++++++++++++ hw/vfio/migration.c | 6 +++++ migration/options.c | 7 ++++++ migration/target.c | 9 ++++++++ 6 files changed, 67 insertions(+)