From patchwork Fri Apr 17 08:53:09 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: chenfan X-Patchwork-Id: 461989 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.gnu.org (lists.gnu.org [IPv6:2001:4830:134:3::11]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id A5E291402B0 for ; Fri, 17 Apr 2015 18:57:57 +1000 (AEST) Received: from localhost ([::1]:40246 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1Yj25z-0006qH-Q6 for incoming@patchwork.ozlabs.org; Fri, 17 Apr 2015 04:57:55 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:43659) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1Yj22H-0007Nq-Bd for qemu-devel@nongnu.org; Fri, 17 Apr 2015 04:54:07 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1Yj22F-00069o-J0 for qemu-devel@nongnu.org; Fri, 17 Apr 2015 04:54:05 -0400 Received: from [59.151.112.132] (port=46403 helo=heian.cn.fujitsu.com) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1Yj22C-00069R-Tb for qemu-devel@nongnu.org; Fri, 17 Apr 2015 04:54:03 -0400 X-IronPort-AV: E=Sophos;i="5.04,848,1406563200"; d="scan'208";a="90829138" Received: from unknown (HELO edo.cn.fujitsu.com) ([10.167.33.5]) by heian.cn.fujitsu.com with ESMTP; 17 Apr 2015 16:50:09 +0800 Received: from G08CNEXCHPEKD02.g08.fujitsu.local (localhost.localdomain [127.0.0.1]) by edo.cn.fujitsu.com (8.14.3/8.13.1) with ESMTP id t3H8qlrn025684; Fri, 17 Apr 2015 16:52:47 +0800 Received: from G08FNSTD131468.g08.fujitsu.local (10.167.226.78) by G08CNEXCHPEKD02.g08.fujitsu.local (10.167.33.89) with Microsoft SMTP Server (TLS) id 14.3.181.6; Fri, 17 Apr 2015 16:53:58 +0800 From: Chen Fan To: Date: Fri, 17 Apr 2015 16:53:09 +0800 Message-ID: <905e612b6113012061e22d22b8f2fce5f95e49c3.1429256889.git.chen.fan.fnst@cn.fujitsu.com> X-Mailer: git-send-email 1.9.3 In-Reply-To: References: MIME-Version: 1.0 X-Originating-IP: [10.167.226.78] X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 59.151.112.132 Cc: izumi.taku@jp.fujitsu.com, qemu-devel@nongnu.org Subject: [Qemu-devel] [RFC 7/7] migrate: add hostdev migrate status to support hostdev migration X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org Sender: qemu-devel-bounces+incoming=patchwork.ozlabs.org@nongnu.org we add a migrate status for hostdev to specify the device don't need to initialze when VM startup, after migration end, we add the migrate status hostdev, so can support hostdev migration. Signed-off-by: Chen Fan --- src/conf/domain_conf.c | 3 ++ src/conf/domain_conf.h | 7 ++++ src/qemu/qemu_command.c | 3 ++ src/qemu/qemu_driver.c | 53 +-------------------------- src/qemu/qemu_hotplug.c | 8 +++-- src/qemu/qemu_migration.c | 92 ++++++++++++++++++++++++++++++++++++++++++++--- src/qemu/qemu_migration.h | 4 +++ src/util/virhostdev.c | 3 ++ 8 files changed, 114 insertions(+), 59 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 7d1cd3e..b56c6fa 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -3035,6 +3035,9 @@ virDomainDeviceInfoIterateInternal(virDomainDefPtr def, device.type = VIR_DOMAIN_DEVICE_HOSTDEV; for (i = 0; i < def->nhostdevs; i++) { device.data.hostdev = def->hostdevs[i]; + if (device.data.hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) + continue; + if (cb(def, &device, def->hostdevs[i]->info, opaque) < 0) return -1; } diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 723f07b..4b7b4c9 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -543,6 +543,12 @@ struct _virDomainHostdevCaps { } u; }; +typedef enum { + VIR_DOMAIN_HOSTDEV_STATE_DEFAULT, + VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE, + + VIR_DOMAIN_HOSTDEV_STATE_LAST +} virDomainHostdevState; /* basic device for direct passthrough */ struct _virDomainHostdevDef { @@ -559,6 +565,7 @@ struct _virDomainHostdevDef { } source; virDomainHostdevOrigStates origstates; virDomainDeviceInfoPtr info; /* Guest address */ + int state; }; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index e7e0937..dc5245a 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -10365,6 +10365,9 @@ qemuBuildCommandLine(virConnectPtr conn, virDomainHostdevDefPtr hostdev = def->hostdevs[i]; char *devstr; + if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) + continue; + if (hostdev->info->bootIndex) { if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS || (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 0ba9e4a..4724171 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -12353,57 +12353,6 @@ qemuDomainMigrateBegin3(virDomainPtr domain, cookieout, cookieoutlen, flags); } -static int -qemuDomainRemovePciPassThruDevices(virConnectPtr conn, - virDomainObjPtr vm) -{ - virQEMUDriverPtr driver = conn->privateData; - virDomainDeviceDef dev; - virDomainDeviceDefPtr dev_copy = NULL; - virCapsPtr caps = NULL; - int ret = -1; - size_t i; - - if (!(caps = virQEMUDriverGetCapabilities(driver, false))) - goto cleanup; - - if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT)) - goto cleanup; - - /* unplug passthrough bond device */ - for (i = 0; i < vm->def->nhostdevs; i++) { - virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i]; - - if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS && - hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && - hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO && - hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND) { - - dev.type = VIR_DOMAIN_DEVICE_HOSTDEV; - dev.data.hostdev = hostdev; - - dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps, driver->xmlopt); - if (!dev_copy) - goto cleanup; - - if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) { - virDomainDeviceDefFree(dev_copy); - goto cleanup; - } - - virDomainDeviceDefFree(dev_copy); - if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) - goto cleanup; - } - } - - ret = 0; - - cleanup: - virObjectUnref(caps); - - return ret; -} static char * qemuDomainMigrateBegin3Params(virDomainPtr domain, @@ -12740,7 +12689,7 @@ qemuDomainMigratePerform3Params(virDomainPtr dom, return -1; } - if (qemuDomainRemovePciPassThruDevices(dom->conn, vm) < 0) { + if (qemuDomainMigratePciPassThruDevices(driver, vm, false) < 0) { qemuDomObjEndAPI(&vm); return -1; } diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index f07c54d..13a7338 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -1239,8 +1239,9 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver, virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver); unsigned int flags = 0; - if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0) - return -1; + if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) + if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0) + return -1; if (!cfg->relaxedACS) flags |= VIR_HOSTDEV_STRICT_ACS_CHECK; @@ -1344,7 +1345,8 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver, if (ret < 0) goto error; - vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; + if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) + vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; VIR_FREE(devstr); VIR_FREE(configfd_name); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 9ea83df..291cb9f 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2001,10 +2001,7 @@ qemuMigrationIsAllowed(virQEMUDriverPtr driver, virDomainObjPtr vm, for (i = 0; i < def->nhostdevs; i++) { virDomainHostdevDefPtr hostdev = def->hostdevs[i]; - if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS && - hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && - hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO && - hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND) + if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) continue; if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS || @@ -2629,6 +2626,80 @@ qemuMigrationCleanup(virDomainObjPtr vm, } +static void +qemuMigrationSetStateForHostdev(virDomainDefPtr def, + int state) +{ + virDomainHostdevDefPtr hostdev; + size_t i; + + if (!def) + return; + + for (i = 0; i < def->nhostdevs; i++) { + hostdev = def->hostdevs[i]; + + if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS && + hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI && + hostdev->source.subsys.u.pci.backend == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO && + hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND) + hostdev->state = state; + } +} + + +int +qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver, + virDomainObjPtr vm, + bool isPlug) +{ + virDomainDeviceDef dev; + virDomainDeviceDefPtr dev_copy = NULL; + virDomainHostdevDefPtr hostdev; + virCapsPtr caps = NULL; + int ret = -1; + int i; + + if (!(caps = virQEMUDriverGetCapabilities(driver, false))) + goto cleanup; + + /* plug/unplug passthrough bond device */ + for (i = vm->def->nhostdevs; i >= 0; i--) { + hostdev = vm->def->hostdevs[i]; + + if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) { + if (!isPlug) { + dev.type = VIR_DOMAIN_DEVICE_HOSTDEV; + dev.data.hostdev = hostdev; + + dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps, driver->xmlopt); + if (!dev_copy) + goto cleanup; + + if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) { + virDomainDeviceDefFree(dev_copy); + goto cleanup; + } + virDomainDeviceDefFree(dev_copy); + } else { + qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_DEFAULT); + if (qemuDomainAttachHostDevice(NULL, driver, vm, hostdev) < 0) + goto cleanup; + } + if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0) + goto cleanup; + } + } + + ret = 0; + + cleanup: + virObjectUnref(caps); + + return ret; +} + + /* The caller is supposed to lock the vm and start a migration job. */ static char *qemuMigrationBeginPhase(virQEMUDriverPtr driver, @@ -2662,6 +2733,8 @@ static char if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT) qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_BEGIN3); + qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE); + if (!qemuMigrationIsAllowed(driver, vm, NULL, true, abort_on_error)) goto cleanup; @@ -2885,6 +2958,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, if (!(caps = virQEMUDriverGetCapabilities(driver, false))) goto cleanup; + qemuMigrationSetStateForHostdev(*def, VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE); + if (!qemuMigrationIsAllowed(driver, NULL, *def, true, abort_on_error)) goto cleanup; @@ -5315,6 +5390,13 @@ qemuMigrationFinish(virQEMUDriverPtr driver, goto endjob; } + /* hotplug previous mark migrate hostdev */ + if (qemuDomainMigratePciPassThruDevices(driver, vm, true) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("passthrough for hostdev failed")); + goto endjob; + } + /* Guest is successfully running, so cancel previous auto destroy */ qemuProcessAutoDestroyRemove(driver, vm); } else if (!(flags & VIR_MIGRATE_OFFLINE)) { @@ -5331,6 +5413,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver, VIR_WARN("Unable to encode migration cookie"); endjob: + qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_DEFAULT); + qemuMigrationJobFinish(driver, vm); if (!vm->persistent && !virDomainObjIsActive(vm)) qemuDomainRemoveInactive(driver, vm); diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 1726455..fa21752 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -177,4 +177,8 @@ int qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm, ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(5) ATTRIBUTE_RETURN_CHECK; +int qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver, + virDomainObjPtr vm, + bool isPlug); + #endif /* __QEMU_MIGRATION_H__ */ diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c index f583e54..4b6152a 100644 --- a/src/util/virhostdev.c +++ b/src/util/virhostdev.c @@ -206,6 +206,9 @@ virHostdevGetPCIHostDeviceList(virDomainHostdevDefPtr *hostdevs, int nhostdevs) virDomainHostdevSubsysPCIPtr pcisrc = &hostdev->source.subsys.u.pci; virPCIDevicePtr dev; + if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) + continue; + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) continue; if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)