| Message ID | 20210219094230.231-4-jiangkunkun@huawei.com |
|---|---|
| State | New |
| Headers | show |
| Series | Add migration support for VFIO PCI devices in SMMUv3 nested stage mode | expand |
Hi Kunkun, On 2/19/21 10:42 AM, Kunkun Jiang wrote: > In nested mode, we call the set_pasid_table() callback on each STE > update to pass the guest stage 1 configuration to the host and > apply it at physical level. > > In the case of live migration, we need to manual call the s/manual/manually > set_pasid_table() to load the guest stage 1 configurations to the > host. If this operation is fail, the migration is fail. s/If this operation is fail, the migration is fail/If this operation fails, the migration fails. > > Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com> > --- > hw/arm/smmuv3.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ > hw/arm/trace-events | 1 + > 2 files changed, 61 insertions(+) > > diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c > index 6c6ed84e78..94ca15375c 100644 > --- a/hw/arm/smmuv3.c > +++ b/hw/arm/smmuv3.c > @@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp) > smmu_init_irq(s, dev); > } > > +static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev) Can't you retrieve the associated sid and then call smmuv3_notify_config_change() > +{ > +#ifdef __linux__ > + IOMMUMemoryRegion *mr = &(sdev->iommu); > + int sid = smmu_get_sid(sdev); > + SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid, > + .inval_ste_allowed = true}; > + IOMMUConfig iommu_config = {}; > + SMMUTransCfg *cfg; > + int ret = -1; > + > + cfg = smmuv3_get_config(sdev, &event); > + if (!cfg) { > + return ret; > + } > + > + iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config); > + iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1; > + iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3; > + iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr; > + iommu_config.pasid_cfg.pasid_bits = 0; > + iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1; > + > + if (cfg->disabled || cfg->bypassed) { > + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS; > + } else if (cfg->aborted) { > + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT; > + } else { > + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE; > + } > + > + ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config); > + if (ret) { > + error_report("Failed to pass PASID table to host for iommu mr %s (%m)", > + mr->parent_obj.name); > + } > + > + return ret; > +#endif > +} > + > +static int smmuv3_post_load(void *opaque, int version_id) > +{ > + SMMUv3State *s3 = opaque; > + SMMUState *s = &(s3->smmu_state); > + SMMUDevice *sdev; > + int ret = 0; > + > + QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { > + trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name); > + ret = smmuv3_manual_set_pci_device_pasid_table(sdev); > + if (ret) { > + break; > + } > + } > + > + return ret; > +} > + > static const VMStateDescription vmstate_smmuv3_queue = { > .name = "smmuv3_queue", > .version_id = 1, > @@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = { > .version_id = 1, > .minimum_version_id = 1, > .priority = MIG_PRI_IOMMU, > + .post_load = smmuv3_post_load, > .fields = (VMStateField[]) { > VMSTATE_UINT32(features, SMMUv3State), > VMSTATE_UINT8(sid_size, SMMUv3State), > diff --git a/hw/arm/trace-events b/hw/arm/trace-events > index 35e562ab74..caa864dd72 100644 > --- a/hw/arm/trace-events > +++ b/hw/arm/trace-events > @@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s > smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s" > smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64 > smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64 > +smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64 > > Thanks Eric
Hi Eric, On 2021/4/12 16:34, Auger Eric wrote: > Hi Kunkun, > On 2/19/21 10:42 AM, Kunkun Jiang wrote: >> In nested mode, we call the set_pasid_table() callback on each STE >> update to pass the guest stage 1 configuration to the host and >> apply it at physical level. >> >> In the case of live migration, we need to manual call the > s/manual/manually You are right. >> set_pasid_table() to load the guest stage 1 configurations to the >> host. If this operation is fail, the migration is fail. > s/If this operation is fail, the migration is fail/If this operation > fails, the migration fails. Thanks for your careful review. >> Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com> >> --- >> hw/arm/smmuv3.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ >> hw/arm/trace-events | 1 + >> 2 files changed, 61 insertions(+) >> >> diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c >> index 6c6ed84e78..94ca15375c 100644 >> --- a/hw/arm/smmuv3.c >> +++ b/hw/arm/smmuv3.c >> @@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp) >> smmu_init_irq(s, dev); >> } >> >> +static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev) > Can't you retrieve the associated sid and then call > smmuv3_notify_config_change() Agreed. It can reuse the code of smmuv3_notify_config_change(). I will modify it in the next version.😁 But I need a return value to judge whether it is successful, which may need to modify the type of smmuv3_notify_config_change(). Thanks, Kunkun Jiang >> +{ >> +#ifdef __linux__ >> + IOMMUMemoryRegion *mr = &(sdev->iommu); >> + int sid = smmu_get_sid(sdev); >> + SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid, >> + .inval_ste_allowed = true}; >> + IOMMUConfig iommu_config = {}; >> + SMMUTransCfg *cfg; >> + int ret = -1; >> + >> + cfg = smmuv3_get_config(sdev, &event); >> + if (!cfg) { >> + return ret; >> + } >> + >> + iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config); >> + iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1; >> + iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3; >> + iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr; >> + iommu_config.pasid_cfg.pasid_bits = 0; >> + iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1; >> + >> + if (cfg->disabled || cfg->bypassed) { >> + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS; >> + } else if (cfg->aborted) { >> + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT; >> + } else { >> + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE; >> + } >> + >> + ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config); >> + if (ret) { >> + error_report("Failed to pass PASID table to host for iommu mr %s (%m)", >> + mr->parent_obj.name); >> + } >> + >> + return ret; >> +#endif >> +} >> + >> +static int smmuv3_post_load(void *opaque, int version_id) >> +{ >> + SMMUv3State *s3 = opaque; >> + SMMUState *s = &(s3->smmu_state); >> + SMMUDevice *sdev; >> + int ret = 0; >> + >> + QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { >> + trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name); >> + ret = smmuv3_manual_set_pci_device_pasid_table(sdev); >> + if (ret) { >> + break; >> + } >> + } >> + >> + return ret; >> +} >> + >> static const VMStateDescription vmstate_smmuv3_queue = { >> .name = "smmuv3_queue", >> .version_id = 1, >> @@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = { >> .version_id = 1, >> .minimum_version_id = 1, >> .priority = MIG_PRI_IOMMU, >> + .post_load = smmuv3_post_load, >> .fields = (VMStateField[]) { >> VMSTATE_UINT32(features, SMMUv3State), >> VMSTATE_UINT8(sid_size, SMMUv3State), >> diff --git a/hw/arm/trace-events b/hw/arm/trace-events >> index 35e562ab74..caa864dd72 100644 >> --- a/hw/arm/trace-events >> +++ b/hw/arm/trace-events >> @@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s >> smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s" >> smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64 >> smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64 >> +smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64 >> >> > Thanks > > Eric > > .
diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c index 6c6ed84e78..94ca15375c 100644 --- a/hw/arm/smmuv3.c +++ b/hw/arm/smmuv3.c @@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp) smmu_init_irq(s, dev); } +static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev) +{ +#ifdef __linux__ + IOMMUMemoryRegion *mr = &(sdev->iommu); + int sid = smmu_get_sid(sdev); + SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid, + .inval_ste_allowed = true}; + IOMMUConfig iommu_config = {}; + SMMUTransCfg *cfg; + int ret = -1; + + cfg = smmuv3_get_config(sdev, &event); + if (!cfg) { + return ret; + } + + iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config); + iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1; + iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3; + iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr; + iommu_config.pasid_cfg.pasid_bits = 0; + iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1; + + if (cfg->disabled || cfg->bypassed) { + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS; + } else if (cfg->aborted) { + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT; + } else { + iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE; + } + + ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config); + if (ret) { + error_report("Failed to pass PASID table to host for iommu mr %s (%m)", + mr->parent_obj.name); + } + + return ret; +#endif +} + +static int smmuv3_post_load(void *opaque, int version_id) +{ + SMMUv3State *s3 = opaque; + SMMUState *s = &(s3->smmu_state); + SMMUDevice *sdev; + int ret = 0; + + QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { + trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name); + ret = smmuv3_manual_set_pci_device_pasid_table(sdev); + if (ret) { + break; + } + } + + return ret; +} + static const VMStateDescription vmstate_smmuv3_queue = { .name = "smmuv3_queue", .version_id = 1, @@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = { .version_id = 1, .minimum_version_id = 1, .priority = MIG_PRI_IOMMU, + .post_load = smmuv3_post_load, .fields = (VMStateField[]) { VMSTATE_UINT32(features, SMMUv3State), VMSTATE_UINT8(sid_size, SMMUv3State), diff --git a/hw/arm/trace-events b/hw/arm/trace-events index 35e562ab74..caa864dd72 100644 --- a/hw/arm/trace-events +++ b/hw/arm/trace-events @@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s" smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64 smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64 +smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64
In nested mode, we call the set_pasid_table() callback on each STE update to pass the guest stage 1 configuration to the host and apply it at physical level. In the case of live migration, we need to manual call the set_pasid_table() to load the guest stage 1 configurations to the host. If this operation is fail, the migration is fail. Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com> --- hw/arm/smmuv3.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ hw/arm/trace-events | 1 + 2 files changed, 61 insertions(+)