| Message ID | 20211022192754.58196-6-yang.zhong@intel.com |
|---|---|
| State | New |
| Headers | show |
| Series | SGX NUMA support plus vepc reset | expand |
On 22/10/21 21:27, Yang Zhong wrote: > + > + for (j = 0; j < num; j++) { > + epc = pcms->sgx_epc.sections[j]; > + hostmem = MEMORY_BACKEND(epc->hostmem); > + fd = memory_region_get_fd(host_memory_backend_get_memory(hostmem)); > + > + failures = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); > + if (failures < 0) { > + return failures; > + } else if (failures > 0) { > + /* Remove SECS pages */ > + sleep(1); > + failures_1 = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); > + } > + > + /* > + * The host or guest can support 8 EPC sections, use the > + * corresponding bit to show each section removal status. > + */ > + if (failures_1) { > + set_bit(j, &ret); > + } > + } This sleep is not necessary, just do two tries on all the regions. So something like int failures; /* * The second pass is needed to remove SECS pages that could not * be removed during the first. */ for (i = 0; i < 2; i++) { failures = 0; for (j = 0; j < pcms->sgx_epc.nr_sections; j++) { epc = pcms->sgx_epc.sections[j]; hostmem = MEMORY_BACKEND(epc->hostmem); fd = memory_region_get_fd(host_memory_backend_get_memory(hostmem)); r = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); if (r < 0) { return r; } if (r > 0) { /* SECS pages remain */ failures++; if (pass == 1) { error_report("cannot reset vEPC section %d\n", j); } } } if (!failures) { return 0; } } return failures; is enough, without any need to do further retries. Paolo
On Fri, Oct 22, 2021 at 11:46:30PM +0200, Paolo Bonzini wrote: > On 22/10/21 21:27, Yang Zhong wrote: > >+ > >+ for (j = 0; j < num; j++) { > >+ epc = pcms->sgx_epc.sections[j]; > >+ hostmem = MEMORY_BACKEND(epc->hostmem); > >+ fd = memory_region_get_fd(host_memory_backend_get_memory(hostmem)); > >+ > >+ failures = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); > >+ if (failures < 0) { > >+ return failures; > >+ } else if (failures > 0) { > >+ /* Remove SECS pages */ > >+ sleep(1); > >+ failures_1 = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); > >+ } > >+ > >+ /* > >+ * The host or guest can support 8 EPC sections, use the > >+ * corresponding bit to show each section removal status. > >+ */ > >+ if (failures_1) { > >+ set_bit(j, &ret); > >+ } > >+ } > > This sleep is not necessary, just do two tries on all the regions. > So something like > > int failures; > > /* > * The second pass is needed to remove SECS pages that could not > * be removed during the first. > */ > for (i = 0; i < 2; i++) { > failures = 0; > for (j = 0; j < pcms->sgx_epc.nr_sections; j++) { > epc = pcms->sgx_epc.sections[j]; > hostmem = MEMORY_BACKEND(epc->hostmem); > fd = > memory_region_get_fd(host_memory_backend_get_memory(hostmem)); > > r = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); > if (r < 0) { > return r; > } > if (r > 0) { > /* SECS pages remain */ > failures++; > if (pass == 1) { > error_report("cannot reset vEPC section %d\n", j); > } > } > } > if (!failures) { > return 0; > } > } > return failures; > > is enough, without any need to do further retries. > Thanks Paolo, i will update it in the next version. Please also help review other patches, thanks! Yang > Paolo
diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h index 23267a3674..e78ca6c156 100644 --- a/include/hw/i386/x86.h +++ b/include/hw/i386/x86.h @@ -141,5 +141,6 @@ qemu_irq x86_allocate_cpu_irq(void); void gsi_handler(void *opaque, int n, int level); void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name); DeviceState *ioapic_init_secondary(GSIState *gsi_state); +void sgx_epc_reset(void *opaque); #endif diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index bcaf66cc4d..ee110e660b 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -887,6 +887,12 @@ struct kvm_ppc_resize_hpt { #define KVM_GET_EMULATED_CPUID _IOWR(KVMIO, 0x09, struct kvm_cpuid2) #define KVM_GET_MSR_FEATURE_INDEX_LIST _IOWR(KVMIO, 0x0a, struct kvm_msr_list) +/* + * ioctl for /dev/sgx_vepc + */ +#define SGX_MAGIC 0xA4 +#define SGX_IOC_VEPC_REMOVE_ALL _IO(SGX_MAGIC, 0x04) + /* * Extension capability list. */ diff --git a/hw/i386/sgx.c b/hw/i386/sgx.c index b5b710a556..3e21094c30 100644 --- a/hw/i386/sgx.c +++ b/hw/i386/sgx.c @@ -22,6 +22,8 @@ #include "exec/address-spaces.h" #include "sysemu/hw_accel.h" #include "hw/acpi/aml-build.h" +#include "hw/i386/x86.h" +#include <sys/ioctl.h> #define SGX_MAX_EPC_SECTIONS 8 #define SGX_CPUID_EPC_INVALID 0x0 @@ -70,6 +72,57 @@ void sgx_epc_build_srat(GArray *table_data) g_slist_free(device_list); } +static int sgx_remove_all_pages(PCMachineState *pcms, int num) +{ + HostMemoryBackend *hostmem; + SGXEPCDevice *epc; + int failures = 0, failures_1 = 0; + unsigned long ret = 0; + int fd, j; + + for (j = 0; j < num; j++) { + epc = pcms->sgx_epc.sections[j]; + hostmem = MEMORY_BACKEND(epc->hostmem); + fd = memory_region_get_fd(host_memory_backend_get_memory(hostmem)); + + failures = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); + if (failures < 0) { + return failures; + } else if (failures > 0) { + /* Remove SECS pages */ + sleep(1); + failures_1 = ioctl(fd, SGX_IOC_VEPC_REMOVE_ALL); + } + + /* + * The host or guest can support 8 EPC sections, use the + * corresponding bit to show each section removal status. + */ + if (failures_1) { + set_bit(j, &ret); + } + } + + return ret; +} + +void sgx_epc_reset(void *opaque) +{ + PCMachineState *pcms = PC_MACHINE(qdev_get_machine()); + GSList *device_list = sgx_epc_get_device_list(); + int len = g_slist_length(device_list); + int ret; + + do { + ret = sgx_remove_all_pages(pcms, len); + if (ret == -ENOTTY) { + break; + } + } while (ret); + + g_slist_free(device_list); +} + static uint64_t sgx_calc_section_metric(uint64_t low, uint64_t high) { return (low & MAKE_64BIT_MASK(12, 20)) + diff --git a/hw/i386/x86.c b/hw/i386/x86.c index 76de7e2265..03d30a487a 100644 --- a/hw/i386/x86.c +++ b/hw/i386/x86.c @@ -39,6 +39,7 @@ #include "sysemu/replay.h" #include "sysemu/sysemu.h" #include "sysemu/cpu-timers.h" +#include "sysemu/reset.h" #include "trace.h" #include "hw/i386/x86.h" @@ -1307,6 +1308,9 @@ static void machine_set_sgx_epc(Object *obj, Visitor *v, const char *name, visit_type_SgxEPCList(v, name, &x86ms->sgx_epc_list, errp); qapi_free_SgxEPCList(list); + + /* register the reset callback for sgx reset */ + qemu_register_reset(sgx_epc_reset, NULL); } static void x86_machine_initfn(Object *obj)
For bare-metal SGX on real hardware, the hardware provides guarantees SGX state at reboot. For instance, all pages start out uninitialized. The vepc driver provides a similar guarantee today for freshly-opened vepc instances, but guests such as Windows expect all pages to be in uninitialized state on startup, including after every guest reboot. Qemu can invoke the ioctl to bring its vEPC pages back to uninitialized state. There is a possibility that some pages fail to be removed if they are SECS pages, and the child and SECS pages could be in separate vEPC regions. Therefore, the ioctl returns the number of EREMOVE failures, telling Qemu to try the ioctl again after it's done with all vEPC regions. The related kernel patches v4 will be merged into kernel release and link: https://lore.kernel.org/all/20211021201155.1523989-1-pbonzini@redhat.com/ Once this kernel patchset is merged, the kernel commit ids will be updated here. Signed-off-by: Yang Zhong <yang.zhong@intel.com> --- include/hw/i386/x86.h | 1 + linux-headers/linux/kvm.h | 6 +++++ hw/i386/sgx.c | 53 +++++++++++++++++++++++++++++++++++++++ hw/i386/x86.c | 4 +++ 4 files changed, 64 insertions(+)