| Message ID | 20191001125845.8793-6-drjones@redhat.com |
|---|---|
| State | New |
| Headers | show |
| Series | target/arm/kvm: enable SVE in guests | expand |
On Tue, Oct 01, 2019 at 02:58:41PM +0200, Andrew Jones wrote: > These are the SVE equivalents to kvm_arch_get/put_fpsimd. Note, the > swabbing is different than it is for fpsmid because the vector format > is a little-endian stream of words. > > Signed-off-by: Andrew Jones <drjones@redhat.com> Hi Eric and Richard, I dropped your tags from this patch because it changed too much. I apologize for requiring a second look. Thanks, drew > --- > target/arm/kvm64.c | 183 ++++++++++++++++++++++++++++++++++++++------- > 1 file changed, 155 insertions(+), 28 deletions(-) > > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c > index 28f6db57d5ee..4c0b11d105a4 100644 > --- a/target/arm/kvm64.c > +++ b/target/arm/kvm64.c > @@ -671,11 +671,12 @@ int kvm_arch_destroy_vcpu(CPUState *cs) > bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) > { > /* Return true if the regidx is a register we should synchronize > - * via the cpreg_tuples array (ie is not a core reg we sync by > - * hand in kvm_arch_get/put_registers()) > + * via the cpreg_tuples array (ie is not a core or sve reg that > + * we sync by hand in kvm_arch_get/put_registers()) > */ > switch (regidx & KVM_REG_ARM_COPROC_MASK) { > case KVM_REG_ARM_CORE: > + case KVM_REG_ARM64_SVE: > return false; > default: > return true; > @@ -721,10 +722,8 @@ int kvm_arm_cpreg_level(uint64_t regidx) > > static int kvm_arch_put_fpsimd(CPUState *cs) > { > - ARMCPU *cpu = ARM_CPU(cs); > - CPUARMState *env = &cpu->env; > + CPUARMState *env = &ARM_CPU(cs)->env; > struct kvm_one_reg reg; > - uint32_t fpr; > int i, ret; > > for (i = 0; i < 32; i++) { > @@ -742,17 +741,73 @@ static int kvm_arch_put_fpsimd(CPUState *cs) > } > } > > - reg.addr = (uintptr_t)(&fpr); > - fpr = vfp_get_fpsr(env); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > - ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > - if (ret) { > - return ret; > + return 0; > +} > + > +/* > + * SVE registers are encoded in KVM's memory in an endianness-invariant format. > + * The byte at offset i from the start of the in-memory representation contains > + * the bits [(7 + 8 * i) : (8 * i)] of the register value. As this means the > + * lowest offsets are stored in the lowest memory addresses, then that nearly > + * matches QEMU's representation, which is to use an array of host-endian > + * uint64_t's, where the lower offsets are at the lower indices. To complete > + * the translation we just need to byte swap the uint64_t's on big-endian hosts. > + */ > +static uint64_t *sve_bswap64(uint64_t *dst, uint64_t *src, int nr) > +{ > +#ifdef HOST_WORDS_BIGENDIAN > + int i; > + > + for (i = 0; i < nr; ++i) { > + dst[i] = bswap64(src[i]); > } > > - reg.addr = (uintptr_t)(&fpr); > - fpr = vfp_get_fpcr(env); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + return dst; > +#else > + return src; > +#endif > +} > + > +/* > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > + * code the slice index to zero for now as it's unlikely we'll need more than > + * one slice for quite some time. > + */ > +static int kvm_arch_put_sve(CPUState *cs) > +{ > + ARMCPU *cpu = ARM_CPU(cs); > + CPUARMState *env = &cpu->env; > + uint64_t tmp[ARM_MAX_VQ * 2]; > + uint64_t *r; > + struct kvm_one_reg reg; > + int n, ret; > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > + r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + } > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > + r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + } > + > + r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > if (ret) { > return ret; > @@ -765,6 +820,7 @@ int kvm_arch_put_registers(CPUState *cs, int level) > { > struct kvm_one_reg reg; > uint64_t val; > + uint32_t fpr; > int i, ret; > unsigned int el; > > @@ -855,7 +911,27 @@ int kvm_arch_put_registers(CPUState *cs, int level) > } > } > > - ret = kvm_arch_put_fpsimd(cs); > + if (cpu_isar_feature(aa64_sve, cpu)) { > + ret = kvm_arch_put_sve(cs); > + } else { > + ret = kvm_arch_put_fpsimd(cs); > + } > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + fpr = vfp_get_fpsr(env); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + fpr = vfp_get_fpcr(env); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > if (ret) { > return ret; > } > @@ -878,10 +954,8 @@ int kvm_arch_put_registers(CPUState *cs, int level) > > static int kvm_arch_get_fpsimd(CPUState *cs) > { > - ARMCPU *cpu = ARM_CPU(cs); > - CPUARMState *env = &cpu->env; > + CPUARMState *env = &ARM_CPU(cs)->env; > struct kvm_one_reg reg; > - uint32_t fpr; > int i, ret; > > for (i = 0; i < 32; i++) { > @@ -899,21 +973,53 @@ static int kvm_arch_get_fpsimd(CPUState *cs) > } > } > > - reg.addr = (uintptr_t)(&fpr); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > - ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > - if (ret) { > - return ret; > + return 0; > +} > + > +/* > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > + * code the slice index to zero for now as it's unlikely we'll need more than > + * one slice for quite some time. > + */ > +static int kvm_arch_get_sve(CPUState *cs) > +{ > + ARMCPU *cpu = ARM_CPU(cs); > + CPUARMState *env = &cpu->env; > + struct kvm_one_reg reg; > + uint64_t *r; > + int n, ret; > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > + r = &env->vfp.zregs[n].d[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + sve_bswap64(r, r, cpu->sve_max_vq * 2); > } > - vfp_set_fpsr(env, fpr); > > - reg.addr = (uintptr_t)(&fpr); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > + r = &env->vfp.pregs[n].p[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + } > + > + r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > if (ret) { > return ret; > } > - vfp_set_fpcr(env, fpr); > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > return 0; > } > @@ -923,6 +1029,7 @@ int kvm_arch_get_registers(CPUState *cs) > struct kvm_one_reg reg; > uint64_t val; > unsigned int el; > + uint32_t fpr; > int i, ret; > > ARMCPU *cpu = ARM_CPU(cs); > @@ -1012,10 +1119,30 @@ int kvm_arch_get_registers(CPUState *cs) > env->spsr = env->banked_spsr[i]; > } > > - ret = kvm_arch_get_fpsimd(cs); > + if (cpu_isar_feature(aa64_sve, cpu)) { > + ret = kvm_arch_get_sve(cs); > + } else { > + ret = kvm_arch_get_fpsimd(cs); > + } > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + vfp_set_fpsr(env, fpr); > + > + reg.addr = (uintptr_t)(&fpr); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > if (ret) { > return ret; > } > + vfp_set_fpcr(env, fpr); > > ret = kvm_get_vcpu_events(cpu); > if (ret) { > -- > 2.20.1 >
On Tue, Oct 01, 2019 at 03:52:46PM +0200, Andrew Jones wrote: > On Tue, Oct 01, 2019 at 02:58:41PM +0200, Andrew Jones wrote: > > These are the SVE equivalents to kvm_arch_get/put_fpsimd. Note, the > > swabbing is different than it is for fpsmid because the vector format > > is a little-endian stream of words. > > > > Signed-off-by: Andrew Jones <drjones@redhat.com> > > Hi Eric and Richard, > > I dropped your tags from this patch because it changed too much. > I apologize for requiring a second look. Also it looks like I should have generated it with --patience, as this diff isn't the easiest thing to review. Let me know if you'd like me to repost it. drew > > Thanks, > drew > > > --- > > target/arm/kvm64.c | 183 ++++++++++++++++++++++++++++++++++++++------- > > 1 file changed, 155 insertions(+), 28 deletions(-) > > > > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c > > index 28f6db57d5ee..4c0b11d105a4 100644 > > --- a/target/arm/kvm64.c > > +++ b/target/arm/kvm64.c > > @@ -671,11 +671,12 @@ int kvm_arch_destroy_vcpu(CPUState *cs) > > bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) > > { > > /* Return true if the regidx is a register we should synchronize > > - * via the cpreg_tuples array (ie is not a core reg we sync by > > - * hand in kvm_arch_get/put_registers()) > > + * via the cpreg_tuples array (ie is not a core or sve reg that > > + * we sync by hand in kvm_arch_get/put_registers()) > > */ > > switch (regidx & KVM_REG_ARM_COPROC_MASK) { > > case KVM_REG_ARM_CORE: > > + case KVM_REG_ARM64_SVE: > > return false; > > default: > > return true; > > @@ -721,10 +722,8 @@ int kvm_arm_cpreg_level(uint64_t regidx) > > > > static int kvm_arch_put_fpsimd(CPUState *cs) > > { > > - ARMCPU *cpu = ARM_CPU(cs); > > - CPUARMState *env = &cpu->env; > > + CPUARMState *env = &ARM_CPU(cs)->env; > > struct kvm_one_reg reg; > > - uint32_t fpr; > > int i, ret; > > > > for (i = 0; i < 32; i++) { > > @@ -742,17 +741,73 @@ static int kvm_arch_put_fpsimd(CPUState *cs) > > } > > } > > > > - reg.addr = (uintptr_t)(&fpr); > > - fpr = vfp_get_fpsr(env); > > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > > - ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > - if (ret) { > > - return ret; > > + return 0; > > +} > > + > > +/* > > + * SVE registers are encoded in KVM's memory in an endianness-invariant format. > > + * The byte at offset i from the start of the in-memory representation contains > > + * the bits [(7 + 8 * i) : (8 * i)] of the register value. As this means the > > + * lowest offsets are stored in the lowest memory addresses, then that nearly > > + * matches QEMU's representation, which is to use an array of host-endian > > + * uint64_t's, where the lower offsets are at the lower indices. To complete > > + * the translation we just need to byte swap the uint64_t's on big-endian hosts. > > + */ > > +static uint64_t *sve_bswap64(uint64_t *dst, uint64_t *src, int nr) > > +{ > > +#ifdef HOST_WORDS_BIGENDIAN > > + int i; > > + > > + for (i = 0; i < nr; ++i) { > > + dst[i] = bswap64(src[i]); > > } > > > > - reg.addr = (uintptr_t)(&fpr); > > - fpr = vfp_get_fpcr(env); > > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > > + return dst; > > +#else > > + return src; > > +#endif > > +} > > + > > +/* > > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > > + * code the slice index to zero for now as it's unlikely we'll need more than > > + * one slice for quite some time. > > + */ > > +static int kvm_arch_put_sve(CPUState *cs) > > +{ > > + ARMCPU *cpu = ARM_CPU(cs); > > + CPUARMState *env = &cpu->env; > > + uint64_t tmp[ARM_MAX_VQ * 2]; > > + uint64_t *r; > > + struct kvm_one_reg reg; > > + int n, ret; > > + > > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > > + r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + } > > + > > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > > + r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], > > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + } > > + > > + r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], > > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > > ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > if (ret) { > > return ret; > > @@ -765,6 +820,7 @@ int kvm_arch_put_registers(CPUState *cs, int level) > > { > > struct kvm_one_reg reg; > > uint64_t val; > > + uint32_t fpr; > > int i, ret; > > unsigned int el; > > > > @@ -855,7 +911,27 @@ int kvm_arch_put_registers(CPUState *cs, int level) > > } > > } > > > > - ret = kvm_arch_put_fpsimd(cs); > > + if (cpu_isar_feature(aa64_sve, cpu)) { > > + ret = kvm_arch_put_sve(cs); > > + } else { > > + ret = kvm_arch_put_fpsimd(cs); > > + } > > + if (ret) { > > + return ret; > > + } > > + > > + reg.addr = (uintptr_t)(&fpr); > > + fpr = vfp_get_fpsr(env); > > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + > > + reg.addr = (uintptr_t)(&fpr); > > + fpr = vfp_get_fpcr(env); > > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > > if (ret) { > > return ret; > > } > > @@ -878,10 +954,8 @@ int kvm_arch_put_registers(CPUState *cs, int level) > > > > static int kvm_arch_get_fpsimd(CPUState *cs) > > { > > - ARMCPU *cpu = ARM_CPU(cs); > > - CPUARMState *env = &cpu->env; > > + CPUARMState *env = &ARM_CPU(cs)->env; > > struct kvm_one_reg reg; > > - uint32_t fpr; > > int i, ret; > > > > for (i = 0; i < 32; i++) { > > @@ -899,21 +973,53 @@ static int kvm_arch_get_fpsimd(CPUState *cs) > > } > > } > > > > - reg.addr = (uintptr_t)(&fpr); > > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > > - ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > - if (ret) { > > - return ret; > > + return 0; > > +} > > + > > +/* > > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > > + * code the slice index to zero for now as it's unlikely we'll need more than > > + * one slice for quite some time. > > + */ > > +static int kvm_arch_get_sve(CPUState *cs) > > +{ > > + ARMCPU *cpu = ARM_CPU(cs); > > + CPUARMState *env = &cpu->env; > > + struct kvm_one_reg reg; > > + uint64_t *r; > > + int n, ret; > > + > > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > > + r = &env->vfp.zregs[n].d[0]; > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + sve_bswap64(r, r, cpu->sve_max_vq * 2); > > } > > - vfp_set_fpsr(env, fpr); > > > > - reg.addr = (uintptr_t)(&fpr); > > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > > + r = &env->vfp.pregs[n].p[0]; > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > + } > > + > > + r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; > > + reg.addr = (uintptr_t)r; > > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > > ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > if (ret) { > > return ret; > > } > > - vfp_set_fpcr(env, fpr); > > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > > > return 0; > > } > > @@ -923,6 +1029,7 @@ int kvm_arch_get_registers(CPUState *cs) > > struct kvm_one_reg reg; > > uint64_t val; > > unsigned int el; > > + uint32_t fpr; > > int i, ret; > > > > ARMCPU *cpu = ARM_CPU(cs); > > @@ -1012,10 +1119,30 @@ int kvm_arch_get_registers(CPUState *cs) > > env->spsr = env->banked_spsr[i]; > > } > > > > - ret = kvm_arch_get_fpsimd(cs); > > + if (cpu_isar_feature(aa64_sve, cpu)) { > > + ret = kvm_arch_get_sve(cs); > > + } else { > > + ret = kvm_arch_get_fpsimd(cs); > > + } > > + if (ret) { > > + return ret; > > + } > > + > > + reg.addr = (uintptr_t)(&fpr); > > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > + if (ret) { > > + return ret; > > + } > > + vfp_set_fpsr(env, fpr); > > + > > + reg.addr = (uintptr_t)(&fpr); > > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > > if (ret) { > > return ret; > > } > > + vfp_set_fpcr(env, fpr); > > > > ret = kvm_get_vcpu_events(cpu); > > if (ret) { > > -- > > 2.20.1 > >
On 10/1/19 5:58 AM, Andrew Jones wrote: > These are the SVE equivalents to kvm_arch_get/put_fpsimd. Note, the > swabbing is different than it is for fpsmid because the vector format > is a little-endian stream of words. > > Signed-off-by: Andrew Jones <drjones@redhat.com> > --- > target/arm/kvm64.c | 183 ++++++++++++++++++++++++++++++++++++++------- > 1 file changed, 155 insertions(+), 28 deletions(-) Reviewed-by: Richard Henderson <richard.henderson@linaro.org> r~
Hi Drew, On 10/1/19 2:58 PM, Andrew Jones wrote: > These are the SVE equivalents to kvm_arch_get/put_fpsimd. Note, the > swabbing is different than it is for fpsmid because the vector format > is a little-endian stream of words. > > Signed-off-by: Andrew Jones <drjones@redhat.com> Reviewed-by: Eric Auger <eric.auger@redhat.com> Eric > --- > target/arm/kvm64.c | 183 ++++++++++++++++++++++++++++++++++++++------- > 1 file changed, 155 insertions(+), 28 deletions(-) > > diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c > index 28f6db57d5ee..4c0b11d105a4 100644 > --- a/target/arm/kvm64.c > +++ b/target/arm/kvm64.c > @@ -671,11 +671,12 @@ int kvm_arch_destroy_vcpu(CPUState *cs) > bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) > { > /* Return true if the regidx is a register we should synchronize > - * via the cpreg_tuples array (ie is not a core reg we sync by > - * hand in kvm_arch_get/put_registers()) > + * via the cpreg_tuples array (ie is not a core or sve reg that > + * we sync by hand in kvm_arch_get/put_registers()) > */ > switch (regidx & KVM_REG_ARM_COPROC_MASK) { > case KVM_REG_ARM_CORE: > + case KVM_REG_ARM64_SVE: > return false; > default: > return true; > @@ -721,10 +722,8 @@ int kvm_arm_cpreg_level(uint64_t regidx) > > static int kvm_arch_put_fpsimd(CPUState *cs) > { > - ARMCPU *cpu = ARM_CPU(cs); > - CPUARMState *env = &cpu->env; > + CPUARMState *env = &ARM_CPU(cs)->env; > struct kvm_one_reg reg; > - uint32_t fpr; > int i, ret; > > for (i = 0; i < 32; i++) { > @@ -742,17 +741,73 @@ static int kvm_arch_put_fpsimd(CPUState *cs) > } > } > > - reg.addr = (uintptr_t)(&fpr); > - fpr = vfp_get_fpsr(env); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > - ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > - if (ret) { > - return ret; > + return 0; > +} > + > +/* > + * SVE registers are encoded in KVM's memory in an endianness-invariant format. > + * The byte at offset i from the start of the in-memory representation contains > + * the bits [(7 + 8 * i) : (8 * i)] of the register value. As this means the > + * lowest offsets are stored in the lowest memory addresses, then that nearly > + * matches QEMU's representation, which is to use an array of host-endian > + * uint64_t's, where the lower offsets are at the lower indices. To complete > + * the translation we just need to byte swap the uint64_t's on big-endian hosts. > + */ > +static uint64_t *sve_bswap64(uint64_t *dst, uint64_t *src, int nr) > +{ > +#ifdef HOST_WORDS_BIGENDIAN > + int i; > + > + for (i = 0; i < nr; ++i) { > + dst[i] = bswap64(src[i]); > } > > - reg.addr = (uintptr_t)(&fpr); > - fpr = vfp_get_fpcr(env); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + return dst; > +#else > + return src; > +#endif > +} > + > +/* > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > + * code the slice index to zero for now as it's unlikely we'll need more than > + * one slice for quite some time. > + */ > +static int kvm_arch_put_sve(CPUState *cs) > +{ > + ARMCPU *cpu = ARM_CPU(cs); > + CPUARMState *env = &cpu->env; > + uint64_t tmp[ARM_MAX_VQ * 2]; > + uint64_t *r; > + struct kvm_one_reg reg; > + int n, ret; > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > + r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + } > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > + r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + } > + > + r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], > + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > if (ret) { > return ret; > @@ -765,6 +820,7 @@ int kvm_arch_put_registers(CPUState *cs, int level) > { > struct kvm_one_reg reg; > uint64_t val; > + uint32_t fpr; > int i, ret; > unsigned int el; > > @@ -855,7 +911,27 @@ int kvm_arch_put_registers(CPUState *cs, int level) > } > } > > - ret = kvm_arch_put_fpsimd(cs); > + if (cpu_isar_feature(aa64_sve, cpu)) { > + ret = kvm_arch_put_sve(cs); > + } else { > + ret = kvm_arch_put_fpsimd(cs); > + } > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + fpr = vfp_get_fpsr(env); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + fpr = vfp_get_fpcr(env); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); > if (ret) { > return ret; > } > @@ -878,10 +954,8 @@ int kvm_arch_put_registers(CPUState *cs, int level) > > static int kvm_arch_get_fpsimd(CPUState *cs) > { > - ARMCPU *cpu = ARM_CPU(cs); > - CPUARMState *env = &cpu->env; > + CPUARMState *env = &ARM_CPU(cs)->env; > struct kvm_one_reg reg; > - uint32_t fpr; > int i, ret; > > for (i = 0; i < 32; i++) { > @@ -899,21 +973,53 @@ static int kvm_arch_get_fpsimd(CPUState *cs) > } > } > > - reg.addr = (uintptr_t)(&fpr); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > - ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > - if (ret) { > - return ret; > + return 0; > +} > + > +/* > + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits > + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard > + * code the slice index to zero for now as it's unlikely we'll need more than > + * one slice for quite some time. > + */ > +static int kvm_arch_get_sve(CPUState *cs) > +{ > + ARMCPU *cpu = ARM_CPU(cs); > + CPUARMState *env = &cpu->env; > + struct kvm_one_reg reg; > + uint64_t *r; > + int n, ret; > + > + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { > + r = &env->vfp.zregs[n].d[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + sve_bswap64(r, r, cpu->sve_max_vq * 2); > } > - vfp_set_fpsr(env, fpr); > > - reg.addr = (uintptr_t)(&fpr); > - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { > + r = &env->vfp.pregs[n].p[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > + } > + > + r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; > + reg.addr = (uintptr_t)r; > + reg.id = KVM_REG_ARM64_SVE_FFR(0); > ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > if (ret) { > return ret; > } > - vfp_set_fpcr(env, fpr); > + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); > > return 0; > } > @@ -923,6 +1029,7 @@ int kvm_arch_get_registers(CPUState *cs) > struct kvm_one_reg reg; > uint64_t val; > unsigned int el; > + uint32_t fpr; > int i, ret; > > ARMCPU *cpu = ARM_CPU(cs); > @@ -1012,10 +1119,30 @@ int kvm_arch_get_registers(CPUState *cs) > env->spsr = env->banked_spsr[i]; > } > > - ret = kvm_arch_get_fpsimd(cs); > + if (cpu_isar_feature(aa64_sve, cpu)) { > + ret = kvm_arch_get_sve(cs); > + } else { > + ret = kvm_arch_get_fpsimd(cs); > + } > + if (ret) { > + return ret; > + } > + > + reg.addr = (uintptr_t)(&fpr); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > + if (ret) { > + return ret; > + } > + vfp_set_fpsr(env, fpr); > + > + reg.addr = (uintptr_t)(&fpr); > + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); > + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); > if (ret) { > return ret; > } > + vfp_set_fpcr(env, fpr); > > ret = kvm_get_vcpu_events(cpu); > if (ret) { >
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index 28f6db57d5ee..4c0b11d105a4 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -671,11 +671,12 @@ int kvm_arch_destroy_vcpu(CPUState *cs) bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) { /* Return true if the regidx is a register we should synchronize - * via the cpreg_tuples array (ie is not a core reg we sync by - * hand in kvm_arch_get/put_registers()) + * via the cpreg_tuples array (ie is not a core or sve reg that + * we sync by hand in kvm_arch_get/put_registers()) */ switch (regidx & KVM_REG_ARM_COPROC_MASK) { case KVM_REG_ARM_CORE: + case KVM_REG_ARM64_SVE: return false; default: return true; @@ -721,10 +722,8 @@ int kvm_arm_cpreg_level(uint64_t regidx) static int kvm_arch_put_fpsimd(CPUState *cs) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUARMState *env = &ARM_CPU(cs)->env; struct kvm_one_reg reg; - uint32_t fpr; int i, ret; for (i = 0; i < 32; i++) { @@ -742,17 +741,73 @@ static int kvm_arch_put_fpsimd(CPUState *cs) } } - reg.addr = (uintptr_t)(&fpr); - fpr = vfp_get_fpsr(env); - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); - ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); - if (ret) { - return ret; + return 0; +} + +/* + * SVE registers are encoded in KVM's memory in an endianness-invariant format. + * The byte at offset i from the start of the in-memory representation contains + * the bits [(7 + 8 * i) : (8 * i)] of the register value. As this means the + * lowest offsets are stored in the lowest memory addresses, then that nearly + * matches QEMU's representation, which is to use an array of host-endian + * uint64_t's, where the lower offsets are at the lower indices. To complete + * the translation we just need to byte swap the uint64_t's on big-endian hosts. + */ +static uint64_t *sve_bswap64(uint64_t *dst, uint64_t *src, int nr) +{ +#ifdef HOST_WORDS_BIGENDIAN + int i; + + for (i = 0; i < nr; ++i) { + dst[i] = bswap64(src[i]); } - reg.addr = (uintptr_t)(&fpr); - fpr = vfp_get_fpcr(env); - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); + return dst; +#else + return src; +#endif +} + +/* + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard + * code the slice index to zero for now as it's unlikely we'll need more than + * one slice for quite some time. + */ +static int kvm_arch_put_sve(CPUState *cs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + uint64_t tmp[ARM_MAX_VQ * 2]; + uint64_t *r; + struct kvm_one_reg reg; + int n, ret; + + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { + r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); + if (ret) { + return ret; + } + } + + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { + r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); + if (ret) { + return ret; + } + } + + r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], + DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_FFR(0); ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); if (ret) { return ret; @@ -765,6 +820,7 @@ int kvm_arch_put_registers(CPUState *cs, int level) { struct kvm_one_reg reg; uint64_t val; + uint32_t fpr; int i, ret; unsigned int el; @@ -855,7 +911,27 @@ int kvm_arch_put_registers(CPUState *cs, int level) } } - ret = kvm_arch_put_fpsimd(cs); + if (cpu_isar_feature(aa64_sve, cpu)) { + ret = kvm_arch_put_sve(cs); + } else { + ret = kvm_arch_put_fpsimd(cs); + } + if (ret) { + return ret; + } + + reg.addr = (uintptr_t)(&fpr); + fpr = vfp_get_fpsr(env); + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); + if (ret) { + return ret; + } + + reg.addr = (uintptr_t)(&fpr); + fpr = vfp_get_fpcr(env); + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); + ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); if (ret) { return ret; } @@ -878,10 +954,8 @@ int kvm_arch_put_registers(CPUState *cs, int level) static int kvm_arch_get_fpsimd(CPUState *cs) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUARMState *env = &ARM_CPU(cs)->env; struct kvm_one_reg reg; - uint32_t fpr; int i, ret; for (i = 0; i < 32; i++) { @@ -899,21 +973,53 @@ static int kvm_arch_get_fpsimd(CPUState *cs) } } - reg.addr = (uintptr_t)(&fpr); - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); - ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); - if (ret) { - return ret; + return 0; +} + +/* + * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits + * and PREGS and the FFR have a slice size of 256 bits. However we simply hard + * code the slice index to zero for now as it's unlikely we'll need more than + * one slice for quite some time. + */ +static int kvm_arch_get_sve(CPUState *cs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + struct kvm_one_reg reg; + uint64_t *r; + int n, ret; + + for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { + r = &env->vfp.zregs[n].d[0]; + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); + if (ret) { + return ret; + } + sve_bswap64(r, r, cpu->sve_max_vq * 2); } - vfp_set_fpsr(env, fpr); - reg.addr = (uintptr_t)(&fpr); - reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); + for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { + r = &env->vfp.pregs[n].p[0]; + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); + if (ret) { + return ret; + } + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); + } + + r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; + reg.addr = (uintptr_t)r; + reg.id = KVM_REG_ARM64_SVE_FFR(0); ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); if (ret) { return ret; } - vfp_set_fpcr(env, fpr); + sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); return 0; } @@ -923,6 +1029,7 @@ int kvm_arch_get_registers(CPUState *cs) struct kvm_one_reg reg; uint64_t val; unsigned int el; + uint32_t fpr; int i, ret; ARMCPU *cpu = ARM_CPU(cs); @@ -1012,10 +1119,30 @@ int kvm_arch_get_registers(CPUState *cs) env->spsr = env->banked_spsr[i]; } - ret = kvm_arch_get_fpsimd(cs); + if (cpu_isar_feature(aa64_sve, cpu)) { + ret = kvm_arch_get_sve(cs); + } else { + ret = kvm_arch_get_fpsimd(cs); + } + if (ret) { + return ret; + } + + reg.addr = (uintptr_t)(&fpr); + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); + if (ret) { + return ret; + } + vfp_set_fpsr(env, fpr); + + reg.addr = (uintptr_t)(&fpr); + reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); + ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); if (ret) { return ret; } + vfp_set_fpcr(env, fpr); ret = kvm_get_vcpu_events(cpu); if (ret) {
These are the SVE equivalents to kvm_arch_get/put_fpsimd. Note, the swabbing is different than it is for fpsmid because the vector format is a little-endian stream of words. Signed-off-by: Andrew Jones <drjones@redhat.com> --- target/arm/kvm64.c | 183 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 155 insertions(+), 28 deletions(-)