| Message ID | 20101004185715.167557459@redhat.com |
|---|---|
| State | New |
| Headers | show |
(2010/10/05 3:54), Marcelo Tosatti wrote: > Port qemu-kvm's > > commit 4b62fff1101a7ad77553147717a8bd3bf79df7ef > Author: Huang Ying <ying.huang@intel.com> > Date: Mon Sep 21 10:43:25 2009 +0800 > > MCE: Relay UCR MCE to guest > > UCR (uncorrected recovery) MCE is supported in recent Intel CPUs, > where some hardware error such as some memory error can be reported > without PCC (processor context corrupted). To recover from such MCE, > the corresponding memory will be unmapped, and all processes accessing > the memory will be killed via SIGBUS. > > For KVM, if QEMU/KVM is killed, all guest processes will be killed > too. So we relay SIGBUS from host OS to guest system via a UCR MCE > injection. Then guest OS can isolate corresponding memory and kill > necessary guest processes only. SIGBUS sent to main thread (not VCPU > threads) will be broadcast to all VCPU threads as UCR MCE. > > Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com> > (snip) > +static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo, > + void *ctx) > +{ > +#if defined(TARGET_I386) > + if (kvm_on_sigbus_vcpu(siginfo->ssi_code, (void *)(intptr_t)siginfo->ssi_addr)) > +#endif > + sigbus_reraise(); > +} > + > static void qemu_kvm_eat_signal(CPUState *env, int timeout) > { > struct timespec ts; > int r, e; > siginfo_t siginfo; > sigset_t waitset; > + sigset_t chkset; > > ts.tv_sec = timeout / 1000; > ts.tv_nsec = (timeout % 1000) * 1000000; > > sigemptyset(&waitset); > sigaddset(&waitset, SIG_IPI); > + sigaddset(&waitset, SIGBUS); > > - qemu_mutex_unlock(&qemu_global_mutex); > - r = sigtimedwait(&waitset, &siginfo, &ts); > - e = errno; > - qemu_mutex_lock(&qemu_global_mutex); > + do { > + qemu_mutex_unlock(&qemu_global_mutex); > > - if (r == -1 && !(e == EAGAIN || e == EINTR)) { > - fprintf(stderr, "sigtimedwait: %s\n", strerror(e)); > - exit(1); > - } > + r = sigtimedwait(&waitset, &siginfo, &ts); > + e = errno; > + > + qemu_mutex_lock(&qemu_global_mutex); > + > + if (r == -1 && !(e == EAGAIN || e == EINTR)) { > + fprintf(stderr, "sigtimedwait: %s\n", strerror(e)); > + exit(1); > + } > + > + switch (r) { > + case SIGBUS: > +#ifdef TARGET_I386 > + if (kvm_on_sigbus(env, siginfo.si_code, siginfo.si_addr)) > +#endif > + sigbus_reraise(); > + break; > + default: > + break; > + } > + > + r = sigpending(&chkset); > + if (r == -1) { > + fprintf(stderr, "sigpending: %s\n", strerror(e)); > + exit(1); > + } > + } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS)); > } > > static void qemu_kvm_wait_io_event(CPUState *env) (snip) > Index: qemu/kvm.h > =================================================================== > --- qemu.orig/kvm.h > +++ qemu/kvm.h > @@ -110,6 +110,9 @@ int kvm_arch_init_vcpu(CPUState *env); > > void kvm_arch_reset_vcpu(CPUState *env); > > +int kvm_on_sigbus(CPUState *env, int code, void *addr); > +int kvm_on_sigbus_vcpu(int code, void *addr); > + > struct kvm_guest_debug; > struct kvm_debug_exit_arch; > So kvm_on_sigbus() is called from qemu_kvm_eat_signal() that is called on vcpu thread, while kvm_on_sigbus_vcpu() is called via sigbus_handler that invoked on iothread using signalfd. ... Inverse naming? Thanks, H.Seto
I got some more question: (2010/10/05 3:54), Marcelo Tosatti wrote: > Index: qemu/target-i386/cpu.h > =================================================================== > --- qemu.orig/target-i386/cpu.h > +++ qemu/target-i386/cpu.h > @@ -250,16 +250,32 @@ > #define PG_ERROR_RSVD_MASK 0x08 > #define PG_ERROR_I_D_MASK 0x10 > > -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ > +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ > +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ > > -#define MCE_CAP_DEF MCG_CTL_P > +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) > #define MCE_BANKS_DEF 10 > It seems that current kvm doesn't support SER_P, so injecting SRAO to guest will mean that guest receives VAL|UC|!PCC and RIPV event from virtual processor that doesn't have SER_P. I think most OSes don't expect that it can receives MCE with !PCC on traditional x86 processor without SER_P. Q1: Is it safe to expect that guests can handle such !PCC event? Q2: What is the expected behavior on the guest? Q3: What happen if guest reboots itself in response to the MCE? Thanks, H.Seto
On Wed, Oct 06, 2010 at 10:10:51AM +0900, Hidetoshi Seto wrote: > > (snip) > > > Index: qemu/kvm.h > > =================================================================== > > --- qemu.orig/kvm.h > > +++ qemu/kvm.h > > @@ -110,6 +110,9 @@ int kvm_arch_init_vcpu(CPUState *env); > > > > void kvm_arch_reset_vcpu(CPUState *env); > > > > +int kvm_on_sigbus(CPUState *env, int code, void *addr); > > +int kvm_on_sigbus_vcpu(int code, void *addr); > > + > > struct kvm_guest_debug; > > struct kvm_debug_exit_arch; > > > > So kvm_on_sigbus() is called from qemu_kvm_eat_signal() that is > called on vcpu thread, while kvm_on_sigbus_vcpu() is called via > sigbus_handler that invoked on iothread using signalfd. > > ... Inverse naming? Yes, fixed.
On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: > I got some more question: > > (2010/10/05 3:54), Marcelo Tosatti wrote: > > Index: qemu/target-i386/cpu.h > > =================================================================== > > --- qemu.orig/target-i386/cpu.h > > +++ qemu/target-i386/cpu.h > > @@ -250,16 +250,32 @@ > > #define PG_ERROR_RSVD_MASK 0x08 > > #define PG_ERROR_I_D_MASK 0x10 > > > > -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ > > +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ > > +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ > > > > -#define MCE_CAP_DEF MCG_CTL_P > > +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) > > #define MCE_BANKS_DEF 10 > > > > It seems that current kvm doesn't support SER_P, so injecting SRAO > to guest will mean that guest receives VAL|UC|!PCC and RIPV event > from virtual processor that doesn't have SER_P. Dean also noted this. I don't think it was deliberate choice to not expose SER_P. Huang? > I think most OSes don't expect that it can receives MCE with !PCC > on traditional x86 processor without SER_P. > > Q1: Is it safe to expect that guests can handle such !PCC event? > Q2: What is the expected behavior on the guest? > Q3: What happen if guest reboots itself in response to the MCE? > > > Thanks, > H.Seto
On 10/06/2010 11:05 AM, Marcelo Tosatti wrote: > On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: >> I got some more question: >> >> (2010/10/05 3:54), Marcelo Tosatti wrote: >>> Index: qemu/target-i386/cpu.h >>> =================================================================== >>> --- qemu.orig/target-i386/cpu.h >>> +++ qemu/target-i386/cpu.h >>> @@ -250,16 +250,32 @@ >>> #define PG_ERROR_RSVD_MASK 0x08 >>> #define PG_ERROR_I_D_MASK 0x10 >>> >>> -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ >>> +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ >>> +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ >>> >>> -#define MCE_CAP_DEF MCG_CTL_P >>> +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) >>> #define MCE_BANKS_DEF 10 >>> >> >> It seems that current kvm doesn't support SER_P, so injecting SRAO >> to guest will mean that guest receives VAL|UC|!PCC and RIPV event >> from virtual processor that doesn't have SER_P. > > Dean also noted this. I don't think it was deliberate choice to not > expose SER_P. Huang? In my testing, I found that MCG_SER_P was not being set (and I was running on a Nehalem-EX system). Injecting a MCE resulted in the guest entering into panic() from mce_panic(). If crash_kexec() finds a kexec_crash_image the system ends up rebooting, otherwise, what happens next requires operator intervention. When I applied a patch to the guest's kernel which forces mce_ser to be set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found that when the memory page was 'owned' by a guest process, the process would be killed (if the page was dirty), and the guest would stay running. The HWPoisoned page would be sidelined and not cause any more issues. >> I think most OSes don't expect that it can receives MCE with !PCC >> on traditional x86 processor without SER_P. >> >> Q1: Is it safe to expect that guests can handle such !PCC event? This might be best answered by Huang, but as I mentioned above, without MCG_SER_P being set, the result was an orderly system panic on the guest. >> Q2: What is the expected behavior on the guest? I think I answered this above. >> Q3: What happen if guest reboots itself in response to the MCE? That depends... And the following issue also holds for a guest that is rebooted at some point having successfully sidelined the bad page. After the guest has panic'd, a system_reset of the guest or a restart initiated by crash_kexec() (called by panic() on the guest), usually results in the guest hanging because the bad page still belongs to qemu-kvm and is now being referenced by the new guest in some way. (It actually may not hang, but successfully reboot and be runnable, with the bad page lurking in the background. It all seems to depend on where the bad page ends up, and whether it's ever referenced.) I believe there was an attempt to deal with this in kvm on the host. See kvm_handle_bad_page(). This function was suppose to result in the sending of a BUS_MCEERR_AR flavored SIGBUS by do_sigbus() to qemu-kvm which in theory would result in the right thing happening. But commit 96054569190bdec375fe824e48ca1f4e3b53dd36 prevents the signal from being sent. So this mechanism needs to be re-worked, and the issue remains. I would think that if the the bad page can't be sidelined, such that the newly booting guest can't use it, then the new guest shouldn't be allowed to boot. But perhaps there is some merit in letting it try to boot and see if one gets 'lucky'. I understand that Huang is looking into what should be done. He can give you better information than I in answer to your questions. Dean
(2010/10/07 3:10), Dean Nelson wrote: > On 10/06/2010 11:05 AM, Marcelo Tosatti wrote: >> On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: >>> I got some more question: >>> >>> (2010/10/05 3:54), Marcelo Tosatti wrote: >>>> Index: qemu/target-i386/cpu.h >>>> =================================================================== >>>> --- qemu.orig/target-i386/cpu.h >>>> +++ qemu/target-i386/cpu.h >>>> @@ -250,16 +250,32 @@ >>>> #define PG_ERROR_RSVD_MASK 0x08 >>>> #define PG_ERROR_I_D_MASK 0x10 >>>> >>>> -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ >>>> +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ >>>> +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ >>>> >>>> -#define MCE_CAP_DEF MCG_CTL_P >>>> +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) >>>> #define MCE_BANKS_DEF 10 >>>> >>> >>> It seems that current kvm doesn't support SER_P, so injecting SRAO >>> to guest will mean that guest receives VAL|UC|!PCC and RIPV event >>> from virtual processor that doesn't have SER_P. >> >> Dean also noted this. I don't think it was deliberate choice to not >> expose SER_P. Huang? > > In my testing, I found that MCG_SER_P was not being set (and I was > running on a Nehalem-EX system). Injecting a MCE resulted in the > guest entering into panic() from mce_panic(). If crash_kexec() > finds a kexec_crash_image the system ends up rebooting, otherwise, > what happens next requires operator intervention. Good to know. What I'm concerning is that if memory scrubbing SRAO event is injected when !SER_P, linux guest with certain mce tolerant level might grade it as "UC" severity and continue running with none of panicking, killing and poisoning because of !PCC and RIPV. Could you provide the panic message of the guest in your test? I think it can tell me why the mce handler decided to go panic. > When I applied a patch to the guest's kernel which forces mce_ser to be > set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found > that when the memory page was 'owned' by a guest process, the process > would be killed (if the page was dirty), and the guest would stay > running. The HWPoisoned page would be sidelined and not cause any more > issues. Excellent. So while guest kernel knows which page is poisoned, guest processes are controlled not to touch the page. ... Therefore rebooting the vm and renewing kernel will lost the information where is poisoned. >>> I think most OSes don't expect that it can receives MCE with !PCC >>> on traditional x86 processor without SER_P. >>> >>> Q1: Is it safe to expect that guests can handle such !PCC event? > > This might be best answered by Huang, but as I mentioned above, without > MCG_SER_P being set, the result was an orderly system panic on the > guest. Though I'll wait Huang (I think he is on holiday), I believe that system panic is just a possible option for AO (Action Optional) event, no matter how the SER_P is. >>> Q2: What is the expected behavior on the guest? > > I think I answered this above. Yeah, thanks. > >>> Q3: What happen if guest reboots itself in response to the MCE? > > That depends... > > And the following issue also holds for a guest that is rebooted at > some point having successfully sidelined the bad page. > > After the guest has panic'd, a system_reset of the guest or a restart > initiated by crash_kexec() (called by panic() on the guest), usually > results in the guest hanging because the bad page still belongs > to qemu-kvm and is now being referenced by the new guest in some way. Yes. In other words my concern about reboot is that new guest kernel including kdump kernel might try to read the bad page. If there is no AR-SIGBUS etc., we need some tricks to inhibit such accesses. > (It actually may not hang, but successfully reboot and be runnable, > with the bad page lurking in the background. It all seems to depend on > where the bad page ends up, and whether it's ever referenced.) I know some tough guys using their PC with buggy DIMMs :-) > > I believe there was an attempt to deal with this in kvm on the host. > See kvm_handle_bad_page(). This function was suppose to result in the > sending of a BUS_MCEERR_AR flavored SIGBUS by do_sigbus() to qemu-kvm > which in theory would result in the right thing happening. But commit > 96054569190bdec375fe824e48ca1f4e3b53dd36 prevents the signal from being > sent. So this mechanism needs to be re-worked, and the issue remains. Definitely. I guess Huang has some plan or hint for rework this point. > > I would think that if the the bad page can't be sidelined, such that > the newly booting guest can't use it, then the new guest shouldn't be > allowed to boot. But perhaps there is some merit in letting it try to > boot and see if one gets 'lucky'. In case of booting a real machine in real world, hardware and firmware usually (or often) do self-test before passing control to OS. Some platform can boot OS with degraded configuration (for example, fewer memory) if it has trouble on its component. Some BIOS may stop booting and show messages like "please reseat [component]" on the screen. So we could implement/request qemu to have such mechanism. I can understand the merit you mentioned here, in some degree. But I think it is hard to say "unlucky" to customer in business... > > I understand that Huang is looking into what should be done. He can > give you better information than I in answer to your questions. Agreed. Thank you very much! Thanks, H.Seto
On 10/06/2010 10:41 PM, Hidetoshi Seto wrote: > (2010/10/07 3:10), Dean Nelson wrote: >> On 10/06/2010 11:05 AM, Marcelo Tosatti wrote: >>> On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: >>>> I got some more question: >>>> >>>> (2010/10/05 3:54), Marcelo Tosatti wrote: >>>>> Index: qemu/target-i386/cpu.h >>>>> =================================================================== >>>>> --- qemu.orig/target-i386/cpu.h >>>>> +++ qemu/target-i386/cpu.h >>>>> @@ -250,16 +250,32 @@ >>>>> #define PG_ERROR_RSVD_MASK 0x08 >>>>> #define PG_ERROR_I_D_MASK 0x10 >>>>> >>>>> -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ >>>>> +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ >>>>> +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ >>>>> >>>>> -#define MCE_CAP_DEF MCG_CTL_P >>>>> +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) >>>>> #define MCE_BANKS_DEF 10 >>>>> >>>> >>>> It seems that current kvm doesn't support SER_P, so injecting SRAO >>>> to guest will mean that guest receives VAL|UC|!PCC and RIPV event >>>> from virtual processor that doesn't have SER_P. >>> >>> Dean also noted this. I don't think it was deliberate choice to not >>> expose SER_P. Huang? >> >> In my testing, I found that MCG_SER_P was not being set (and I was >> running on a Nehalem-EX system). Injecting a MCE resulted in the >> guest entering into panic() from mce_panic(). If crash_kexec() >> finds a kexec_crash_image the system ends up rebooting, otherwise, >> what happens next requires operator intervention. > > Good to know. > What I'm concerning is that if memory scrubbing SRAO event is > injected when !SER_P, linux guest with certain mce tolerant level > might grade it as "UC" severity and continue running with none of > panicking, killing and poisoning because of !PCC and RIPV. > > Could you provide the panic message of the guest in your test? > I think it can tell me why the mce handler decided to go panic. Sure, I'll add the info below at the end of this email. >> When I applied a patch to the guest's kernel which forces mce_ser to be >> set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found >> that when the memory page was 'owned' by a guest process, the process >> would be killed (if the page was dirty), and the guest would stay >> running. The HWPoisoned page would be sidelined and not cause any more >> issues. > > Excellent. > So while guest kernel knows which page is poisoned, guest processes > are controlled not to touch the page. > > ... Therefore rebooting the vm and renewing kernel will lost the > information where is poisoned. Correct. >>>> I think most OSes don't expect that it can receives MCE with !PCC >>>> on traditional x86 processor without SER_P. >>>> >>>> Q1: Is it safe to expect that guests can handle such !PCC event? >> >> This might be best answered by Huang, but as I mentioned above, without >> MCG_SER_P being set, the result was an orderly system panic on the >> guest. > > Though I'll wait Huang (I think he is on holiday), I believe that > system panic is just a possible option for AO (Action Optional) > event, no matter how the SER_P is. I think you may be correct, but Huang will know for sure. >>>> Q2: What is the expected behavior on the guest? >> >> I think I answered this above. > > Yeah, thanks. > >> >>>> Q3: What happen if guest reboots itself in response to the MCE? >> >> That depends... >> >> And the following issue also holds for a guest that is rebooted at >> some point having successfully sidelined the bad page. >> >> After the guest has panic'd, a system_reset of the guest or a restart >> initiated by crash_kexec() (called by panic() on the guest), usually >> results in the guest hanging because the bad page still belongs >> to qemu-kvm and is now being referenced by the new guest in some way. > > Yes. In other words my concern about reboot is that new guest kernel > including kdump kernel might try to read the bad page. If there is > no AR-SIGBUS etc., we need some tricks to inhibit such accesses. Agreed. >> (It actually may not hang, but successfully reboot and be runnable, >> with the bad page lurking in the background. It all seems to depend on >> where the bad page ends up, and whether it's ever referenced.) > > I know some tough guys using their PC with buggy DIMMs :-) > >> >> I believe there was an attempt to deal with this in kvm on the host. >> See kvm_handle_bad_page(). This function was suppose to result in the >> sending of a BUS_MCEERR_AR flavored SIGBUS by do_sigbus() to qemu-kvm >> which in theory would result in the right thing happening. But commit >> 96054569190bdec375fe824e48ca1f4e3b53dd36 prevents the signal from being >> sent. So this mechanism needs to be re-worked, and the issue remains. > > Definitely. > I guess Huang has some plan or hint for rework this point. Yeah, as far as I know Huang is looking into this. >> I would think that if the the bad page can't be sidelined, such that >> the newly booting guest can't use it, then the new guest shouldn't be >> allowed to boot. But perhaps there is some merit in letting it try to >> boot and see if one gets 'lucky'. > > In case of booting a real machine in real world, hardware and firmware > usually (or often) do self-test before passing control to OS. > Some platform can boot OS with degraded configuration (for example, > fewer memory) if it has trouble on its component. Some BIOS may > stop booting and show messages like "please reseat [component]" on the > screen. So we could implement/request qemu to have such mechanism. > > I can understand the merit you mentioned here, in some degree. But I > think it is hard to say "unlucky" to customer in business... I totally agree. >> I understand that Huang is looking into what should be done. He can >> give you better information than I in answer to your questions. > > Agreed. Thank you very much! You're welcome. Dean > Thanks, > H.Seto :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: The test I'm running is the mce-test suite's kvm test. A portion of the messages it outputted (to stdout) follows: > Guest physical address is 0x71220000 > Host virtual address is 7f9dc5020 > Host physical address is 0x1051620000 > Guest physical klog address is 0x71220 And it called mce-inject with the following data file: > [root@intel-s3e36-02 test]# cat SRAO > CPU 0 BANK 2 > STATUS UNCORRECTED SRAO 0x17a > MCGSTATUS MCIP RIPV > MISC 0x8c > ADDR 0x1051620000 > [root@intel-s3e36-02 test]# The following is from the host's /var/log/messages: > Oct 7 09:42:48 intel-s3e36-02 kernel: Triggering MCE exception on CPU 0 > Oct 7 09:42:48 intel-s3e36-02 kernel: Machine check events logged > Oct 7 09:42:48 intel-s3e36-02 kernel: MCE exception done on CPU 0 > Oct 7 09:42:48 intel-s3e36-02 kernel: MCE 0x1051620: Killing qemu-system-x86:6867 early due to hardware memory corruption > Oct 7 09:42:48 intel-s3e36-02 kernel: MCE 0x1051620: dirty LRU page recovery: Recovered Lastly, the following is a screen grab from the guest's serial console: > HARDWARE ERROR > CPU 0: Machine Check Exception: 5 Bank 9: bd000000000000c0 > RIP !INEXACT! 33:<0000000000400428> > TSC 17a67acd14 ADDR 71220000 MISC 8c > PROCESSOR 0:6d3 TIME 1286458966 SOCKET 0 APIC 0 > No human readable MCE decoding support on this CPU type. > Run the message through 'mcelog --ascii' to decode. > This is not a software problem! > Machine check: Uncorrected > Kernel panic - not syncing: Fatal machine check on current CPU > Pid:1493, comm: simple_process Tainted: B M ---------------- 2.6.32.dnelson_test #48 > > Call Trace: > <#MC> [<ffffffff814c7c8d>] panic+0x78/0x137 > [<ffffffff81027382>] mce_panic+0x1e2/0x210 > [<ffffffff81028873>] do_machine_check+0x843/0xa70 > [<ffffffff814cb0cc>] machine_check+0x1c/0x30 > <<EOE>>
On Thu, 2010-10-07 at 00:05 +0800, Marcelo Tosatti wrote: > On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: > > I got some more question: > > > > (2010/10/05 3:54), Marcelo Tosatti wrote: > > > Index: qemu/target-i386/cpu.h > > > =================================================================== > > > --- qemu.orig/target-i386/cpu.h > > > +++ qemu/target-i386/cpu.h > > > @@ -250,16 +250,32 @@ > > > #define PG_ERROR_RSVD_MASK 0x08 > > > #define PG_ERROR_I_D_MASK 0x10 > > > > > > -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ > > > +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ > > > +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ > > > > > > -#define MCE_CAP_DEF MCG_CTL_P > > > +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) > > > #define MCE_BANKS_DEF 10 > > > > > > > It seems that current kvm doesn't support SER_P, so injecting SRAO > > to guest will mean that guest receives VAL|UC|!PCC and RIPV event > > from virtual processor that doesn't have SER_P. > > Dean also noted this. I don't think it was deliberate choice to not > expose SER_P. Huang? In fact, that should be a BUG. I will fix it as soon as possible. Best Regards, Huang Ying
Hi, Seto, On Thu, 2010-10-07 at 11:41 +0800, Hidetoshi Seto wrote: > (2010/10/07 3:10), Dean Nelson wrote: > > On 10/06/2010 11:05 AM, Marcelo Tosatti wrote: > >> On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: > >>> I got some more question: > >>> > >>> (2010/10/05 3:54), Marcelo Tosatti wrote: > >>>> Index: qemu/target-i386/cpu.h > >>>> =================================================================== > >>>> --- qemu.orig/target-i386/cpu.h > >>>> +++ qemu/target-i386/cpu.h > >>>> @@ -250,16 +250,32 @@ > >>>> #define PG_ERROR_RSVD_MASK 0x08 > >>>> #define PG_ERROR_I_D_MASK 0x10 > >>>> > >>>> -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ > >>>> +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ > >>>> +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ > >>>> > >>>> -#define MCE_CAP_DEF MCG_CTL_P > >>>> +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) > >>>> #define MCE_BANKS_DEF 10 > >>>> > >>> > >>> It seems that current kvm doesn't support SER_P, so injecting SRAO > >>> to guest will mean that guest receives VAL|UC|!PCC and RIPV event > >>> from virtual processor that doesn't have SER_P. > >> > >> Dean also noted this. I don't think it was deliberate choice to not > >> expose SER_P. Huang? > > > > In my testing, I found that MCG_SER_P was not being set (and I was > > running on a Nehalem-EX system). Injecting a MCE resulted in the > > guest entering into panic() from mce_panic(). If crash_kexec() > > finds a kexec_crash_image the system ends up rebooting, otherwise, > > what happens next requires operator intervention. > > Good to know. > What I'm concerning is that if memory scrubbing SRAO event is > injected when !SER_P, linux guest with certain mce tolerant level > might grade it as "UC" severity and continue running with none of > panicking, killing and poisoning because of !PCC and RIPV. > > Could you provide the panic message of the guest in your test? > I think it can tell me why the mce handler decided to go panic. That is a bug that the SER_P is not in KVM_MCE_CAP_SUPPORTED in kernel. I will fix it as soon as possible. And SRAO MCE should not be sent when !SER_P, we should add that condition in qemu-kvm. > > When I applied a patch to the guest's kernel which forces mce_ser to be > > set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found > > that when the memory page was 'owned' by a guest process, the process > > would be killed (if the page was dirty), and the guest would stay > > running. The HWPoisoned page would be sidelined and not cause any more > > issues. > > Excellent. > So while guest kernel knows which page is poisoned, guest processes > are controlled not to touch the page. > > ... Therefore rebooting the vm and renewing kernel will lost the > information where is poisoned. Yes. That is an issue. Dean suggests that make qemu-kvm to refuse reboot the guest if there is poisoned page and ask for user to intervention. I have another idea to replace the poison pages with good pages when reboot, that is, recover without user intervention. > >>> I think most OSes don't expect that it can receives MCE with !PCC > >>> on traditional x86 processor without SER_P. > >>> > >>> Q1: Is it safe to expect that guests can handle such !PCC event? > > > > This might be best answered by Huang, but as I mentioned above, without > > MCG_SER_P being set, the result was an orderly system panic on the > > guest. > > Though I'll wait Huang (I think he is on holiday), I believe that > system panic is just a possible option for AO (Action Optional) > event, no matter how the SER_P is. We should fix this as I said above. > >>> Q2: What is the expected behavior on the guest? > > > > I think I answered this above. > > Yeah, thanks. > > > > >>> Q3: What happen if guest reboots itself in response to the MCE? > > > > That depends... > > > > And the following issue also holds for a guest that is rebooted at > > some point having successfully sidelined the bad page. > > > > After the guest has panic'd, a system_reset of the guest or a restart > > initiated by crash_kexec() (called by panic() on the guest), usually > > results in the guest hanging because the bad page still belongs > > to qemu-kvm and is now being referenced by the new guest in some way. > > Yes. In other words my concern about reboot is that new guest kernel > including kdump kernel might try to read the bad page. If there is > no AR-SIGBUS etc., we need some tricks to inhibit such accesses. > > > (It actually may not hang, but successfully reboot and be runnable, > > with the bad page lurking in the background. It all seems to depend on > > where the bad page ends up, and whether it's ever referenced.) > > I know some tough guys using their PC with buggy DIMMs :-) > > > > > I believe there was an attempt to deal with this in kvm on the host. > > See kvm_handle_bad_page(). This function was suppose to result in the > > sending of a BUS_MCEERR_AR flavored SIGBUS by do_sigbus() to qemu-kvm > > which in theory would result in the right thing happening. But commit > > 96054569190bdec375fe824e48ca1f4e3b53dd36 prevents the signal from being > > sent. So this mechanism needs to be re-worked, and the issue remains. > > Definitely. > I guess Huang has some plan or hint for rework this point. Yes. This should be fixed. The SRAR SIGBUS should be sent directly instead of being sent via touching poisoned virtual address. > > I would think that if the the bad page can't be sidelined, such that > > the newly booting guest can't use it, then the new guest shouldn't be > > allowed to boot. But perhaps there is some merit in letting it try to > > boot and see if one gets 'lucky'. > > In case of booting a real machine in real world, hardware and firmware > usually (or often) do self-test before passing control to OS. > Some platform can boot OS with degraded configuration (for example, > fewer memory) if it has trouble on its component. Some BIOS may > stop booting and show messages like "please reseat [component]" on the > screen. So we could implement/request qemu to have such mechanism. > > I can understand the merit you mentioned here, in some degree. But I > think it is hard to say "unlucky" to customer in business... Because the contents of poisoned pages are not relevant after reboot. Qemu can replace the poisoned pages with good pages when reboot guest. Do you think that is good. Best Regards, Huang Ying
Hi, Huang-san, (2010/10/08 12:15), Huang Ying wrote: > Hi, Seto, > > On Thu, 2010-10-07 at 11:41 +0800, Hidetoshi Seto wrote: >> (2010/10/07 3:10), Dean Nelson wrote: >>> On 10/06/2010 11:05 AM, Marcelo Tosatti wrote: >>>> On Wed, Oct 06, 2010 at 10:58:36AM +0900, Hidetoshi Seto wrote: >>>>> I got some more question: >>>>> >>>>> (2010/10/05 3:54), Marcelo Tosatti wrote: >>>>>> Index: qemu/target-i386/cpu.h >>>>>> =================================================================== >>>>>> --- qemu.orig/target-i386/cpu.h >>>>>> +++ qemu/target-i386/cpu.h >>>>>> @@ -250,16 +250,32 @@ >>>>>> #define PG_ERROR_RSVD_MASK 0x08 >>>>>> #define PG_ERROR_I_D_MASK 0x10 >>>>>> >>>>>> -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ >>>>>> +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ >>>>>> +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ >>>>>> >>>>>> -#define MCE_CAP_DEF MCG_CTL_P >>>>>> +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) >>>>>> #define MCE_BANKS_DEF 10 >>>>>> >>>>> >>>>> It seems that current kvm doesn't support SER_P, so injecting SRAO >>>>> to guest will mean that guest receives VAL|UC|!PCC and RIPV event >>>>> from virtual processor that doesn't have SER_P. >>>> >>>> Dean also noted this. I don't think it was deliberate choice to not >>>> expose SER_P. Huang? >>> >>> In my testing, I found that MCG_SER_P was not being set (and I was >>> running on a Nehalem-EX system). Injecting a MCE resulted in the >>> guest entering into panic() from mce_panic(). If crash_kexec() >>> finds a kexec_crash_image the system ends up rebooting, otherwise, >>> what happens next requires operator intervention. >> >> Good to know. >> What I'm concerning is that if memory scrubbing SRAO event is >> injected when !SER_P, linux guest with certain mce tolerant level >> might grade it as "UC" severity and continue running with none of >> panicking, killing and poisoning because of !PCC and RIPV. >> >> Could you provide the panic message of the guest in your test? >> I think it can tell me why the mce handler decided to go panic. > > That is a bug that the SER_P is not in KVM_MCE_CAP_SUPPORTED in kernel. > I will fix it as soon as possible. And SRAO MCE should not be sent > when !SER_P, we should add that condition in qemu-kvm. That makes sense. I think it is qemu's responsibility for what follows the AO-SIGBUS, what action should be taken depends on the KVM's capability. >>> When I applied a patch to the guest's kernel which forces mce_ser to be >>> set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found >>> that when the memory page was 'owned' by a guest process, the process >>> would be killed (if the page was dirty), and the guest would stay >>> running. The HWPoisoned page would be sidelined and not cause any more >>> issues. >> >> Excellent. >> So while guest kernel knows which page is poisoned, guest processes >> are controlled not to touch the page. >> >> ... Therefore rebooting the vm and renewing kernel will lost the >> information where is poisoned. > > Yes. That is an issue. Dean suggests that make qemu-kvm to refuse reboot > the guest if there is poisoned page and ask for user to intervention. I > have another idea to replace the poison pages with good pages when > reboot, that is, recover without user intervention. Sounds good. I think it may be worth something to reserve pages for the replacement before reboot is requested; at least we really don't want to fail rebooting with 'no memory'. >>>>> I think most OSes don't expect that it can receives MCE with !PCC >>>>> on traditional x86 processor without SER_P. >>>>> >>>>> Q1: Is it safe to expect that guests can handle such !PCC event? >>> >>> This might be best answered by Huang, but as I mentioned above, without >>> MCG_SER_P being set, the result was an orderly system panic on the >>> guest. >> >> Though I'll wait Huang (I think he is on holiday), I believe that >> system panic is just a possible option for AO (Action Optional) >> event, no matter how the SER_P is. > > We should fix this as I said above. > >>>>> Q2: What is the expected behavior on the guest? >>> >>> I think I answered this above. >> >> Yeah, thanks. >> >>> >>>>> Q3: What happen if guest reboots itself in response to the MCE? >>> >>> That depends... >>> >>> And the following issue also holds for a guest that is rebooted at >>> some point having successfully sidelined the bad page. >>> >>> After the guest has panic'd, a system_reset of the guest or a restart >>> initiated by crash_kexec() (called by panic() on the guest), usually >>> results in the guest hanging because the bad page still belongs >>> to qemu-kvm and is now being referenced by the new guest in some way. >> >> Yes. In other words my concern about reboot is that new guest kernel >> including kdump kernel might try to read the bad page. If there is >> no AR-SIGBUS etc., we need some tricks to inhibit such accesses. >> >>> (It actually may not hang, but successfully reboot and be runnable, >>> with the bad page lurking in the background. It all seems to depend on >>> where the bad page ends up, and whether it's ever referenced.) >> >> I know some tough guys using their PC with buggy DIMMs :-) >> >>> >>> I believe there was an attempt to deal with this in kvm on the host. >>> See kvm_handle_bad_page(). This function was suppose to result in the >>> sending of a BUS_MCEERR_AR flavored SIGBUS by do_sigbus() to qemu-kvm >>> which in theory would result in the right thing happening. But commit >>> 96054569190bdec375fe824e48ca1f4e3b53dd36 prevents the signal from being >>> sent. So this mechanism needs to be re-worked, and the issue remains. >> >> Definitely. >> I guess Huang has some plan or hint for rework this point. > > Yes. This should be fixed. The SRAR SIGBUS should be sent directly > instead of being sent via touching poisoned virtual address. Good. It should work. >>> I would think that if the the bad page can't be sidelined, such that >>> the newly booting guest can't use it, then the new guest shouldn't be >>> allowed to boot. But perhaps there is some merit in letting it try to >>> boot and see if one gets 'lucky'. >> >> In case of booting a real machine in real world, hardware and firmware >> usually (or often) do self-test before passing control to OS. >> Some platform can boot OS with degraded configuration (for example, >> fewer memory) if it has trouble on its component. Some BIOS may >> stop booting and show messages like "please reseat [component]" on the >> screen. So we could implement/request qemu to have such mechanism. >> >> I can understand the merit you mentioned here, in some degree. But I >> think it is hard to say "unlucky" to customer in business... > > Because the contents of poisoned pages are not relevant after reboot. > Qemu can replace the poisoned pages with good pages when reboot guest. > Do you think that is good. Sure. Of course this trick will not needed if user has done migration or save/restore the guest before a reboot. Thank you for answering! Thanks, H.Seto
On 10/07/2010 10:15 PM, Huang Ying wrote: > Hi, Seto, > > On Thu, 2010-10-07 at 11:41 +0800, Hidetoshi Seto wrote: >> (2010/10/07 3:10), Dean Nelson wrote: <snip> >>> When I applied a patch to the guest's kernel which forces mce_ser to be >>> set, as if MCG_SER_P was set (see __mcheck_cpu_cap_init()), I found >>> that when the memory page was 'owned' by a guest process, the process >>> would be killed (if the page was dirty), and the guest would stay >>> running. The HWPoisoned page would be sidelined and not cause any more >>> issues. >> >> Excellent. >> So while guest kernel knows which page is poisoned, guest processes >> are controlled not to touch the page. >> >> ... Therefore rebooting the vm and renewing kernel will lost the >> information where is poisoned. > > Yes. That is an issue. Dean suggests that make qemu-kvm to refuse reboot > the guest if there is poisoned page and ask for user to intervention. I > have another idea to replace the poison pages with good pages when > reboot, that is, recover without user intervention. Hi, Huang, I much prefer the replacing of the poisoned pages with good pages on reboot, over the refusing to reboot. So definitely go with your idea. Thanks, Dean
Index: qemu/cpus.c =================================================================== --- qemu.orig/cpus.c +++ qemu/cpus.c @@ -34,6 +34,10 @@ #include "cpus.h" #include "compatfd.h" +#ifdef CONFIG_LINUX +#include <sys/prctl.h> +#include <sys/signalfd.h> +#endif #ifdef SIGRTMIN #define SIG_IPI (SIGRTMIN+4) @@ -41,6 +45,10 @@ #define SIG_IPI SIGUSR1 #endif +#ifndef PR_MCE_KILL +#define PR_MCE_KILL 33 +#endif + static CPUState *next_cpu; /***********************************************************/ @@ -498,28 +506,77 @@ static void qemu_tcg_wait_io_event(void) } } +static void sigbus_reraise(void) +{ + sigset_t set; + struct sigaction action; + + memset(&action, 0, sizeof(action)); + action.sa_handler = SIG_DFL; + if (!sigaction(SIGBUS, &action, NULL)) { + raise(SIGBUS); + sigemptyset(&set); + sigaddset(&set, SIGBUS); + sigprocmask(SIG_UNBLOCK, &set, NULL); + } + perror("Failed to re-raise SIGBUS!\n"); + abort(); +} + +static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo, + void *ctx) +{ +#if defined(TARGET_I386) + if (kvm_on_sigbus_vcpu(siginfo->ssi_code, (void *)(intptr_t)siginfo->ssi_addr)) +#endif + sigbus_reraise(); +} + static void qemu_kvm_eat_signal(CPUState *env, int timeout) { struct timespec ts; int r, e; siginfo_t siginfo; sigset_t waitset; + sigset_t chkset; ts.tv_sec = timeout / 1000; ts.tv_nsec = (timeout % 1000) * 1000000; sigemptyset(&waitset); sigaddset(&waitset, SIG_IPI); + sigaddset(&waitset, SIGBUS); - qemu_mutex_unlock(&qemu_global_mutex); - r = sigtimedwait(&waitset, &siginfo, &ts); - e = errno; - qemu_mutex_lock(&qemu_global_mutex); + do { + qemu_mutex_unlock(&qemu_global_mutex); - if (r == -1 && !(e == EAGAIN || e == EINTR)) { - fprintf(stderr, "sigtimedwait: %s\n", strerror(e)); - exit(1); - } + r = sigtimedwait(&waitset, &siginfo, &ts); + e = errno; + + qemu_mutex_lock(&qemu_global_mutex); + + if (r == -1 && !(e == EAGAIN || e == EINTR)) { + fprintf(stderr, "sigtimedwait: %s\n", strerror(e)); + exit(1); + } + + switch (r) { + case SIGBUS: +#ifdef TARGET_I386 + if (kvm_on_sigbus(env, siginfo.si_code, siginfo.si_addr)) +#endif + sigbus_reraise(); + break; + default: + break; + } + + r = sigpending(&chkset); + if (r == -1) { + fprintf(stderr, "sigpending: %s\n", strerror(e)); + exit(1); + } + } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS)); } static void qemu_kvm_wait_io_event(CPUState *env) @@ -645,6 +702,7 @@ static void kvm_init_ipi(CPUState *env) pthread_sigmask(SIG_BLOCK, NULL, &set); sigdelset(&set, SIG_IPI); + sigdelset(&set, SIGBUS); r = kvm_set_signal_mask(env, &set); if (r) { fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(r)); @@ -655,6 +713,7 @@ static void kvm_init_ipi(CPUState *env) static sigset_t block_io_signals(void) { sigset_t set; + struct sigaction action; /* SIGUSR2 used by posix-aio-compat.c */ sigemptyset(&set); @@ -665,8 +724,15 @@ static sigset_t block_io_signals(void) sigaddset(&set, SIGIO); sigaddset(&set, SIGALRM); sigaddset(&set, SIG_IPI); + sigaddset(&set, SIGBUS); pthread_sigmask(SIG_BLOCK, &set, NULL); + memset(&action, 0, sizeof(action)); + action.sa_flags = SA_SIGINFO; + action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler; + sigaction(SIGBUS, &action, NULL); + prctl(PR_MCE_KILL, 1, 1, 0, 0); + return set; } Index: qemu/kvm.h =================================================================== --- qemu.orig/kvm.h +++ qemu/kvm.h @@ -110,6 +110,9 @@ int kvm_arch_init_vcpu(CPUState *env); void kvm_arch_reset_vcpu(CPUState *env); +int kvm_on_sigbus(CPUState *env, int code, void *addr); +int kvm_on_sigbus_vcpu(int code, void *addr); + struct kvm_guest_debug; struct kvm_debug_exit_arch; Index: qemu/target-i386/cpu.h =================================================================== --- qemu.orig/target-i386/cpu.h +++ qemu/target-i386/cpu.h @@ -250,16 +250,32 @@ #define PG_ERROR_RSVD_MASK 0x08 #define PG_ERROR_I_D_MASK 0x10 -#define MCG_CTL_P (1UL<<8) /* MCG_CAP register available */ +#define MCG_CTL_P (1ULL<<8) /* MCG_CAP register available */ +#define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ -#define MCE_CAP_DEF MCG_CTL_P +#define MCE_CAP_DEF (MCG_CTL_P|MCG_SER_P) #define MCE_BANKS_DEF 10 +#define MCG_STATUS_RIPV (1ULL<<0) /* restart ip valid */ +#define MCG_STATUS_EIPV (1ULL<<1) /* ip points to correct instruction */ #define MCG_STATUS_MCIP (1ULL<<2) /* machine check in progress */ #define MCI_STATUS_VAL (1ULL<<63) /* valid error */ #define MCI_STATUS_OVER (1ULL<<62) /* previous errors lost */ #define MCI_STATUS_UC (1ULL<<61) /* uncorrected error */ +#define MCI_STATUS_EN (1ULL<<60) /* error enabled */ +#define MCI_STATUS_MISCV (1ULL<<59) /* misc error reg. valid */ +#define MCI_STATUS_ADDRV (1ULL<<58) /* addr reg. valid */ +#define MCI_STATUS_PCC (1ULL<<57) /* processor context corrupt */ +#define MCI_STATUS_S (1ULL<<56) /* Signaled machine check */ +#define MCI_STATUS_AR (1ULL<<55) /* Action required */ + +/* MISC register defines */ +#define MCM_ADDR_SEGOFF 0 /* segment offset */ +#define MCM_ADDR_LINEAR 1 /* linear address */ +#define MCM_ADDR_PHYS 2 /* physical address */ +#define MCM_ADDR_MEM 3 /* memory address */ +#define MCM_ADDR_GENERIC 7 /* generic */ #define MSR_IA32_TSC 0x10 #define MSR_IA32_APICBASE 0x1b Index: qemu/target-i386/kvm.c =================================================================== --- qemu.orig/target-i386/kvm.c +++ qemu/target-i386/kvm.c @@ -46,6 +46,13 @@ #define MSR_KVM_WALL_CLOCK 0x11 #define MSR_KVM_SYSTEM_TIME 0x12 +#ifndef BUS_MCEERR_AR +#define BUS_MCEERR_AR 4 +#endif +#ifndef BUS_MCEERR_AO +#define BUS_MCEERR_AO 5 +#endif + #ifdef KVM_CAP_EXT_CPUID static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max) @@ -192,10 +199,39 @@ static int kvm_set_mce(CPUState *env, st return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m); } +static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n) +{ + struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs); + int r; + + kmsrs->nmsrs = n; + memcpy(kmsrs->entries, msrs, n * sizeof *msrs); + r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs); + memcpy(msrs, kmsrs->entries, n * sizeof *msrs); + free(kmsrs); + return r; +} + +/* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */ +static int kvm_mce_in_exception(CPUState *env) +{ + struct kvm_msr_entry msr_mcg_status = { + .index = MSR_MCG_STATUS, + }; + int r; + + r = kvm_get_msr(env, &msr_mcg_status, 1); + if (r == -1 || r == 0) { + return -1; + } + return !!(msr_mcg_status.data & MCG_STATUS_MCIP); +} + struct kvm_x86_mce_data { CPUState *env; struct kvm_x86_mce *mce; + int abort_on_error; }; static void kvm_do_inject_x86_mce(void *_data) @@ -203,14 +239,26 @@ static void kvm_do_inject_x86_mce(void * struct kvm_x86_mce_data *data = _data; int r; + /* If there is an MCE excpetion being processed, ignore this SRAO MCE */ + r = kvm_mce_in_exception(data->env); + if (r == -1) + fprintf(stderr, "Failed to get MCE status\n"); + else if (r && !(data->mce->status & MCI_STATUS_AR)) + return; + r = kvm_set_mce(data->env, data->mce); - if (r < 0) + if (r < 0) { perror("kvm_set_mce FAILED"); + if (data->abort_on_error) { + abort(); + } + } } #endif void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status, - uint64_t mcg_status, uint64_t addr, uint64_t misc) + uint64_t mcg_status, uint64_t addr, uint64_t misc, + int abort_on_error) { #ifdef KVM_CAP_MCE struct kvm_x86_mce mce = { @@ -225,7 +273,15 @@ void kvm_inject_x86_mce(CPUState *cenv, .mce = &mce, }; + if (!cenv->mcg_cap) { + fprintf(stderr, "MCE support is not enabled!\n"); + return; + } + run_on_cpu(cenv, kvm_do_inject_x86_mce, &data); +#else + if (abort_on_error) + abort(); #endif } @@ -1525,3 +1581,122 @@ bool kvm_arch_stop_on_emulation_error(CP ((env->segs[R_CS].selector & 3) != 3); } +static void hardware_memory_error(void) +{ + fprintf(stderr, "Hardware memory error!\n"); + exit(1); +} + +int kvm_on_sigbus(CPUState *env, int code, void *addr) +{ +#if defined(KVM_CAP_MCE) + struct kvm_x86_mce mce = { + .bank = 9, + }; + void *vaddr; + ram_addr_t ram_addr; + unsigned long paddr; + int r; + + if (env->mcg_cap && addr + && (code == BUS_MCEERR_AR + || code == BUS_MCEERR_AO)) { + if (code == BUS_MCEERR_AR) { + /* Fake an Intel architectural Data Load SRAR UCR */ + mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN + | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S + | MCI_STATUS_AR | 0x134; + mce.misc = (MCM_ADDR_PHYS << 6) | 0xc; + mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV; + } else { + /* + * If there is an MCE excpetion being processed, ignore + * this SRAO MCE + */ + r = kvm_mce_in_exception(env); + if (r == -1) { + fprintf(stderr, "Failed to get MCE status\n"); + } else if (r) { + return 0; + } + /* Fake an Intel architectural Memory scrubbing UCR */ + mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN + | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S + | 0xc0; + mce.misc = (MCM_ADDR_PHYS << 6) | 0xc; + mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV; + } + vaddr = (void *)addr; + if (do_qemu_ram_addr_from_host(vaddr, &ram_addr) || + !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) { + fprintf(stderr, "Hardware memory error for memory used by " + "QEMU itself instaed of guest system!\n"); + /* Hope we are lucky for AO MCE */ + if (code == BUS_MCEERR_AO) { + return 0; + } else { + hardware_memory_error(); + } + } + mce.addr = paddr; + r = kvm_set_mce(env, &mce); + if (r < 0) { + fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno)); + abort(); + } + } else +#endif + { + if (code == BUS_MCEERR_AO) { + return 0; + } else if (code == BUS_MCEERR_AR) { + hardware_memory_error(); + } else { + return 1; + } + } + return 0; +} + +int kvm_on_sigbus_vcpu(int code, void *addr) +{ +#if defined(KVM_CAP_MCE) + if (first_cpu->mcg_cap && addr && code == BUS_MCEERR_AO) { + uint64_t status; + void *vaddr; + ram_addr_t ram_addr; + unsigned long paddr; + CPUState *cenv; + + /* Hope we are lucky for AO MCE */ + vaddr = addr; + if (do_qemu_ram_addr_from_host(vaddr, &ram_addr) || + !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr, &paddr)) { + fprintf(stderr, "Hardware memory error for memory used by " + "QEMU itself instead of guest system!: %p\n", addr); + return 0; + } + status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN + | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S + | 0xc0; + kvm_inject_x86_mce(first_cpu, 9, status, + MCG_STATUS_MCIP | MCG_STATUS_RIPV, paddr, + (MCM_ADDR_PHYS << 6) | 0xc, 1); + for (cenv = first_cpu->next_cpu; cenv != NULL; cenv = cenv->next_cpu) { + kvm_inject_x86_mce(cenv, 1, MCI_STATUS_VAL | MCI_STATUS_UC, + MCG_STATUS_MCIP | MCG_STATUS_RIPV, 0, 0, 1); + } + } else +#endif + { + if (code == BUS_MCEERR_AO) { + return 0; + } else if (code == BUS_MCEERR_AR) { + hardware_memory_error(); + } else { + return 1; + } + } + return 0; +} + Index: qemu/target-i386/helper.c =================================================================== --- qemu.orig/target-i386/helper.c +++ qemu/target-i386/helper.c @@ -1032,7 +1032,7 @@ void cpu_inject_x86_mce(CPUState *cenv, return; if (kvm_enabled()) { - kvm_inject_x86_mce(cenv, bank, status, mcg_status, addr, misc); + kvm_inject_x86_mce(cenv, bank, status, mcg_status, addr, misc, 0); return; } Index: qemu/target-i386/kvm_x86.h =================================================================== --- qemu.orig/target-i386/kvm_x86.h +++ qemu/target-i386/kvm_x86.h @@ -16,6 +16,7 @@ #define __KVM_X86_H__ void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status, - uint64_t mcg_status, uint64_t addr, uint64_t misc); + uint64_t mcg_status, uint64_t addr, uint64_t misc, + int abort_on_error); #endif Index: qemu/kvm-stub.c =================================================================== --- qemu.orig/kvm-stub.c +++ qemu/kvm-stub.c @@ -141,3 +141,9 @@ int kvm_set_ioeventfd_mmio_long(int fd, { return -ENOSYS; } + +int kvm_on_sigbus_vcpu(int code, void *addr) +{ + return 1; +} +
Port qemu-kvm's commit 4b62fff1101a7ad77553147717a8bd3bf79df7ef Author: Huang Ying <ying.huang@intel.com> Date: Mon Sep 21 10:43:25 2009 +0800 MCE: Relay UCR MCE to guest UCR (uncorrected recovery) MCE is supported in recent Intel CPUs, where some hardware error such as some memory error can be reported without PCC (processor context corrupted). To recover from such MCE, the corresponding memory will be unmapped, and all processes accessing the memory will be killed via SIGBUS. For KVM, if QEMU/KVM is killed, all guest processes will be killed too. So we relay SIGBUS from host OS to guest system via a UCR MCE injection. Then guest OS can isolate corresponding memory and kill necessary guest processes only. SIGBUS sent to main thread (not VCPU threads) will be broadcast to all VCPU threads as UCR MCE. Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>