diff mbox

[v5,3/4] Plumb the HAXM-based hardware acceleration support

Message ID 85250fb0450b5b95232b39c82ab310a0632fd23f.1482164622.git.vpalatin@chromium.org
State New
Headers show

Commit Message

Vincent Palatin Dec. 19, 2016, 4:24 p.m. UTC
Use the Intel HAX is kernel-based hardware acceleration module for
Windows (similar to KVM on Linux).

Based on the "target-i386: Add Intel HAX to android emulator" patch
from David Chou <david.j.chou@intel.com>

Signed-off-by: Vincent Palatin <vpalatin@chromium.org>
---
 Makefile.target           |  1 +
 configure                 | 18 ++++++++++
 cpus.c                    | 87 ++++++++++++++++++++++++++++++++++++++++++++++-
 exec.c                    | 16 +++++++++
 hw/intc/apic_common.c     |  3 +-
 include/qom/cpu.h         |  5 +++
 include/sysemu/hw_accel.h |  9 +++++
 qemu-options.hx           | 11 ++++++
 target-i386/Makefile.objs |  4 +++
 vl.c                      | 15 ++++++--
 10 files changed, 164 insertions(+), 5 deletions(-)

Comments

Paolo Bonzini Dec. 22, 2016, 9:57 a.m. UTC | #1
On 19/12/2016 17:24, Vincent Palatin wrote:
>  #else /* _WIN32 */
> -    abort();
> +    if (!qemu_cpu_is_self(cpu)) {
> +        CONTEXT context;
> +        if (SuspendThread(cpu->hThread) == (DWORD)(-1)) {
> +            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
> +                    GetLastError());
> +            exit(1);
> +        }
> +
> +        /* On multi-core systems, we are not sure that the thread is actually
> +         * suspended until we can get the context.
> +         */
> +        context.ContextFlags = CONTEXT_CONTROL;
> +        while (GetThreadContext(cpu->hThread, &context) != 0) {
> +            continue;
> +        }
> +
> +        if (hax_enabled()) {
> +            cpu->exit_request = 1;
> +        }

As mentioned in the reply to patch 4, please leave the cpu->exit_request
= 1 assignment to the caller.

Apart from the above change, can you check if there are some less
heavyeight methods to force an exit?  I can think of QueueUserAPC with
an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
before qemu_wait_io_event_common.

> +        if (ResumeThread(cpu->hThread) == (DWORD)(-1)) {
> +            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
> +                    GetLastError());
> +            exit(1);
> +        }
> +    }

[...]

> 
> +            /*
> +             * In Hax, the qemu allocate the virtual address, and HAX kernel
> +             * populate the memory with physical memory. Currently we have no
> +             * paging, so user should make sure enough free memory in advance
> +             */
> +            if (hax_enabled()) {
> +                int ret;
> +                ret = hax_populate_ram((uint64_t)(uintptr_t)new_block->host,
> +                                       new_block->max_length);
> +                if (ret < 0) {
> +                    error_setg(errp, "Hax failed to populate ram");
> +                    return;
> +                }
> +            }
> +

Can you check if the interface at
http://marc.info/?l=qemu-devel&m=148225154320642&q=raw would be good for
your purposes, and if so include that patch in your v6?

Otherwise looks great, so I'm confident we can add this in QEMU 2.9.

Paolo
Vincent Palatin Jan. 5, 2017, 1:50 p.m. UTC | #2
On Thu, Dec 22, 2016 at 10:57 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 19/12/2016 17:24, Vincent Palatin wrote:
>>  #else /* _WIN32 */
>> -    abort();
>> +    if (!qemu_cpu_is_self(cpu)) {
>> +        CONTEXT context;
>> +        if (SuspendThread(cpu->hThread) == (DWORD)(-1)) {
>> +            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
>> +                    GetLastError());
>> +            exit(1);
>> +        }
>> +
>> +        /* On multi-core systems, we are not sure that the thread is actually
>> +         * suspended until we can get the context.
>> +         */
>> +        context.ContextFlags = CONTEXT_CONTROL;
>> +        while (GetThreadContext(cpu->hThread, &context) != 0) {
>> +            continue;
>> +        }
>> +
>> +        if (hax_enabled()) {
>> +            cpu->exit_request = 1;
>> +        }
>
> As mentioned in the reply to patch 4, please leave the cpu->exit_request
> = 1 assignment to the caller.

Sorry I missed it.
I move it to qemu_cpu_kick() as asked in the Darwin patch.

>
> Apart from the above change, can you check if there are some less
> heavyeight methods to force an exit?  I can think of QueueUserAPC with
> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
> before qemu_wait_io_event_common.


Actually I don't know a good test case to verify such a change, any advice ?


>
>> +        if (ResumeThread(cpu->hThread) == (DWORD)(-1)) {
>> +            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
>> +                    GetLastError());
>> +            exit(1);
>> +        }
>> +    }
>
> [...]
>
>>
>> +            /*
>> +             * In Hax, the qemu allocate the virtual address, and HAX kernel
>> +             * populate the memory with physical memory. Currently we have no
>> +             * paging, so user should make sure enough free memory in advance
>> +             */
>> +            if (hax_enabled()) {
>> +                int ret;
>> +                ret = hax_populate_ram((uint64_t)(uintptr_t)new_block->host,
>> +                                       new_block->max_length);
>> +                if (ret < 0) {
>> +                    error_setg(errp, "Hax failed to populate ram");
>> +                    return;
>> +                }
>> +            }
>> +
>
> Can you check if the interface at
> http://marc.info/?l=qemu-devel&m=148225154320642&q=raw would be good for
> your purposes, and if so include that patch in your v6?

Seems to work, updated for v6.
Paolo Bonzini Jan. 5, 2017, 2:01 p.m. UTC | #3
On 05/01/2017 14:50, Vincent Palatin wrote:
> Sorry I missed it.
> I move it to qemu_cpu_kick() as asked in the Darwin patch.
> 
>> Apart from the above change, can you check if there are some less
>> heavyeight methods to force an exit?  I can think of QueueUserAPC with
>> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
>> before qemu_wait_io_event_common.
> 
> Actually I don't know a good test case to verify such a change, any advice ?

Try computing the latency between qemu_cpu_kick and the exit from HAXM.
That is, something like

    int64_t kick_time = -1;
    ...

    atomic_set(&kick_time, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));

before SuspendThread or respectively QueueUserAPC, and

    extern volatile int64_t kick_time;
    ...

    int64_t kicked = atomic_xchg(&kick_time, -1);
    if (kicked != -1) {
        printf("%lld\n",
               qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - kicked);
    }

right after hax_vcpu_run.    As long as the printed values remain
smallish, it's okay.

If you don't get any output, passing "info cpus" repeatedly to the QEMU
monitor will force a kick.

Paolo
Paolo Bonzini Jan. 5, 2017, 9:38 p.m. UTC | #4
On 05/01/2017 15:01, Paolo Bonzini wrote:
> 
> 
> On 05/01/2017 14:50, Vincent Palatin wrote:
>> Sorry I missed it.
>> I move it to qemu_cpu_kick() as asked in the Darwin patch.
>>
>>> Apart from the above change, can you check if there are some less
>>> heavyeight methods to force an exit?  I can think of QueueUserAPC with
>>> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
>>> before qemu_wait_io_event_common.
>>
>> Actually I don't know a good test case to verify such a change, any advice ?

In fact there is a race anyway:

        if (cpu->exit_request) {
            ret = 1;
            break;
        }
					cpu->exit_request
					SuspendThread
					ResumeThread
        hax_vcpu_interrupt(env);
        qemu_mutex_unlock_iothread();
        hax_ret = hax_vcpu_run(vcpu);

and the same race is true for QueueUserAPC.  It's rare enough that I
guess we can accept the patches with just a FIXME comment, but...  Yu
Ning, can you tell us what user_event_pending is for? :)  My hunch is
that we should call hax_raise_event after setting cpu->exit_request, like

	hax_raise_event();
	/* write user_event_pending before exit_request */
	smp_wmb();
	cpu->exit_request = 1;
	SuspendThread/ResumeThread
		(or QueueUserAPC)

and in the hax thread:

        if (cpu->exit_request) {
	    cpu->hax_vcpu->tunnel->user_event_pending = 0;
            ret = 1;
            break;
        }

        hax_vcpu_interrupt(env);
        qemu_mutex_unlock_iothread();

	/* read exit_request before user_event_pending */
	smp_rmb();
        hax_ret = hax_vcpu_run(vcpu);

but I would like some more official documentation than my own reverse
engineering of the brain of whoever wrote the interface (I have not
looked at the HAXM driver binary).

Paolo
Vincent Palatin Jan. 6, 2017, 2:08 p.m. UTC | #5
On Thu, Jan 5, 2017 at 10:38 PM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 05/01/2017 15:01, Paolo Bonzini wrote:
>>
>>
>> On 05/01/2017 14:50, Vincent Palatin wrote:
>>> Sorry I missed it.
>>> I move it to qemu_cpu_kick() as asked in the Darwin patch.
>>>
>>>> Apart from the above change, can you check if there are some less
>>>> heavyeight methods to force an exit?  I can think of QueueUserAPC with
>>>> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
>>>> before qemu_wait_io_event_common.
>>>
>>> Actually I don't know a good test case to verify such a change, any advice ?
>
> In fact there is a race anyway:

Thanks for the detailed examples and thoughts.
The timing/benchmarking code might actually need some kind of per-vcpu
time storage, but that's a detail.
I have experimented with it and so far, I have mainly generated random
numbers ...
I have yet to find a use-case where the current code (with
SuspendThread/ResumeThread) yields a better latency than just nothing
instead :(



>
>         if (cpu->exit_request) {
>             ret = 1;
>             break;
>         }
>                                         cpu->exit_request
>                                         SuspendThread
>                                         ResumeThread
>         hax_vcpu_interrupt(env);
>         qemu_mutex_unlock_iothread();
>         hax_ret = hax_vcpu_run(vcpu);
>
> and the same race is true for QueueUserAPC.  It's rare enough that I
> guess we can accept the patches with just a FIXME comment, but...  Yu
> Ning, can you tell us what user_event_pending is for? :)  My hunch is
> that we should call hax_raise_event after setting cpu->exit_request, like
>
>         hax_raise_event();
>         /* write user_event_pending before exit_request */
>         smp_wmb();
>         cpu->exit_request = 1;
>         SuspendThread/ResumeThread
>                 (or QueueUserAPC)
>
> and in the hax thread:
>
>         if (cpu->exit_request) {
>             cpu->hax_vcpu->tunnel->user_event_pending = 0;
>             ret = 1;
>             break;
>         }
>
>         hax_vcpu_interrupt(env);
>         qemu_mutex_unlock_iothread();
>
>         /* read exit_request before user_event_pending */
>         smp_rmb();
>         hax_ret = hax_vcpu_run(vcpu);
>
> but I would like some more official documentation than my own reverse
> engineering of the brain of whoever wrote the interface (I have not
> looked at the HAXM driver binary).
>
> Paolo
Yu Ning Jan. 9, 2017, 6:17 a.m. UTC | #6
On 1/6/2017 5:38, Paolo Bonzini wrote:
>
> On 05/01/2017 15:01, Paolo Bonzini wrote:
>>
>> In fact there is a race anyway:
>>
>>          if (cpu->exit_request) {
>>              ret = 1;
>>              break;
>>          }
>> 					cpu->exit_request
>> 					SuspendThread
>> 					ResumeThread
>>          hax_vcpu_interrupt(env);
>>          qemu_mutex_unlock_iothread();
>>          hax_ret = hax_vcpu_run(vcpu);
>>
>> and the same race is true for QueueUserAPC.  It's rare enough that I
>> guess we can accept the patches with just a FIXME comment, but...  Yu
>> Ning, can you tell us what user_event_pending is for? :)  My hunch is
>> that we should call hax_raise_event after setting cpu->exit_request, like
>>
>> 	hax_raise_event();
>> 	/* write user_event_pending before exit_request */
>> 	smp_wmb();
>> 	cpu->exit_request = 1;
>> 	SuspendThread/ResumeThread
>> 		(or QueueUserAPC)
>>
>> and in the hax thread:
>>
>>          if (cpu->exit_request) {
>> 	    cpu->hax_vcpu->tunnel->user_event_pending = 0;
>>              ret = 1;
>>              break;
>>          }
>>
>>          hax_vcpu_interrupt(env);
>>          qemu_mutex_unlock_iothread();
>>
>> 	/* read exit_request before user_event_pending */
>> 	smp_rmb();
>>          hax_ret = hax_vcpu_run(vcpu);
>>
>> but I would like some more official documentation than my own reverse
>> engineering of the brain of whoever wrote the interface (I have not
>> looked at the HAXM driver binary).

Unfortunately, user_event_pending was introduced in 2011 without proper 
documentation, so I guess even the original author might not have an 
answer now (even if I could find him).  I need more time to analyze the 
HAXM driver code to understand what it's about.  Please feel free to add 
a FIXME for now.

But one thing I've noticed is that the Darwin driver does not test or 
reset this flag properly - it will remain 1 after the first 
hax_raise_event() call.  So removing user_event_pending from the QEMU 
side should not affect Mac.
Paolo Bonzini Jan. 9, 2017, 1:03 p.m. UTC | #7
On 06/01/2017 15:08, Vincent Palatin wrote:
>>>>> Apart from the above change, can you check if there are some less
>>>>> heavyeight methods to force an exit?  I can think of QueueUserAPC with
>>>>> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
>>>>> before qemu_wait_io_event_common.
>>>> Actually I don't know a good test case to verify such a change, any advice ?
>> In fact there is a race anyway:
> Thanks for the detailed examples and thoughts.
> The timing/benchmarking code might actually need some kind of per-vcpu
> time storage, but that's a detail.
> I have experimented with it and so far, I have mainly generated random
> numbers ...
> I have yet to find a use-case where the current code (with
> SuspendThread/ResumeThread) yields a better latency than just nothing
> instead :(

:)  Does QueueUserAPC generate better latency?

Windows delivers the scheduler tick to the first physical CPU.  Try
pinning QEMU away from the first CPU.

Paolo
Vincent Palatin Jan. 9, 2017, 4:54 p.m. UTC | #8
On Mon, Jan 9, 2017 at 2:03 PM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 06/01/2017 15:08, Vincent Palatin wrote:
>>>>>> Apart from the above change, can you check if there are some less
>>>>>> heavyeight methods to force an exit?  I can think of QueueUserAPC with
>>>>>> an empty pfnAPC here, and SleepEx(0, TRUE) in qemu_hax_cpu_thread_fn
>>>>>> before qemu_wait_io_event_common.
>>>>> Actually I don't know a good test case to verify such a change, any advice ?
>>> In fact there is a race anyway:
>> Thanks for the detailed examples and thoughts.
>> The timing/benchmarking code might actually need some kind of per-vcpu
>> time storage, but that's a detail.
>> I have experimented with it and so far, I have mainly generated random
>> numbers ...
>> I have yet to find a use-case where the current code (with
>> SuspendThread/ResumeThread) yields a better latency than just nothing
>> instead :(
>
> :)  Does QueueUserAPC generate better latency?


The same kind of random numbers so far.

By the way I have added  the THREAD_SET_CONTEXT  flag to the
OpenThread call in qemu_thread_get_handle() function, as I was getting
ACCESS_DENIED on the QueueUserApc call. Probably not terribly harmful.
I will publish the v6 series with this and continue my benchmarking quest.

>
> Windows delivers the scheduler tick to the first physical CPU.  Try
> pinning QEMU away from the first CPU.

Ok interesting, I will give it a try.
diff mbox

Patch

diff --git a/Makefile.target b/Makefile.target
index 7a5080e..dab81e7 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -96,6 +96,7 @@  obj-y += target-$(TARGET_BASE_ARCH)/
 obj-y += disas.o
 obj-y += tcg-runtime.o
 obj-$(call notempty,$(TARGET_XML_FILES)) += gdbstub-xml.o
+obj-$(call lnot,$(CONFIG_HAX)) += hax-stub.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 
 obj-$(CONFIG_LIBDECNUMBER) += libdecnumber/decContext.o
diff --git a/configure b/configure
index 3770d7c..ba32bea 100755
--- a/configure
+++ b/configure
@@ -230,6 +230,7 @@  vhost_net="no"
 vhost_scsi="no"
 vhost_vsock="no"
 kvm="no"
+hax="no"
 colo="yes"
 rdma=""
 gprof="no"
@@ -563,6 +564,7 @@  CYGWIN*)
 ;;
 MINGW32*)
   mingw32="yes"
+  hax="yes"
   audio_possible_drivers="dsound sdl"
   if check_include dsound.h; then
     audio_drv_list="dsound"
@@ -612,6 +614,7 @@  OpenBSD)
 Darwin)
   bsd="yes"
   darwin="yes"
+  hax="yes"
   LDFLAGS_SHARED="-bundle -undefined dynamic_lookup"
   if [ "$cpu" = "x86_64" ] ; then
     QEMU_CFLAGS="-arch x86_64 $QEMU_CFLAGS"
@@ -921,6 +924,10 @@  for opt do
   ;;
   --enable-kvm) kvm="yes"
   ;;
+  --disable-hax) hax="no"
+  ;;
+  --enable-hax) hax="yes"
+  ;;
   --disable-colo) colo="no"
   ;;
   --enable-colo) colo="yes"
@@ -1373,6 +1380,7 @@  disabled with --disable-FEATURE, default is enabled if available:
   fdt             fdt device tree
   bluez           bluez stack connectivity
   kvm             KVM acceleration support
+  hax             HAX acceleration support
   colo            COarse-grain LOck-stepping VM for Non-stop Service
   rdma            RDMA-based migration support
   vde             support for vde network
@@ -5051,6 +5059,7 @@  echo "ATTR/XATTR support $attr"
 echo "Install blobs     $blobs"
 echo "KVM support       $kvm"
 echo "COLO support      $colo"
+echo "HAX support       $hax"
 echo "RDMA support      $rdma"
 echo "TCG interpreter   $tcg_interpreter"
 echo "fdt support       $fdt"
@@ -6035,6 +6044,15 @@  case "$target_name" in
       fi
     fi
 esac
+if test "$hax" = "yes" ; then
+  if test "$target_softmmu" = "yes" ; then
+    case "$target_name" in
+    i386|x86_64)
+      echo "CONFIG_HAX=y" >> $config_target_mak
+    ;;
+    esac
+  fi
+fi
 if test "$target_bigendian" = "yes" ; then
   echo "TARGET_WORDS_BIGENDIAN=y" >> $config_target_mak
 fi
diff --git a/cpus.c b/cpus.c
index fc78502..0e01791 100644
--- a/cpus.c
+++ b/cpus.c
@@ -35,6 +35,7 @@ 
 #include "sysemu/dma.h"
 #include "sysemu/hw_accel.h"
 #include "sysemu/kvm.h"
+#include "sysemu/hax.h"
 #include "qmp-commands.h"
 #include "exec/exec-all.h"
 
@@ -1221,6 +1222,39 @@  static void *qemu_tcg_cpu_thread_fn(void *arg)
     return NULL;
 }
 
+static void *qemu_hax_cpu_thread_fn(void *arg)
+{
+    CPUState *cpu = arg;
+    int r;
+    qemu_thread_get_self(cpu->thread);
+    qemu_mutex_lock(&qemu_global_mutex);
+
+    cpu->thread_id = qemu_get_thread_id();
+    cpu->created = true;
+    cpu->halted = 0;
+    current_cpu = cpu;
+
+    hax_init_vcpu(cpu);
+    qemu_cond_signal(&qemu_cpu_cond);
+
+    while (1) {
+        if (cpu_can_run(cpu)) {
+            r = hax_smp_cpu_exec(cpu);
+            if (r == EXCP_DEBUG) {
+                cpu_handle_guest_debug(cpu);
+            }
+        }
+
+        while (cpu_thread_is_idle(cpu)) {
+            qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
+        }
+
+        qemu_wait_io_event_common(cpu);
+    }
+    return NULL;
+}
+
+
 static void qemu_cpu_kick_thread(CPUState *cpu)
 {
 #ifndef _WIN32
@@ -1236,7 +1270,33 @@  static void qemu_cpu_kick_thread(CPUState *cpu)
         exit(1);
     }
 #else /* _WIN32 */
-    abort();
+    if (!qemu_cpu_is_self(cpu)) {
+        CONTEXT context;
+
+        if (SuspendThread(cpu->hThread) == (DWORD)(-1)) {
+            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
+                    GetLastError());
+            exit(1);
+        }
+
+        /* On multi-core systems, we are not sure that the thread is actually
+         * suspended until we can get the context.
+         */
+        context.ContextFlags = CONTEXT_CONTROL;
+        while (GetThreadContext(cpu->hThread, &context) != 0) {
+            continue;
+        }
+
+        if (hax_enabled()) {
+            cpu->exit_request = 1;
+        }
+
+        if (ResumeThread(cpu->hThread) == (DWORD)(-1)) {
+            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
+                    GetLastError());
+            exit(1);
+        }
+    }
 #endif
 }
 
@@ -1396,6 +1456,9 @@  static void qemu_tcg_init_vcpu(CPUState *cpu)
     static QemuCond *tcg_halt_cond;
     static QemuThread *tcg_cpu_thread;
 
+    if (hax_enabled())
+        hax_init_vcpu(cpu);
+
     /* share a single thread for all cpus with TCG */
     if (!tcg_cpu_thread) {
         cpu->thread = g_malloc0(sizeof(QemuThread));
@@ -1419,6 +1482,26 @@  static void qemu_tcg_init_vcpu(CPUState *cpu)
     }
 }
 
+static void qemu_hax_start_vcpu(CPUState *cpu)
+{
+    char thread_name[VCPU_THREAD_NAME_SIZE];
+
+    cpu->thread = g_malloc0(sizeof(QemuThread));
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
+
+    snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/HAX",
+             cpu->cpu_index);
+    qemu_thread_create(cpu->thread, thread_name, qemu_hax_cpu_thread_fn,
+                       cpu, QEMU_THREAD_JOINABLE);
+#ifdef _WIN32
+    cpu->hThread = qemu_thread_get_handle(cpu->thread);
+#endif
+    while (!cpu->created) {
+        qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
+    }
+}
+
 static void qemu_kvm_start_vcpu(CPUState *cpu)
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
@@ -1469,6 +1552,8 @@  void qemu_init_vcpu(CPUState *cpu)
 
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(cpu);
+    } else if (hax_enabled()) {
+        qemu_hax_start_vcpu(cpu);
     } else if (tcg_enabled()) {
         qemu_tcg_init_vcpu(cpu);
     } else {
diff --git a/exec.c b/exec.c
index 08c558e..25e393d 100644
--- a/exec.c
+++ b/exec.c
@@ -31,6 +31,7 @@ 
 #include "hw/xen/xen.h"
 #endif
 #include "sysemu/kvm.h"
+#include "sysemu/hax.h"
 #include "sysemu/sysemu.h"
 #include "qemu/timer.h"
 #include "qemu/config-file.h"
@@ -1611,6 +1612,21 @@  static void ram_block_add(RAMBlock *new_block, Error **errp)
                 qemu_mutex_unlock_ramlist();
                 return;
             }
+            /*
+             * In Hax, the qemu allocate the virtual address, and HAX kernel
+             * populate the memory with physical memory. Currently we have no
+             * paging, so user should make sure enough free memory in advance
+             */
+            if (hax_enabled()) {
+                int ret;
+                ret = hax_populate_ram((uint64_t)(uintptr_t)new_block->host,
+                                       new_block->max_length);
+                if (ret < 0) {
+                    error_setg(errp, "Hax failed to populate ram");
+                    return;
+                }
+            }
+
             memory_try_enable_merging(new_block->host, new_block->max_length);
         }
     }
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index d78c885..3945dfd 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -26,6 +26,7 @@ 
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "trace.h"
+#include "sysemu/hax.h"
 #include "sysemu/kvm.h"
 #include "hw/qdev.h"
 #include "hw/sysbus.h"
@@ -316,7 +317,7 @@  static void apic_common_realize(DeviceState *dev, Error **errp)
 
     /* Note: We need at least 1M to map the VAPIC option ROM */
     if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK &&
-        ram_size >= 1024 * 1024) {
+        !hax_enabled() && ram_size >= 1024 * 1024) {
         vapic = sysbus_create_simple("kvmvapic", -1, NULL);
     }
     s->vapic = vapic;
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index 3f79a8e..ca4d0fb 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -227,6 +227,8 @@  struct CPUWatchpoint {
 struct KVMState;
 struct kvm_run;
 
+struct hax_vcpu_state;
+
 #define TB_JMP_CACHE_BITS 12
 #define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS)
 
@@ -392,6 +394,9 @@  struct CPUState {
        (absolute value) offset as small as possible.  This reduces code
        size, especially for hosts without large memory offsets.  */
     uint32_t tcg_exit_req;
+
+    bool hax_vcpu_dirty;
+    struct hax_vcpu_state *hax_vcpu;
 };
 
 QTAILQ_HEAD(CPUTailQ, CPUState);
diff --git a/include/sysemu/hw_accel.h b/include/sysemu/hw_accel.h
index 03812cf..c9b3105 100644
--- a/include/sysemu/hw_accel.h
+++ b/include/sysemu/hw_accel.h
@@ -20,6 +20,9 @@  static inline void cpu_synchronize_state(CPUState *cpu)
     if (kvm_enabled()) {
         kvm_cpu_synchronize_state(cpu);
     }
+    if (hax_enabled()) {
+        hax_cpu_synchronize_state(cpu);
+    }
 }
 
 static inline void cpu_synchronize_post_reset(CPUState *cpu)
@@ -27,6 +30,9 @@  static inline void cpu_synchronize_post_reset(CPUState *cpu)
     if (kvm_enabled()) {
         kvm_cpu_synchronize_post_reset(cpu);
     }
+    if (hax_enabled()) {
+        hax_cpu_synchronize_post_reset(cpu);
+    }
 }
 
 static inline void cpu_synchronize_post_init(CPUState *cpu)
@@ -34,6 +40,9 @@  static inline void cpu_synchronize_post_init(CPUState *cpu)
     if (kvm_enabled()) {
         kvm_cpu_synchronize_post_init(cpu);
     }
+    if (hax_enabled()) {
+        hax_cpu_synchronize_post_init(cpu);
+    }
 }
 
 #endif /* QEMU_HW_ACCEL_H */
diff --git a/qemu-options.hx b/qemu-options.hx
index c534a2f..7fc2807 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -3280,6 +3280,17 @@  Enable KVM full virtualization support. This option is only available
 if KVM support is enabled when compiling.
 ETEXI
 
+DEF("enable-hax", 0, QEMU_OPTION_enable_hax, \
+    "-enable-hax     enable HAX virtualization support\n", QEMU_ARCH_I386)
+STEXI
+@item -enable-hax
+@findex -enable-hax
+Enable HAX (Hardware-based Acceleration eXecution) support. This option
+is only available if HAX support is enabled when compiling. HAX is only
+applicable to MAC and Windows platform, and thus does not conflict with
+KVM.
+ETEXI
+
 DEF("xen-domid", HAS_ARG, QEMU_OPTION_xen_domid,
     "-xen-domid id   specify xen guest domain id\n", QEMU_ARCH_ALL)
 DEF("xen-create", 0, QEMU_OPTION_xen_create,
diff --git a/target-i386/Makefile.objs b/target-i386/Makefile.objs
index b223d79..acbe7b0 100644
--- a/target-i386/Makefile.objs
+++ b/target-i386/Makefile.objs
@@ -5,3 +5,7 @@  obj-y += gdbstub.o
 obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o monitor.o
 obj-$(CONFIG_KVM) += kvm.o hyperv.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
+# HAX support
+ifdef CONFIG_WIN32
+obj-$(CONFIG_HAX) += hax-all.o hax-mem.o hax-windows.o
+endif
diff --git a/vl.c b/vl.c
index d77dd86..47bbfe7 100644
--- a/vl.c
+++ b/vl.c
@@ -92,6 +92,7 @@  int main(int argc, char **argv)
 #include "sysemu/cpus.h"
 #include "migration/colo.h"
 #include "sysemu/kvm.h"
+#include "sysemu/hax.h"
 #include "qapi/qmp/qjson.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
@@ -1959,7 +1960,7 @@  static void main_loop(void)
     int64_t ti;
 #endif
     do {
-        nonblocking = !kvm_enabled() && !xen_enabled() && last_io > 0;
+        nonblocking = tcg_enabled() && last_io > 0;
 #ifdef CONFIG_PROFILER
         ti = profile_getclock();
 #endif
@@ -3724,6 +3725,10 @@  int main(int argc, char **argv, char **envp)
                 olist = qemu_find_opts("machine");
                 qemu_opts_parse_noisily(olist, "accel=kvm", false);
                 break;
+            case QEMU_OPTION_enable_hax:
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse_noisily(olist, "accel=hax", false);
+                break;
             case QEMU_OPTION_M:
             case QEMU_OPTION_machine:
                 olist = qemu_find_opts("machine");
@@ -4418,8 +4423,8 @@  int main(int argc, char **argv, char **envp)
 
     cpu_ticks_init();
     if (icount_opts) {
-        if (kvm_enabled() || xen_enabled()) {
-            error_report("-icount is not allowed with kvm or xen");
+        if (!tcg_enabled()) {
+            error_report("-icount is not allowed with hardware virtualization");
             exit(1);
         }
         configure_icount(icount_opts, &error_abort);
@@ -4555,6 +4560,10 @@  int main(int argc, char **argv, char **envp)
 
     numa_post_machine_init();
 
+    if (hax_enabled()) {
+        hax_sync_vcpus();
+    }
+
     if (qemu_opts_foreach(qemu_find_opts("fw_cfg"),
                           parse_fw_cfg, fw_cfg_find(), NULL) != 0) {
         exit(1);