| Message ID | 20200522160755.886-14-robert.foley@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series | Add Thread Sanitizer support to QEMU | expand |
On Fri, May 22, 2020 at 12:07:49 -0400, Robert Foley wrote: > For example: > WARNING: ThreadSanitizer: data race (pid=35425) > Write of size 4 at 0x7bbc000000ac by main thread (mutexes: write M875): > #0 cpu_reset_interrupt hw/core/cpu.c:107:28 (qemu-system-aarch64+0x843790) > #1 arm_cpu_set_irq target/arm/cpu.c (qemu-system-aarch64+0x616265) > #2 qemu_set_irq hw/core/irq.c:44:5 (qemu-system-aarch64+0x8462ca) > Previous atomic read of size 4 at 0x7bbc000000ac by thread T6: > #0 __tsan_atomic32_load <null> (qemu-system-aarch64+0x394c1c) > #1 cpu_handle_interrupt accel/tcg/cpu-exec.c:534:9 (qemu-system-aarch64+0x4b7e79) > #2 cpu_exec accel/tcg/cpu-exec.c:720:17 (qemu-system-aarch64+0x4b7e79) > or > WARNING: ThreadSanitizer: data race (pid=25425) > Read of size 8 at 0x7f8ad8e138d0 by thread T10: > #0 tb_lookup_cmp accel/tcg/cpu-exec.c:307:13 (qemu-system-aarch64+0x4ac4d2) > #1 qht_do_lookup util/qht.c:502:34 (qemu-system-aarch64+0xd05264) > Previous write of size 8 at 0x7f8ad8e138d0 by thread T15 (mutexes: write M728311726235541804): > #0 tb_link_page accel/tcg/translate-all.c:1625:26 (qemu-system-aarch64+0x4b0bf2) > #1 tb_gen_code accel/tcg/translate-all.c:1865:19 (qemu-system-aarch64+0x4b0bf2) > #2 tb_find accel/tcg/cpu-exec.c:407:14 (qemu-system-aarch64+0x4ad77c) I see you're working through the warnings in this file, but I think it would be better to forget about files and focus on the data itself. Therefore this patch should be split in two: (1) cpu-<interrupt_request and (2) gen_code_buf. (1) requires a lot of changes with a proper audit; the per-cpu-lock series has a possible solution for that, so I will ignore those hunks and just comment on (2) below. > Cc: Richard Henderson <richard.henderson@linaro.org> > Cc: Paolo Bonzini <pbonzini@redhat.com> > Signed-off-by: Robert Foley <robert.foley@linaro.org> > --- > accel/tcg/tcg-all.c | 4 ++-- > accel/tcg/tcg-runtime.c | 7 ++++++- > accel/tcg/translate-all.c | 6 +++++- > hw/core/cpu.c | 2 +- > 4 files changed, 14 insertions(+), 5 deletions(-) > (snip) > diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c > index 446465a09a..bd0cd77450 100644 > --- a/accel/tcg/tcg-runtime.c > +++ b/accel/tcg/tcg-runtime.c > @@ -31,6 +31,7 @@ > #include "disas/disas.h" > #include "exec/log.h" > #include "tcg/tcg.h" > +#include "qemu/tsan.h" > > /* 32-bit helpers */ > > @@ -151,6 +152,7 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) > TranslationBlock *tb; > target_ulong cs_base, pc; > uint32_t flags; > + void *tc_ptr; > > tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, curr_cflags()); > if (tb == NULL) { > @@ -161,7 +163,10 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) > TARGET_FMT_lx "/" TARGET_FMT_lx "/%#x] %s\n", > cpu->cpu_index, tb->tc.ptr, cs_base, pc, flags, > lookup_symbol(pc)); > - return tb->tc.ptr; > + TSAN_ANNOTATE_IGNORE_READS_BEGIN(); > + tc_ptr = tb->tc.ptr; > + TSAN_ANNOTATE_IGNORE_READS_END(); > + return tc_ptr; I'm not sure these are needed. At least after applying all other patches in this series, I don't get a warning here. > } > > void HELPER(exit_atomic)(CPUArchState *env) > diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c > index 3fb71a1503..6c0e61994c 100644 > --- a/accel/tcg/translate-all.c > +++ b/accel/tcg/translate-all.c > @@ -58,6 +58,7 @@ > #include "exec/log.h" > #include "sysemu/cpus.h" > #include "sysemu/tcg.h" > +#include "qemu/tsan.h" > > /* #define DEBUG_TB_INVALIDATE */ > /* #define DEBUG_TB_FLUSH */ > @@ -1704,6 +1705,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, > max_insns = 1; > } > > + TSAN_ANNOTATE_IGNORE_WRITES_BEGIN(); Same here, I don't get a warning in this hunk if I remove these, except for: --- WARNING: ThreadSanitizer: data race (pid=445867) Atomic read of size 1 at 0x7f906e050158 by thread T7: #0 __tsan_mutex_post_lock <null> (qemu-system-aarch64+0x481721) #1 qemu_spin_lock /home/cota/src/qemu/include/qemu/thread.h:244:5 (qemu-system-aarch64+0x5578e9) #2 tb_add_jump /home/cota/src/qemu/accel/tcg/cpu-exec.c:363:5 (qemu-system-aarch64+0x5578e9) #3 tb_find /home/cota/src/qemu/accel/tcg/cpu-exec.c:423:9 (qemu-system-aarch64+0x5578e9) Previous write of size 1 at 0x7f906e050158 by thread T8: #0 __tsan_mutex_create <null> (qemu-system-aarch64+0x481589) #1 qemu_spin_init /home/cota/src/qemu/include/qemu/thread.h:221:5 (qemu-system-aarch64+0x559a71) #2 tb_gen_code /home/cota/src/qemu/accel/tcg/translate-all.c:1875:5 (qemu-system-aarch64+0x559a71) Thread T7 'CPU 0/TCG' (tid=445875, running) created by main thread at: #0 pthread_create <null> (qemu-system-aarch64+0x43915b) #1 qemu_thread_create /home/cota/src/qemu/util/qemu-thread-posix.c:558:11 (qemu-system-aarch64+0xaf91ff) Thread T8 'CPU 1/TCG' (tid=445876, running) created by main thread at: #0 pthread_create <null> (qemu-system-aarch64+0x43915b) #1 qemu_thread_create /home/cota/src/qemu/util/qemu-thread-posix.c:558:11 (qemu-system-aarch64+0xaf91ff) SUMMARY: ThreadSanitizer: data race (/home/cota/src/qemu/build/aarch64-softmmu/qemu-system-aarch64+0x481721) in __tsan_mutex_post_lock --- Seems like tsan is confusing itself here. Thanks, E.
On Sat, 23 May 2020 at 16:06, Emilio G. Cota <cota@braap.org> wrote: > > On Fri, May 22, 2020 at 12:07:49 -0400, Robert Foley wrote: > > For example: > > WARNING: ThreadSanitizer: data race (pid=35425) > > Write of size 4 at 0x7bbc000000ac by main thread (mutexes: write M875): > > #0 cpu_reset_interrupt hw/core/cpu.c:107:28 (qemu-system-aarch64+0x843790) > > #1 arm_cpu_set_irq target/arm/cpu.c (qemu-system-aarch64+0x616265) > > #2 qemu_set_irq hw/core/irq.c:44:5 (qemu-system-aarch64+0x8462ca) > > Previous atomic read of size 4 at 0x7bbc000000ac by thread T6: > > #0 __tsan_atomic32_load <null> (qemu-system-aarch64+0x394c1c) > > #1 cpu_handle_interrupt accel/tcg/cpu-exec.c:534:9 (qemu-system-aarch64+0x4b7e79) > > #2 cpu_exec accel/tcg/cpu-exec.c:720:17 (qemu-system-aarch64+0x4b7e79) > > or > > WARNING: ThreadSanitizer: data race (pid=25425) > > Read of size 8 at 0x7f8ad8e138d0 by thread T10: > > #0 tb_lookup_cmp accel/tcg/cpu-exec.c:307:13 (qemu-system-aarch64+0x4ac4d2) > > #1 qht_do_lookup util/qht.c:502:34 (qemu-system-aarch64+0xd05264) > > Previous write of size 8 at 0x7f8ad8e138d0 by thread T15 (mutexes: write M728311726235541804): > > #0 tb_link_page accel/tcg/translate-all.c:1625:26 (qemu-system-aarch64+0x4b0bf2) > > #1 tb_gen_code accel/tcg/translate-all.c:1865:19 (qemu-system-aarch64+0x4b0bf2) > > #2 tb_find accel/tcg/cpu-exec.c:407:14 (qemu-system-aarch64+0x4ad77c) > > I see you're working through the warnings in this file, but I think it would > be better to forget about files and focus on the data itself. > Therefore this patch should be split in two: (1) cpu-<interrupt_request > and (2) gen_code_buf. (1) requires a lot of changes with a proper audit; > the per-cpu-lock series has a possible solution for that, so I will > ignore those hunks and just comment on (2) below. We will be dropping the changes in this file which overlap with the per-cpu-locks patch. > > > Cc: Richard Henderson <richard.henderson@linaro.org> > > Cc: Paolo Bonzini <pbonzini@redhat.com> > > Signed-off-by: Robert Foley <robert.foley@linaro.org> > > --- > > accel/tcg/tcg-all.c | 4 ++-- > > accel/tcg/tcg-runtime.c | 7 ++++++- > > accel/tcg/translate-all.c | 6 +++++- > > hw/core/cpu.c | 2 +- > > 4 files changed, 14 insertions(+), 5 deletions(-) > > > (snip) > > diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c > > index 446465a09a..bd0cd77450 100644 > > --- a/accel/tcg/tcg-runtime.c > > +++ b/accel/tcg/tcg-runtime.c > > @@ -31,6 +31,7 @@ > > #include "disas/disas.h" > > #include "exec/log.h" > > #include "tcg/tcg.h" > > +#include "qemu/tsan.h" > > > > /* 32-bit helpers */ > > > > @@ -151,6 +152,7 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) > > TranslationBlock *tb; > > target_ulong cs_base, pc; > > uint32_t flags; > > + void *tc_ptr; > > > > tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, curr_cflags()); > > if (tb == NULL) { > > @@ -161,7 +163,10 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) > > TARGET_FMT_lx "/" TARGET_FMT_lx "/%#x] %s\n", > > cpu->cpu_index, tb->tc.ptr, cs_base, pc, flags, > > lookup_symbol(pc)); > > - return tb->tc.ptr; > > + TSAN_ANNOTATE_IGNORE_READS_BEGIN(); > > + tc_ptr = tb->tc.ptr; > > + TSAN_ANNOTATE_IGNORE_READS_END(); > > + return tc_ptr; > > I'm not sure these are needed. At least after applying all other patches > in this series, I don't get a warning here. I think we will also be dropping the ANNOTATE calls here as we re-focus the patch series. Thanks & Regards, -Rob > > > } > > > > void HELPER(exit_atomic)(CPUArchState *env) > > diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c > > index 3fb71a1503..6c0e61994c 100644 > > --- a/accel/tcg/translate-all.c > > +++ b/accel/tcg/translate-all.c > > @@ -58,6 +58,7 @@ > > #include "exec/log.h" > > #include "sysemu/cpus.h" > > #include "sysemu/tcg.h" > > +#include "qemu/tsan.h" > > > > /* #define DEBUG_TB_INVALIDATE */ > > /* #define DEBUG_TB_FLUSH */ > > @@ -1704,6 +1705,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, > > max_insns = 1; > > } > > > > + TSAN_ANNOTATE_IGNORE_WRITES_BEGIN(); > > Same here, I don't get a warning in this hunk if I remove these, > except for: > --- > WARNING: ThreadSanitizer: data race (pid=445867) > Atomic read of size 1 at 0x7f906e050158 by thread T7: > #0 __tsan_mutex_post_lock <null> (qemu-system-aarch64+0x481721) > #1 qemu_spin_lock /home/cota/src/qemu/include/qemu/thread.h:244:5 (qemu-system-aarch64+0x5578e9) > #2 tb_add_jump /home/cota/src/qemu/accel/tcg/cpu-exec.c:363:5 (qemu-system-aarch64+0x5578e9) > #3 tb_find /home/cota/src/qemu/accel/tcg/cpu-exec.c:423:9 (qemu-system-aarch64+0x5578e9) > > Previous write of size 1 at 0x7f906e050158 by thread T8: > #0 __tsan_mutex_create <null> (qemu-system-aarch64+0x481589) > #1 qemu_spin_init /home/cota/src/qemu/include/qemu/thread.h:221:5 (qemu-system-aarch64+0x559a71) > #2 tb_gen_code /home/cota/src/qemu/accel/tcg/translate-all.c:1875:5 (qemu-system-aarch64+0x559a71) > > Thread T7 'CPU 0/TCG' (tid=445875, running) created by main thread at: > #0 pthread_create <null> (qemu-system-aarch64+0x43915b) > #1 qemu_thread_create /home/cota/src/qemu/util/qemu-thread-posix.c:558:11 (qemu-system-aarch64+0xaf91ff) > > Thread T8 'CPU 1/TCG' (tid=445876, running) created by main thread at: > #0 pthread_create <null> (qemu-system-aarch64+0x43915b) > #1 qemu_thread_create /home/cota/src/qemu/util/qemu-thread-posix.c:558:11 (qemu-system-aarch64+0xaf91ff) > > SUMMARY: ThreadSanitizer: data race (/home/cota/src/qemu/build/aarch64-softmmu/qemu-system-aarch64+0x481721) in __tsan_mutex_post_lock > --- > > Seems like tsan is confusing itself here. > > Thanks, > E.
On 22/05/20 18:07, Robert Foley wrote: > For example: > WARNING: ThreadSanitizer: data race (pid=35425) > Write of size 4 at 0x7bbc000000ac by main thread (mutexes: write M875): > #0 cpu_reset_interrupt hw/core/cpu.c:107:28 (qemu-system-aarch64+0x843790) > #1 arm_cpu_set_irq target/arm/cpu.c (qemu-system-aarch64+0x616265) > #2 qemu_set_irq hw/core/irq.c:44:5 (qemu-system-aarch64+0x8462ca) > Previous atomic read of size 4 at 0x7bbc000000ac by thread T6: > #0 __tsan_atomic32_load <null> (qemu-system-aarch64+0x394c1c) > #1 cpu_handle_interrupt accel/tcg/cpu-exec.c:534:9 (qemu-system-aarch64+0x4b7e79) > #2 cpu_exec accel/tcg/cpu-exec.c:720:17 (qemu-system-aarch64+0x4b7e79) > or > WARNING: ThreadSanitizer: data race (pid=25425) > Read of size 8 at 0x7f8ad8e138d0 by thread T10: > #0 tb_lookup_cmp accel/tcg/cpu-exec.c:307:13 (qemu-system-aarch64+0x4ac4d2) > #1 qht_do_lookup util/qht.c:502:34 (qemu-system-aarch64+0xd05264) > Previous write of size 8 at 0x7f8ad8e138d0 by thread T15 (mutexes: write M728311726235541804): > #0 tb_link_page accel/tcg/translate-all.c:1625:26 (qemu-system-aarch64+0x4b0bf2) > #1 tb_gen_code accel/tcg/translate-all.c:1865:19 (qemu-system-aarch64+0x4b0bf2) > #2 tb_find accel/tcg/cpu-exec.c:407:14 (qemu-system-aarch64+0x4ad77c) > > Cc: Richard Henderson <richard.henderson@linaro.org> > Cc: Paolo Bonzini <pbonzini@redhat.com> > Signed-off-by: Robert Foley <robert.foley@linaro.org> > --- > accel/tcg/tcg-all.c | 4 ++-- > accel/tcg/tcg-runtime.c | 7 ++++++- > accel/tcg/translate-all.c | 6 +++++- > hw/core/cpu.c | 2 +- > 4 files changed, 14 insertions(+), 5 deletions(-) > > diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c > index 3b4fda5640..f94ea4c4b3 100644 > --- a/accel/tcg/tcg-all.c > +++ b/accel/tcg/tcg-all.c > @@ -54,8 +54,8 @@ static void tcg_handle_interrupt(CPUState *cpu, int mask) > int old_mask; > g_assert(qemu_mutex_iothread_locked()); > > - old_mask = cpu->interrupt_request; > - cpu->interrupt_request |= mask; > + old_mask = atomic_read(&cpu->interrupt_request); > + atomic_or(&cpu->interrupt_request, mask); You can use atomic_fetch_or here. Paolo
diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index 3b4fda5640..f94ea4c4b3 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -54,8 +54,8 @@ static void tcg_handle_interrupt(CPUState *cpu, int mask) int old_mask; g_assert(qemu_mutex_iothread_locked()); - old_mask = cpu->interrupt_request; - cpu->interrupt_request |= mask; + old_mask = atomic_read(&cpu->interrupt_request); + atomic_or(&cpu->interrupt_request, mask); /* * If called from iothread context, wake the target cpu in diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c index 446465a09a..bd0cd77450 100644 --- a/accel/tcg/tcg-runtime.c +++ b/accel/tcg/tcg-runtime.c @@ -31,6 +31,7 @@ #include "disas/disas.h" #include "exec/log.h" #include "tcg/tcg.h" +#include "qemu/tsan.h" /* 32-bit helpers */ @@ -151,6 +152,7 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) TranslationBlock *tb; target_ulong cs_base, pc; uint32_t flags; + void *tc_ptr; tb = tb_lookup__cpu_state(cpu, &pc, &cs_base, &flags, curr_cflags()); if (tb == NULL) { @@ -161,7 +163,10 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env) TARGET_FMT_lx "/" TARGET_FMT_lx "/%#x] %s\n", cpu->cpu_index, tb->tc.ptr, cs_base, pc, flags, lookup_symbol(pc)); - return tb->tc.ptr; + TSAN_ANNOTATE_IGNORE_READS_BEGIN(); + tc_ptr = tb->tc.ptr; + TSAN_ANNOTATE_IGNORE_READS_END(); + return tc_ptr; } void HELPER(exit_atomic)(CPUArchState *env) diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index 3fb71a1503..6c0e61994c 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -58,6 +58,7 @@ #include "exec/log.h" #include "sysemu/cpus.h" #include "sysemu/tcg.h" +#include "qemu/tsan.h" /* #define DEBUG_TB_INVALIDATE */ /* #define DEBUG_TB_FLUSH */ @@ -1704,6 +1705,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu, max_insns = 1; } + TSAN_ANNOTATE_IGNORE_WRITES_BEGIN(); buffer_overflow: tb = tcg_tb_alloc(tcg_ctx); if (unlikely(!tb)) { @@ -1902,9 +1904,11 @@ TranslationBlock *tb_gen_code(CPUState *cpu, orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); tb_destroy(tb); + TSAN_ANNOTATE_IGNORE_WRITES_END(); return existing_tb; } tcg_tb_insert(tb); + TSAN_ANNOTATE_IGNORE_WRITES_END(); return tb; } @@ -2409,7 +2413,7 @@ void dump_opcount_info(void) void cpu_interrupt(CPUState *cpu, int mask) { g_assert(qemu_mutex_iothread_locked()); - cpu->interrupt_request |= mask; + atomic_or(&cpu->interrupt_request, mask); atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1); } diff --git a/hw/core/cpu.c b/hw/core/cpu.c index 77703d62b7..6c16ccc426 100644 --- a/hw/core/cpu.c +++ b/hw/core/cpu.c @@ -104,7 +104,7 @@ void cpu_reset_interrupt(CPUState *cpu, int mask) if (need_lock) { qemu_mutex_lock_iothread(); } - cpu->interrupt_request &= ~mask; + atomic_and(&cpu->interrupt_request, ~mask); if (need_lock) { qemu_mutex_unlock_iothread(); }
For example: WARNING: ThreadSanitizer: data race (pid=35425) Write of size 4 at 0x7bbc000000ac by main thread (mutexes: write M875): #0 cpu_reset_interrupt hw/core/cpu.c:107:28 (qemu-system-aarch64+0x843790) #1 arm_cpu_set_irq target/arm/cpu.c (qemu-system-aarch64+0x616265) #2 qemu_set_irq hw/core/irq.c:44:5 (qemu-system-aarch64+0x8462ca) Previous atomic read of size 4 at 0x7bbc000000ac by thread T6: #0 __tsan_atomic32_load <null> (qemu-system-aarch64+0x394c1c) #1 cpu_handle_interrupt accel/tcg/cpu-exec.c:534:9 (qemu-system-aarch64+0x4b7e79) #2 cpu_exec accel/tcg/cpu-exec.c:720:17 (qemu-system-aarch64+0x4b7e79) or WARNING: ThreadSanitizer: data race (pid=25425) Read of size 8 at 0x7f8ad8e138d0 by thread T10: #0 tb_lookup_cmp accel/tcg/cpu-exec.c:307:13 (qemu-system-aarch64+0x4ac4d2) #1 qht_do_lookup util/qht.c:502:34 (qemu-system-aarch64+0xd05264) Previous write of size 8 at 0x7f8ad8e138d0 by thread T15 (mutexes: write M728311726235541804): #0 tb_link_page accel/tcg/translate-all.c:1625:26 (qemu-system-aarch64+0x4b0bf2) #1 tb_gen_code accel/tcg/translate-all.c:1865:19 (qemu-system-aarch64+0x4b0bf2) #2 tb_find accel/tcg/cpu-exec.c:407:14 (qemu-system-aarch64+0x4ad77c) Cc: Richard Henderson <richard.henderson@linaro.org> Cc: Paolo Bonzini <pbonzini@redhat.com> Signed-off-by: Robert Foley <robert.foley@linaro.org> --- accel/tcg/tcg-all.c | 4 ++-- accel/tcg/tcg-runtime.c | 7 ++++++- accel/tcg/translate-all.c | 6 +++++- hw/core/cpu.c | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-)