@@ -299,6 +299,13 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
env->tlb_v_table[mmu_idx][vidx] = *te;
env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
+ if (unlikely(!(te->addr_write & TLB_MMIO) && (te->addr_write & TLB_EXCL))) {
+ /* We are removing an exclusive entry, set the page to dirty. */
+ hwaddr hw_addr = (env->iotlb[mmu_idx][index].addr & TARGET_PAGE_MASK) +
+ (te->addr_write & TARGET_PAGE_MASK);
+ cpu_physical_memory_set_excl_dirty(hw_addr, cpu->cpu_index);
+ }
+
/* refill the tlb */
env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
env->iotlb[mmu_idx][index].attrs = attrs;
@@ -324,7 +331,15 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
+ xlat)) {
te->addr_write = address | TLB_NOTDIRTY;
} else {
- te->addr_write = address;
+ if (!(address & TLB_MMIO) &&
+ cpu_physical_memory_excl_atleast_one_clean(section->mr->ram_addr
+ + xlat)) {
+ /* There is at least one vCPU that has flagged the address as
+ * exclusive. */
+ te->addr_write = address | TLB_EXCL;
+ } else {
+ te->addr_write = address;
+ }
}
} else {
te->addr_write = -1;
@@ -376,6 +391,28 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
return qemu_ram_addr_from_host_nofail(p);
}
+/* Atomic insn translation TLB support. */
+#define EXCLUSIVE_RESET_ADDR ULLONG_MAX
+/* For every vCPU compare the exclusive address and reset it in case of a
+ * match. Since only one vCPU is running at once, no lock has to be held to
+ * guard this operation. */
+static inline void lookup_and_reset_cpus_ll_addr(hwaddr addr, hwaddr size)
+{
+ CPUState *cpu;
+ CPUArchState *acpu;
+
+ CPU_FOREACH(cpu) {
+ acpu = (CPUArchState *)cpu->env_ptr;
+
+ if (acpu->excl_protected_range.begin != EXCLUSIVE_RESET_ADDR &&
+ ranges_overlap(acpu->excl_protected_range.begin,
+ acpu->excl_protected_range.end - acpu->excl_protected_range.begin,
+ addr, size)) {
+ acpu->excl_protected_range.begin = EXCLUSIVE_RESET_ADDR;
+ }
+ }
+}
+
#define MMUSUFFIX _mmu
#define SHIFT 0
@@ -320,6 +320,14 @@ extern RAMList ram_list;
#define TLB_NOTDIRTY (1 << 4)
/* Set if TLB entry is an IO callback. */
#define TLB_MMIO (1 << 5)
+/* Set if TLB entry references a page that requires exclusive access. */
+#define TLB_EXCL (1 << 6)
+
+/* Do not allow a TARGET_PAGE_MASK which covers one or more bits defined
+ * above. */
+#if TLB_EXCL >= TARGET_PAGE_SIZE
+#error TARGET_PAGE_MASK covering the low bits of the TLB virtual address
+#endif
void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf);
@@ -27,6 +27,7 @@
#include <inttypes.h>
#include "qemu/osdep.h"
#include "qemu/queue.h"
+#include "qemu/range.h"
#include "tcg-target.h"
#ifndef CONFIG_USER_ONLY
#include "exec/hwaddr.h"
@@ -150,5 +151,16 @@ typedef struct CPUIOTLBEntry {
#define CPU_COMMON \
/* soft mmu support */ \
CPU_COMMON_TLB \
+ \
+ /* Used by the atomic insn translation backend. */ \
+ int ll_sc_context; \
+ /* vCPU current exclusive addresses range.
+ * The address is set to EXCLUSIVE_RESET_ADDR if the vCPU is not.
+ * in the middle of a LL/SC. */ \
+ struct Range excl_protected_range; \
+ /* Used to carry the SC result but also to flag a normal (legacy)
+ * store access made by a stcond (see softmmu_template.h). */ \
+ int excl_succeeded; \
+
#endif
@@ -409,19 +409,53 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
}
- /* Handle an IO access. */
+ /* Handle an IO access or exclusive access. */
if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
- CPUIOTLBEntry *iotlbentry;
- if ((addr & (DATA_SIZE - 1)) != 0) {
- goto do_unaligned_access;
+ CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
+
+ if ((tlb_addr & ~TARGET_PAGE_MASK) == TLB_EXCL) {
+ /* The slow-path has been forced since we are writing to
+ * exclusive-protected memory. */
+ hwaddr hw_addr = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
+
+ /* The function lookup_and_reset_cpus_ll_addr could have reset the
+ * exclusive address. Fail the SC in this case.
+ * N.B.: Here excl_succeeded == 0 means that helper_le_st_name has
+ * not been called by a softmmu_llsc_template.h. */
+ if(env->excl_succeeded) {
+ if (env->excl_protected_range.begin != hw_addr) {
+ /* The vCPU is SC-ing to an unprotected address. */
+ env->excl_protected_range.begin = EXCLUSIVE_RESET_ADDR;
+ env->excl_succeeded = 0;
+
+ return;
+ }
+
+ cpu_physical_memory_set_excl_dirty(hw_addr, ENV_GET_CPU(env)->cpu_index);
+ }
+
+ haddr = addr + env->tlb_table[mmu_idx][index].addend;
+ #if DATA_SIZE == 1
+ glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
+ #else
+ glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
+ #endif
+
+ lookup_and_reset_cpus_ll_addr(hw_addr, DATA_SIZE);
+
+ return;
+ } else {
+ if ((addr & (DATA_SIZE - 1)) != 0) {
+ goto do_unaligned_access;
+ }
+ iotlbentry = &env->iotlb[mmu_idx][index];
+
+ /* ??? Note that the io helpers always read data in the target
+ byte ordering. We should push the LE/BE request down into io. */
+ val = TGT_LE(val);
+ glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
+ return;
}
- iotlbentry = &env->iotlb[mmu_idx][index];
-
- /* ??? Note that the io helpers always read data in the target
- byte ordering. We should push the LE/BE request down into io. */
- val = TGT_LE(val);
- glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
- return;
}
/* Handle slow unaligned access (it spans two pages or IO). */
@@ -489,19 +523,53 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
}
- /* Handle an IO access. */
+ /* Handle an IO access or exclusive access. */
if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
- CPUIOTLBEntry *iotlbentry;
- if ((addr & (DATA_SIZE - 1)) != 0) {
- goto do_unaligned_access;
+ CPUIOTLBEntry *iotlbentry = &env->iotlb[mmu_idx][index];
+
+ if ((tlb_addr & ~TARGET_PAGE_MASK) == TLB_EXCL) {
+ /* The slow-path has been forced since we are writing to
+ * exclusive-protected memory. */
+ hwaddr hw_addr = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
+
+ /* The function lookup_and_reset_cpus_ll_addr could have reset the
+ * exclusive address. Fail the SC in this case.
+ * N.B.: Here excl_succeeded == 0 means that helper_le_st_name has
+ * not been called by a softmmu_llsc_template.h. */
+ if(env->excl_succeeded) {
+ if (env->excl_protected_range.begin != hw_addr) {
+ /* The vCPU is SC-ing to an unprotected address. */
+ env->excl_protected_range.begin = EXCLUSIVE_RESET_ADDR;
+ env->excl_succeeded = 0;
+
+ return;
+ }
+
+ cpu_physical_memory_set_excl_dirty(hw_addr, ENV_GET_CPU(env)->cpu_index);
+ }
+
+ haddr = addr + env->tlb_table[mmu_idx][index].addend;
+ #if DATA_SIZE == 1
+ glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
+ #else
+ glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
+ #endif
+
+ lookup_and_reset_cpus_ll_addr(hw_addr, DATA_SIZE);
+
+ return;
+ } else {
+ if ((addr & (DATA_SIZE - 1)) != 0) {
+ goto do_unaligned_access;
+ }
+ iotlbentry = &env->iotlb[mmu_idx][index];
+
+ /* ??? Note that the io helpers always read data in the target
+ byte ordering. We should push the LE/BE request down into io. */
+ val = TGT_BE(val);
+ glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
+ return;
}
- iotlbentry = &env->iotlb[mmu_idx][index];
-
- /* ??? Note that the io helpers always read data in the target
- byte ordering. We should push the LE/BE request down into io. */
- val = TGT_BE(val);
- glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
- return;
}
/* Handle slow unaligned access (it spans two pages or IO). */
Add a new TLB flag to force all the accesses made to a page to follow the slow-path. In the case we remove a TLB entry marked as EXCL, we unset the corresponding exclusive bit in the bitmap. Suggested-by: Jani Kokkonen <jani.kokkonen@huawei.com> Suggested-by: Claudio Fontana <claudio.fontana@huawei.com> Signed-off-by: Alvise Rigo <a.rigo@virtualopensystems.com> --- cputlb.c | 39 ++++++++++++++++- include/exec/cpu-all.h | 8 ++++ include/exec/cpu-defs.h | 12 ++++++ softmmu_template.h | 112 ++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 148 insertions(+), 23 deletions(-)