@@ -416,11 +416,20 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
|| memory_region_is_romd(section->mr)) {
/* Write access calls the I/O callback. */
te->addr_write = address | TLB_MMIO;
- } else if (memory_region_is_ram(section->mr)
- && cpu_physical_memory_is_clean(section->mr->ram_addr
- + xlat)) {
- te->addr_write = address | TLB_NOTDIRTY;
} else {
+ if (memory_region_is_ram(section->mr)
+ && cpu_physical_memory_is_clean(section->mr->ram_addr
+ + xlat)) {
+ address |= TLB_NOTDIRTY;
+ }
+ /* Only normal RAM accesses need the TLB_EXCL flag to handle
+ * exclusive store operatoins. */
+ if (!(address & TLB_MMIO) &&
+ cpu_physical_memory_is_excl(section->mr->ram_addr + xlat)) {
+ /* There is at least one vCPU that has flagged the address as
+ * exclusive. */
+ address |= TLB_EXCL;
+ }
te->addr_write = address;
}
} else {
@@ -496,6 +505,25 @@ static inline void excl_history_put_addr(hwaddr addr)
excl_history.c_array[excl_history.last_idx] = addr & TARGET_PAGE_MASK;
}
+/* For every vCPU compare the exclusive address and reset it in case of a
+ * match. Since only one vCPU is running at once, no lock has to be held to
+ * guard this operation. */
+static inline void reset_other_cpus_colliding_ll_addr(hwaddr addr, hwaddr size)
+{
+ CPUState *cpu;
+
+ CPU_FOREACH(cpu) {
+ if (current_cpu != cpu &&
+ cpu->excl_protected_range.begin != EXCLUSIVE_RESET_ADDR &&
+ ranges_overlap(cpu->excl_protected_range.begin,
+ cpu->excl_protected_range.end -
+ cpu->excl_protected_range.begin,
+ addr, size)) {
+ cpu->excl_protected_range.begin = EXCLUSIVE_RESET_ADDR;
+ }
+ }
+}
+
#define MMUSUFFIX _mmu
/* Generates LoadLink/StoreConditional helpers in softmmu_template.h */
@@ -469,6 +469,43 @@ static inline void smmu_helper(do_ram_store)(CPUArchState *env,
#endif
}
+static inline void smmu_helper(do_excl_store)(CPUArchState *env,
+ bool little_endian,
+ DATA_TYPE val, target_ulong addr,
+ TCGMemOpIdx oi, int index,
+ uintptr_t retaddr)
+{
+ CPUIOTLBEntry *iotlbentry = &env->iotlb[get_mmuidx(oi)][index];
+ CPUState *cpu = ENV_GET_CPU(env);
+ CPUClass *cc = CPU_GET_CLASS(cpu);
+ /* The slow-path has been forced since we are writing to
+ * exclusive-protected memory. */
+ hwaddr hw_addr = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
+
+ /* The function reset_other_cpus_colliding_ll_addr could have reset
+ * the exclusive address. Fail the SC in this case.
+ * N.B.: here excl_succeed == true means that the caller is
+ * helper_stcond_name in softmmu_llsc_template.
+ * On the contrary, excl_succeeded == false occurs when a VCPU is
+ * writing through normal store to a page with TLB_EXCL bit set. */
+ if (cpu->excl_succeeded) {
+ if (!cc->cpu_valid_excl_access(cpu, hw_addr, DATA_SIZE)) {
+ /* The vCPU is SC-ing to an unprotected address. */
+ cpu->excl_protected_range.begin = EXCLUSIVE_RESET_ADDR;
+ cpu->excl_succeeded = false;
+
+ return;
+ }
+ }
+
+ smmu_helper(do_ram_store)(env, little_endian, val, addr, oi,
+ get_mmuidx(oi), index, retaddr);
+
+ reset_other_cpus_colliding_ll_addr(hw_addr, DATA_SIZE);
+
+ return;
+}
+
void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
TCGMemOpIdx oi, uintptr_t retaddr)
{
@@ -493,11 +530,17 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
}
- /* Handle an IO access. */
+ /* Handle an IO access or exclusive access. */
if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
- smmu_helper(do_mmio_store)(env, true, val, addr, oi, mmu_idx, index,
- retaddr);
- return;
+ if (tlb_addr & TLB_EXCL) {
+ smmu_helper(do_excl_store)(env, true, val, addr, oi, index,
+ retaddr);
+ return;
+ } else {
+ smmu_helper(do_mmio_store)(env, true, val, addr, oi, mmu_idx,
+ index, retaddr);
+ return;
+ }
}
smmu_helper(do_ram_store)(env, true, val, addr, oi, mmu_idx, index,
@@ -529,11 +572,17 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
}
- /* Handle an IO access. */
+ /* Handle an IO access or exclusive access. */
if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
- smmu_helper(do_mmio_store)(env, false, val, addr, oi, mmu_idx, index,
- retaddr);
- return;
+ if (tlb_addr & TLB_EXCL) {
+ smmu_helper(do_excl_store)(env, false, val, addr, oi, index,
+ retaddr);
+ return;
+ } else {
+ smmu_helper(do_mmio_store)(env, false, val, addr, oi, mmu_idx,
+ index, retaddr);
+ return;
+ }
}
smmu_helper(do_ram_store)(env, false, val, addr, oi, mmu_idx, index,
The pages set as exclusive (clean) in the DIRTY_MEMORY_EXCLUSIVE bitmap have to have their TLB entries flagged with TLB_EXCL. The accesses to pages with TLB_EXCL flag set have to be properly handled in that they can potentially invalidate an open LL/SC transaction. Modify the TLB entries generation to honor the new bitmap and extend the softmmu_template to handle the accesses made to guest pages marked as exclusive. The TLB_EXCL flag is used only for normal RAM memory. Exclusive accesses to MMIO memory are still not supported, but they will with the next patch. Suggested-by: Jani Kokkonen <jani.kokkonen@huawei.com> Suggested-by: Claudio Fontana <claudio.fontana@huawei.com> Signed-off-by: Alvise Rigo <a.rigo@virtualopensystems.com> --- cputlb.c | 36 ++++++++++++++++++++++++++---- softmmu_template.h | 65 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 12 deletions(-)