@@ -3650,7 +3650,7 @@ echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
echo "ARCH=$ARCH" >> $config_host_mak
case "$cpu" in
- arm|i386|x86_64|ppc|aarch64)
+ aarch64 | arm | i386 | x86_64 | ppc*)
# The TCG interpreter currently does not support ld/st optimization.
if test "$tcg_interpreter" = "no" ; then
echo "CONFIG_QEMU_LDST_OPTIMIZATION=y" >> $config_host_mak
@@ -326,11 +326,11 @@ extern uintptr_t tci_tb_ptr;
(5) post-process (e.g. stack adjust)
(6) jump to corresponding code of the next of fast path
*/
-# if defined(__i386__) || defined(__x86_64__)
+# if defined(__i386__) || defined(__x86_64__) || defined(_ARCH_PPC64)
# define GETRA() ((uintptr_t)__builtin_return_address(0))
/* The return address argument for ldst is passed directly. */
# define GETPC_LDST() (abort(), 0)
-# elif defined (_ARCH_PPC) && !defined (_ARCH_PPC64)
+# elif defined(_ARCH_PPC)
# define GETRA() ((uintptr_t)__builtin_return_address(0))
# define GETPC_LDST() ((uintptr_t) ((*(int32_t *)(GETRA() - 4)) - 1))
# elif defined(__arm__)
@@ -830,26 +830,50 @@ static void tcg_out_call(TCGContext *s, tcg_target_long arg,
#endif
}
+static const PowerOpcode qemu_ldx_opc[8] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+ LBZX, LHZX, LWZX, LDX,
+ 0, LHAX, LWAX, LDX
+#else
+ LBZX, LHBRX, LWBRX, LDBRX,
+ 0, 0, 0, LDBRX,
+#endif
+};
+
+static const PowerOpcode qemu_stx_opc[4] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+ STBX, STHX, STWX, STDX
+#else
+ STBX, STHBRX, STWBRX, STDBRX,
+#endif
+};
+
+static const PowerOpcode qemu_exts_opc[4] = {
+ EXTSB, EXTSH, EXTSW, 0
+};
+
#if defined (CONFIG_SOFTMMU)
#include "exec/softmmu_defs.h"
/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
- int mmu_idx) */
+ * int mmu_idx, uintptr_t ra)
+ */
static const void * const qemu_ld_helpers[4] = {
- helper_ldb_mmu,
- helper_ldw_mmu,
- helper_ldl_mmu,
- helper_ldq_mmu,
+ helper_ret_ldb_mmu,
+ helper_ret_ldw_mmu,
+ helper_ret_ldl_mmu,
+ helper_ret_ldq_mmu,
};
/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
- uintxx_t val, int mmu_idx) */
+ * uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
static const void * const qemu_st_helpers[4] = {
- helper_stb_mmu,
- helper_stw_mmu,
- helper_stl_mmu,
- helper_stq_mmu,
+ helper_ret_stb_mmu,
+ helper_ret_stw_mmu,
+ helper_ret_stl_mmu,
+ helper_ret_stq_mmu,
};
/* Perform the TLB load and compare. Places the result of the comparison
@@ -911,29 +935,108 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, int s_bits, TCGReg addr_reg,
return addr_reg;
}
-#endif
-static const PowerOpcode qemu_ldx_opc[8] = {
-#ifdef TARGET_WORDS_BIGENDIAN
- LBZX, LHZX, LWZX, LDX,
- 0, LHAX, LWAX, LDX
-#else
- LBZX, LHBRX, LWBRX, LDBRX,
- 0, 0, 0, LDBRX,
-#endif
-};
+/* Record the context of a call to the out of line helper code for the slow
+ path for a load or store, so that we can later generate the correct
+ helper code. */
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, int opc,
+ int data_reg, int addr_reg, int mem_index,
+ uint8_t *raddr, uint8_t *label_ptr)
+{
+ int idx;
+ TCGLabelQemuLdst *label;
-static const PowerOpcode qemu_stx_opc[4] = {
-#ifdef TARGET_WORDS_BIGENDIAN
- STBX, STHX, STWX, STDX
-#else
- STBX, STHBRX, STWBRX, STDBRX,
-#endif
-};
+ if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
+ tcg_abort();
+ }
-static const PowerOpcode qemu_exts_opc[4] = {
- EXTSB, EXTSH, EXTSW, 0
-};
+ idx = s->nb_qemu_ldst_labels++;
+ label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
+ label->is_ld = is_ld;
+ label->opc = opc;
+ label->datalo_reg = data_reg;
+ label->addrlo_reg = addr_reg;
+ label->mem_index = mem_index;
+ label->raddr = raddr;
+ label->label_ptr[0] = label_ptr;
+}
+
+/* See the GETPC definition in include/exec/exec-all.h. */
+static inline uintptr_t do_getpc(uint8_t *raddr)
+{
+ return (uintptr_t)raddr - 1;
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+ int opc = lb->opc;
+ int s_bits = opc & 3;
+ PowerOpcode insn;
+
+ reloc_pc14(lb->label_ptr[0], (uintptr_t)s->code_ptr);
+
+ tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_AREG0);
+
+ /* If the address needed to be zero-extended, we'll have already
+ placed it in R4. The only remaining case is 64-bit guest. */
+ if (lb->addrlo_reg != TCG_REG_R4) {
+ tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, lb->addrlo_reg);
+ }
+
+ tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, lb->mem_index);
+ tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, do_getpc(lb->raddr));
+
+ tcg_out_call(s, (tcg_target_long)qemu_ld_helpers[s_bits], 1, LK);
+
+ if (opc & 4) {
+ insn = qemu_exts_opc[s_bits];
+ tcg_out32(s, insn | RA(lb->datalo_reg) | RS(TCG_REG_R3));
+ } else {
+ tcg_out_mov(s, TCG_TYPE_I64, lb->datalo_reg, TCG_REG_R3);
+ }
+
+ tcg_out_b(s, 0, (uintptr_t)lb->raddr);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+ int opc = lb->opc;
+
+ reloc_pc14(lb->label_ptr[0], (uintptr_t)s->code_ptr);
+
+ tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, TCG_AREG0);
+
+ /* If the address needed to be zero-extended, we'll have already
+ placed it in R4. The only remaining case is 64-bit guest. */
+ if (lb->addrlo_reg != TCG_REG_R4) {
+ tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, lb->addrlo_reg);
+ }
+
+ tcg_out_rld(s, RLDICL, TCG_REG_R5, lb->datalo_reg,
+ 0, 64 - (1 << (3 + opc)));
+ tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R6, lb->mem_index);
+ tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R7, do_getpc(lb->raddr));
+
+ tcg_out_call(s, (tcg_target_long)qemu_st_helpers[opc], 1, LK);
+
+ tcg_out_b(s, 0, (uintptr_t)lb->raddr);
+}
+
+void tcg_out_tb_finalize(TCGContext *s)
+{
+ int i, n = s->nb_qemu_ldst_labels;
+
+ /* qemu_ld/st slow paths */
+ for (i = 0; i < n; i++) {
+ TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
+ if (label->is_ld) {
+ tcg_out_qemu_ld_slow_path(s, label);
+ } else {
+ tcg_out_qemu_st_slow_path(s, label);
+ }
+ }
+}
+#endif /* SOFTMMU */
static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
{
@@ -941,9 +1044,8 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
PowerOpcode insn;
int s_bits;
#ifdef CONFIG_SOFTMMU
- TCGReg ir;
int mem_index;
- void *label1_ptr, *label2_ptr;
+ void *label_ptr;
#endif
data_reg = *args++;
@@ -955,29 +1057,8 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
r0 = tcg_out_tlb_read(s, s_bits, addr_reg, mem_index, true);
- label1_ptr = s->code_ptr;
- tcg_out32(s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-
- /* slow path */
- ir = TCG_REG_R3;
- tcg_out_mov(s, TCG_TYPE_I64, ir++, TCG_AREG0);
- tcg_out_mov(s, TCG_TYPE_I64, ir++, addr_reg);
- tcg_out_movi(s, TCG_TYPE_I64, ir++, mem_index);
-
- tcg_out_call(s, (tcg_target_long)qemu_ld_helpers[s_bits], 1, LK);
-
- if (opc & 4) {
- insn = qemu_exts_opc[s_bits];
- tcg_out32(s, insn | RA(data_reg) | RS(3));
- } else if (data_reg != 3) {
- tcg_out_mov(s, TCG_TYPE_I64, data_reg, 3);
- }
-
- label2_ptr = s->code_ptr;
- tcg_out32(s, B);
-
- /* label1: fast path */
- reloc_pc14(label1_ptr, (tcg_target_long)s->code_ptr);
+ label_ptr = s->code_ptr;
+ tcg_out32(s, BC | BI (7, CR_EQ) | BO_COND_FALSE);
rbase = TCG_REG_R3;
r1 = TCG_REG_R0;
@@ -1007,7 +1088,8 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
}
#ifdef CONFIG_SOFTMMU
- reloc_pc24(label2_ptr, (tcg_target_long)s->code_ptr);
+ add_qemu_ldst_label(s, true, opc, data_reg, r0, mem_index,
+ s->code_ptr, label_ptr);
#endif
}
@@ -1016,9 +1098,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
TCGReg addr_reg, r0, r1, rbase, data_reg;
PowerOpcode insn;
#ifdef CONFIG_SOFTMMU
- TCGReg ir;
int mem_index;
- void *label1_ptr, *label2_ptr;
+ void *label_ptr;
#endif
data_reg = *args++;
@@ -1029,23 +1110,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
r0 = tcg_out_tlb_read(s, opc, addr_reg, mem_index, false);
- label1_ptr = s->code_ptr;
- tcg_out32(s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-
- /* slow path */
- ir = TCG_REG_R3;
- tcg_out_mov(s, TCG_TYPE_I64, ir++, TCG_AREG0);
- tcg_out_mov(s, TCG_TYPE_I64, ir++, addr_reg);
- tcg_out_rld(s, RLDICL, ir++, data_reg, 0, 64 - (1 << (3 + opc)));
- tcg_out_movi(s, TCG_TYPE_I64, ir++, mem_index);
-
- tcg_out_call(s, (tcg_target_long)qemu_st_helpers[opc], 1, LK);
-
- label2_ptr = s->code_ptr;
- tcg_out32(s, B);
-
- /* label1: fast path */
- reloc_pc14(label1_ptr, (tcg_target_long) s->code_ptr);
+ label_ptr = s->code_ptr;
+ tcg_out32(s, BC | BI (7, CR_EQ) | BO_COND_FALSE);
rbase = TCG_REG_R3;
r1 = TCG_REG_R2;
@@ -1070,7 +1136,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
}
#ifdef CONFIG_SOFTMMU
- reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
+ add_qemu_ldst_label(s, false, opc, data_reg, r0, mem_index,
+ s->code_ptr, label_ptr);
#endif
}
Signed-off-by: Richard Henderson <rth@twiddle.net> --- configure | 2 +- include/exec/exec-all.h | 4 +- tcg/ppc64/tcg-target.c | 219 +++++++++++++++++++++++++++++++----------------- 3 files changed, 146 insertions(+), 79 deletions(-)