@@ -716,6 +716,16 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
arg, arg1, arg2);
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ if (val == 0) {
+ tcg_out_st(s, type, TCG_REG_XZR, base, ofs);
+ return true;
+ }
+ return false;
+}
+
static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
TCGReg rn, unsigned int a, unsigned int b)
{
@@ -2046,6 +2046,12 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
tcg_out_st32(s, COND_AL, arg, arg1, arg2);
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ return false;
+}
+
static inline void tcg_out_mov(TCGContext *s, TCGType type,
TCGReg ret, TCGReg arg)
{
@@ -710,12 +710,19 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
tcg_out_modrm_offset(s, opc, arg, arg1, arg2);
}
-static inline void tcg_out_sti(TCGContext *s, TCGType type, TCGReg base,
- tcg_target_long ofs, tcg_target_long val)
+static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
{
- int opc = OPC_MOVL_EvIz + (type == TCG_TYPE_I64 ? P_REXW : 0);
- tcg_out_modrm_offset(s, opc, 0, base, ofs);
+ int rexw = 0;
+ if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) {
+ if (val != (int32_t)val) {
+ return false;
+ }
+ rexw = P_REXW;
+ }
+ tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs);
tcg_out32(s, val);
+ return true;
}
static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
@@ -1321,10 +1328,10 @@ static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
ofs += 4;
}
- tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
+ tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
ofs += 4;
- tcg_out_sti(s, TCG_TYPE_PTR, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
+ tcg_out_sti(s, TCG_TYPE_PTR, (uintptr_t)l->raddr, TCG_REG_ESP, ofs);
} else {
tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
/* The second argument is already loaded with addrlo. */
@@ -1413,7 +1420,7 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
ofs += 4;
}
- tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
+ tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
ofs += 4;
retaddr = TCG_REG_EAX;
@@ -973,6 +973,16 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
}
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ if (val == 0) {
+ tcg_out_st(s, type, TCG_REG_R0, base, ofs);
+ return true;
+ }
+ return false;
+}
+
static inline void tcg_out_alu(TCGContext *s, uint64_t opc_a1, uint64_t opc_a3,
TCGReg ret, TCGArg arg1, int const_arg1,
TCGArg arg2, int const_arg2)
@@ -576,6 +576,16 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ if (val == 0) {
+ tcg_out_st(s, type, TCG_REG_ZERO, base, ofs);
+ return true;
+ }
+ return false;
+}
+
static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
{
if (val == (int16_t)val) {
@@ -857,6 +857,12 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
tcg_out_mem_long(s, opi, opx, arg, arg1, arg2);
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ return false;
+}
+
static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
int const_arg2, int cr, TCGType type)
{
@@ -798,6 +798,12 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
}
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ return false;
+}
+
/* load data from an absolute host address */
static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs)
{
@@ -504,6 +504,16 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ if (val == 0) {
+ tcg_out_st(s, type, TCG_REG_G0, base, ofs);
+ return true;
+ }
+ return false;
+}
+
static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, uintptr_t arg)
{
tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
@@ -108,6 +108,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
const int *const_args);
static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
intptr_t arg2);
+static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs);
static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
static int tcg_target_const_match(tcg_target_long val, TCGType type,
const TCGArgConstraint *arg_ct);
@@ -1680,35 +1682,89 @@ static void temp_allocate_frame(TCGContext *s, int temp)
static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet);
-/* sync register 'reg' by saving it to the corresponding temporary */
-static void tcg_reg_sync(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
+/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
+ mark it free; otherwise mark it dead. */
+static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
{
- TCGTemp *ts = s->reg_to_temp[reg];
+ if (ts->fixed_reg) {
+ return;
+ }
+ if (ts->val_type == TEMP_VAL_REG) {
+ s->reg_to_temp[ts->reg] = NULL;
+ }
+ ts->val_type = (free_or_dead < 0
+ || ts->temp_local
+ || temp_idx(s, ts) < s->nb_globals
+ ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
+}
- tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
- if (!ts->mem_coherent && !ts->fixed_reg) {
+/* Mark a temporary as dead. */
+static inline void temp_dead(TCGContext *s, TCGTemp *ts)
+{
+ temp_free_or_dead(s, ts, 1);
+}
+
+/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
+ registers needs to be allocated to store a constant. If 'free_or_dead'
+ is non-zero, subsequently release the temporary; if it is positive, the
+ temp is dead; if it is negative, the temp is free. */
+static void temp_sync(TCGContext *s, TCGTemp *ts,
+ TCGRegSet allocated_regs, int free_or_dead)
+{
+ if (ts->fixed_reg) {
+ return;
+ }
+ if (!ts->mem_coherent) {
if (!ts->mem_allocated) {
temp_allocate_frame(s, temp_idx(s, ts));
- } else if (ts->indirect_reg) {
- tcg_regset_set_reg(allocated_regs, ts->reg);
+ }
+ if (ts->indirect_reg) {
+ if (ts->val_type == TEMP_VAL_REG) {
+ tcg_regset_set_reg(allocated_regs, ts->reg);
+ }
temp_load(s, ts->mem_base,
tcg_target_available_regs[TCG_TYPE_PTR],
allocated_regs);
}
- tcg_out_st(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
+ switch (ts->val_type) {
+ case TEMP_VAL_CONST:
+ /* If we're going to free the temp immediately, then we won't
+ require it later in a register, so attempt to store the
+ constant to memory directly. */
+ if (free_or_dead
+ && tcg_out_sti(s, ts->type, ts->val,
+ ts->mem_base->reg, ts->mem_offset)) {
+ break;
+ }
+ temp_load(s, ts, tcg_target_available_regs[ts->type],
+ allocated_regs);
+ /* fallthrough */
+
+ case TEMP_VAL_REG:
+ tcg_out_st(s, ts->type, ts->reg,
+ ts->mem_base->reg, ts->mem_offset);
+ break;
+
+ case TEMP_VAL_MEM:
+ break;
+
+ case TEMP_VAL_DEAD:
+ default:
+ tcg_abort();
+ }
+ ts->mem_coherent = 1;
+ }
+ if (free_or_dead) {
+ temp_free_or_dead(s, ts, free_or_dead);
}
- ts->mem_coherent = 1;
}
/* free register 'reg' by spilling the corresponding temporary if necessary */
static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
{
TCGTemp *ts = s->reg_to_temp[reg];
-
if (ts != NULL) {
- tcg_reg_sync(s, reg, allocated_regs);
- ts->val_type = TEMP_VAL_MEM;
- s->reg_to_temp[reg] = NULL;
+ temp_sync(s, ts, allocated_regs, -1);
}
}
@@ -1756,7 +1812,6 @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
case TEMP_VAL_CONST:
reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
tcg_out_movi(s, ts->type, reg, ts->val);
- ts->mem_coherent = 0;
break;
case TEMP_VAL_MEM:
reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
@@ -1778,41 +1833,6 @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
s->reg_to_temp[reg] = ts;
}
-/* mark a temporary as dead. */
-static inline void temp_dead(TCGContext *s, TCGTemp *ts)
-{
- if (ts->fixed_reg) {
- return;
- }
- if (ts->val_type == TEMP_VAL_REG) {
- s->reg_to_temp[ts->reg] = NULL;
- }
- ts->val_type = (temp_idx(s, ts) < s->nb_globals || ts->temp_local
- ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
-}
-
-/* sync a temporary to memory. 'allocated_regs' is used in case a
- temporary registers needs to be allocated to store a constant. */
-static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
-{
- if (ts->fixed_reg) {
- return;
- }
- switch (ts->val_type) {
- case TEMP_VAL_CONST:
- temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs);
- /* fallthrough */
- case TEMP_VAL_REG:
- tcg_reg_sync(s, ts->reg, allocated_regs);
- break;
- case TEMP_VAL_DEAD:
- case TEMP_VAL_MEM:
- break;
- default:
- tcg_abort();
- }
-}
-
/* save a temporary to memory. 'allocated_regs' is used in case a
temporary registers needs to be allocated to store a constant. */
static inline void temp_save(TCGContext *s, TCGTemp *ts,
@@ -1827,8 +1847,7 @@ static inline void temp_save(TCGContext *s, TCGTemp *ts,
return;
}
#endif
- temp_sync(s, ts, allocated_regs);
- temp_dead(s, ts);
+ temp_sync(s, ts, allocated_regs, 1);
}
/* save globals to their canonical location and assume they can be
@@ -1861,7 +1880,7 @@ static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
continue;
}
#endif
- temp_sync(s, ts, allocated_regs);
+ temp_sync(s, ts, allocated_regs, 0);
}
}
@@ -1905,21 +1924,21 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
val = args[1];
if (ots->fixed_reg) {
- /* for fixed registers, we do not do any constant
- propagation */
+ /* For fixed registers, we do not do any constant propagation. */
tcg_out_movi(s, ots->type, ots->reg, val);
- } else {
- /* The movi is not explicitly generated here */
- if (ots->val_type == TEMP_VAL_REG) {
- s->reg_to_temp[ots->reg] = NULL;
- }
- ots->val_type = TEMP_VAL_CONST;
- ots->val = val;
+ return;
}
- if (NEED_SYNC_ARG(0)) {
- temp_sync(s, ots, s->reserved_regs);
+
+ /* The movi is not explicitly generated here. */
+ if (ots->val_type == TEMP_VAL_REG) {
+ s->reg_to_temp[ots->reg] = NULL;
}
- if (IS_DEAD_ARG(0)) {
+ ots->val_type = TEMP_VAL_CONST;
+ ots->val = val;
+ ots->mem_coherent = 0;
+ if (NEED_SYNC_ARG(0)) {
+ temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0));
+ } else if (IS_DEAD_ARG(0)) {
temp_dead(s, ots);
}
}
@@ -2004,7 +2023,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
ots->mem_coherent = 0;
s->reg_to_temp[ots->reg] = ots;
if (NEED_SYNC_ARG(0)) {
- tcg_reg_sync(s, ots->reg, allocated_regs);
+ temp_sync(s, ots, allocated_regs, 0);
}
}
}
@@ -2163,9 +2182,8 @@ static void tcg_reg_alloc_op(TCGContext *s,
tcg_out_mov(s, ts->type, ts->reg, reg);
}
if (NEED_SYNC_ARG(i)) {
- tcg_reg_sync(s, reg, allocated_regs);
- }
- if (IS_DEAD_ARG(i)) {
+ temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i));
+ } else if (IS_DEAD_ARG(i)) {
temp_dead(s, ts);
}
}
@@ -2298,9 +2316,8 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
ts->mem_coherent = 0;
s->reg_to_temp[reg] = ts;
if (NEED_SYNC_ARG(i)) {
- tcg_reg_sync(s, reg, allocated_regs);
- }
- if (IS_DEAD_ARG(i)) {
+ temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i));
+ } else if (IS_DEAD_ARG(i)) {
temp_dead(s, ts);
}
}
@@ -834,6 +834,12 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
old_code_ptr[1] = s->code_ptr - old_code_ptr;
}
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+ TCGReg base, intptr_t ofs)
+{
+ return false;
+}
+
/* Test if a constant matches the constraint. */
static int tcg_target_const_match(tcg_target_long val, TCGType type,
const TCGArgConstraint *arg_ct)
While we can store constants via constrants on INDEX_op_st_i32 et al, we weren't able to spill constants to backing store. Add a new backend interface, tcg_out_sti, which may store the constant (and is allowed to fail). Rearrange the temp_* helpers so that we only attempt to directly store a constant when the temp is becoming dead/free. Signed-off-by: Richard Henderson <rth@twiddle.net> --- tcg/aarch64/tcg-target.inc.c | 10 +++ tcg/arm/tcg-target.inc.c | 6 ++ tcg/i386/tcg-target.inc.c | 21 ++++-- tcg/ia64/tcg-target.inc.c | 10 +++ tcg/mips/tcg-target.inc.c | 10 +++ tcg/ppc/tcg-target.inc.c | 6 ++ tcg/s390/tcg-target.inc.c | 6 ++ tcg/sparc/tcg-target.inc.c | 10 +++ tcg/tcg.c | 159 ++++++++++++++++++++++++------------------- tcg/tci/tcg-target.inc.c | 6 ++ 10 files changed, 166 insertions(+), 78 deletions(-)