commit 44fbc9c6e02ca5b8f98f25b514ed7588e7ba733d
Author: Vladimir N. Makarov <vmakarov@redhat.com>
Date: Fri Oct 30 15:05:22 2020 -0400
Take insn scratch RA requirements into account in IRA.
The patch changes insn scratches which require registers for all
insn alternatives (in other words w/o X constraint in scratch
constraint string). This is done before IRA staring its work. LRA
still continue to change the rest scratches (with X constraint and in
insn created during IRA) into pseudos. As before the patch at the end
of LRA work, spilled scratch pseudos (for which X constraint was
chosen) changed into scratches back.
gcc/ChangeLog:
* lra.c (get_scratch_reg): New function.
(remove_scratches_1): Rename remove_insn_scratches. Use
ira_remove_insn_scratches and get_scratch_reg.
(remove_scratches): Do not
initialize scratches, scratch_bitmap, and scratch_operand_bitmap.
(lra): Call ira_restore_scratches instead of restore_scratches.
(struct sloc, sloc_t, scratches, scratch_bitmap)
(scratch_operand_bitmap, lra_former_scratch_p)
(lra_former_scratch_operand_p, lra_register_new_scratch_op)
(restore_scratches): Move them to ...
* ira.c: ... here.
(former_scratch_p, former_scratch_operand_p): Rename to
ira_former_scratch_p and ira_former_scratch_operand_p.
(contains_X_constraint_p): New function.
(register_new_scratch_op): Rename to ira_register_new_scratch_op.
Change it to work for IRA and LRA.
(restore_scratches): Rename to ira_restore_scratches.
(get_scratch_reg, ira_remove_insn_scratches): New functions.
(ira): Call ira_remove_scratches if we use LRA.
* ira.h (ira_former_scratch_p, ira_former_scratch_operand_p): New
prototypes.
(ira_register_new_scratch_op, ira_restore_scratches): New prototypes.
(ira_remove_insn_scratches): New prototype.
* lra-int.h (lra_former_scratch_p, lra_former_scratch_operand_p):
Remove prototypes.
(lra_register_new_scratch_op): Ditto.
* lra-constraints.c: Rename lra_former_scratch_p and
lra_former_scratch_p to ira_former_scratch_p and to
ira_former_scratch_p.
* lra-remat.c: Ditto.
* lra-spills.c: Rename lra_former_scratch_p to ira_former_scratch_p.
@@ -5133,7 +5133,191 @@ move_unallocated_pseudos (void)
first_moveable_pseudo = last_moveable_pseudo = 0;
}
+
+
+
+/* Code dealing with scratches (changing them onto
+ pseudos and restoring them from the pseudos).
+
+ We change scratches into pseudos at the beginning of IRA to
+ simplify dealing with them (conflicts, hard register assignments).
+
+ If the pseudo denoting scratch was spilled it means that we do not
+ need a hard register for it. Such pseudos are transformed back to
+ scratches at the end of LRA. */
+
+/* Description of location of a former scratch operand. */
+struct sloc
+{
+ rtx_insn *insn; /* Insn where the scratch was. */
+ int nop; /* Number of the operand which was a scratch. */
+ unsigned regno; /* regno gnerated instead of scratch */
+ int icode; /* Original icode from which scratch was removed. */
+};
+
+typedef struct sloc *sloc_t;
+
+/* Locations of the former scratches. */
+static vec<sloc_t> scratches;
+
+/* Bitmap of scratch regnos. */
+static bitmap_head scratch_bitmap;
+
+/* Bitmap of scratch operands. */
+static bitmap_head scratch_operand_bitmap;
+
+/* Return true if pseudo REGNO is made of SCRATCH. */
+bool
+ira_former_scratch_p (int regno)
+{
+ return bitmap_bit_p (&scratch_bitmap, regno);
+}
+
+/* Return true if the operand NOP of INSN is a former scratch. */
+bool
+ira_former_scratch_operand_p (rtx_insn *insn, int nop)
+{
+ return bitmap_bit_p (&scratch_operand_bitmap,
+ INSN_UID (insn) * MAX_RECOG_OPERANDS + nop) != 0;
+}
+
+/* Register operand NOP in INSN as a former scratch. It will be
+ changed to scratch back, if it is necessary, at the LRA end. */
+void
+ira_register_new_scratch_op (rtx_insn *insn, int nop, int icode)
+{
+ rtx op = *recog_data.operand_loc[nop];
+ sloc_t loc = XNEW (struct sloc);
+ ira_assert (REG_P (op));
+ loc->insn = insn;
+ loc->nop = nop;
+ loc->regno = REGNO (op);
+ loc->icode = icode;
+ scratches.safe_push (loc);
+ bitmap_set_bit (&scratch_bitmap, REGNO (op));
+ bitmap_set_bit (&scratch_operand_bitmap,
+ INSN_UID (insn) * MAX_RECOG_OPERANDS + nop);
+ add_reg_note (insn, REG_UNUSED, op);
+}
+
+/* Return true if string STR contains constraint 'X'. */
+static bool
+contains_X_constraint_p (const char *str)
+{
+ int c;
+
+ while ((c = *str))
+ {
+ str += CONSTRAINT_LEN (c, str);
+ if (c == 'X') return true;
+ }
+ return false;
+}
+
+/* Change INSN's scratches into pseudos and save their location. */
+bool
+ira_remove_insn_scratches (rtx_insn *insn, bool all_p, FILE *dump_file,
+ rtx (*get_reg) (rtx original))
+{
+ int i;
+ bool insn_changed_p;
+ rtx reg, *loc;
+
+ extract_insn (insn);
+ insn_changed_p = false;
+ for (i = 0; i < recog_data.n_operands; i++)
+ {
+ loc = recog_data.operand_loc[i];
+ if (GET_CODE (*loc) == SCRATCH && GET_MODE (*loc) != VOIDmode)
+ {
+ if (! all_p && contains_X_constraint_p (recog_data.constraints[i]))
+ continue;
+ insn_changed_p = true;
+ *loc = reg = get_reg (*loc);
+ ira_register_new_scratch_op (insn, i, INSN_CODE (insn));
+ if (ira_dump_file != NULL)
+ fprintf (dump_file,
+ "Removing SCRATCH to p%u in insn #%u (nop %d)\n",
+ REGNO (reg), INSN_UID (insn), i);
+ }
+ }
+ return insn_changed_p;
+}
+
+/* Return new register of the same mode as ORIGINAL. Used in
+ ira_remove_scratches. */
+static rtx
+get_scratch_reg (rtx original)
+{
+ return gen_reg_rtx (GET_MODE (original));
+}
+
+/* Change scratches into pseudos and save their location. */
+void
+ira_remove_scratches (void)
+{
+ basic_block bb;
+ rtx_insn *insn;
+
+ scratches.create (get_max_uid ());
+ bitmap_initialize (&scratch_bitmap, ®_obstack);
+ bitmap_initialize (&scratch_operand_bitmap, ®_obstack);
+ FOR_EACH_BB_FN (bb, cfun)
+ FOR_BB_INSNS (bb, insn)
+ if (INSN_P (insn)
+ && ira_remove_insn_scratches (insn, false, ira_dump_file, get_scratch_reg))
+ /* Because we might use DF, we need to keep DF info up to date. */
+ df_insn_rescan (insn);
+}
+
+/* Changes pseudos created by function remove_scratches onto scratches. */
+void
+ira_restore_scratches (FILE *dump_file)
+{
+ int regno, n;
+ unsigned i;
+ rtx *op_loc;
+ sloc_t loc;
+
+ for (i = 0; scratches.iterate (i, &loc); i++)
+ {
+ /* Ignore already deleted insns. */
+ if (NOTE_P (loc->insn)
+ && NOTE_KIND (loc->insn) == NOTE_INSN_DELETED)
+ continue;
+ extract_insn (loc->insn);
+ if (loc->icode != INSN_CODE (loc->insn))
+ {
+ /* The icode doesn't match, which means the insn has been modified
+ (e.g. register elimination). The scratch cannot be restored. */
+ continue;
+ }
+ op_loc = recog_data.operand_loc[loc->nop];
+ if (REG_P (*op_loc)
+ && ((regno = REGNO (*op_loc)) >= FIRST_PSEUDO_REGISTER)
+ && reg_renumber[regno] < 0)
+ {
+ /* It should be only case when scratch register with chosen
+ constraint 'X' did not get memory or hard register. */
+ ira_assert (ira_former_scratch_p (regno));
+ *op_loc = gen_rtx_SCRATCH (GET_MODE (*op_loc));
+ for (n = 0; n < recog_data.n_dups; n++)
+ *recog_data.dup_loc[n]
+ = *recog_data.operand_loc[(int) recog_data.dup_num[n]];
+ if (dump_file != NULL)
+ fprintf (dump_file, "Restoring SCRATCH in insn #%u(nop %d)\n",
+ INSN_UID (loc->insn), loc->nop);
+ }
+ }
+ for (i = 0; scratches.iterate (i, &loc); i++)
+ free (loc);
+ scratches.release ();
+ bitmap_clear (&scratch_bitmap);
+ bitmap_clear (&scratch_operand_bitmap);
+}
+
+
/* If the backend knows where to allocate pseudos for hard
register initial values, register these allocations now. */
static void
@@ -5182,8 +5366,10 @@ allocate_initial_values (void)
&hreg, &preg));
}
}
+
+
/* True when we use LRA instead of reload pass for the current
function. */
bool ira_use_lra_p;
@@ -5204,6 +5390,17 @@ ira (FILE *f)
bool saved_flag_caller_saves = flag_caller_saves;
enum ira_region saved_flag_ira_region = flag_ira_region;
+ if (flag_ira_verbose < 10)
+ {
+ internal_flag_ira_verbose = flag_ira_verbose;
+ ira_dump_file = f;
+ }
+ else
+ {
+ internal_flag_ira_verbose = flag_ira_verbose - 10;
+ ira_dump_file = stderr;
+ }
+
clear_bb_flags ();
/* Determine if the current function is a leaf before running IRA
@@ -5250,17 +5447,6 @@ ira (FILE *f)
if (flag_caller_saves && !ira_use_lra_p)
init_caller_save ();
- if (flag_ira_verbose < 10)
- {
- internal_flag_ira_verbose = flag_ira_verbose;
- ira_dump_file = f;
- }
- else
- {
- internal_flag_ira_verbose = flag_ira_verbose - 10;
- ira_dump_file = stderr;
- }
-
setup_prohibited_mode_move_regs ();
decrease_live_ranges_number ();
df_note_add_problem ();
@@ -5305,9 +5491,6 @@ ira (FILE *f)
if (warn_clobbered)
generate_setjmp_warnings ();
- if (resize_reg_info () && flag_ira_loop_pressure)
- ira_set_pseudo_classes (true, ira_dump_file);
-
init_alias_analysis ();
loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
reg_equiv = XCNEWVEC (struct equivalence, max_reg_num ());
@@ -5331,6 +5514,12 @@ ira (FILE *f)
end_alias_analysis ();
free (reg_equiv);
+ if (ira_use_lra_p)
+ ira_remove_scratches ();
+
+ if (resize_reg_info () && flag_ira_loop_pressure)
+ ira_set_pseudo_classes (true, ira_dump_file);
+
setup_reg_equiv ();
grow_reg_equivs ();
setup_reg_equiv_init ();
@@ -207,6 +207,13 @@ extern bool ira_bad_reload_regno (int, rtx, rtx);
extern void ira_adjust_equiv_reg_cost (unsigned, int);
+extern bool ira_former_scratch_p (int regno);
+extern bool ira_former_scratch_operand_p (rtx_insn *insn, int nop);
+extern void ira_register_new_scratch_op (rtx_insn *insn, int nop, int icode);
+extern bool ira_remove_insn_scratches (rtx_insn *insn, bool all_p, FILE *dump_file,
+ rtx (*get_reg) (rtx original));
+extern void ira_restore_scratches (FILE *dump_file);
+
/* ira-costs.c */
extern void ira_costs_c_finalize (void);
@@ -2505,7 +2505,7 @@ process_alt_operands (int only_alternative)
while ((p += len), c);
scratch_p = (operand_reg[nop] != NULL_RTX
- && lra_former_scratch_p (REGNO (operand_reg[nop])));
+ && ira_former_scratch_p (REGNO (operand_reg[nop])));
/* Record which operands fit this alternative. */
if (win)
{
@@ -4354,8 +4354,8 @@ curr_insn_transform (bool check_only_p)
assigment pass and the scratch pseudo will be
spilled. Spilled scratch pseudos are transformed
back to scratches at the LRA end. */
- && lra_former_scratch_operand_p (curr_insn, i)
- && lra_former_scratch_p (REGNO (op)))
+ && ira_former_scratch_operand_p (curr_insn, i)
+ && ira_former_scratch_p (REGNO (op)))
{
int regno = REGNO (op);
lra_change_class (regno, NO_REGS, " Change to", true);
@@ -4376,7 +4376,7 @@ curr_insn_transform (bool check_only_p)
&& goal_alt[i] != NO_REGS && REG_P (op)
&& (regno = REGNO (op)) >= FIRST_PSEUDO_REGISTER
&& regno < new_regno_start
- && ! lra_former_scratch_p (regno)
+ && ! ira_former_scratch_p (regno)
&& reg_renumber[regno] < 0
/* Check that the optional reload pseudo will be able to
hold given mode value. */
@@ -319,9 +319,6 @@ extern struct lra_insn_reg *lra_get_insn_regs (int);
extern void lra_free_copies (void);
extern void lra_create_copy (int, int, int);
extern lra_copy_t lra_get_copy (int);
-extern bool lra_former_scratch_p (int);
-extern bool lra_former_scratch_operand_p (rtx_insn *, int);
-extern void lra_register_new_scratch_op (rtx_insn *, int, int);
extern int lra_new_regno_start;
extern int lra_constraint_new_regno_start;
@@ -1031,12 +1031,12 @@ update_scratch_ops (rtx_insn *remat_insn)
if (! REG_P (*loc))
continue;
int regno = REGNO (*loc);
- if (! lra_former_scratch_p (regno))
+ if (! ira_former_scratch_p (regno))
continue;
*loc = lra_create_new_reg (GET_MODE (*loc), *loc,
lra_get_allocno_class (regno),
"scratch pseudo copy");
- lra_register_new_scratch_op (remat_insn, i, id->icode);
+ ira_register_new_scratch_op (remat_insn, i, id->icode);
}
}
@@ -446,7 +446,7 @@ remove_pseudos (rtx *loc, rtx_insn *insn)
it might result in an address reload for some targets. In
any case we transform such pseudos not getting hard registers
into scratches back. */
- && ! lra_former_scratch_p (i))
+ && ! ira_former_scratch_p (i))
{
if (lra_reg_info[i].nrefs == 0
&& pseudo_slots[i].mem == NULL && spill_hard_reg[i] == NULL)
@@ -494,7 +494,7 @@ spill_pseudos (void)
for (i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
{
if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
- && ! lra_former_scratch_p (i))
+ && ! ira_former_scratch_p (i))
{
bitmap_set_bit (spilled_pseudos, i);
bitmap_ior_into (changed_insns, &lra_reg_info[i].insn_bitmap);
@@ -578,7 +578,7 @@ lra_need_for_scratch_reg_p (void)
for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
- && lra_former_scratch_p (i))
+ && ira_former_scratch_p (i))
return true;
return false;
}
@@ -591,7 +591,7 @@ lra_need_for_spills_p (void)
for (i = FIRST_PSEUDO_REGISTER; i < max_regno; i++)
if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
- && ! lra_former_scratch_p (i))
+ && ! ira_former_scratch_p (i))
return true;
return false;
}
@@ -612,7 +612,7 @@ lra_spill (void)
for (n = 0, i = FIRST_PSEUDO_REGISTER; i < regs_num; i++)
if (lra_reg_info[i].nrefs != 0 && lra_get_regno_hard_regno (i) < 0
/* We do not want to assign memory for former scratches. */
- && ! lra_former_scratch_p (i))
+ && ! ira_former_scratch_p (i))
pseudo_regnos[n++] = i;
lra_assert (n > 0);
pseudo_slots = XNEWVEC (struct pseudo_slot, regs_num);
@@ -160,8 +160,6 @@ static void invalidate_insn_recog_data (int);
static int get_insn_freq (rtx_insn *);
static void invalidate_insn_data_regno_info (lra_insn_recog_data_t,
rtx_insn *, int);
-static void remove_scratches_1 (rtx_insn *);
-
/* Expand all regno related info needed for LRA. */
static void
expand_reg_data (int old)
@@ -482,6 +480,8 @@ lra_emit_add (rtx x, rtx y, rtx z)
/* The number of emitted reload insns so far. */
int lra_curr_reload_num;
+static void remove_insn_scratches (rtx_insn *insn);
+
/* Emit x := y, processing special case when y = u + v or y = u + v *
scale + w through emit_add (Y can be an address which is base +
index reg * scale + displacement in general case). X may be used
@@ -503,7 +503,7 @@ lra_emit_move (rtx x, rtx y)
/* The move pattern may require scratch registers, so convert them
into real registers now. */
if (insn != NULL_RTX)
- remove_scratches_1 (insn);
+ remove_insn_scratches (insn);
if (REG_P (x))
lra_reg_info[ORIGINAL_REGNO (x)].last_reload = ++lra_curr_reload_num;
/* Function emit_move can create pseudos -- so expand the pseudo
@@ -1988,170 +1988,35 @@ lra_substitute_pseudo_within_insn (rtx_insn *insn, int old_regno,
-/* This page contains code dealing with scratches (changing them onto
- pseudos and restoring them from the pseudos).
-
- We change scratches into pseudos at the beginning of LRA to
- simplify dealing with them (conflicts, hard register assignments).
-
- If the pseudo denoting scratch was spilled it means that we do need
- a hard register for it. Such pseudos are transformed back to
- scratches at the end of LRA. */
-
-/* Description of location of a former scratch operand. */
-struct sloc
+/* Return new register of the same mode as ORIGINAL of class ALL_REGS.
+ Used in ira_remove_scratches. */
+static rtx
+get_scratch_reg (rtx original)
{
- rtx_insn *insn; /* Insn where the scratch was. */
- int nop; /* Number of the operand which was a scratch. */
- int icode; /* Original icode from which scratch was removed. */
-};
-
-typedef struct sloc *sloc_t;
-
-/* Locations of the former scratches. */
-static vec<sloc_t> scratches;
-
-/* Bitmap of scratch regnos. */
-static bitmap_head scratch_bitmap;
-
-/* Bitmap of scratch operands. */
-static bitmap_head scratch_operand_bitmap;
-
-/* Return true if pseudo REGNO is made of SCRATCH. */
-bool
-lra_former_scratch_p (int regno)
-{
- return bitmap_bit_p (&scratch_bitmap, regno);
+ return lra_create_new_reg (GET_MODE (original), original, ALL_REGS, NULL);
}
-/* Return true if the operand NOP of INSN is a former scratch. */
-bool
-lra_former_scratch_operand_p (rtx_insn *insn, int nop)
-{
- return bitmap_bit_p (&scratch_operand_bitmap,
- INSN_UID (insn) * MAX_RECOG_OPERANDS + nop) != 0;
-}
-
-/* Register operand NOP in INSN as a former scratch. It will be
- changed to scratch back, if it is necessary, at the LRA end. */
-void
-lra_register_new_scratch_op (rtx_insn *insn, int nop, int icode)
-{
- lra_insn_recog_data_t id = lra_get_insn_recog_data (insn);
- rtx op = *id->operand_loc[nop];
- sloc_t loc = XNEW (struct sloc);
- lra_assert (REG_P (op));
- loc->insn = insn;
- loc->nop = nop;
- loc->icode = icode;
- scratches.safe_push (loc);
- bitmap_set_bit (&scratch_bitmap, REGNO (op));
- bitmap_set_bit (&scratch_operand_bitmap,
- INSN_UID (insn) * MAX_RECOG_OPERANDS + nop);
- add_reg_note (insn, REG_UNUSED, op);
-}
-
-/* Change INSN's scratches into pseudos and save their location. */
+/* Remove all insn scratches in INSN. */
static void
-remove_scratches_1 (rtx_insn *insn)
+remove_insn_scratches (rtx_insn *insn)
{
- int i;
- bool insn_changed_p;
- rtx reg;
- lra_insn_recog_data_t id;
- struct lra_static_insn_data *static_id;
-
- id = lra_get_insn_recog_data (insn);
- static_id = id->insn_static_data;
- insn_changed_p = false;
- for (i = 0; i < static_id->n_operands; i++)
- if (GET_CODE (*id->operand_loc[i]) == SCRATCH
- && GET_MODE (*id->operand_loc[i]) != VOIDmode)
- {
- insn_changed_p = true;
- *id->operand_loc[i] = reg
- = lra_create_new_reg (static_id->operand[i].mode,
- *id->operand_loc[i], ALL_REGS, NULL);
- lra_register_new_scratch_op (insn, i, id->icode);
- if (lra_dump_file != NULL)
- fprintf (lra_dump_file,
- "Removing SCRATCH in insn #%u (nop %d)\n",
- INSN_UID (insn), i);
- }
- if (insn_changed_p)
- /* Because we might use DF right after caller-saves sub-pass
- we need to keep DF info up to date. */
+ if (ira_remove_insn_scratches (insn, true, lra_dump_file, get_scratch_reg))
df_insn_rescan (insn);
}
-/* Change scratches into pseudos and save their location. */
+/* Remove all insn scratches in the current function. */
static void
remove_scratches (void)
{
basic_block bb;
rtx_insn *insn;
- scratches.create (get_max_uid ());
- bitmap_initialize (&scratch_bitmap, ®_obstack);
- bitmap_initialize (&scratch_operand_bitmap, ®_obstack);
FOR_EACH_BB_FN (bb, cfun)
FOR_BB_INSNS (bb, insn)
- if (INSN_P (insn))
- remove_scratches_1 (insn);
-}
-
-/* Changes pseudos created by function remove_scratches onto scratches. */
-static void
-restore_scratches (void)
-{
- int regno;
- unsigned i;
- sloc_t loc;
- rtx_insn *last = NULL;
- lra_insn_recog_data_t id = NULL;
-
- for (i = 0; scratches.iterate (i, &loc); i++)
- {
- /* Ignore already deleted insns. */
- if (NOTE_P (loc->insn)
- && NOTE_KIND (loc->insn) == NOTE_INSN_DELETED)
- continue;
- if (last != loc->insn)
- {
- last = loc->insn;
- id = lra_get_insn_recog_data (last);
- }
- if (loc->icode != id->icode)
- {
- /* The icode doesn't match, which means the insn has been modified
- (e.g. register elimination). The scratch cannot be restored. */
- continue;
- }
- if (REG_P (*id->operand_loc[loc->nop])
- && ((regno = REGNO (*id->operand_loc[loc->nop]))
- >= FIRST_PSEUDO_REGISTER)
- && lra_get_regno_hard_regno (regno) < 0)
- {
- /* It should be only case when scratch register with chosen
- constraint 'X' did not get memory or hard register. */
- lra_assert (lra_former_scratch_p (regno));
- *id->operand_loc[loc->nop]
- = gen_rtx_SCRATCH (GET_MODE (*id->operand_loc[loc->nop]));
- lra_update_dup (id, loc->nop);
- if (lra_dump_file != NULL)
- fprintf (lra_dump_file, "Restoring SCRATCH in insn #%u(nop %d)\n",
- INSN_UID (loc->insn), loc->nop);
- }
- }
- for (i = 0; scratches.iterate (i, &loc); i++)
- free (loc);
- scratches.release ();
- bitmap_clear (&scratch_bitmap);
- bitmap_clear (&scratch_operand_bitmap);
+ if (INSN_P (insn))
+ remove_insn_scratches (insn);
}
-
-
/* Function checks RTL for correctness. If FINAL_P is true, it is
done at the end of LRA and the check is more rigorous. */
static void
@@ -2543,7 +2408,7 @@ lra (FILE *f)
lra_bad_spill_regno_start = lra_constraint_new_regno_start;
lra_assignment_iter_after_spill = 0;
}
- restore_scratches ();
+ ira_restore_scratches (lra_dump_file);
lra_eliminate (true, false);
lra_final_code_change ();
lra_in_progress = 0;