===================================================================
@@ -204,7 +204,7 @@ extern void copy_bbs (basic_block *, unsigned, bas
void account_profile_record (struct profile_record *, int);
extern void cfg_layout_initialize (unsigned int);
-extern void cfg_layout_finalize (void);
+extern void cfg_layout_finalize (bool);
/* Hooks containers. */
extern struct cfg_hooks gimple_cfg_hooks;
@@ -218,4 +218,3 @@ extern void cfg_layout_rtl_register_cfg_hooks (voi
extern void gimple_register_cfg_hooks (void);
extern struct cfg_hooks get_cfg_hooks (void);
extern void set_cfg_hooks (struct cfg_hooks);
-
===================================================================
@@ -3347,7 +3347,7 @@ rest_of_handle_sms (void)
if (bb->next_bb != EXIT_BLOCK_PTR)
bb->aux = bb->next_bb;
free_dominance_info (CDI_DOMINATORS);
- cfg_layout_finalize ();
+ cfg_layout_finalize (false);
#endif /* INSN_SCHEDULING */
return 0;
}
===================================================================
@@ -3899,10 +3899,9 @@ find_if_case_1 (basic_block test_bb, edge then_edg
if (new_bb)
{
df_bb_replace (then_bb_index, new_bb);
- /* Since the fallthru edge was redirected from test_bb to new_bb,
- we need to ensure that new_bb is in the same partition as
- test bb (you can not fall through across section boundaries). */
- BB_COPY_PARTITION (new_bb, test_bb);
+ /* This should have been done above via force_nonfallthru_and_redirect
+ (possibly called from redirect_edge_and_branch_force). */
+ gcc_assert (BB_PARTITION (new_bb) == BB_PARTITION (test_bb));
}
num_true_changes++;
===================================================================
@@ -6246,8 +6246,10 @@ thread_prologue_and_epilogue_insns (void)
break;
if (e)
{
- copy_bb = create_basic_block (NEXT_INSN (BB_END (e->src)),
- NULL_RTX, e->src);
+ /* Make sure we insert after any barriers. */
+ rtx end = get_last_bb_insn (e->src);
+ copy_bb = create_basic_block (NEXT_INSN (end),
+ NULL_RTX, e->src);
BB_COPY_PARTITION (copy_bb, e->src);
}
else
@@ -6472,7 +6474,7 @@ thread_prologue_and_epilogue_insns (void)
if (cur_bb->index >= NUM_FIXED_BLOCKS
&& cur_bb->next_bb->index >= NUM_FIXED_BLOCKS)
cur_bb->aux = cur_bb->next_bb;
- cfg_layout_finalize ();
+ cfg_layout_finalize (false);
}
epilogue_done:
@@ -6514,7 +6516,7 @@ epilogue_done:
basic_block simple_return_block_cold = NULL;
edge pending_edge_hot = NULL;
edge pending_edge_cold = NULL;
- basic_block exit_pred = EXIT_BLOCK_PTR->prev_bb;
+ basic_block exit_pred;
int i;
gcc_assert (entry_edge != orig_entry_edge);
@@ -6542,6 +6544,12 @@ epilogue_done:
else
pending_edge_cold = e;
}
+
+ /* Save a pointer to the exit's predecessor BB for use in
+ inserting new BBs at the end of the function. Do this
+ after the call to split_block above which may split
+ the original exit pred. */
+ exit_pred = EXIT_BLOCK_PTR->prev_bb;
FOR_EACH_VEC_ELT (unconverted_simple_returns, i, e)
{
===================================================================
@@ -451,6 +451,11 @@ struct GTY(()) rtl_data {
sched2) and is useful only if the port defines LEAF_REGISTERS. */
bool uses_only_leaf_regs;
+ /* Nonzero if the function being compiled has undergone hot/cold partitioning
+ (under flag_reorder_blocks_and_partition) and has at least one cold
+ block. */
+ bool has_bb_partition;
+
/* Like regs_ever_live, but 1 if a reg is set or clobbered from an
asm. Unlike regs_ever_live, elements of this array corresponding
to eliminable regs (like the frame pointer) are set if an asm
===================================================================
@@ -547,7 +547,7 @@ reorder_loops (hwloop_info loops)
else
bb->aux = NULL;
}
- cfg_layout_finalize ();
+ cfg_layout_finalize (false);
clear_aux_for_blocks ();
df_analyze ();
}
===================================================================
@@ -1846,7 +1846,7 @@ try_crossjump_to_edge (int mode, edge e1, edge e2,
partition boundaries). See the comments at the top of
bb-reorder.c:partition_hot_cold_basic_blocks for complete details. */
- if (flag_reorder_blocks_and_partition && reload_completed)
+ if (crtl->has_bb_partition && reload_completed)
return false;
/* Search backward through forwarder blocks. We don't need to worry
@@ -2789,10 +2789,21 @@ try_optimize_cfg (int mode)
df_analyze ();
}
+ if (changed)
+ {
+ /* Edge forwarding in particular can cause hot blocks previously
+ reached by both hot and cold blocks to become dominated only
+ by cold blocks. This will cause the verification
below to fail,
+ and lead to now cold code in the hot section. This is not easy
+ to detect and fix during edge forwarding, and in some cases
+ is only visible after newly unreachable blocks are deleted,
+ which will be done in fixup_partitions. */
+ fixup_partitions ();
+
#ifdef ENABLE_CHECKING
- if (changed)
- verify_flow_info ();
+ verify_flow_info ();
#endif
+ }
changed_overall |= changed;
first_pass = false;
===================================================================
@@ -1054,7 +1054,7 @@ connect_traces (int n_traces, struct trace *traces
current_partition = BB_PARTITION (traces[0].first);
two_passes = false;
- if (flag_reorder_blocks_and_partition)
+ if (crtl->has_bb_partition)
for (i = 0; i < n_traces && !two_passes; i++)
if (BB_PARTITION (traces[0].first)
!= BB_PARTITION (traces[i].first))
@@ -1263,7 +1263,7 @@ connect_traces (int n_traces, struct trace *traces
}
}
- if (flag_reorder_blocks_and_partition)
+ if (crtl->has_bb_partition)
try_copy = false;
/* Copy tiny blocks always; copy larger blocks only when the
@@ -1381,13 +1381,14 @@ get_uncond_jump_length (void)
return length;
}
-/* Emit a barrier into the footer of BB. */
+/* Emit a barrier after BB, into the footer if we are in CFGLAYOUT mode. */
-static void
+void
emit_barrier_after_bb (basic_block bb)
{
rtx barrier = emit_barrier_after (BB_END (bb));
- BB_FOOTER (bb) = unlink_insn_chain (barrier, barrier);
+ if (current_ir_type () == IR_RTL_CFGLAYOUT)
+ BB_FOOTER (bb) = unlink_insn_chain (barrier, barrier);
}
/* The landing pad OLD_LP, in block OLD_BB, has edges from both partitions.
@@ -1463,18 +1464,109 @@ find_rarely_executed_basic_blocks_and_crossing_edg
{
vec<edge> crossing_edges = vNULL;
basic_block bb;
- edge e;
- edge_iterator ei;
+ edge e, e2;
+ edge_iterator ei, ei2;
+ unsigned int cold_bb_count = 0;
+ vec<basic_block> bbs_in_hot_partition = vNULL;
+ vec<basic_block> bbs_newly_hot = vNULL;
/* Mark which partition (hot/cold) each basic block belongs in. */
FOR_EACH_BB (bb)
{
if (probably_never_executed_bb_p (cfun, bb))
- BB_SET_PARTITION (bb, BB_COLD_PARTITION);
+ {
+ BB_SET_PARTITION (bb, BB_COLD_PARTITION);
+ cold_bb_count++;
+ }
else
- BB_SET_PARTITION (bb, BB_HOT_PARTITION);
+ {
+ BB_SET_PARTITION (bb, BB_HOT_PARTITION);
+ bbs_in_hot_partition.safe_push (bb);
+ }
}
+ /* Ensure that no cold bbs dominate hot bbs. This could happen as a result of
+ several different possibilities. One is that there are edge
weight insanities
+ due to optimization phases that do not properly update basic block profile
+ counts. The second is that the entry of the function may not be
hot, because
+ it is entered fewer times than the number of profile training
runs, but there
+ is a loop inside the function that causes blocks within the function to be
+ above the threshold for hotness. */
+ if (cold_bb_count)
+ {
+ bool dom_calculated_here = !dom_info_available_p (CDI_DOMINATORS);
+
+ if (dom_calculated_here)
+ calculate_dominance_info (CDI_DOMINATORS);
+
+ /* Keep examining hot bbs until we have either checked them all, or
+ re-marked all cold bbs hot. */
+ while (! bbs_in_hot_partition.is_empty ()
+ && cold_bb_count)
+ {
+ basic_block dom_bb;
+
+ bb = bbs_in_hot_partition.pop ();
+ dom_bb = get_immediate_dominator (CDI_DOMINATORS, bb);
+
+ /* If bb's immediate dominator is also hot then it is ok. */
+ if (BB_PARTITION (dom_bb) != BB_COLD_PARTITION)
+ continue;
+
+ /* We have a hot bb with an immediate dominator that is cold.
+ The dominator needs to be re-marked to hot. */
+ BB_SET_PARTITION (dom_bb, BB_HOT_PARTITION);
+ cold_bb_count--;
+
+ /* Now we need to examine newly-hot dom_bb to see if it is also
+ dominated by a cold bb. */
+ bbs_in_hot_partition.safe_push (dom_bb);
+
+ /* We should also adjust any cold blocks that the newly-hot bb
+ feeds and see if it makes sense to re-mark those as hot as
+ well. */
+ bbs_newly_hot.safe_push (dom_bb);
+ while (! bbs_newly_hot.is_empty ())
+ {
+ basic_block new_hot_bb = bbs_newly_hot.pop ();
+ /* Examine all successors of this newly-hot bb to see if they
+ are cold and should be re-marked as hot. */
+ FOR_EACH_EDGE (e, ei, new_hot_bb->succs)
+ {
+ bool any_cold_preds = false;
+ basic_block succ = e->dest;
+ if (BB_PARTITION (succ) != BB_COLD_PARTITION)
+ continue;
+ /* Does this block have any cold predecessors now? */
+ FOR_EACH_EDGE (e2, ei2, succ->preds)
+ {
+ if (BB_PARTITION (e2->src) == BB_COLD_PARTITION)
+ {
+ any_cold_preds = true;
+ break;
+ }
+ }
+ if (any_cold_preds)
+ continue;
+
+ /* Here we have a successor of newly-hot bb that is cold
+ but no longer has any cold precessessors. Since
the original
+ assignment of our newly-hot bb was incorrect,
this successor's
+ assignment as cold is also suspect. Go ahead and
re-mark it
+ as hot now too. Better heuristics may be in
order here. */
+ BB_SET_PARTITION (succ, BB_HOT_PARTITION);
+ cold_bb_count--;
+ bbs_in_hot_partition.safe_push (succ);
+ /* Examine this successor as a newly-hot bb. */
+ bbs_newly_hot.safe_push (succ);
+ }
+ }
+ }
+
+ if (dom_calculated_here)
+ free_dominance_info (CDI_DOMINATORS);
+ }
+
/* The format of .gcc_except_table does not allow landing pads to
be in a different partition as the throw. Fix this by either
moving or duplicating the landing pads. */
@@ -1766,10 +1858,10 @@ fix_up_fall_thru_edges (void)
new_bb->aux = cur_bb->aux;
cur_bb->aux = new_bb;
- /* Make sure new fall-through bb is in same
- partition as bb it's falling through from. */
+ /* This is done by force_nonfallthru_and_redirect. */
+ gcc_assert (BB_PARTITION (new_bb)
+ == BB_PARTITION (cur_bb));
- BB_COPY_PARTITION (new_bb, cur_bb);
single_succ_edge (new_bb)->flags |= EDGE_CROSSING;
}
else
@@ -2067,7 +2159,10 @@ add_reg_crossing_jump_notes (void)
FOR_EACH_BB (bb)
FOR_EACH_EDGE (e, ei, bb->succs)
if ((e->flags & EDGE_CROSSING)
- && JUMP_P (BB_END (e->src)))
+ && JUMP_P (BB_END (e->src))
+ /* Some notes were added during fix_up_fall_thru_edges, via
+ force_nonfallthru_and_redirect. */
+ && !find_reg_note (BB_END (e->src), REG_CROSSING_JUMP, NULL_RTX))
add_reg_note (BB_END (e->src), REG_CROSSING_JUMP, NULL_RTX);
}
@@ -2160,7 +2255,7 @@ reorder_basic_blocks (void)
dump_flow_info (dump_file, dump_flags);
}
- if (flag_reorder_blocks_and_partition)
+ if (crtl->has_bb_partition)
verify_hot_cold_block_grouping ();
}
@@ -2172,14 +2267,14 @@ reorder_basic_blocks (void)
encountering this note will make the compiler switch between the
hot and cold text sections. */
-static void
+void
insert_section_boundary_note (void)
{
basic_block bb;
rtx new_note;
int first_partition = 0;
- if (!flag_reorder_blocks_and_partition)
+ if (!crtl->has_bb_partition)
return;
FOR_EACH_BB (bb)
@@ -2222,10 +2317,8 @@ rest_of_handle_reorder_blocks (void)
FOR_EACH_BB (bb)
if (bb->next_bb != EXIT_BLOCK_PTR)
bb->aux = bb->next_bb;
- cfg_layout_finalize ();
+ cfg_layout_finalize (true);
- /* Add NOTE_INSN_SWITCH_TEXT_SECTIONS notes. */
- insert_section_boundary_note ();
return 0;
}
@@ -2366,7 +2459,7 @@ duplicate_computed_gotos (void)
}
done:
- cfg_layout_finalize ();
+ cfg_layout_finalize (false);
BITMAP_FREE (candidates);
return 0;
@@ -2511,6 +2604,8 @@ partition_hot_cold_basic_blocks (void)
if (!crossing_edges.exists ())
return 0;
+ crtl->has_bb_partition = true;
+
/* Make sure the source of any crossing edge ends in a jump and the
destination of any crossing edge has a label. */
add_labels_and_missing_jumps (crossing_edges);
===================================================================
@@ -36,4 +36,8 @@ extern struct target_bb_reorder *this_target_bb_re
extern int get_uncond_jump_length (void);
+extern void insert_section_boundary_note (void);
+
+extern void emit_barrier_after_bb (basic_block bb);
+
#endif
===================================================================
@@ -800,6 +800,7 @@ extern basic_block force_nonfallthru_and_redirect
extern bool contains_no_active_insn_p (const_basic_block);
extern bool forwarder_block_p (const_basic_block);
extern bool can_fallthru (basic_block, basic_block);
+extern void fixup_partitions (void);
/* In cfgbuild.c. */
extern void find_many_sub_basic_blocks (sbitmap);
===================================================================
@@ -46,6 +46,7 @@ along with GCC; see the file COPYING3. If not see
#include "tree.h"
#include "hard-reg-set.h"
#include "basic-block.h"
+#include "bb-reorder.h"
#include "regs.h"
#include "flags.h"
#include "function.h"
@@ -67,11 +68,12 @@ along with GCC; see the file COPYING3. If not see
Only applicable if the CFG is in cfglayout mode. */
static GTY(()) rtx cfg_layout_function_footer;
static GTY(()) rtx cfg_layout_function_header;
+static bool had_sec_boundary_notes;
static rtx skip_insns_after_block (basic_block);
static void record_effective_endpoints (void);
static rtx label_for_bb (basic_block);
-static void fixup_reorder_chain (void);
+static void fixup_reorder_chain (bool finalize_reorder_blocks);
void verify_insn_chain (void);
static void fixup_fallthru_exit_predecessor (void);
@@ -976,8 +978,7 @@ try_redirect_by_replacing_jump (edge e, basic_bloc
partition boundaries). See the comments at the top of
bb-reorder.c:partition_hot_cold_basic_blocks for complete details. */
- if (find_reg_note (insn, REG_CROSSING_JUMP, NULL_RTX)
- || BB_PARTITION (src) != BB_PARTITION (target))
+ if (BB_PARTITION (src) != BB_PARTITION (target))
return NULL;
/* We can replace or remove a complex jump only when we have exactly
@@ -1286,6 +1287,71 @@ redirect_branch_edge (edge e, basic_block target)
return e;
}
+/* Called when edge E has been redirected to a new destination,
+ in order to update the region crossing flag on the edge and
+ jump. */
+
+static void
+fixup_partition_crossing (edge e, basic_block target)
+{
+ rtx note;
+
+ gcc_assert (e->dest == target);
+
+ if (e->src == ENTRY_BLOCK_PTR || target == EXIT_BLOCK_PTR)
+ return;
+ /* If we redirected an existing edge, it may already be marked
+ crossing, even though the new src is missing a reg crossing note.
+ But make sure reg crossing note doesn't already exist before
+ inserting. */
+ if (BB_PARTITION (e->src) != BB_PARTITION (target))
+ {
+ e->flags |= EDGE_CROSSING;
+ note = find_reg_note (BB_END (e->src), REG_CROSSING_JUMP, NULL_RTX);
+ if (JUMP_P (BB_END (e->src))
+ && !note)
+ add_reg_note (BB_END (e->src), REG_CROSSING_JUMP, NULL_RTX);
+ }
+ else if (BB_PARTITION (e->src) == BB_PARTITION (target))
+ {
+ e->flags &= ~EDGE_CROSSING;
+ /* Remove the region crossing note from jump at end of
+ e->src if it exists. */
+ note = find_reg_note (BB_END (e->src), REG_CROSSING_JUMP, NULL_RTX);
+ if (note)
+ remove_note (BB_END (e->src), note);
+ }
+}
+
+/* Called when block BB has been reassigned to a different partition,
+ to ensure that the region crossing attributes are updated. */
+
+static void
+fixup_bb_partition (basic_block bb)
+{
+ edge e;
+ edge_iterator ei;
+
+ /* Now need to make bb's pred edges non-region crossing. */
+ FOR_EACH_EDGE (e, ei, bb->preds)
+ {
+ fixup_partition_crossing (e, e->dest);
+ }
+
+ /* Possibly need to make bb's successor edges region crossing,
+ or remove stale region crossing. */
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ if ((e->flags & EDGE_FALLTHRU)
+ && BB_PARTITION (bb) != BB_PARTITION (e->dest)
+ && e->dest != EXIT_BLOCK_PTR)
+ /* force_nonfallthru_and_redirect calls fixup_partition_crossing. */
+ force_nonfallthru (e);
+ else
+ fixup_partition_crossing (e, e->dest);
+ }
+}
+
/* Attempt to change code to redirect edge E to TARGET. Don't do that on
expense of adding new instructions or reordering basic blocks.
@@ -1302,16 +1368,18 @@ rtl_redirect_edge_and_branch (edge e, basic_block
{
edge ret;
basic_block src = e->src;
+ basic_block dest = e->dest;
if (e->flags & (EDGE_ABNORMAL_CALL | EDGE_EH))
return NULL;
- if (e->dest == target)
+ if (dest == target)
return e;
if ((ret = try_redirect_by_replacing_jump (e, target, false)) != NULL)
{
df_set_bb_dirty (src);
+ fixup_partition_crossing (ret, target);
return ret;
}
@@ -1320,6 +1388,7 @@ rtl_redirect_edge_and_branch (edge e, basic_block
return NULL;
df_set_bb_dirty (src);
+ fixup_partition_crossing (ret, target);
return ret;
}
@@ -1486,18 +1555,16 @@ force_nonfallthru_and_redirect (edge e, basic_bloc
/* Make sure new block ends up in correct hot/cold section. */
BB_COPY_PARTITION (jump_block, e->src);
- if (flag_reorder_blocks_and_partition
- && targetm_common.have_named_sections
- && JUMP_P (BB_END (jump_block))
- && !any_condjump_p (BB_END (jump_block))
- && (EDGE_SUCC (jump_block, 0)->flags & EDGE_CROSSING))
- add_reg_note (BB_END (jump_block), REG_CROSSING_JUMP, NULL_RTX);
/* Wire edge in. */
new_edge = make_edge (e->src, jump_block, EDGE_FALLTHRU);
new_edge->probability = probability;
new_edge->count = count;
+ /* If e->src was previously region crossing, it no longer is
+ and the reg crossing note should be removed. */
+ fixup_partition_crossing (new_edge, jump_block);
+
/* Redirect old edge. */
redirect_edge_pred (e, jump_block);
e->probability = REG_BR_PROB_BASE;
@@ -1553,13 +1620,16 @@ force_nonfallthru_and_redirect (edge e, basic_bloc
LABEL_NUSES (label)++;
}
- emit_barrier_after (BB_END (jump_block));
+ /* We might be in cfg layout mode, and if so, the following routine will
+ insert the barrier correctly. */
+ emit_barrier_after_bb (jump_block);
redirect_edge_succ_nodup (e, target);
if (abnormal_edge_flags)
make_edge (src, target, abnormal_edge_flags);
df_mark_solutions_dirty ();
+ fixup_partition_crossing (e, target);
return new_bb;
}
@@ -1658,7 +1734,7 @@ rtl_move_block_after (basic_block bb ATTRIBUTE_UNU
static basic_block
rtl_split_edge (edge edge_in)
{
- basic_block bb;
+ basic_block bb, new_bb;
rtx before;
/* Abnormal edges cannot be split. */
@@ -1691,12 +1767,26 @@ rtl_split_edge (edge edge_in)
else
{
bb = create_basic_block (before, NULL, edge_in->dest->prev_bb);
- /* ??? Why not edge_in->dest->prev_bb here? */
- BB_COPY_PARTITION (bb, edge_in->dest);
+ if (edge_in->src == ENTRY_BLOCK_PTR)
+ BB_COPY_PARTITION (bb, edge_in->dest);
+ else
+ /* Put the split bb into the src partition, to avoid creating
+ a situation where a cold bb dominates a hot bb, in the case
+ where src is cold and dest is hot. The src will dominate
+ the new bb (whereas it might not have dominated dest). */
+ BB_COPY_PARTITION (bb, edge_in->src);
}
make_single_succ_edge (bb, edge_in->dest, EDGE_FALLTHRU);
+ /* Can't allow a region crossing edge to be fallthrough. */
+ if (BB_PARTITION (bb) != BB_PARTITION (edge_in->dest)
+ && edge_in->dest != EXIT_BLOCK_PTR)
+ {
+ new_bb = force_nonfallthru (single_succ_edge (bb));
+ gcc_assert (!new_bb);
+ }
+
/* For non-fallthru edges, we must adjust the predecessor's
jump instruction to target our new block. */
if ((edge_in->flags & EDGE_FALLTHRU) == 0)
@@ -1809,17 +1899,13 @@ commit_one_edge_insertion (edge e)
else
{
bb = split_edge (e);
- after = BB_END (bb);
- if (flag_reorder_blocks_and_partition
- && targetm_common.have_named_sections
- && e->src != ENTRY_BLOCK_PTR
- && BB_PARTITION (e->src) == BB_COLD_PARTITION
- && !(e->flags & EDGE_CROSSING)
- && JUMP_P (after)
- && !any_condjump_p (after)
- && (single_succ_edge (bb)->flags & EDGE_CROSSING))
- add_reg_note (after, REG_CROSSING_JUMP, NULL_RTX);
+ /* If e crossed a partition boundary, we needed to make bb end in
+ a region-crossing jump, even though it was originally fallthru. */
+ if (JUMP_P (BB_END (bb)))
+ before = BB_END (bb);
+ else
+ after = BB_END (bb);
}
/* Now that we've found the spot, do the insertion. */
@@ -1859,6 +1945,14 @@ commit_edge_insertions (void)
{
basic_block bb;
+ /* Optimization passes that invoke this routine can cause hot blocks
+ previously reached by both hot and cold blocks to become dominated only
+ by cold blocks. This will cause the verification below to fail,
+ and lead to now cold code in the hot section. In some cases this
+ may only be visible after newly unreachable blocks are deleted,
+ which will be done by fixup_partitions. */
+ fixup_partitions ();
+
#ifdef ENABLE_CHECKING
verify_flow_info ();
#endif
@@ -2060,7 +2154,75 @@ get_last_bb_insn (basic_block bb)
return end;
}
-
+
+/* Perform cleanup on the hot/cold bb partitioning after optimization
+ passes that modify the cfg. */
+
+void
+fixup_partitions (void)
+{
+ basic_block bb;
+
+ if (!crtl->has_bb_partition)
+ return;
+
+ /* Delete any blocks that became unreachable and weren't
+ already cleaned up, for example during edge forwarding
+ and convert_jumps_to_returns. This will expose more
+ opportunities for fixing the partition boundaries here.
+ Also, the calculation of the dominance graph during verification
+ will assert if there are unreachable nodes. */
+ delete_unreachable_blocks ();
+
+ /* If there are partitions, do a sanity check on them: A basic block in
+ a cold partition cannot dominate a basic block in a hot partition.
+ Fixup any that now violate this requirement, as a result of edge
+ forwarding and unreachable block deletion. */
+ vec<basic_block> bbs_in_cold_partition = vNULL;
+ vec<basic_block> bbs_to_fix = vNULL;
+ FOR_EACH_BB (bb)
+ if ((BB_PARTITION (bb) == BB_COLD_PARTITION))
+ bbs_in_cold_partition.safe_push (bb);
+ if (! bbs_in_cold_partition.is_empty ())
+ {
+ bool dom_calculated_here = !dom_info_available_p (CDI_DOMINATORS);
+ basic_block son;
+
+ if (dom_calculated_here)
+ calculate_dominance_info (CDI_DOMINATORS);
+
+ while (! bbs_in_cold_partition.is_empty ())
+ {
+ bb = bbs_in_cold_partition.pop ();
+ /* If bb is not yet cold (because it was added below as
+ a block dominated by a cold bb) then mark it cold here. */
+ if ((BB_PARTITION (bb) != BB_COLD_PARTITION))
+ {
+ BB_SET_PARTITION (bb, BB_COLD_PARTITION);
+ bbs_to_fix.safe_push (bb);
+ }
+ /* Any blocks dominated by a block in the cold section
+ must also be cold. */
+ for (son = first_dom_son (CDI_DOMINATORS, bb);
+ son;
+ son = next_dom_son (CDI_DOMINATORS, son))
+ bbs_in_cold_partition.safe_push (son);
+ }
+
+ if (dom_calculated_here)
+ free_dominance_info (CDI_DOMINATORS);
+ }
+
+ /* Do the partition fixup after all necessary blocks have been converted to
+ cold, so that we only update the region crossings the minimum number of
+ places, which can require forcing edges to be non fallthru. */
+ while (! bbs_to_fix.is_empty ())
+ {
+ bb = bbs_to_fix.pop ();
+ fixup_bb_partition (bb);
+ }
+}
+
/* Verify the CFG and RTL consistency common for both underlying RTL and
cfglayout RTL.