===================================================================
@@ -1061,7 +1061,7 @@ afdo_indirect_call (gimple_stmt_iterator
/* FIXME: Count should be initialized. */
struct cgraph_edge *new_edge
= indirect_edge->make_speculative (direct_call,
- profile_count::uninitialized (), 0);
+ profile_count::uninitialized ());
new_edge->redirect_call_stmt_to_callee ();
gimple_remove_histogram_value (cfun, stmt, hist);
inline_call (new_edge, true, NULL, NULL, false);
===================================================================
@@ -822,7 +822,7 @@ cgraph_edge::set_call_stmt (gcall *new_s
cgraph_edge *
symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee,
- gcall *call_stmt, profile_count count, int freq,
+ gcall *call_stmt, profile_count count,
bool indir_unknown_callee)
{
cgraph_edge *edge;
@@ -862,10 +862,7 @@ symbol_table::create_edge (cgraph_node *
edge->next_callee = NULL;
edge->lto_stmt_uid = 0;
- edge->count = count.ipa ();
- edge->frequency = freq;
- gcc_checking_assert (freq >= 0);
- gcc_checking_assert (freq <= CGRAPH_FREQ_MAX);
+ edge->count = count;
edge->call_stmt = call_stmt;
push_cfun (DECL_STRUCT_FUNCTION (caller->decl));
@@ -907,10 +904,10 @@ symbol_table::create_edge (cgraph_node *
cgraph_edge *
cgraph_node::create_edge (cgraph_node *callee,
- gcall *call_stmt, profile_count count, int freq)
+ gcall *call_stmt, profile_count count)
{
cgraph_edge *edge = symtab->create_edge (this, callee, call_stmt, count,
- freq, false);
+ false);
initialize_inline_failed (edge);
@@ -944,11 +941,11 @@ cgraph_allocate_init_indirect_info (void
cgraph_edge *
cgraph_node::create_indirect_edge (gcall *call_stmt, int ecf_flags,
- profile_count count, int freq,
+ profile_count count,
bool compute_indirect_info)
{
cgraph_edge *edge = symtab->create_edge (this, NULL, call_stmt,
- count, freq, true);
+ count, true);
tree target;
initialize_inline_failed (edge);
@@ -1060,8 +1057,7 @@ cgraph_edge::remove (void)
Return direct edge created. */
cgraph_edge *
-cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count,
- int direct_frequency)
+cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count)
{
cgraph_node *n = caller;
ipa_ref *ref = NULL;
@@ -1071,7 +1067,7 @@ cgraph_edge::make_speculative (cgraph_no
fprintf (dump_file, "Indirect call -> speculative call %s => %s\n",
n->dump_name (), n2->dump_name ());
speculative = true;
- e2 = n->create_edge (n2, call_stmt, direct_count, direct_frequency);
+ e2 = n->create_edge (n2, call_stmt, direct_count);
initialize_inline_failed (e2);
e2->speculative = true;
if (TREE_NOTHROW (n2->decl))
@@ -1081,7 +1077,6 @@ cgraph_edge::make_speculative (cgraph_no
e2->lto_stmt_uid = lto_stmt_uid;
e2->in_polymorphic_cdtor = in_polymorphic_cdtor;
count -= e2->count;
- frequency -= e2->frequency;
symtab->call_edge_duplication_hooks (this, e2);
ref = n->create_reference (n2, IPA_REF_ADDR, call_stmt);
ref->lto_stmt_uid = lto_stmt_uid;
@@ -1198,9 +1193,6 @@ cgraph_edge::resolve_speculation (tree c
in the functions inlined through it. */
}
edge->count += e2->count;
- edge->frequency += e2->frequency;
- if (edge->frequency > CGRAPH_FREQ_MAX)
- edge->frequency = CGRAPH_FREQ_MAX;
edge->speculative = false;
e2->speculative = false;
ref->remove_reference ();
@@ -1308,9 +1300,7 @@ cgraph_edge::redirect_call_stmt_to_calle
/* We are producing the final function body and will throw away the
callgraph edges really soon. Reset the counts/frequencies to
keep verifier happy in the case of roundoff errors. */
- e->count = gimple_bb (e->call_stmt)->count.ipa ();
- e->frequency = compute_call_stmt_bb_frequency
- (e->caller->decl, gimple_bb (e->call_stmt));
+ e->count = gimple_bb (e->call_stmt)->count;
}
/* Expand speculation into GIMPLE code. */
else
@@ -1329,12 +1319,7 @@ cgraph_edge::redirect_call_stmt_to_calle
profile_probability prob = e->count.probability_in (e->count
+ e2->count);
- if (prob.initialized_p ())
- ;
- else if (e->frequency || e2->frequency)
- prob = profile_probability::probability_in_gcov_type
- (e->frequency, e->frequency + e2->frequency).guessed ();
- else
+ if (!prob.initialized_p ())
prob = profile_probability::even ();
new_stmt = gimple_ic (e->call_stmt,
dyn_cast<cgraph_node *> (ref->referred),
@@ -1355,24 +1340,11 @@ cgraph_edge::redirect_call_stmt_to_calle
gcall *ibndret = chkp_retbnd_call_by_val (iresult);
struct cgraph_edge *iedge
= e2->caller->cgraph_node::get_edge (ibndret);
- struct cgraph_edge *dedge;
if (dbndret)
- {
- dedge = iedge->caller->create_edge (iedge->callee,
- dbndret, e->count,
- e->frequency);
- dedge->frequency = compute_call_stmt_bb_frequency
- (dedge->caller->decl, gimple_bb (dedge->call_stmt));
- }
- iedge->frequency = compute_call_stmt_bb_frequency
- (iedge->caller->decl, gimple_bb (iedge->call_stmt));
+ iedge->caller->create_edge (iedge->callee, dbndret, e->count);
}
- e->frequency = compute_call_stmt_bb_frequency
- (e->caller->decl, gimple_bb (e->call_stmt));
- e2->frequency = compute_call_stmt_bb_frequency
- (e2->caller->decl, gimple_bb (e2->call_stmt));
e2->speculative = false;
ref->speculative = false;
ref->stmt = NULL;
@@ -1610,7 +1582,6 @@ cgraph_update_edges_for_call_stmt_node (
cgraph_edge *e = node->get_edge (old_stmt);
cgraph_edge *ne = NULL;
profile_count count;
- int frequency;
if (e)
{
@@ -1644,8 +1615,7 @@ cgraph_update_edges_for_call_stmt_node (
/* Otherwise remove edge and create new one; we can't simply redirect
since function has changed, so inline plan and other information
attached to edge is invalid. */
- count = e->count.ipa ();
- frequency = e->frequency;
+ count = e->count;
if (e->indirect_unknown_callee || e->inline_failed)
e->remove ();
else
@@ -1655,16 +1625,13 @@ cgraph_update_edges_for_call_stmt_node (
{
/* We are seeing new direct call; compute profile info based on BB. */
basic_block bb = gimple_bb (new_stmt);
- count = bb->count.ipa ();
- frequency = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
+ count = bb->count;
}
if (new_call)
{
ne = node->create_edge (cgraph_node::get_create (new_call),
- as_a <gcall *> (new_stmt), count,
- frequency);
+ as_a <gcall *> (new_stmt), count);
gcc_assert (ne->inline_failed);
}
}
@@ -2056,10 +2023,9 @@ cgraph_edge::dump_edge_flags (FILE *f)
{
fprintf (f, "(");
count.dump (f);
- fprintf (f, ")");
+ fprintf (f, ",");
+ fprintf (f, "%.2f per call) ", frequency () / (double)CGRAPH_FREQ_BASE);
}
- if (frequency)
- fprintf (f, "(%.2f per call) ", frequency / (double)CGRAPH_FREQ_BASE);
if (can_throw_external)
fprintf (f, "(can throw external) ");
}
@@ -2205,7 +2171,7 @@ cgraph_node::dump (FILE *f)
}
fprintf (f, "\n");
- if (count.initialized_p ())
+ if (count.ipa ().initialized_p ())
{
bool ok = true;
bool min = false;
@@ -2213,14 +2179,14 @@ cgraph_node::dump (FILE *f)
FOR_EACH_ALIAS (this, ref)
if (dyn_cast <cgraph_node *> (ref->referring)->count.initialized_p ())
- sum += dyn_cast <cgraph_node *> (ref->referring)->count;
+ sum += dyn_cast <cgraph_node *> (ref->referring)->count.ipa ();
if (global.inlined_to
|| (symtab->state < EXPANSION
&& ultimate_alias_target () == this && only_called_directly_p ()))
ok = !count.differs_from_p (sum);
- else if (count > profile_count::from_gcov_type (100)
- && count < sum.apply_scale (99, 100))
+ else if (count.ipa () > profile_count::from_gcov_type (100)
+ && count.ipa () < sum.apply_scale (99, 100))
ok = false, min = true;
if (!ok)
{
@@ -2826,7 +2792,7 @@ cgraph_edge::cannot_lead_to_return_p (vo
bool
cgraph_edge::maybe_hot_p (void)
{
- if (!maybe_hot_count_p (NULL, count))
+ if (!maybe_hot_count_p (NULL, count.ipa ()))
return false;
if (caller->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED
|| (callee
@@ -2845,12 +2811,12 @@ cgraph_edge::maybe_hot_p (void)
if (symtab->state < IPA_SSA)
return true;
if (caller->frequency == NODE_FREQUENCY_EXECUTED_ONCE
- && frequency < CGRAPH_FREQ_BASE * 3 / 2)
+ && frequency () < CGRAPH_FREQ_BASE * 3 / 2)
return false;
if (opt_for_fn (caller->decl, flag_guess_branch_prob))
{
if (PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION) == 0
- || frequency <= (CGRAPH_FREQ_BASE
+ || frequency () <= (CGRAPH_FREQ_BASE
/ PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION)))
return false;
}
@@ -3079,7 +3045,7 @@ clone_of_p (cgraph_node *node, cgraph_no
/* Verify edge count and frequency. */
bool
-cgraph_edge::verify_count_and_frequency ()
+cgraph_edge::verify_count ()
{
bool error_found = false;
if (!count.verify ())
@@ -3087,21 +3053,6 @@ cgraph_edge::verify_count_and_frequency
error ("caller edge count invalid");
error_found = true;
}
- if (count.initialized_p () && !(count.ipa () == count))
- {
- error ("caller edge count is local");
- error_found = true;
- }
- if (frequency < 0)
- {
- error ("caller edge frequency is negative");
- error_found = true;
- }
- if (frequency > CGRAPH_FREQ_MAX)
- {
- error ("caller edge frequency is too large");
- error_found = true;
- }
return error_found;
}
@@ -3193,11 +3144,6 @@ cgraph_node::verify_node (void)
error ("cgraph count invalid");
error_found = true;
}
- if (count.initialized_p () && !(count.ipa () == count))
- {
- error ("cgraph count is local");
- error_found = true;
- }
if (global.inlined_to && same_comdat_group)
{
error ("inline clone in same comdat group list");
@@ -3244,7 +3190,7 @@ cgraph_node::verify_node (void)
bool check_comdat = comdat_local_p ();
for (e = callers; e; e = e->next_caller)
{
- if (e->verify_count_and_frequency ())
+ if (e->verify_count ())
error_found = true;
if (check_comdat
&& !in_same_comdat_group_p (e->caller))
@@ -3277,46 +3223,49 @@ cgraph_node::verify_node (void)
}
for (e = callees; e; e = e->next_callee)
{
- if (e->verify_count_and_frequency ())
+ if (e->verify_count ())
error_found = true;
- /* FIXME: re-enable once cgraph is converted to counts. */
if (gimple_has_body_p (e->caller->decl)
- && 0
&& !e->caller->global.inlined_to
&& !e->speculative
/* Optimized out calls are redirected to __builtin_unreachable. */
- && (e->frequency
+ && (e->count.nonzero_p ()
|| ! e->callee->decl
|| DECL_BUILT_IN_CLASS (e->callee->decl) != BUILT_IN_NORMAL
|| DECL_FUNCTION_CODE (e->callee->decl) != BUILT_IN_UNREACHABLE)
- && (e->frequency
- != compute_call_stmt_bb_frequency (e->caller->decl,
- gimple_bb (e->call_stmt))))
- {
- error ("caller edge frequency %i does not match BB frequency %i",
- e->frequency,
- compute_call_stmt_bb_frequency (e->caller->decl,
- gimple_bb (e->call_stmt)));
+ && count
+ == ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (decl))->count
+ && (!e->count.ipa_p ()
+ && e->count.differs_from_p (gimple_bb (e->call_stmt)->count)))
+ {
+ error ("caller edge count does not match BB count");
+ fprintf (stderr, "edge count: ");
+ e->count.dump (stderr);
+ fprintf (stderr, "\n bb count: ");
+ gimple_bb (e->call_stmt)->count.dump (stderr);
+ fprintf (stderr, "\n");
error_found = true;
}
}
for (e = indirect_calls; e; e = e->next_callee)
{
- if (e->verify_count_and_frequency ())
+ if (e->verify_count ())
error_found = true;
- /* FIXME: re-enable once cgraph is converted to counts. */
if (gimple_has_body_p (e->caller->decl)
&& !e->caller->global.inlined_to
&& !e->speculative
- && 0
- && (e->frequency
- != compute_call_stmt_bb_frequency (e->caller->decl,
- gimple_bb (e->call_stmt))))
- {
- error ("indirect call frequency %i does not match BB frequency %i",
- e->frequency,
- compute_call_stmt_bb_frequency (e->caller->decl,
- gimple_bb (e->call_stmt)));
+ && e->count.ipa_p ()
+ && count
+ == ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (decl))->count
+ && (!e->count.ipa_p ()
+ && e->count.differs_from_p (gimple_bb (e->call_stmt)->count)))
+ {
+ error ("indirect call count does not match BB count");
+ fprintf (stderr, "edge count: ");
+ e->count.dump (stderr);
+ fprintf (stderr, "\n bb count: ");
+ gimple_bb (e->call_stmt)->count.dump (stderr);
+ fprintf (stderr, "\n");
error_found = true;
}
}
===================================================================
@@ -942,7 +942,7 @@ public:
All hooks will see this in node's global.inlined_to, when invoked.
Can be NULL if the node is not inlined. SUFFIX is string that is appended
to the original name. */
- cgraph_node *create_clone (tree decl, profile_count count, int freq,
+ cgraph_node *create_clone (tree decl, profile_count count,
bool update_original,
vec<cgraph_edge *> redirect_callers,
bool call_duplication_hook,
@@ -1110,14 +1110,13 @@ public:
/* Create edge from a given function to CALLEE in the cgraph. */
cgraph_edge *create_edge (cgraph_node *callee,
- gcall *call_stmt, profile_count count,
- int freq);
+ gcall *call_stmt, profile_count count);
/* Create an indirect edge with a yet-undetermined callee where the call
statement destination is a formal parameter of the caller with index
PARAM_INDEX. */
cgraph_edge *create_indirect_edge (gcall *call_stmt, int ecf_flags,
- profile_count count, int freq,
+ profile_count count,
bool compute_indirect_info = true);
/* Like cgraph_create_edge walk the clone tree and update all clones sharing
@@ -1126,7 +1125,6 @@ public:
void create_edge_including_clones (cgraph_node *callee,
gimple *old_stmt, gcall *stmt,
profile_count count,
- int freq,
cgraph_inline_failed_t reason);
/* Return the callgraph edge representing the GIMPLE_CALL statement
@@ -1665,8 +1663,7 @@ struct GTY((chain_next ("%h.next_caller"
/* Turn edge into speculative call calling N2. Update
the profile so the direct call is taken COUNT times
with FREQUENCY. */
- cgraph_edge *make_speculative (cgraph_node *n2, profile_count direct_count,
- int direct_frequency);
+ cgraph_edge *make_speculative (cgraph_node *n2, profile_count direct_count);
/* Given speculative call edge, return all three components. */
void speculative_call_info (cgraph_edge *&direct, cgraph_edge *&indirect,
@@ -1684,11 +1681,11 @@ struct GTY((chain_next ("%h.next_caller"
/* Create clone of edge in the node N represented
by CALL_EXPR the callgraph. */
cgraph_edge * clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
- profile_count num, profile_count den, int freq_scale,
+ profile_count num, profile_count den,
bool update_original);
/* Verify edge count and frequency. */
- bool verify_count_and_frequency ();
+ bool verify_count ();
/* Return true when call of edge can not lead to return from caller
and thus it is safe to ignore its side effects for IPA analysis
@@ -1728,10 +1725,6 @@ struct GTY((chain_next ("%h.next_caller"
/* The stmt_uid of call_stmt. This is used by LTO to recover the call_stmt
when the function is serialized in. */
unsigned int lto_stmt_uid;
- /* Expected frequency of executions within the function.
- When set to CGRAPH_FREQ_BASE, the edge is expected to be called once
- per function call. The range is 0 to CGRAPH_FREQ_MAX. */
- int frequency;
/* Unique id of the edge. */
int uid;
/* Whether this edge was made direct by indirect inlining. */
@@ -1769,6 +1762,10 @@ struct GTY((chain_next ("%h.next_caller"
/* Return true if call must bind to current definition. */
bool binds_to_current_def_p ();
+ /* Expected frequency of executions within the function.
+ When set to CGRAPH_FREQ_BASE, the edge is expected to be called once
+ per function call. The range is 0 to CGRAPH_FREQ_MAX. */
+ int frequency ();
private:
/* Remove the edge from the list of the callers of the callee. */
void remove_caller (void);
@@ -2287,7 +2284,7 @@ private:
parameters of which only CALLEE can be NULL (when creating an indirect call
edge). */
cgraph_edge *create_edge (cgraph_node *caller, cgraph_node *callee,
- gcall *call_stmt, profile_count count, int freq,
+ gcall *call_stmt, profile_count count,
bool indir_unknown_callee);
/* Put the edge onto the free list. */
@@ -3111,6 +3108,18 @@ cgraph_edge::binds_to_current_def_p ()
return false;
}
+/* Expected frequency of executions within the function.
+ When set to CGRAPH_FREQ_BASE, the edge is expected to be called once
+ per function call. The range is 0 to CGRAPH_FREQ_MAX. */
+
+inline int
+cgraph_edge::frequency ()
+{
+ return count.to_cgraph_frequency (caller->global.inlined_to
+ ? caller->global.inlined_to->count
+ : caller->count);
+}
+
/* Return true if the TM_CLONE bit is set for a given FNDECL. */
static inline bool
decl_is_tm_clone (const_tree fndecl)
===================================================================
@@ -317,17 +317,15 @@ pass_build_cgraph_edges::execute (functi
if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
{
- int freq = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
decl = gimple_call_fndecl (call_stmt);
if (decl)
- node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count, freq);
+ node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count);
else if (gimple_call_internal_p (call_stmt))
;
else
node->create_indirect_edge (call_stmt,
gimple_call_flags (call_stmt),
- bb->count, freq);
+ bb->count);
}
node->record_stmt_references (stmt);
if (gomp_parallel *omp_par_stmt = dyn_cast <gomp_parallel *> (stmt))
@@ -402,7 +400,7 @@ cgraph_edge::rebuild_edges (void)
node->remove_callees ();
node->remove_all_references ();
- node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ();
+ node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
FOR_EACH_BB_FN (bb, cfun)
{
@@ -413,18 +411,16 @@ cgraph_edge::rebuild_edges (void)
if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
{
- int freq = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
decl = gimple_call_fndecl (call_stmt);
if (decl)
node->create_edge (cgraph_node::get_create (decl), call_stmt,
- bb->count, freq);
+ bb->count);
else if (gimple_call_internal_p (call_stmt))
;
else
node->create_indirect_edge (call_stmt,
gimple_call_flags (call_stmt),
- bb->count, freq);
+ bb->count);
}
node->record_stmt_references (stmt);
}
===================================================================
@@ -87,20 +87,11 @@ along with GCC; see the file COPYING3.
cgraph_edge *
cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid,
profile_count num, profile_count den,
- int freq_scale, bool update_original)
+ bool update_original)
{
cgraph_edge *new_edge;
- profile_count gcov_count
- = (num == profile_count::zero () || den > 0)
- ? count.apply_scale (num, den) : count;
- gcov_type freq;
-
- /* We do not want to ignore loop nest after frequency drops to 0. */
- if (!freq_scale)
- freq_scale = 1;
- freq = frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
- if (freq > CGRAPH_FREQ_MAX)
- freq = CGRAPH_FREQ_MAX;
+ profile_count::adjust_for_ipa_scaling (&num, &den);
+ profile_count gcov_count = count.apply_scale (num, den);
if (indirect_unknown_callee)
{
@@ -113,19 +104,19 @@ cgraph_edge::clone (cgraph_node *n, gcal
{
cgraph_node *callee = cgraph_node::get (decl);
gcc_checking_assert (callee);
- new_edge = n->create_edge (callee, call_stmt, gcov_count, freq);
+ new_edge = n->create_edge (callee, call_stmt, gcov_count);
}
else
{
new_edge = n->create_indirect_edge (call_stmt,
indirect_info->ecf_flags,
- gcov_count, freq, false);
+ gcov_count, false);
*new_edge->indirect_info = *indirect_info;
}
}
else
{
- new_edge = n->create_edge (callee, call_stmt, gcov_count, freq);
+ new_edge = n->create_edge (callee, call_stmt, gcov_count);
if (indirect_info)
{
new_edge->indirect_info
@@ -142,10 +133,14 @@ cgraph_edge::clone (cgraph_node *n, gcal
new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p;
new_edge->speculative = speculative;
new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor;
- if (update_original)
- {
- count -= new_edge->count;
- }
+
+ /* Update IPA profile. Local profiles need no updating in original. */
+ if (update_original
+ && count.ipa () == count && new_edge->count.ipa () == new_edge->count)
+ count -= new_edge->count;
+ else if (caller->count.global0 () == caller->count
+ && !(count == profile_count::zero ()))
+ count = count.global0 ();
symtab->call_edge_duplication_hooks (this, new_edge);
return new_edge;
}
@@ -337,8 +332,7 @@ duplicate_thunk_for_node (cgraph_node *t
new_thunk->clone.args_to_skip = node->clone.args_to_skip;
new_thunk->clone.combined_args_to_skip = node->clone.combined_args_to_skip;
- cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count,
- CGRAPH_FREQ_BASE);
+ cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count);
symtab->call_edge_duplication_hooks (thunk->callees, e);
symtab->call_cgraph_duplication_hooks (thunk, new_thunk);
return new_thunk;
@@ -422,7 +416,7 @@ dump_callgraph_transformation (const cgr
node is not inlined. */
cgraph_node *
-cgraph_node::create_clone (tree new_decl, profile_count prof_count, int freq,
+cgraph_node::create_clone (tree new_decl, profile_count prof_count,
bool update_original,
vec<cgraph_edge *> redirect_callers,
bool call_duplication_hook,
@@ -432,11 +426,27 @@ cgraph_node::create_clone (tree new_decl
cgraph_node *new_node = symtab->create_empty ();
cgraph_edge *e;
unsigned i;
+ profile_count old_count = count;
if (new_inlined_to)
dump_callgraph_transformation (this, new_inlined_to, "inlining to");
+ if (prof_count == profile_count::zero ()
+ && !(count == profile_count::zero ()))
+ prof_count = count.global0 ();
+
new_node->count = prof_count;
+
+ /* Update IPA profile. Local profiles need no updating in original. */
+ if (update_original && !(count == profile_count::zero ())
+ && count.ipa () == count && prof_count.ipa () == prof_count)
+ {
+ if (count.nonzero_p ()
+ && !(count - prof_count).nonzero_p ())
+ count = count.global0 ();
+ else
+ count -= prof_count;
+ }
new_node->decl = new_decl;
new_node->register_symbol ();
new_node->origin = origin;
@@ -489,12 +499,12 @@ cgraph_node::create_clone (tree new_decl
new_node->expand_all_artificial_thunks ();
for (e = callees;e; e=e->next_callee)
- e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, count,
- freq, update_original);
+ e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count,
+ update_original);
for (e = indirect_calls; e; e = e->next_callee)
e->clone (new_node, e->call_stmt, e->lto_stmt_uid,
- new_node->count, count, freq, update_original);
+ new_node->count, old_count, update_original);
new_node->clone_references (this);
new_node->next_sibling_clone = clones;
@@ -503,9 +513,6 @@ cgraph_node::create_clone (tree new_decl
clones = new_node;
new_node->clone_of = this;
- if (update_original)
- count -= prof_count;
-
if (call_duplication_hook)
symtab->call_cgraph_duplication_hooks (this, new_node);
@@ -591,7 +598,7 @@ cgraph_node::create_virtual_clone (vec<c
SET_DECL_ASSEMBLER_NAME (new_decl, clone_function_name (old_decl, suffix));
SET_DECL_RTL (new_decl, NULL);
- new_node = create_clone (new_decl, count, CGRAPH_FREQ_BASE, false,
+ new_node = create_clone (new_decl, count, false,
redirect_callers, false, NULL, args_to_skip, suffix);
/* Update the properties.
@@ -773,7 +780,6 @@ void
cgraph_node::create_edge_including_clones (cgraph_node *callee,
gimple *old_stmt, gcall *stmt,
profile_count count,
- int freq,
cgraph_inline_failed_t reason)
{
cgraph_node *node;
@@ -781,7 +787,7 @@ cgraph_node::create_edge_including_clone
if (!get_edge (stmt))
{
- edge = create_edge (callee, stmt, count, freq);
+ edge = create_edge (callee, stmt, count);
edge->inline_failed = reason;
}
@@ -801,7 +807,7 @@ cgraph_node::create_edge_including_clone
edge->set_call_stmt (stmt);
else if (! node->get_edge (stmt))
{
- edge = node->create_edge (callee, stmt, count, freq);
+ edge = node->create_edge (callee, stmt, count);
edge->inline_failed = reason;
}
@@ -904,14 +910,12 @@ cgraph_node::create_version_clone (tree
|| bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
e->clone (new_version, e->call_stmt,
e->lto_stmt_uid, count, count,
- CGRAPH_FREQ_BASE,
true);
for (e = indirect_calls; e; e=e->next_callee)
if (!bbs_to_copy
|| bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
e->clone (new_version, e->call_stmt,
e->lto_stmt_uid, count, count,
- CGRAPH_FREQ_BASE,
true);
FOR_EACH_VEC_ELT (redirect_callers, i, e)
{
===================================================================
@@ -620,7 +620,7 @@ cgraph_node::analyze (void)
{
cgraph_node *t = cgraph_node::get (thunk.alias);
- create_edge (t, NULL, t->count, CGRAPH_FREQ_BASE);
+ create_edge (t, NULL, t->count);
callees->can_throw_external = !TREE_NOTHROW (t->decl);
/* Target code in expand_thunk may need the thunk's target
to be analyzed, so recurse here. */
@@ -1950,7 +1950,7 @@ cgraph_node::expand_thunk (bool output_a
resbnd = chkp_insert_retbnd_call (NULL, restmp, &bsi);
create_edge (get_create (gimple_call_fndecl (gsi_stmt (bsi))),
as_a <gcall *> (gsi_stmt (bsi)),
- callees->count, callees->frequency);
+ callees->count);
}
if (restmp && !this_adjusting
@@ -2759,7 +2759,7 @@ cgraph_node::create_wrapper (cgraph_node
memset (&thunk, 0, sizeof (cgraph_thunk_info));
thunk.thunk_p = true;
- create_edge (target, NULL, count, CGRAPH_FREQ_BASE);
+ create_edge (target, NULL, count);
callees->can_throw_external = !TREE_NOTHROW (target->decl);
tree arguments = DECL_ARGUMENTS (decl);
===================================================================
@@ -83,7 +83,6 @@ static void
update_call_edge_frequencies (gimple_seq_node first, basic_block bb)
{
struct cgraph_node *cfun_node = NULL;
- int bb_freq = 0;
gimple_seq_node n;
for (n = first; n ; n = n->next)
@@ -94,15 +93,11 @@ update_call_edge_frequencies (gimple_seq
/* These function calls are expensive enough that we want
to avoid calling them if we never see any calls. */
if (cfun_node == NULL)
- {
- cfun_node = cgraph_node::get (current_function_decl);
- bb_freq = (compute_call_stmt_bb_frequency
- (current_function_decl, bb));
- }
+ cfun_node = cgraph_node::get (current_function_decl);
e = cfun_node->get_edge (n);
if (e != NULL)
- e->frequency = bb_freq;
+ e->count = bb->count;
}
}
===================================================================
@@ -264,8 +264,11 @@ input_bb (struct lto_input_block *ib, en
index = streamer_read_uhwi (ib);
bb = BASIC_BLOCK_FOR_FN (fn, index);
- bb->count = profile_count::stream_in (ib).apply_scale
- (count_materialization_scale, REG_BR_PROB_BASE);
+ bb->count = profile_count::stream_in (ib);
+ if (count_materialization_scale != REG_BR_PROB_BASE
+ && bb->count.ipa ().nonzero_p ())
+ bb->count
+ = bb->count.apply_scale (count_materialization_scale, REG_BR_PROB_BASE);
bb->flags = streamer_read_hwi (ib);
/* LTO_bb1 has statements. LTO_bb0 does not. */
===================================================================
@@ -717,7 +717,7 @@ chkp_produce_thunks (bool early)
node->thunk.thunk_p = true;
node->thunk.add_pointer_bounds_args = true;
node->create_edge (node->instrumented_version, NULL,
- node->count, CGRAPH_FREQ_BASE);
+ node->count);
node->create_reference (node->instrumented_version,
IPA_REF_CHKP, NULL);
/* Thunk shouldn't be a cdtor. */
===================================================================
@@ -498,7 +498,7 @@ ipcp_lattice<valtype>::print (FILE * f,
fprintf (f, " [from:");
for (s = val->sources; s; s = s->next)
fprintf (f, " %i(%i)", s->cs->caller->order,
- s->cs->frequency);
+ s->cs->frequency ());
fprintf (f, "]");
}
@@ -677,9 +677,9 @@ gather_caller_stats (struct cgraph_node
for (cs = node->callers; cs; cs = cs->next_caller)
if (!cs->caller->thunk.thunk_p)
{
- if (cs->count.initialized_p ())
- stats->count_sum += cs->count;
- stats->freq_sum += cs->frequency;
+ if (cs->count.ipa ().initialized_p ())
+ stats->count_sum += cs->count.ipa ();
+ stats->freq_sum += cs->frequency ();
stats->n_calls++;
if (cs->maybe_hot_p ())
stats->n_hot_calls ++;
@@ -731,7 +731,7 @@ ipcp_cloning_candidate_p (struct cgraph_
significantly. */
if (max_count > profile_count::zero ())
{
- if (stats.count_sum > node->count.apply_scale (90, 100))
+ if (stats.count_sum > node->count.ipa ().apply_scale (90, 100))
{
if (dump_file)
fprintf (dump_file, "Considering %s for cloning; "
@@ -3272,7 +3272,7 @@ ipcp_propagate_stage (struct ipa_topo_in
}
if (node->definition && !node->alias)
overall_size += ipa_fn_summaries->get (node)->self_size;
- max_count = max_count.max (node->count);
+ max_count = max_count.max (node->count.ipa ());
}
max_new_size = overall_size;
@@ -3550,9 +3550,9 @@ get_info_about_necessary_edges (ipcp_val
if (cgraph_edge_brings_value_p (cs, src, dest))
{
count++;
- freq += cs->frequency;
- if (cs->count.initialized_p ())
- cnt += cs->count;
+ freq += cs->frequency ();
+ if (cs->count.ipa ().initialized_p ())
+ cnt += cs->count.ipa ();
hot |= cs->maybe_hot_p ();
}
cs = get_next_cgraph_edge_clone (cs);
@@ -3662,7 +3662,7 @@ update_profiling_info (struct cgraph_nod
profile_count new_sum, orig_sum;
profile_count remainder, orig_node_count = orig_node->count;
- if (!(orig_node_count > profile_count::zero ()))
+ if (!(orig_node_count.ipa () > profile_count::zero ()))
return;
init_caller_stats (&stats);
@@ -3701,7 +3701,7 @@ update_profiling_info (struct cgraph_nod
for (cs = new_node->callees; cs; cs = cs->next_callee)
/* FIXME: why we care about non-zero frequency here? */
- if (cs->frequency)
+ if (cs->frequency ())
cs->count = cs->count.apply_scale (new_sum, orig_node_count);
else
cs->count = profile_count::zero ();
@@ -3741,7 +3741,7 @@ update_specialized_profile (struct cgrap
orig_node->count -= redirected_sum;
for (cs = new_node->callees; cs; cs = cs->next_callee)
- if (cs->frequency)
+ if (cs->frequency ())
cs->count += cs->count.apply_scale (redirected_sum, new_node_count);
else
cs->count = profile_count::zero ();
@@ -4463,8 +4463,8 @@ perhaps_add_new_callers (cgraph_node *no
cs->redirect_callee_duplicating_thunks (val->spec_node);
val->spec_node->expand_all_artificial_thunks ();
- if (cs->count.initialized_p ())
- redirected_sum = redirected_sum + cs->count;
+ if (cs->count.ipa ().initialized_p ())
+ redirected_sum = redirected_sum + cs->count.ipa ();
}
cs = get_next_cgraph_edge_clone (cs);
}
===================================================================
@@ -3566,7 +3566,7 @@ ipa_devirt (void)
bool final;
if (final_warning_records)
- final_warning_records->dyn_count = e->count;
+ final_warning_records->dyn_count = e->count.ipa ();
vec <cgraph_node *>targets
= possible_polymorphic_call_targets
@@ -3727,8 +3727,7 @@ ipa_devirt (void)
nconverted++;
update = true;
e->make_speculative
- (likely_target, e->count.apply_scale (8, 10),
- e->frequency * 8 / 10);
+ (likely_target, e->count.apply_scale (8, 10));
}
}
if (update)
===================================================================
@@ -244,7 +244,6 @@ redirect_to_unreachable (struct cgraph_e
e->redirect_callee (target);
struct ipa_call_summary *es = ipa_call_summaries->get (e);
e->inline_failed = CIF_UNREACHABLE;
- e->frequency = 0;
e->count = profile_count::zero ();
es->call_stmt_size = 0;
es->call_stmt_time = 0;
@@ -823,7 +822,7 @@ dump_ipa_call_summary (FILE *f, int inde
indent, "", callee->name (), callee->order,
!edge->inline_failed
? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
- indent, "", es->loop_depth, edge->frequency,
+ indent, "", es->loop_depth, edge->frequency (),
es->call_stmt_size, es->call_stmt_time,
(int) ipa_fn_summaries->get (callee)->size / ipa_fn_summary::size_scale,
(int) ipa_fn_summaries->get (callee)->estimated_stack_size);
@@ -865,7 +864,7 @@ dump_ipa_call_summary (FILE *f, int inde
" time: %2i",
indent, "",
es->loop_depth,
- edge->frequency, es->call_stmt_size, es->call_stmt_time);
+ edge->frequency (), es->call_stmt_size, es->call_stmt_time);
if (es->predicate)
{
fprintf (f, "predicate: ");
@@ -2579,9 +2578,9 @@ estimate_edge_size_and_time (struct cgra
if (min_size)
*min_size += cur_size;
if (prob == REG_BR_PROB_BASE)
- *time += ((sreal)(call_time * e->frequency)) / CGRAPH_FREQ_BASE;
+ *time += ((sreal)(call_time * e->frequency ())) / CGRAPH_FREQ_BASE;
else
- *time += ((sreal)call_time) * (prob * e->frequency)
+ *time += ((sreal)call_time) * (prob * e->frequency ())
/ (CGRAPH_FREQ_BASE * REG_BR_PROB_BASE);
}
@@ -3059,7 +3058,7 @@ ipa_merge_fn_summary_after_inlining (str
toplev_predicate);
if (p != false && nonconstp != false)
{
- sreal add_time = ((sreal)e->time * edge->frequency) / CGRAPH_FREQ_BASE;
+ sreal add_time = ((sreal)e->time * edge->frequency ()) / CGRAPH_FREQ_BASE;
int prob = e->nonconst_predicate.probability (callee_info->conds,
clause, es->param);
add_time = add_time * prob / REG_BR_PROB_BASE;
===================================================================
@@ -175,11 +175,11 @@ do_estimate_edge_time (struct cgraph_edg
edges and for those we disable size limits. Don't do that when
probability that caller will call the callee is low however, since it
may hurt optimization of the caller's hot path. */
- if (edge->count.initialized_p () && edge->maybe_hot_p ()
- && (edge->count.apply_scale (2, 1)
+ if (edge->count.ipa ().initialized_p () && edge->maybe_hot_p ()
+ && (edge->count.ipa ().apply_scale (2, 1)
> (edge->caller->global.inlined_to
- ? edge->caller->global.inlined_to->count
- : edge->caller->count)))
+ ? edge->caller->global.inlined_to->count.ipa ()
+ : edge->caller->count.ipa ())))
hints |= INLINE_HINT_known_hot;
known_vals.release ();
===================================================================
@@ -51,36 +51,24 @@ along with GCC; see the file COPYING3.
int ncalls_inlined;
int nfunctions_inlined;
-/* Scale frequency of NODE edges by FREQ_SCALE. */
+/* Scale counts of NODE edges by NUM/DEN. */
static void
-update_noncloned_frequencies (struct cgraph_node *node,
- int freq_scale, profile_count num,
- profile_count den)
+update_noncloned_counts (struct cgraph_node *node,
+ profile_count num, profile_count den)
{
struct cgraph_edge *e;
profile_count::adjust_for_ipa_scaling (&num, &den);
- /* We do not want to ignore high loop nest after freq drops to 0. */
- if (!freq_scale)
- freq_scale = 1;
for (e = node->callees; e; e = e->next_callee)
{
- e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
- if (e->frequency > CGRAPH_FREQ_MAX)
- e->frequency = CGRAPH_FREQ_MAX;
if (!e->inline_failed)
- update_noncloned_frequencies (e->callee, freq_scale, num, den);
+ update_noncloned_counts (e->callee, num, den);
e->count = e->count.apply_scale (num, den);
}
for (e = node->indirect_calls; e; e = e->next_callee)
- {
- e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
- if (e->frequency > CGRAPH_FREQ_MAX)
- e->frequency = CGRAPH_FREQ_MAX;
- e->count = e->count.apply_scale (num, den);
- }
+ e->count = e->count.apply_scale (num, den);
node->count = node->count.apply_scale (num, den);
}
@@ -169,12 +157,11 @@ master_clone_with_noninline_clones_p (st
By default the offline copy is removed, when it appears dead after inlining.
UPDATE_ORIGINAL prevents this transformation.
If OVERALL_SIZE is non-NULL, the size is updated to reflect the
- transformation.
- FREQ_SCALE specify the scaling of frequencies of call sites. */
+ transformation. */
void
clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
- bool update_original, int *overall_size, int freq_scale)
+ bool update_original, int *overall_size)
{
struct cgraph_node *inlining_into;
struct cgraph_edge *next;
@@ -218,8 +205,7 @@ clone_inlined_nodes (struct cgraph_edge
}
duplicate = false;
e->callee->externally_visible = false;
- update_noncloned_frequencies (e->callee, e->frequency,
- e->count, e->callee->count);
+ update_noncloned_counts (e->callee, e->count, e->callee->count);
dump_callgraph_transformation (e->callee, inlining_into,
"inlining to");
@@ -228,11 +214,8 @@ clone_inlined_nodes (struct cgraph_edge
{
struct cgraph_node *n;
- if (freq_scale == -1)
- freq_scale = e->frequency;
n = e->callee->create_clone (e->callee->decl,
- MIN (e->count, e->callee->count),
- freq_scale,
+ e->count,
update_original, vNULL, true,
inlining_into,
NULL);
@@ -250,7 +233,7 @@ clone_inlined_nodes (struct cgraph_edge
{
next = e->next_callee;
if (!e->inline_failed)
- clone_inlined_nodes (e, duplicate, update_original, overall_size, freq_scale);
+ clone_inlined_nodes (e, duplicate, update_original, overall_size);
}
}
@@ -457,7 +440,7 @@ inline_call (struct cgraph_edge *e, bool
}
}
- clone_inlined_nodes (e, true, update_original, overall_size, e->frequency);
+ clone_inlined_nodes (e, true, update_original, overall_size);
gcc_assert (curr->callee->global.inlined_to == to);
===================================================================
@@ -640,12 +640,12 @@ compute_uninlined_call_time (struct cgra
? edge->caller->global.inlined_to
: edge->caller);
- if (edge->count.nonzero_p ()
- && caller->count.nonzero_p ())
- uninlined_call_time *= (sreal)edge->count.to_gcov_type ()
- / caller->count.to_gcov_type ();
- if (edge->frequency)
- uninlined_call_time *= cgraph_freq_base_rec * edge->frequency;
+ if (edge->count.ipa ().nonzero_p ()
+ && caller->count.ipa ().nonzero_p ())
+ uninlined_call_time *= (sreal)edge->count.ipa ().to_gcov_type ()
+ / caller->count.ipa ().to_gcov_type ();
+ if (edge->frequency ())
+ uninlined_call_time *= cgraph_freq_base_rec * edge->frequency ();
else
uninlined_call_time = uninlined_call_time >> 11;
@@ -665,17 +665,17 @@ compute_inlined_call_time (struct cgraph
: edge->caller);
sreal caller_time = ipa_fn_summaries->get (caller)->time;
- if (edge->count.nonzero_p ()
- && caller->count.nonzero_p ())
+ if (edge->count.ipa ().nonzero_p ()
+ && caller->count.ipa ().nonzero_p ())
time *= (sreal)edge->count.to_gcov_type () / caller->count.to_gcov_type ();
- if (edge->frequency)
- time *= cgraph_freq_base_rec * edge->frequency;
+ if (edge->frequency ())
+ time *= cgraph_freq_base_rec * edge->frequency ();
else
time = time >> 11;
/* This calculation should match one in ipa-inline-analysis.c
(estimate_edge_size_and_time). */
- time -= (sreal) edge->frequency
+ time -= (sreal) edge->frequency ()
* ipa_call_summaries->get (edge)->call_stmt_time / CGRAPH_FREQ_BASE;
time += caller_time;
if (time <= 0)
@@ -724,7 +724,7 @@ want_inline_small_function_p (struct cgr
promote non-inline functions to inline and we increase
MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */
else if ((!DECL_DECLARED_INLINE_P (callee->decl)
- && (!e->count.initialized_p () || !e->maybe_hot_p ()))
+ && (!e->count.ipa ().initialized_p () || !e->maybe_hot_p ()))
&& ipa_fn_summaries->get (callee)->min_size
- ipa_call_summaries->get (e)->call_stmt_size
> MAX (MAX_INLINE_INSNS_SINGLE, MAX_INLINE_INSNS_AUTO))
@@ -733,7 +733,7 @@ want_inline_small_function_p (struct cgr
want_inline = false;
}
else if ((DECL_DECLARED_INLINE_P (callee->decl)
- || e->count.nonzero_p ())
+ || e->count.ipa ().nonzero_p ())
&& ipa_fn_summaries->get (callee)->min_size
- ipa_call_summaries->get (e)->call_stmt_size
> 16 * MAX_INLINE_INSNS_SINGLE)
@@ -843,7 +843,7 @@ want_inline_self_recursive_call_p (struc
reason = "recursive call is cold";
want_inline = false;
}
- else if (!outer_node->count.nonzero_p ())
+ else if (!outer_node->count.ipa ().nonzero_p ())
{
reason = "not executed in profile";
want_inline = false;
@@ -855,7 +855,7 @@ want_inline_self_recursive_call_p (struc
}
if (outer_node->global.inlined_to)
- caller_freq = outer_node->callers->frequency;
+ caller_freq = outer_node->callers->frequency ();
if (!caller_freq)
{
@@ -881,16 +881,16 @@ want_inline_self_recursive_call_p (struc
int i;
for (i = 1; i < depth; i++)
max_prob = max_prob * max_prob / CGRAPH_FREQ_BASE;
- if (max_count.nonzero_p () && edge->count.nonzero_p ()
- && (edge->count.to_gcov_type () * CGRAPH_FREQ_BASE
- / outer_node->count.to_gcov_type ()
+ if (max_count.nonzero_p () && edge->count.ipa ().nonzero_p ()
+ && (edge->count.ipa ().to_gcov_type () * CGRAPH_FREQ_BASE
+ / outer_node->count.ipa ().to_gcov_type ()
>= max_prob))
{
reason = "profile of recursive call is too large";
want_inline = false;
}
if (!max_count.nonzero_p ()
- && (edge->frequency * CGRAPH_FREQ_BASE / caller_freq
+ && (edge->frequency () * CGRAPH_FREQ_BASE / caller_freq
>= max_prob))
{
reason = "frequency of recursive call is too large";
@@ -915,17 +915,17 @@ want_inline_self_recursive_call_p (struc
methods. */
else
{
- if (max_count.nonzero_p () && edge->count.initialized_p ()
- && (edge->count.to_gcov_type () * 100
- / outer_node->count.to_gcov_type ()
+ if (max_count.nonzero_p () && edge->count.ipa ().initialized_p ()
+ && (edge->count.ipa ().to_gcov_type () * 100
+ / outer_node->count.ipa ().to_gcov_type ()
<= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
{
reason = "profile of recursive call is too small";
want_inline = false;
}
else if ((!max_count.nonzero_p ()
- || !edge->count.initialized_p ())
- && (edge->frequency * 100 / caller_freq
+ || !edge->count.ipa ().initialized_p ())
+ && (edge->frequency () * 100 / caller_freq
<= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
{
reason = "frequency of recursive call is too small";
@@ -1023,8 +1023,11 @@ edge_badness (struct cgraph_edge *edge,
edge_time = estimate_edge_time (edge, &unspec_edge_time);
hints = estimate_edge_hints (edge);
gcc_checking_assert (edge_time >= 0);
- /* Check that inlined time is better, but tolerate some roundoff issues. */
- gcc_checking_assert ((edge_time - callee_info->time).to_int () <= 0);
+ /* Check that inlined time is better, but tolerate some roundoff issues.
+ FIXME: When callee profile drops to 0 we account calls more. This
+ should be fixed by never doing that. */
+ gcc_checking_assert ((edge_time - callee_info->time).to_int () <= 0
+ || !callee->count.nonzero_p ());
gcc_checking_assert (growth <= callee_info->size);
if (dump)
@@ -1070,7 +1073,7 @@ edge_badness (struct cgraph_edge *edge,
then calls without.
*/
else if (opt_for_fn (caller->decl, flag_guess_branch_prob)
- || caller->count.nonzero_p ())
+ || caller->count.ipa ().nonzero_p ())
{
sreal numerator, denominator;
int overall_growth;
@@ -1080,9 +1083,9 @@ edge_badness (struct cgraph_edge *edge,
- inlined_time);
if (numerator == 0)
numerator = ((sreal) 1 >> 8);
- if (caller->count.nonzero_p ())
- numerator *= caller->count.to_gcov_type ();
- else if (caller->count.initialized_p ())
+ if (caller->count.ipa ().nonzero_p ())
+ numerator *= caller->count.ipa ().to_gcov_type ();
+ else if (caller->count.ipa ().initialized_p ())
numerator = numerator >> 11;
denominator = growth;
@@ -1108,14 +1111,14 @@ edge_badness (struct cgraph_edge *edge,
&& callee_info->single_caller
&& !edge->caller->global.inlined_to
/* ... and edges executed only conditionally ... */
- && edge->frequency < CGRAPH_FREQ_BASE
+ && edge->frequency () < CGRAPH_FREQ_BASE
/* ... consider case where callee is not inline but caller is ... */
&& ((!DECL_DECLARED_INLINE_P (edge->callee->decl)
&& DECL_DECLARED_INLINE_P (caller->decl))
/* ... or when early optimizers decided to split and edge
frequency still indicates splitting is a win ... */
|| (callee->split_part && !caller->split_part
- && edge->frequency
+ && edge->frequency ()
< CGRAPH_FREQ_BASE
* PARAM_VALUE
(PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100
@@ -1166,9 +1169,9 @@ edge_badness (struct cgraph_edge *edge,
" overall growth %i (current) %i (original)"
" %i (compensated)\n",
badness.to_double (),
- (double)edge->frequency / CGRAPH_FREQ_BASE,
- edge->count.initialized_p () ? edge->count.to_gcov_type () : -1,
- caller->count.initialized_p () ? caller->count.to_gcov_type () : -1,
+ (double)edge->frequency () / CGRAPH_FREQ_BASE,
+ edge->count.ipa ().initialized_p () ? edge->count.ipa ().to_gcov_type () : -1,
+ caller->count.ipa ().initialized_p () ? caller->count.ipa ().to_gcov_type () : -1,
compute_uninlined_call_time (edge,
unspec_edge_time).to_double (),
compute_inlined_call_time (edge, edge_time).to_double (),
@@ -1430,8 +1433,8 @@ lookup_recursive_calls (struct cgraph_no
{
/* When profile feedback is available, prioritize by expected number
of calls. */
- heap->insert (!(max_count > 0) || !e->count.initialized_p () ? -e->frequency
- : -(e->count.to_gcov_type ()
+ heap->insert (!(max_count > 0) || !e->count.ipa ().initialized_p () ? -e->frequency ()
+ : -(e->count.ipa ().to_gcov_type ()
/ ((max_count.to_gcov_type () + (1<<24) - 1)
/ (1<<24))),
e);
@@ -1533,11 +1536,10 @@ recursive_inlining (struct cgraph_edge *
{
/* We need original clone to copy around. */
master_clone = node->create_clone (node->decl, node->count,
- CGRAPH_FREQ_BASE, false, vNULL,
- true, NULL, NULL);
+ false, vNULL, true, NULL, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
- clone_inlined_nodes (e, true, false, NULL, CGRAPH_FREQ_BASE);
+ clone_inlined_nodes (e, true, false, NULL);
curr->redirect_callee (master_clone);
reset_edge_growth_cache (curr);
}
@@ -1684,8 +1686,8 @@ resolve_noninline_speculation (edge_heap
? node->global.inlined_to : node;
auto_bitmap updated_nodes;
- if (edge->count.initialized_p ())
- spec_rem += edge->count;
+ if (edge->count.ipa ().initialized_p ())
+ spec_rem += edge->count.ipa ();
edge->resolve_speculation ();
reset_edge_caches (where);
ipa_update_overall_fn_summary (where);
@@ -1790,7 +1792,7 @@ inline_small_functions (void)
}
for (edge = node->callers; edge; edge = edge->next_caller)
- max_count = max_count.max (edge->count);
+ max_count = max_count.max (edge->count.ipa ());
}
ipa_free_postorder_info ();
initialize_growth_caches ();
@@ -1874,35 +1876,40 @@ inline_small_functions (void)
continue;
#if CHECKING_P
- /* Be sure that caches are maintained consistent. */
- sreal cached_badness = edge_badness (edge, false);
-
- int old_size_est = estimate_edge_size (edge);
- sreal old_time_est = estimate_edge_time (edge);
- int old_hints_est = estimate_edge_hints (edge);
-
- reset_edge_growth_cache (edge);
- gcc_assert (old_size_est == estimate_edge_size (edge));
- gcc_assert (old_time_est == estimate_edge_time (edge));
- /* FIXME:
-
- gcc_assert (old_hints_est == estimate_edge_hints (edge));
-
- fails with profile feedback because some hints depends on
- maybe_hot_edge_p predicate and because callee gets inlined to other
- calls, the edge may become cold.
- This ought to be fixed by computing relative probabilities
- for given invocation but that will be better done once whole
- code is converted to sreals. Disable for now and revert to "wrong"
- value so enable/disable checking paths agree. */
- edge_growth_cache[edge->uid].hints = old_hints_est + 1;
-
- /* When updating the edge costs, we only decrease badness in the keys.
- Increases of badness are handled lazilly; when we see key with out
- of date value on it, we re-insert it now. */
- current_badness = edge_badness (edge, false);
- gcc_assert (cached_badness == current_badness);
- gcc_assert (current_badness >= badness);
+ /* Be sure that caches are maintained consistent.
+ This check is affected by scaling roundoff errors when compiling for
+ IPA this we skip it in that case. */
+ if (!edge->callee->count.ipa_p ())
+ {
+ sreal cached_badness = edge_badness (edge, false);
+
+ int old_size_est = estimate_edge_size (edge);
+ sreal old_time_est = estimate_edge_time (edge);
+ int old_hints_est = estimate_edge_hints (edge);
+
+ reset_edge_growth_cache (edge);
+ gcc_assert (old_size_est == estimate_edge_size (edge));
+ gcc_assert (old_time_est == estimate_edge_time (edge));
+ /* FIXME:
+
+ gcc_assert (old_hints_est == estimate_edge_hints (edge));
+
+ fails with profile feedback because some hints depends on
+ maybe_hot_edge_p predicate and because callee gets inlined to other
+ calls, the edge may become cold.
+ This ought to be fixed by computing relative probabilities
+ for given invocation but that will be better done once whole
+ code is converted to sreals. Disable for now and revert to "wrong"
+ value so enable/disable checking paths agree. */
+ edge_growth_cache[edge->uid].hints = old_hints_est + 1;
+
+ /* When updating the edge costs, we only decrease badness in the keys.
+ Increases of badness are handled lazilly; when we see key with out
+ of date value on it, we re-insert it now. */
+ current_badness = edge_badness (edge, false);
+ gcc_assert (cached_badness == current_badness);
+ gcc_assert (current_badness >= badness);
+ }
#else
current_badness = edge_badness (edge, false);
#endif
@@ -1945,11 +1952,11 @@ inline_small_functions (void)
? gimple_lineno ((const gimple *) edge->call_stmt)
: -1,
badness.to_double (),
- edge->frequency / (double)CGRAPH_FREQ_BASE);
- if (edge->count.initialized_p ())
+ edge->frequency () / (double)CGRAPH_FREQ_BASE);
+ if (edge->count.ipa ().initialized_p ())
{
fprintf (dump_file, " Called ");
- edge->count.dump (dump_file);
+ edge->count.ipa ().dump (dump_file);
fprintf (dump_file, "times\n");
}
if (dump_flags & TDF_DETAILS)
@@ -2255,8 +2262,8 @@ dump_overall_stats (void)
{
sreal time = ipa_fn_summaries->get (node)->time;
sum += time;
- if (node->count.initialized_p ())
- sum_weighted += time * node->count.to_gcov_type ();
+ if (node->count.ipa ().initialized_p ())
+ sum_weighted += time * node->count.ipa ().to_gcov_type ();
}
fprintf (dump_file, "Overall time estimate: "
"%f weighted by profile: "
@@ -2286,57 +2293,57 @@ dump_inline_stats (void)
{
if (e->inline_failed)
{
- if (e->count.initialized_p ())
- reason[(int) e->inline_failed][0] += e->count.to_gcov_type ();
- reason[(int) e->inline_failed][1] += e->frequency;
+ if (e->count.ipa ().initialized_p ())
+ reason[(int) e->inline_failed][0] += e->count.ipa ().to_gcov_type ();
+ reason[(int) e->inline_failed][1] += e->frequency ();
reason[(int) e->inline_failed][2] ++;
if (DECL_VIRTUAL_P (e->callee->decl)
- && e->count.initialized_p ())
+ && e->count.ipa ().initialized_p ())
{
if (e->indirect_inlining_edge)
- noninlined_virt_indir_cnt += e->count.to_gcov_type ();
+ noninlined_virt_indir_cnt += e->count.ipa ().to_gcov_type ();
else
- noninlined_virt_cnt += e->count.to_gcov_type ();
+ noninlined_virt_cnt += e->count.ipa ().to_gcov_type ();
}
- else if (e->count.initialized_p ())
+ else if (e->count.ipa ().initialized_p ())
{
if (e->indirect_inlining_edge)
- noninlined_indir_cnt += e->count.to_gcov_type ();
+ noninlined_indir_cnt += e->count.ipa ().to_gcov_type ();
else
- noninlined_cnt += e->count.to_gcov_type ();
+ noninlined_cnt += e->count.ipa ().to_gcov_type ();
}
}
- else if (e->count.initialized_p ())
+ else if (e->count.ipa ().initialized_p ())
{
if (e->speculative)
{
if (DECL_VIRTUAL_P (e->callee->decl))
- inlined_speculative_ply += e->count.to_gcov_type ();
+ inlined_speculative_ply += e->count.ipa ().to_gcov_type ();
else
- inlined_speculative += e->count.to_gcov_type ();
+ inlined_speculative += e->count.ipa ().to_gcov_type ();
}
else if (DECL_VIRTUAL_P (e->callee->decl))
{
if (e->indirect_inlining_edge)
- inlined_virt_indir_cnt += e->count.to_gcov_type ();
+ inlined_virt_indir_cnt += e->count.ipa ().to_gcov_type ();
else
- inlined_virt_cnt += e->count.to_gcov_type ();
+ inlined_virt_cnt += e->count.ipa ().to_gcov_type ();
}
else
{
if (e->indirect_inlining_edge)
- inlined_indir_cnt += e->count.to_gcov_type ();
+ inlined_indir_cnt += e->count.ipa ().to_gcov_type ();
else
- inlined_cnt += e->count.to_gcov_type ();
+ inlined_cnt += e->count.ipa ().to_gcov_type ();
}
}
}
for (e = node->indirect_calls; e; e = e->next_callee)
if (e->indirect_info->polymorphic
- & e->count.initialized_p ())
- indirect_poly_cnt += e->count.to_gcov_type ();
- else if (e->count.initialized_p ())
- indirect_cnt += e->count.to_gcov_type ();
+ & e->count.ipa ().initialized_p ())
+ indirect_poly_cnt += e->count.ipa ().to_gcov_type ();
+ else if (e->count.ipa ().initialized_p ())
+ indirect_cnt += e->count.ipa ().to_gcov_type ();
}
if (max_count.initialized_p ())
{
@@ -2488,8 +2495,8 @@ ipa_inline (void)
next = edge->next_callee;
if (edge->speculative && !speculation_useful_p (edge, false))
{
- if (edge->count.initialized_p ())
- spec_rem += edge->count;
+ if (edge->count.ipa ().initialized_p ())
+ spec_rem += edge->count.ipa ();
edge->resolve_speculation ();
update = true;
remove_functions = true;
===================================================================
@@ -59,8 +59,7 @@ bool inline_account_function_p (struct c
bool inline_call (struct cgraph_edge *, bool, vec<cgraph_edge *> *, int *, bool,
bool *callee_removed = NULL);
unsigned int inline_transform (struct cgraph_node *);
-void clone_inlined_nodes (struct cgraph_edge *e, bool, bool, int *,
- int freq_scale);
+void clone_inlined_nodes (struct cgraph_edge *e, bool, bool, int *);
extern int ncalls_inlined;
extern int nfunctions_inlined;
===================================================================
@@ -340,7 +340,7 @@ ipa_propagate_frequency_1 (struct cgraph
&& edge->caller->global.inlined_to->frequency
!= NODE_FREQUENCY_UNLIKELY_EXECUTED)))
d->maybe_unlikely_executed = false;
- if (!edge->frequency)
+ if (!edge->frequency ())
continue;
switch (edge->caller->frequency)
{
@@ -431,11 +431,11 @@ ipa_propagate_frequency (struct cgraph_n
}
/* With profile we can decide on hot/normal based on count. */
- if (node->count.initialized_p ())
+ if (node->count. ipa().initialized_p ())
{
bool hot = false;
- if (!(node->count == profile_count::zero ())
- && node->count >= get_hot_bb_threshold ())
+ if (!(node->count. ipa() == profile_count::zero ())
+ && node->count. ipa() >= get_hot_bb_threshold ())
hot = true;
if (!hot)
hot |= contains_hot_call_p (node);
@@ -667,9 +667,7 @@ ipa_profile (void)
e->make_speculative
(n2,
e->count.apply_probability
- (e->indirect_info->common_target_probability),
- apply_scale (e->frequency,
- e->indirect_info->common_target_probability));
+ (e->indirect_info->common_target_probability));
update = true;
}
}
===================================================================
@@ -2982,7 +2982,7 @@ ipa_make_edge_direct_to_target (struct c
}
/* make_speculative will update ie's cost to direct call cost. */
ie = ie->make_speculative
- (callee, ie->count.apply_scale (8, 10), ie->frequency * 8 / 10);
+ (callee, ie->count.apply_scale (8, 10));
}
return ie;
===================================================================
@@ -564,18 +564,12 @@ ipa_merge_profiles (struct cgraph_node *
{
if (e->speculative)
continue;
- e->count = gimple_bb (e->call_stmt)->count.ipa ();
- e->frequency = compute_call_stmt_bb_frequency
- (dst->decl,
- gimple_bb (e->call_stmt));
+ e->count = gimple_bb (e->call_stmt)->count;
}
for (e = dst->indirect_calls, e2 = src->indirect_calls; e;
e2 = (e2 ? e2->next_callee : NULL), e = e->next_callee)
{
profile_count count = gimple_bb (e->call_stmt)->count;
- int freq = compute_call_stmt_bb_frequency
- (dst->decl,
- gimple_bb (e->call_stmt));
/* When call is speculative, we need to re-distribute probabilities
the same way as they was. This is not really correct because
in the other copy the speculation may differ; but probably it
@@ -624,12 +618,6 @@ ipa_merge_profiles (struct cgraph_node *
indirect->count += indirect2->count;
}
}
- int prob = direct->count.probability_in (direct->count
- + indirect->count).
- to_reg_br_prob_base ();
- direct->frequency = RDIV (freq * prob, REG_BR_PROB_BASE);
- indirect->frequency = RDIV (freq * (REG_BR_PROB_BASE - prob),
- REG_BR_PROB_BASE);
}
else
/* At the moment we should have only profile feedback based
@@ -642,18 +630,11 @@ ipa_merge_profiles (struct cgraph_node *
ipa_ref *ref;
e2->speculative_call_info (direct, indirect, ref);
- e->count = count.ipa ();
- e->frequency = freq;
- int prob = direct->count.probability_in (e->count)
- .to_reg_br_prob_base ();
- e->make_speculative (direct->callee, direct->count,
- RDIV (freq * prob, REG_BR_PROB_BASE));
+ e->count = count;
+ e->make_speculative (direct->callee, direct->count);
}
else
- {
- e->count = count.ipa ();
- e->frequency = freq;
- }
+ e->count = count;
}
if (!preserve_body)
src->release_body ();
===================================================================
@@ -587,7 +587,7 @@ lto_balanced_map (int n_lto_partitions,
for (edge = node->callees; edge; edge = edge->next_callee)
if (edge->callee->definition)
{
- int edge_cost = edge->frequency;
+ int edge_cost = edge->frequency ();
int index;
if (!edge_cost)
@@ -603,7 +603,7 @@ lto_balanced_map (int n_lto_partitions,
}
for (edge = node->callers; edge; edge = edge->next_caller)
{
- int edge_cost = edge->frequency;
+ int edge_cost = edge->frequency ();
int index;
gcc_assert (edge->caller->definition);
===================================================================
@@ -266,7 +266,6 @@ lto_output_edge (struct lto_simple_outpu
bp_pack_enum (&bp, cgraph_inline_failed_t,
CIF_N_REASONS, edge->inline_failed);
bp_pack_var_len_unsigned (&bp, uid);
- bp_pack_var_len_unsigned (&bp, edge->frequency);
bp_pack_value (&bp, edge->indirect_inlining_edge, 1);
bp_pack_value (&bp, edge->speculative, 1);
bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1);
@@ -1248,7 +1247,7 @@ input_node (struct lto_file_decl_data *f
if (clone_ref != LCC_NOT_FOUND)
{
node = dyn_cast<cgraph_node *> (nodes[clone_ref])->create_clone (fn_decl,
- profile_count::uninitialized (), CGRAPH_FREQ_BASE, false,
+ profile_count::uninitialized (), false,
vNULL, false, NULL, NULL);
}
else
@@ -1464,7 +1463,6 @@ input_edge (struct lto_input_block *ib,
struct cgraph_edge *edge;
unsigned int stmt_id;
profile_count count;
- int freq;
cgraph_inline_failed_t inline_failed;
struct bitpack_d bp;
int ecf_flags = 0;
@@ -1487,12 +1485,11 @@ input_edge (struct lto_input_block *ib,
bp = streamer_read_bitpack (ib);
inline_failed = bp_unpack_enum (&bp, cgraph_inline_failed_t, CIF_N_REASONS);
stmt_id = bp_unpack_var_len_unsigned (&bp);
- freq = (int) bp_unpack_var_len_unsigned (&bp);
if (indirect)
- edge = caller->create_indirect_edge (NULL, 0, count, freq);
+ edge = caller->create_indirect_edge (NULL, 0, count);
else
- edge = caller->create_edge (callee, NULL, count, freq);
+ edge = caller->create_edge (callee, NULL, count);
edge->indirect_inlining_edge = bp_unpack_value (&bp, 1);
edge->speculative = bp_unpack_value (&bp, 1);
@@ -1823,8 +1820,13 @@ merge_profile_summaries (struct lto_file
if (scale == REG_BR_PROB_BASE)
continue;
for (edge = node->callees; edge; edge = edge->next_callee)
- edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE);
- node->count = node->count.apply_scale (scale, REG_BR_PROB_BASE);
+ if (edge->count.ipa ().nonzero_p ())
+ edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE);
+ for (edge = node->indirect_calls; edge; edge = edge->next_callee)
+ if (edge->count.ipa ().nonzero_p ())
+ edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE);
+ if (node->count.ipa ().nonzero_p ())
+ node->count = node->count.apply_scale (scale, REG_BR_PROB_BASE);
}
}
===================================================================
@@ -1389,10 +1389,8 @@ simd_clone_adjust (struct cgraph_node *n
(single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- int freq = compute_call_stmt_bb_frequency (current_function_decl,
- entry_bb);
node->create_edge (cgraph_node::get_create (fn),
- call, entry_bb->count, freq);
+ call, entry_bb->count);
imm_use_iterator iter;
use_operand_p use_p;
===================================================================
@@ -3222,11 +3222,10 @@ drop_profile (struct cgraph_node *node,
pop_cfun ();
struct cgraph_edge *e;
- for (e = node->callees; e; e = e->next_caller)
- {
- e->frequency = compute_call_stmt_bb_frequency (e->caller->decl,
- gimple_bb (e->call_stmt));
- }
+ for (e = node->callees; e; e = e->next_callee)
+ e->count = gimple_bb (e->call_stmt)->count;
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ e->count = gimple_bb (e->call_stmt)->count;
profile_status_for_fn (fn)
= (flag_guess_branch_prob ? PROFILE_GUESSED : PROFILE_ABSENT);
===================================================================
@@ -5065,9 +5065,7 @@ ipa_tm_insert_irr_call (struct cgraph_no
node->create_edge (cgraph_node::get_create
(builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE)),
- g, gimple_bb (g)->count,
- compute_call_stmt_bb_frequency (node->decl,
- gimple_bb (g)));
+ g, gimple_bb (g)->count);
}
/* Construct a call to TM_GETTMCLONE and insert it before GSI. */
@@ -5116,9 +5114,7 @@ ipa_tm_insert_gettmclone_call (struct cg
gsi_insert_before (gsi, g, GSI_SAME_STMT);
- node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count,
- compute_call_stmt_bb_frequency (node->decl,
- gimple_bb (g)));
+ node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count);
/* Cast return value from tm_gettmclone* into appropriate function
pointer. */
===================================================================
@@ -9225,7 +9225,7 @@ execute_fixup_cfg (void)
gimple_stmt_iterator gsi;
int todo = 0;
cgraph_node *node = cgraph_node::get (current_function_decl);
- profile_count num = node->count;
+ profile_count num = node->count.ipa ();
profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
bool scale = num.initialized_p () && den.ipa_p ()
&& (den.nonzero_p () || num == profile_count::zero ())
@@ -9241,7 +9241,15 @@ execute_fixup_cfg (void)
FOR_EACH_BB_FN (bb, cfun)
{
if (scale)
- bb->count = bb->count.apply_scale (num, den);
+ {
+ if (num == profile_count::zero ())
+ {
+ if (!(bb->count == profile_count::zero ()))
+ bb->count = bb->count.global0 ();
+ }
+ else
+ bb->count = bb->count.apply_scale (num, den);
+ }
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
{
gimple *stmt = gsi_stmt (gsi);
@@ -9331,11 +9339,8 @@ execute_fixup_cfg (void)
if (!cfun->after_inlining)
{
gcall *call_stmt = dyn_cast <gcall *> (stmt);
- int freq
- = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
node->create_edge (cgraph_node::get_create (fndecl),
- call_stmt, bb->count, freq);
+ call_stmt, bb->count);
}
}
}
===================================================================
@@ -4231,18 +4231,12 @@ chkp_copy_bounds_for_assign (gimple *ass
{
tree fndecl = gimple_call_fndecl (stmt);
struct cgraph_node *callee = cgraph_node::get_create (fndecl);
- struct cgraph_edge *new_edge;
gcc_assert (chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDSTX)
|| chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDLDX)
|| chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDRET));
- new_edge = edge->caller->create_edge (callee,
- as_a <gcall *> (stmt),
- edge->count,
- edge->frequency);
- new_edge->frequency = compute_call_stmt_bb_frequency
- (edge->caller->decl, gimple_bb (stmt));
+ edge->caller->create_edge (callee, as_a <gcall *> (stmt), edge->count);
}
gsi_prev (&iter);
}
===================================================================
@@ -417,7 +417,7 @@ gen_emutls_addr (tree decl, struct lower
gimple_seq_add_stmt (&d->seq, x);
- d->cfun_node->create_edge (d->builtin_node, x, d->bb->count, d->bb_freq);
+ d->cfun_node->create_edge (d->builtin_node, x, d->bb->count);
/* We may be adding a new reference to a new variable to the function.
This means we have to play with the ipa-reference web. */
===================================================================
@@ -2001,23 +2001,16 @@ copy_bb (copy_body_data *id, basic_block
edge = id->src_node->get_edge (orig_stmt);
if (edge)
{
- int edge_freq = edge->frequency;
- int new_freq;
struct cgraph_edge *old_edge = edge;
+ profile_count old_cnt = edge->count;
edge = edge->clone (id->dst_node, call_stmt,
gimple_uid (stmt),
- profile_count::one (),
- profile_count::one (),
- CGRAPH_FREQ_BASE,
+ num, den,
true);
- /* We could also just rescale the frequency, but
- doing so would introduce roundoff errors and make
- verifier unhappy. */
- new_freq = compute_call_stmt_bb_frequency (id->dst_node->decl,
- copy_basic_block);
- /* Speculative calls consist of two edges - direct and indirect.
- Duplicate the whole thing and distribute frequencies accordingly. */
+ /* Speculative calls consist of two edges - direct and
+ indirect. Duplicate the whole thing and distribute
+ frequencies accordingly. */
if (edge->speculative)
{
struct cgraph_edge *direct, *indirect;
@@ -2025,42 +2018,22 @@ copy_bb (copy_body_data *id, basic_block
gcc_assert (!edge->indirect_unknown_callee);
old_edge->speculative_call_info (direct, indirect, ref);
+
+ profile_count indir_cnt = indirect->count;
indirect = indirect->clone (id->dst_node, call_stmt,
gimple_uid (stmt),
- profile_count::one (),
- profile_count::one (),
- CGRAPH_FREQ_BASE,
+ num, den,
true);
- if (old_edge->frequency + indirect->frequency)
- {
- edge->frequency = MIN (RDIV ((gcov_type)new_freq * old_edge->frequency,
- (old_edge->frequency + indirect->frequency)),
- CGRAPH_FREQ_MAX);
- indirect->frequency = MIN (RDIV ((gcov_type)new_freq * indirect->frequency,
- (old_edge->frequency + indirect->frequency)),
- CGRAPH_FREQ_MAX);
- }
+
+ profile_probability prob
+ = indir_cnt.probability_in (old_cnt + indir_cnt);
+ indirect->count
+ = copy_basic_block->count.apply_probability (prob);
+ edge->count = copy_basic_block->count - indirect->count;
id->dst_node->clone_reference (ref, stmt);
}
else
- {
- edge->frequency = new_freq;
- if (dump_file
- && profile_status_for_fn (cfun) != PROFILE_ABSENT
- && (edge_freq > edge->frequency + 10
- || edge_freq < edge->frequency - 10))
- {
- fprintf (dump_file, "Edge frequency estimated by "
- "cgraph %i diverge from inliner's estimate %i\n",
- edge_freq,
- edge->frequency);
- fprintf (dump_file,
- "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
- bb->index,
- bb->count.to_frequency (cfun),
- copy_basic_block->count.to_frequency (cfun));
- }
- }
+ edge->count = copy_basic_block->count;
}
break;
@@ -2103,15 +2076,10 @@ copy_bb (copy_body_data *id, basic_block
if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
id->dst_node->create_edge_including_clones
(dest, orig_stmt, call_stmt, bb->count,
- compute_call_stmt_bb_frequency (id->dst_node->decl,
- copy_basic_block),
CIF_ORIGINALLY_INDIRECT_CALL);
else
id->dst_node->create_edge (dest, call_stmt,
- bb->count,
- compute_call_stmt_bb_frequency
- (id->dst_node->decl,
- copy_basic_block))->inline_failed
+ bb->count)->inline_failed
= CIF_ORIGINALLY_INDIRECT_CALL;
if (dump_file)
{
@@ -4475,7 +4443,6 @@ expand_call_inline (basic_block bb, gimp
{
cgraph_edge *edge;
tree virtual_offset = NULL;
- int freq = cg_edge->frequency;
profile_count count = cg_edge->count;
tree op;
gimple_stmt_iterator iter = gsi_for_stmt (stmt);
@@ -4485,9 +4452,7 @@ expand_call_inline (basic_block bb, gimp
gimple_uid (stmt),
profile_count::one (),
profile_count::one (),
- CGRAPH_FREQ_BASE,
true);
- edge->frequency = freq;
edge->count = count;
if (id->src_node->thunk.virtual_offset_p)
virtual_offset = size_int (id->src_node->thunk.virtual_value);
@@ -4709,7 +4674,7 @@ expand_call_inline (basic_block bb, gimp
print_generic_expr (dump_file, id->src_fn);
fprintf (dump_file, " to ");
print_generic_expr (dump_file, id->dst_fn);
- fprintf (dump_file, " with frequency %i\n", cg_edge->frequency);
+ fprintf (dump_file, " with frequency %i\n", cg_edge->frequency ());
}
/* This is it. Duplicate the callee body. Assume callee is
@@ -6089,20 +6054,16 @@ tree_function_versioning (tree old_decl,
struct cgraph_edge *e;
rebuild_frequencies ();
- new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa ();
+ new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
for (e = new_version_node->callees; e; e = e->next_callee)
{
basic_block bb = gimple_bb (e->call_stmt);
- e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
- e->count = bb->count.ipa ();
+ e->count = bb->count;
}
for (e = new_version_node->indirect_calls; e; e = e->next_callee)
{
basic_block bb = gimple_bb (e->call_stmt);
- e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
- bb);
- e->count = bb->count.ipa ();
+ e->count = bb->count;
}
}
===================================================================
@@ -675,12 +675,10 @@ ubsan_create_edge (gimple *stmt)
{
gcall *call_stmt = dyn_cast <gcall *> (stmt);
basic_block bb = gimple_bb (stmt);
- int freq = compute_call_stmt_bb_frequency (current_function_decl, bb);
cgraph_node *node = cgraph_node::get (current_function_decl);
tree decl = gimple_call_fndecl (call_stmt);
if (decl)
- node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count,
- freq);
+ node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count);
}
/* Expand the UBSAN_BOUNDS special builtin function. */