diff options
| -rw-r--r-- | gcc/ChangeLog | 49 | ||||
| -rw-r--r-- | gcc/cgraph.c | 50 | ||||
| -rw-r--r-- | gcc/cgraph.h | 15 | ||||
| -rw-r--r-- | gcc/cgraphbuild.c | 10 | ||||
| -rw-r--r-- | gcc/cgraphunit.c | 4 | ||||
| -rw-r--r-- | gcc/ipa-inline-analysis.c | 177 | ||||
| -rw-r--r-- | gcc/ipa-inline-transform.c | 40 | ||||
| -rw-r--r-- | gcc/ipa-inline.c | 11 | ||||
| -rw-r--r-- | gcc/ipa-inline.h | 23 | ||||
| -rw-r--r-- | gcc/lto-cgraph.c | 16 | ||||
| -rw-r--r-- | gcc/tree-emutls.c | 2 | ||||
| -rw-r--r-- | gcc/tree-inline.c | 7 |
12 files changed, 291 insertions, 113 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index f234c714a61..1b1f4d26f72 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,52 @@ +2011-04-26 Jan Hubicka <jh@suse.cz> + + * cgraphbuild.c (build_cgraph_edges): Update call + of cgraph_create_edge and cgraph_create_indirect_edge. + * cgraph.c (cgraph_create_edge_including_clones, + cgraph_create_edge_1, cgraph_allocate_init_indirect_info, + cgraph_update_edges_for_call_stmt_node): Do not take nest + argument; do not initialize call_stmt_size/time. + (dump_cgraph_node): Do not dump nest. + (cgraph_clone_edge): Do not take loop_nest argument; + do not propagate it; do not clone call_stmt_size/time. + (cgraph_clone_node): Likewise. + (cgraph_create_virtual_clone): Update. + * cgraph.h (struct cgraph_edge): Remove + call_stmt_size/call_stmt_time/loop_nest. + (cgraph_create_edge, cgraph_create_indirect_edge, + cgraph_create_edge_including_clones, cgraph_clone_node): Update + prototype. + * tree-emutls.c (gen_emutls_addr): Update. + * ipa-inline-transform.c (update_noncloned_frequencies): Do not handle + loop_nest; handle indirect calls, too. + (clone_inlined_nodes): Do not care about updating inline summaries. + * cgraphunit.c (cgraph_copy_node_for_versioning): Update. + * lto-cgraph.c (lto_output_edge, input_node, input_edge): Do not + stream call_stmt_size/call_stmt_time/loop_nest. + * ipa-inline.c (edge_badness): Update. + (ipa_inline): dump summaries after inlining. + * ipa-inline.h (struct inline_edge_summary, inline_edge_summary_t): + new. + (inline_edge_summary): New function. + * ipa-inline-analysis.c (edge_duplication_hook_holder): New holder. + (inline_edge_removal_hook): Handle edge summaries. + (inline_edge_duplication_hook): New hook. + (inline_summary_alloc): Alloc hooks. + (initialize_growth_caches): Do not register removal hooks. + (free_growth_caches); Do not free removal hook. + (dump_inline_edge_summary): New function. + (dump_inline_summary): Use it. + (estimate_function_body_sizes, estimate_edge_size_and_time): Update. + (inline_update_callee_summaries): New function. + (inline_merge_summary): Use it. + (do_estimate_edge_time, do_estimate_edge_growth): Update. + (read_inline_edge_summary): New function. + (inline_read_section): Use it. + (write_inline_edge_summary): New function. + (inline_write_summary): Use it. + (inline_free_summary): Free edge new holders. + * tree-inline.c (copy_bb): Update. + 2011-04-26 Jason Merrill <jason@redhat.com> * tree-eh.c (lower_try_finally_switch): Create the label along with diff --git a/gcc/cgraph.c b/gcc/cgraph.c index 85266467284..da020fe1ab2 100644 --- a/gcc/cgraph.c +++ b/gcc/cgraph.c @@ -921,7 +921,7 @@ cgraph_create_edge_including_clones (struct cgraph_node *orig, struct cgraph_node *callee, gimple old_stmt, gimple stmt, gcov_type count, - int freq, int loop_depth, + int freq, cgraph_inline_failed_t reason) { struct cgraph_node *node; @@ -929,7 +929,7 @@ cgraph_create_edge_including_clones (struct cgraph_node *orig, if (!cgraph_edge (orig, stmt)) { - edge = cgraph_create_edge (orig, callee, stmt, count, freq, loop_depth); + edge = cgraph_create_edge (orig, callee, stmt, count, freq); edge->inline_failed = reason; } @@ -948,7 +948,7 @@ cgraph_create_edge_including_clones (struct cgraph_node *orig, else if (!cgraph_edge (node, stmt)) { edge = cgraph_create_edge (node, callee, stmt, count, - freq, loop_depth); + freq); edge->inline_failed = reason; } @@ -972,7 +972,7 @@ cgraph_create_edge_including_clones (struct cgraph_node *orig, static struct cgraph_edge * cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee, - gimple call_stmt, gcov_type count, int freq, int nest) + gimple call_stmt, gcov_type count, int freq) { struct cgraph_edge *edge; @@ -1011,11 +1011,8 @@ cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee, edge->frequency = freq; gcc_assert (freq >= 0); gcc_assert (freq <= CGRAPH_FREQ_MAX); - edge->loop_nest = nest; edge->call_stmt = call_stmt; - edge->call_stmt_size = 0; - edge->call_stmt_time = 0; push_cfun (DECL_STRUCT_FUNCTION (caller->decl)); edge->can_throw_external = call_stmt ? stmt_can_throw_external (call_stmt) : false; @@ -1035,10 +1032,10 @@ cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee, struct cgraph_edge * cgraph_create_edge (struct cgraph_node *caller, struct cgraph_node *callee, - gimple call_stmt, gcov_type count, int freq, int nest) + gimple call_stmt, gcov_type count, int freq) { struct cgraph_edge *edge = cgraph_create_edge_1 (caller, callee, call_stmt, - count, freq, nest); + count, freq); edge->indirect_unknown_callee = 0; initialize_inline_failed (edge); @@ -1074,10 +1071,10 @@ cgraph_allocate_init_indirect_info (void) struct cgraph_edge * cgraph_create_indirect_edge (struct cgraph_node *caller, gimple call_stmt, int ecf_flags, - gcov_type count, int freq, int nest) + gcov_type count, int freq) { struct cgraph_edge *edge = cgraph_create_edge_1 (caller, NULL, call_stmt, - count, freq, nest); + count, freq); edge->indirect_unknown_callee = 1; initialize_inline_failed (edge); @@ -1244,7 +1241,6 @@ cgraph_update_edges_for_call_stmt_node (struct cgraph_node *node, struct cgraph_edge *ne = NULL; gcov_type count; int frequency; - int loop_nest; if (e) { @@ -1268,7 +1264,6 @@ cgraph_update_edges_for_call_stmt_node (struct cgraph_node *node, attached to edge is invalid. */ count = e->count; frequency = e->frequency; - loop_nest = e->loop_nest; cgraph_remove_edge (e); } else @@ -1278,14 +1273,12 @@ cgraph_update_edges_for_call_stmt_node (struct cgraph_node *node, count = bb->count; frequency = compute_call_stmt_bb_frequency (current_function_decl, bb); - loop_nest = bb->loop_depth; } if (new_call) { ne = cgraph_create_edge (node, cgraph_get_create_node (new_call), - new_stmt, count, frequency, - loop_nest); + new_stmt, count, frequency); gcc_assert (ne->inline_failed); } } @@ -1915,8 +1908,6 @@ dump_cgraph_node (FILE *f, struct cgraph_node *node) if (edge->frequency) fprintf (f, "(%.2f per call) ", edge->frequency / (double)CGRAPH_FREQ_BASE); - if (edge->loop_nest) - fprintf (f, "(nested in %i loops) ", edge->loop_nest); if (edge->can_throw_external) fprintf(f, "(can throw external) "); } @@ -2064,7 +2055,7 @@ cgraph_function_possibly_inlined_p (tree decl) struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n, gimple call_stmt, unsigned stmt_uid, gcov_type count_scale, - int freq_scale, int loop_nest, bool update_original) + int freq_scale, bool update_original) { struct cgraph_edge *new_edge; gcov_type count = e->count * count_scale / REG_BR_PROB_BASE; @@ -2085,22 +2076,19 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n, { struct cgraph_node *callee = cgraph_get_node (decl); gcc_checking_assert (callee); - new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq, - e->loop_nest + loop_nest); + new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq); } else { new_edge = cgraph_create_indirect_edge (n, call_stmt, e->indirect_info->ecf_flags, - count, freq, - e->loop_nest + loop_nest); + count, freq); *new_edge->indirect_info = *e->indirect_info; } } else { - new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq, - e->loop_nest + loop_nest); + new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq); if (e->indirect_info) { new_edge->indirect_info @@ -2109,8 +2097,6 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n, } } - new_edge->call_stmt_size = e->call_stmt_size; - new_edge->call_stmt_time = e->call_stmt_time; new_edge->inline_failed = e->inline_failed; new_edge->indirect_inlining_edge = e->indirect_inlining_edge; new_edge->lto_stmt_uid = stmt_uid; @@ -2137,7 +2123,7 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n, by node. */ struct cgraph_node * cgraph_clone_node (struct cgraph_node *n, tree decl, gcov_type count, int freq, - int loop_nest, bool update_original, + bool update_original, VEC(cgraph_edge_p,heap) *redirect_callers) { struct cgraph_node *new_node = cgraph_create_node_1 (); @@ -2188,11 +2174,11 @@ cgraph_clone_node (struct cgraph_node *n, tree decl, gcov_type count, int freq, for (e = n->callees;e; e=e->next_callee) cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid, - count_scale, freq, loop_nest, update_original); + count_scale, freq, update_original); for (e = n->indirect_calls; e; e = e->next_callee) cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid, - count_scale, freq, loop_nest, update_original); + count_scale, freq, update_original); ipa_clone_references (new_node, NULL, &n->ref_list); new_node->next_sibling_clone = n->clones; @@ -2285,7 +2271,7 @@ cgraph_create_virtual_clone (struct cgraph_node *old_node, SET_DECL_RTL (new_decl, NULL); new_node = cgraph_clone_node (old_node, new_decl, old_node->count, - CGRAPH_FREQ_BASE, 0, false, + CGRAPH_FREQ_BASE, false, redirect_callers); /* Update the properties. Make clone visible only within this translation unit. Make sure @@ -2700,7 +2686,7 @@ cgraph_propagate_frequency (struct cgraph_node *node) fprintf (dump_file, " Called by %s that is executed once\n", cgraph_node_name (edge->caller)); maybe_unlikely_executed = false; - if (edge->loop_nest) + if (inline_edge_summary (edge)->loop_depth) { maybe_executed_once = false; if (dump_file && (dump_flags & TDF_DETAILS)) diff --git a/gcc/cgraph.h b/gcc/cgraph.h index 54e75943cf3..14848b5e6b9 100644 --- a/gcc/cgraph.h +++ b/gcc/cgraph.h @@ -381,11 +381,6 @@ struct GTY((chain_next ("%h.next_caller"), chain_prev ("%h.prev_caller"))) cgrap int frequency; /* Unique id of the edge. */ int uid; - /* Estimated size and time of the call statement. */ - int call_stmt_size; - int call_stmt_time; - /* Depth of loop nest, 1 means no loop nest. */ - unsigned short int loop_nest; /* Whether this edge was made direct by indirect inlining. */ unsigned int indirect_inlining_edge : 1; /* Whether this edge describes an indirect call with an undetermined @@ -504,9 +499,9 @@ void cgraph_release_function_body (struct cgraph_node *); void cgraph_node_remove_callees (struct cgraph_node *node); struct cgraph_edge *cgraph_create_edge (struct cgraph_node *, struct cgraph_node *, - gimple, gcov_type, int, int); + gimple, gcov_type, int); struct cgraph_edge *cgraph_create_indirect_edge (struct cgraph_node *, gimple, - int, gcov_type, int, int); + int, gcov_type, int); struct cgraph_indirect_call_info *cgraph_allocate_init_indirect_info (void); struct cgraph_node * cgraph_get_node (const_tree); struct cgraph_node * cgraph_get_node_or_alias (const_tree); @@ -522,7 +517,7 @@ void cgraph_set_call_stmt (struct cgraph_edge *, gimple); void cgraph_set_call_stmt_including_clones (struct cgraph_node *, gimple, gimple); void cgraph_create_edge_including_clones (struct cgraph_node *, struct cgraph_node *, - gimple, gimple, gcov_type, int, int, + gimple, gimple, gcov_type, int, cgraph_inline_failed_t); void cgraph_update_edges_for_call_stmt (gimple, tree, gimple); struct cgraph_local_info *cgraph_local_info (tree); @@ -531,8 +526,8 @@ struct cgraph_rtl_info *cgraph_rtl_info (tree); const char * cgraph_node_name (struct cgraph_node *); struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *, struct cgraph_node *, gimple, - unsigned, gcov_type, int, int, bool); -struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type, int, + unsigned, gcov_type, int, bool); +struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type, int, bool, VEC(cgraph_edge_p,heap) *); void cgraph_redirect_edge_callee (struct cgraph_edge *, struct cgraph_node *); diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c index 3a2cb676679..eb9da7fc16c 100644 --- a/gcc/cgraphbuild.c +++ b/gcc/cgraphbuild.c @@ -354,12 +354,11 @@ build_cgraph_edges (void) decl = gimple_call_fndecl (stmt); if (decl) cgraph_create_edge (node, cgraph_get_create_node (decl), - stmt, bb->count, freq, bb->loop_depth); + stmt, bb->count, freq); else cgraph_create_indirect_edge (node, stmt, gimple_call_flags (stmt), - bb->count, freq, - bb->loop_depth); + bb->count, freq); } walk_stmt_load_store_addr_ops (stmt, node, mark_load, mark_store, mark_address); @@ -464,12 +463,11 @@ rebuild_cgraph_edges (void) decl = gimple_call_fndecl (stmt); if (decl) cgraph_create_edge (node, cgraph_get_create_node (decl), stmt, - bb->count, freq, bb->loop_depth); + bb->count, freq); else cgraph_create_indirect_edge (node, stmt, gimple_call_flags (stmt), - bb->count, freq, - bb->loop_depth); + bb->count, freq); } walk_stmt_load_store_addr_ops (stmt, node, mark_load, mark_store, mark_address); diff --git a/gcc/cgraphunit.c b/gcc/cgraphunit.c index 8803cf6b8a2..1d40d5a49fd 100644 --- a/gcc/cgraphunit.c +++ b/gcc/cgraphunit.c @@ -2001,14 +2001,14 @@ cgraph_copy_node_for_versioning (struct cgraph_node *old_version, cgraph_clone_edge (e, new_version, e->call_stmt, e->lto_stmt_uid, REG_BR_PROB_BASE, CGRAPH_FREQ_BASE, - e->loop_nest, true); + true); for (e = old_version->indirect_calls; e; e=e->next_callee) if (!bbs_to_copy || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) cgraph_clone_edge (e, new_version, e->call_stmt, e->lto_stmt_uid, REG_BR_PROB_BASE, CGRAPH_FREQ_BASE, - e->loop_nest, true); + true); FOR_EACH_VEC_ELT (cgraph_edge_p, redirect_callers, i, e) { /* Redirect calls to the old version node to point to its new diff --git a/gcc/ipa-inline-analysis.c b/gcc/ipa-inline-analysis.c index bedd9633da8..8cf9bc3d924 100644 --- a/gcc/ipa-inline-analysis.c +++ b/gcc/ipa-inline-analysis.c @@ -110,14 +110,20 @@ enum predicate_conditions static struct cgraph_node_hook_list *function_insertion_hook_holder; static struct cgraph_node_hook_list *node_removal_hook_holder; static struct cgraph_2node_hook_list *node_duplication_hook_holder; +static struct cgraph_2edge_hook_list *edge_duplication_hook_holder; static struct cgraph_edge_hook_list *edge_removal_hook_holder; static void inline_node_removal_hook (struct cgraph_node *, void *); static void inline_node_duplication_hook (struct cgraph_node *, struct cgraph_node *, void *); +static void inline_edge_removal_hook (struct cgraph_edge *, void *); +static void inline_edge_duplication_hook (struct cgraph_edge *, + struct cgraph_edge *, + void *); /* VECtor holding inline summaries. In GGC memory because conditions might point to constant trees. */ VEC(inline_summary_t,gc) *inline_summary_vec; +VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec; /* Cached node/edge growths. */ VEC(int,heap) *node_growth_cache; @@ -512,14 +518,24 @@ inline_summary_alloc (void) if (!node_removal_hook_holder) node_removal_hook_holder = cgraph_add_node_removal_hook (&inline_node_removal_hook, NULL); + if (!edge_removal_hook_holder) + edge_removal_hook_holder = + cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL); if (!node_duplication_hook_holder) node_duplication_hook_holder = cgraph_add_node_duplication_hook (&inline_node_duplication_hook, NULL); + if (!edge_duplication_hook_holder) + edge_duplication_hook_holder = + cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook, NULL); if (VEC_length (inline_summary_t, inline_summary_vec) <= (unsigned) cgraph_max_uid) VEC_safe_grow_cleared (inline_summary_t, gc, inline_summary_vec, cgraph_max_uid + 1); + if (VEC_length (inline_edge_summary_t, inline_edge_summary_vec) + <= (unsigned) cgraph_edge_max_uid) + VEC_safe_grow_cleared (inline_edge_summary_t, heap, + inline_edge_summary_vec, cgraph_edge_max_uid + 1); } /* Hook that is called by cgraph.c when a node is removed. */ @@ -540,6 +556,7 @@ inline_node_removal_hook (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED) memset (info, 0, sizeof (inline_summary_t)); } + /* Hook that is called by cgraph.c when a node is duplicated. */ static void @@ -556,12 +573,29 @@ inline_node_duplication_hook (struct cgraph_node *src, struct cgraph_node *dst, } +/* Hook that is called by cgraph.c when a node is duplicated. */ + +static void +inline_edge_duplication_hook (struct cgraph_edge *src, struct cgraph_edge *dst, + ATTRIBUTE_UNUSED void *data) +{ + struct inline_edge_summary *info; + inline_summary_alloc (); + info = inline_edge_summary (dst); + memcpy (info, inline_edge_summary (src), + sizeof (struct inline_edge_summary)); +} + + /* Keep edge cache consistent across edge removal. */ static void inline_edge_removal_hook (struct cgraph_edge *edge, void *data ATTRIBUTE_UNUSED) { - reset_edge_growth_cache (edge); + if (edge_growth_cache) + reset_edge_growth_cache (edge); + if (edge->uid < (int)VEC_length (inline_edge_summary_t, inline_edge_summary_vec)) + memset (inline_edge_summary (edge), 0, sizeof (struct inline_edge_summary)); } @@ -570,9 +604,6 @@ inline_edge_removal_hook (struct cgraph_edge *edge, void *data ATTRIBUTE_UNUSED) void initialize_growth_caches (void) { - if (!edge_removal_hook_holder) - edge_removal_hook_holder = - cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL); if (cgraph_edge_max_uid) VEC_safe_grow_cleared (edge_growth_cache_entry, heap, edge_growth_cache, cgraph_edge_max_uid); @@ -586,8 +617,6 @@ initialize_growth_caches (void) void free_growth_caches (void) { - if (edge_removal_hook_holder) - cgraph_remove_edge_removal_hook (edge_removal_hook_holder); VEC_free (edge_growth_cache_entry, heap, edge_growth_cache); edge_growth_cache = 0; VEC_free (int, heap, node_growth_cache); @@ -595,6 +624,42 @@ free_growth_caches (void) } +/* Dump edge summaries associated to NODE and recursively to all clones. + Indent by INDENT. */ + +static void +dump_inline_edge_summary (FILE * f, int indent, struct cgraph_node *node) +{ + struct cgraph_edge *edge; + for (edge = node->callees; edge; edge = edge->next_callee) + { + struct inline_edge_summary *es = inline_edge_summary (edge); + fprintf (f, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i\n", + indent, "", cgraph_node_name (edge->callee), + edge->callee->uid, + edge->inline_failed ? "inlined" + : cgraph_inline_failed_string (edge->inline_failed), + indent, "", + es->loop_depth, + edge->frequency, + es->call_stmt_size, + es->call_stmt_time); + if (!edge->inline_failed) + dump_inline_edge_summary (f, indent+2, edge->callee); + } + for (edge = node->indirect_calls; edge; edge = edge->next_callee) + { + struct inline_edge_summary *es = inline_edge_summary (edge); + fprintf (f, "%*sindirect call loop depth:%2i freq:%4i size:%2i time: %2i\n", + indent, "", + es->loop_depth, + edge->frequency, + es->call_stmt_size, + es->call_stmt_time); + } +} + + static void dump_inline_summary (FILE * f, struct cgraph_node *node) { @@ -630,6 +695,8 @@ dump_inline_summary (FILE * f, struct cgraph_node *node) (double) e->time / INLINE_TIME_SCALE); dump_predicate (f, s->conds, &e->predicate); } + fprintf (f, " calls:\n"); + dump_inline_edge_summary (f, 4, node); fprintf (f, "\n"); } } @@ -646,7 +713,7 @@ dump_inline_summaries (FILE *f) struct cgraph_node *node; for (node = cgraph_nodes; node; node = node->next) - if (node->analyzed) + if (node->analyzed && !node->global.inlined_to) dump_inline_summary (f, node); } @@ -919,8 +986,11 @@ estimate_function_body_sizes (struct cgraph_node *node, bool early) if (is_gimple_call (stmt)) { struct cgraph_edge *edge = cgraph_edge (node, stmt); - edge->call_stmt_size = this_size; - edge->call_stmt_time = this_time; + struct inline_edge_summary *es = inline_edge_summary (edge); + + es->call_stmt_size = this_size; + es->call_stmt_time = this_time; + es->loop_depth = bb->loop_depth; /* Do not inline calls where we cannot triviall work around mismatches in argument or return types. */ @@ -1076,8 +1146,9 @@ struct gimple_opt_pass pass_inline_parameters = static void estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time) { - *size += e->call_stmt_size * INLINE_SIZE_SCALE; - *time += (e->call_stmt_time + struct inline_edge_summary *es = inline_edge_summary (e); + *size += es->call_stmt_size * INLINE_SIZE_SCALE; + *time += (es->call_stmt_time * e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE)); if (*time > MAX_TIME * INLINE_TIME_SCALE) *time = MAX_TIME * INLINE_TIME_SCALE; @@ -1222,6 +1293,38 @@ remap_predicate (struct inline_summary *info, struct inline_summary *callee_info } +/* Update summary information of inline clones after inlining. + Compute peak stack usage. */ + +static void +inline_update_callee_summaries (struct cgraph_node *node, + int depth) +{ + struct cgraph_edge *e; + struct inline_summary *callee_info = inline_summary (node); + struct inline_summary *caller_info = inline_summary (node->callers->caller); + HOST_WIDE_INT peak; + + callee_info->stack_frame_offset + = caller_info->stack_frame_offset + + caller_info->estimated_self_stack_size; + peak = callee_info->stack_frame_offset + + callee_info->estimated_self_stack_size; + if (inline_summary (node->global.inlined_to)->estimated_stack_size + < peak) + inline_summary (node->global.inlined_to)->estimated_stack_size = peak; + cgraph_propagate_frequency (node); + for (e = node->callees; e; e = e->next_callee) + { + if (!e->inline_failed) + inline_update_callee_summaries (e->callee, depth); + inline_edge_summary (e)->loop_depth += depth; + } + for (e = node->indirect_calls; e; e = e->next_callee) + inline_edge_summary (e)->loop_depth += depth; +} + + /* We inlined EDGE. Update summary of the function we inlined into. */ void @@ -1275,6 +1378,10 @@ inline_merge_summary (struct cgraph_edge *edge) for (i = 0; VEC_iterate (size_time_entry, info->entry, i, e); i++) info->size += e->size, info->time += e->time; estimate_calls_size_and_time (to, &info->size, &info->time); + + inline_update_callee_summaries (edge->callee, + inline_edge_summary (edge)->loop_depth); + info->time = (info->time + INLINE_TIME_SCALE / 2) / INLINE_TIME_SCALE; info->size = (info->size + INLINE_SIZE_SCALE / 2) / INLINE_SIZE_SCALE; } @@ -1293,11 +1400,12 @@ do_estimate_edge_time (struct cgraph_edge *edge) int time; int size; gcov_type ret; + struct inline_edge_summary *es = inline_edge_summary (edge); gcc_checking_assert (edge->inline_failed); estimate_callee_size_and_time (edge, true, &size, &time); - ret = (((gcov_type)time - edge->call_stmt_time) * edge->frequency + ret = (((gcov_type)time - es->call_stmt_time) * edge->frequency + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE; if (ret > MAX_TIME) ret = MAX_TIME; @@ -1313,8 +1421,8 @@ do_estimate_edge_time (struct cgraph_edge *edge) VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->time = ret + (ret >= 0); - ret_size = size - edge->call_stmt_size; - gcc_checking_assert (edge->call_stmt_size); + ret_size = size - es->call_stmt_size; + gcc_checking_assert (es->call_stmt_size); VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->size = ret_size + (ret_size >= 0); } @@ -1345,8 +1453,8 @@ do_estimate_edge_growth (struct cgraph_edge *edge) /* Early inliner runs without caching, go ahead and do the dirty work. */ gcc_checking_assert (edge->inline_failed); estimate_callee_size_and_time (edge, true, &size, NULL); - gcc_checking_assert (edge->call_stmt_size); - return size - edge->call_stmt_size; + gcc_checking_assert (inline_edge_summary (edge)->call_stmt_size); + return size - inline_edge_summary (edge)->call_stmt_size; } @@ -1494,6 +1602,18 @@ inline_generate_summary (void) } +/* Write inline summary for edge E to OB. */ + +static void +read_inline_edge_summary (struct lto_input_block *ib, struct cgraph_edge *e) +{ + struct inline_edge_summary *es = inline_edge_summary (e); + es->call_stmt_size = lto_input_uleb128 (ib); + es->call_stmt_time = lto_input_uleb128 (ib); + es->loop_depth = lto_input_uleb128 (ib); +} + + /* Stream in inline summaries from the section. */ static void @@ -1524,6 +1644,7 @@ inline_read_section (struct lto_file_decl_data *file_data, const char *data, struct inline_summary *info; lto_cgraph_encoder_t encoder; struct bitpack_d bp; + struct cgraph_edge *e; index = lto_input_uleb128 (&ib); encoder = file_data->cgraph_node_encoder; @@ -1568,6 +1689,10 @@ inline_read_section (struct lto_file_decl_data *file_data, const char *data, VEC_safe_push (size_time_entry, gc, info->entry, &e); } + for (e = node->callees; e; e = e->next_callee) + read_inline_edge_summary (&ib, e); + for (e = node->indirect_calls; e; e = e->next_callee) + read_inline_edge_summary (&ib, e); } lto_free_section_data (file_data, LTO_section_inline_summary, NULL, data, @@ -1611,6 +1736,17 @@ inline_read_summary (void) cgraph_add_function_insertion_hook (&add_new_function, NULL); } +/* Write inline summary for edge E to OB. */ + +static void +write_inline_edge_summary (struct output_block *ob, struct cgraph_edge *e) +{ + struct inline_edge_summary *es = inline_edge_summary (e); + lto_output_uleb128_stream (ob->main_stream, es->call_stmt_size); + lto_output_uleb128_stream (ob->main_stream, es->call_stmt_time); + lto_output_uleb128_stream (ob->main_stream, es->loop_depth); +} + /* Write inline summary for node in SET. Jump functions are shared among ipa-cp and inliner, so when ipa-cp is @@ -1638,6 +1774,7 @@ inline_write_summary (cgraph_node_set set, { struct inline_summary *info = inline_summary (node); struct bitpack_d bp; + struct cgraph_edge *edge; int i; size_time_entry *e; struct condition *c; @@ -1684,6 +1821,10 @@ inline_write_summary (cgraph_node_set set, } lto_output_uleb128_stream (ob->main_stream, 0); } + for (edge = node->callees; edge; edge = edge->next_callee) + write_inline_edge_summary (ob, edge); + for (edge = node->indirect_calls; edge; edge = edge->next_callee) + write_inline_edge_summary (ob, edge); } } lto_output_1_stream (ob->main_stream, 0); @@ -1705,9 +1846,13 @@ inline_free_summary (void) function_insertion_hook_holder = NULL; if (node_removal_hook_holder) cgraph_remove_node_removal_hook (node_removal_hook_holder); + if (edge_removal_hook_holder) + cgraph_remove_edge_removal_hook (edge_removal_hook_holder); node_removal_hook_holder = NULL; if (node_duplication_hook_holder) cgraph_remove_node_duplication_hook (node_duplication_hook_holder); + if (edge_duplication_hook_holder) + cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder); node_duplication_hook_holder = NULL; VEC_free (inline_summary_t, gc, inline_summary_vec); inline_summary_vec = NULL; diff --git a/gcc/ipa-inline-transform.c b/gcc/ipa-inline-transform.c index 1fdb6d01ad7..7465325a35b 100644 --- a/gcc/ipa-inline-transform.c +++ b/gcc/ipa-inline-transform.c @@ -49,12 +49,11 @@ along with GCC; see the file COPYING3. If not see int ncalls_inlined; int nfunctions_inlined; -/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest - by NEST. */ +/* Scale frequency of NODE edges by FREQ_SCALE. */ static void update_noncloned_frequencies (struct cgraph_node *node, - int freq_scale, int nest) + int freq_scale) { struct cgraph_edge *e; @@ -63,12 +62,17 @@ update_noncloned_frequencies (struct cgraph_node *node, freq_scale = 1; for (e = node->callees; e; e = e->next_callee) { - e->loop_nest += nest; e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; if (e->frequency > CGRAPH_FREQ_MAX) e->frequency = CGRAPH_FREQ_MAX; if (!e->inline_failed) - update_noncloned_frequencies (e->callee, freq_scale, nest); + update_noncloned_frequencies (e->callee, freq_scale); + } + for (e = node->indirect_calls; e; e = e->next_callee) + { + e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; + if (e->frequency > CGRAPH_FREQ_MAX) + e->frequency = CGRAPH_FREQ_MAX; } } @@ -83,9 +87,6 @@ void clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, bool update_original, int *overall_size) { - HOST_WIDE_INT peak; - struct inline_summary *caller_info, *callee_info; - if (duplicate) { /* We may eliminate the need for out-of-line copy to be output. @@ -125,34 +126,22 @@ clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, } duplicate = false; e->callee->local.externally_visible = false; - update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest); + update_noncloned_frequencies (e->callee, e->frequency); } else { struct cgraph_node *n; n = cgraph_clone_node (e->callee, e->callee->decl, - e->count, e->frequency, e->loop_nest, + e->count, e->frequency, update_original, NULL); cgraph_redirect_edge_callee (e, n); } } - callee_info = inline_summary (e->callee); - caller_info = inline_summary (e->caller); - if (e->caller->global.inlined_to) e->callee->global.inlined_to = e->caller->global.inlined_to; else e->callee->global.inlined_to = e->caller; - callee_info->stack_frame_offset - = caller_info->stack_frame_offset - + caller_info->estimated_self_stack_size; - peak = callee_info->stack_frame_offset - + callee_info->estimated_self_stack_size; - if (inline_summary (e->callee->global.inlined_to)->estimated_stack_size - < peak) - inline_summary (e->callee->global.inlined_to)->estimated_stack_size = peak; - cgraph_propagate_frequency (e->callee); /* Recursively clone all bodies. */ for (e = e->callee->callees; e; e = e->next_callee) @@ -187,13 +176,14 @@ inline_call (struct cgraph_edge *e, bool update_original, to = e->caller; if (to->global.inlined_to) to = to->global.inlined_to; - old_size = inline_summary (to)->size; - inline_merge_summary (e); - new_size = inline_summary (to)->size; clone_inlined_nodes (e, true, update_original, overall_size); gcc_assert (curr->callee->global.inlined_to == to); + + old_size = inline_summary (to)->size; + inline_merge_summary (e); + new_size = inline_summary (to)->size; if (overall_size && new_size > old_size) *overall_size += new_size - old_size; ncalls_inlined++; diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c index 02cc7738960..d4052dfa294 100644 --- a/gcc/ipa-inline.c +++ b/gcc/ipa-inline.c @@ -739,7 +739,7 @@ edge_badness (struct cgraph_edge *edge, bool dump) of functions fully inlined in program. */ else { - int nest = MIN (edge->loop_nest, 8); + int nest = MIN (inline_edge_summary (edge)->loop_depth, 8); badness = estimate_growth (edge->callee) * 256; /* Decrease badness if call is nested. */ @@ -1027,7 +1027,7 @@ recursive_inlining (struct cgraph_edge *edge, { /* We need original clone to copy around. */ master_clone = cgraph_clone_node (node, node->decl, - node->count, CGRAPH_FREQ_BASE, 1, + node->count, CGRAPH_FREQ_BASE, false, NULL); for (e = master_clone->callees; e; e = e->next_callee) if (!e->inline_failed) @@ -1555,6 +1555,8 @@ ipa_inline (void) "\nInlined %i calls, eliminated %i functions\n\n", ncalls_inlined, nfunctions_inlined); + if (dump_file) + dump_inline_summaries (dump_file); /* In WPA we use inline summaries for partitioning process. */ if (!flag_wpa) inline_free_summary (); @@ -1709,9 +1711,10 @@ early_inliner (void) info that might be cleared out for newly discovered edges. */ for (edge = node->callees; edge; edge = edge->next_callee) { - edge->call_stmt_size + struct inline_edge_summary *es = inline_edge_summary (edge); + es->call_stmt_size = estimate_num_insns (edge->call_stmt, &eni_size_weights); - edge->call_stmt_time + es->call_stmt_time = estimate_num_insns (edge->call_stmt, &eni_time_weights); } timevar_pop (TV_INTEGRATION); diff --git a/gcc/ipa-inline.h b/gcc/ipa-inline.h index 4fe4489e8cf..d27fa46b7b8 100644 --- a/gcc/ipa-inline.h +++ b/gcc/ipa-inline.h @@ -101,6 +101,8 @@ struct GTY(()) inline_summary int time; int size; + /* Conditional size/time information. The summaries are being + merged during inlining. */ conditions conds; VEC(size_time_entry,gc) *entry; }; @@ -110,6 +112,21 @@ DEF_VEC_O(inline_summary_t); DEF_VEC_ALLOC_O(inline_summary_t,gc); extern GTY(()) VEC(inline_summary_t,gc) *inline_summary_vec; +/* Information kept about callgraph edges. */ +struct inline_edge_summary +{ + /* Estimated size and time of the call statement. */ + int call_stmt_size; + int call_stmt_time; + /* Depth of loop nest, 0 means no nesting. */ + unsigned short int loop_depth; +}; + +typedef struct inline_edge_summary inline_edge_summary_t; +DEF_VEC_O(inline_edge_summary_t); +DEF_VEC_ALLOC_O(inline_edge_summary_t,heap); +extern VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec; + typedef struct edge_growth_cache_entry { int time, size; @@ -152,6 +169,12 @@ inline_summary (struct cgraph_node *node) return VEC_index (inline_summary_t, inline_summary_vec, node->uid); } +static inline struct inline_edge_summary * +inline_edge_summary (struct cgraph_edge *edge) +{ + return VEC_index (inline_edge_summary_t, + inline_edge_summary_vec, edge->uid); +} /* Return estimated unit growth after inlning all calls to NODE. Quick accesors to the inline growth caches. diff --git a/gcc/lto-cgraph.c b/gcc/lto-cgraph.c index f560357b508..73911f32b83 100644 --- a/gcc/lto-cgraph.c +++ b/gcc/lto-cgraph.c @@ -285,9 +285,6 @@ lto_output_edge (struct lto_simple_output_block *ob, struct cgraph_edge *edge, bp_pack_value (&bp, uid, HOST_BITS_PER_INT); bp_pack_value (&bp, edge->inline_failed, HOST_BITS_PER_INT); bp_pack_value (&bp, edge->frequency, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->call_stmt_size, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->call_stmt_time, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->loop_nest, 30); bp_pack_value (&bp, edge->indirect_inlining_edge, 1); bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1); bp_pack_value (&bp, edge->can_throw_external, 1); @@ -1003,7 +1000,7 @@ input_node (struct lto_file_decl_data *file_data, if (clone_ref != LCC_NOT_FOUND) { node = cgraph_clone_node (VEC_index (cgraph_node_ptr, nodes, clone_ref), fn_decl, - 0, CGRAPH_FREQ_BASE, 0, false, NULL); + 0, CGRAPH_FREQ_BASE, false, NULL); } else node = cgraph_get_create_node (fn_decl); @@ -1164,11 +1161,9 @@ input_edge (struct lto_input_block *ib, VEC(cgraph_node_ptr, heap) *nodes, unsigned int stmt_id; gcov_type count; int freq; - unsigned int nest; cgraph_inline_failed_t inline_failed; struct bitpack_d bp; int ecf_flags = 0; - int call_stmt_time, call_stmt_size; caller = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib)); if (caller == NULL || caller->decl == NULL_TREE) @@ -1190,22 +1185,17 @@ input_edge (struct lto_input_block *ib, VEC(cgraph_node_ptr, heap) *nodes, inline_failed = (cgraph_inline_failed_t) bp_unpack_value (&bp, HOST_BITS_PER_INT); freq = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT); - call_stmt_size = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT); - call_stmt_time = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT); - nest = (unsigned) bp_unpack_value (&bp, 30); if (indirect) - edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq, nest); + edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq); else - edge = cgraph_create_edge (caller, callee, NULL, count, freq, nest); + edge = cgraph_create_edge (caller, callee, NULL, count, freq); edge->indirect_inlining_edge = bp_unpack_value (&bp, 1); edge->lto_stmt_uid = stmt_id; edge->inline_failed = inline_failed; edge->call_stmt_cannot_inline_p = bp_unpack_value (&bp, 1); edge->can_throw_external = bp_unpack_value (&bp, 1); - edge->call_stmt_size = call_stmt_size; - edge->call_stmt_time = call_stmt_time; if (indirect) { if (bp_unpack_value (&bp, 1)) diff --git a/gcc/tree-emutls.c b/gcc/tree-emutls.c index d29524997db..19f627e16d0 100644 --- a/gcc/tree-emutls.c +++ b/gcc/tree-emutls.c @@ -437,7 +437,7 @@ gen_emutls_addr (tree decl, struct lower_emutls_data *d) gimple_seq_add_stmt (&d->seq, x); cgraph_create_edge (d->cfun_node, d->builtin_node, x, - d->bb->count, d->bb_freq, d->bb->loop_depth); + d->bb->count, d->bb_freq); /* We may be adding a new reference to a new variable to the function. This means we have to play with the ipa-reference web. */ diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index 797ea8bd572..440699f7719 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -1678,7 +1678,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, edge = cgraph_clone_edge (edge, id->dst_node, stmt, gimple_uid (stmt), REG_BR_PROB_BASE, CGRAPH_FREQ_BASE, - edge->frequency, true); + true); /* We could also just rescale the frequency, but doing so would introduce roundoff errors and make verifier unhappy. */ @@ -1745,13 +1745,12 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, (id->dst_node, dest, orig_stmt, stmt, bb->count, compute_call_stmt_bb_frequency (id->dst_node->decl, copy_basic_block), - bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL); + CIF_ORIGINALLY_INDIRECT_CALL); else cgraph_create_edge (id->dst_node, dest, stmt, bb->count, compute_call_stmt_bb_frequency - (id->dst_node->decl, copy_basic_block), - bb->loop_depth)->inline_failed + (id->dst_node->decl, copy_basic_block))->inline_failed = CIF_ORIGINALLY_INDIRECT_CALL; if (dump_file) { |