diff options
Diffstat (limited to 'gcc/tree-vect-stmts.c')
| -rw-r--r-- | gcc/tree-vect-stmts.c | 497 |
1 files changed, 150 insertions, 347 deletions
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index ec4ba060df0..06d51e2d318 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -116,13 +116,14 @@ vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo, use_operand_p use_p; def_operand_p def_p; - *relevant = vect_unused_in_loop; + *relevant = vect_unused_in_scope; *live_p = false; /* cond stmt other than loop exit cond. */ if (is_ctrl_stmt (stmt) - && STMT_VINFO_TYPE (vinfo_for_stmt (stmt)) != loop_exit_ctrl_vec_info_type) - *relevant = vect_used_in_loop; + && STMT_VINFO_TYPE (vinfo_for_stmt (stmt)) + != loop_exit_ctrl_vec_info_type) + *relevant = vect_used_in_scope; /* changing memory. */ if (gimple_code (stmt) != GIMPLE_PHI) @@ -130,7 +131,7 @@ vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo, { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "vec_stmt_relevant_p: stmt has vdefs."); - *relevant = vect_used_in_loop; + *relevant = vect_used_in_scope; } /* uses outside the loop. */ @@ -249,7 +250,7 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt)) { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) fprintf (vect_dump, "not vectorized: unsupported use in stmt."); return false; } @@ -284,7 +285,7 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo)); gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction); gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo) - || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_loop); + || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope); return true; } @@ -301,18 +302,18 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, fprintf (vect_dump, "outer-loop def-stmt defining inner-loop stmt."); switch (relevant) { - case vect_unused_in_loop: + case vect_unused_in_scope: relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ? - vect_used_by_reduction : vect_unused_in_loop; + vect_used_by_reduction : vect_unused_in_scope; break; case vect_used_in_outer_by_reduction: relevant = vect_used_by_reduction; break; case vect_used_in_outer: - relevant = vect_used_in_loop; + relevant = vect_used_in_scope; break; case vect_used_by_reduction: - case vect_used_in_loop: + case vect_used_in_scope: break; default: @@ -333,9 +334,9 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, fprintf (vect_dump, "inner-loop def-stmt defining outer-loop stmt."); switch (relevant) { - case vect_unused_in_loop: + case vect_unused_in_scope: relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) ? - vect_used_in_outer_by_reduction : vect_unused_in_loop; + vect_used_in_outer_by_reduction : vect_unused_in_scope; break; case vect_used_in_outer_by_reduction: @@ -346,7 +347,7 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, relevant = vect_used_in_outer_by_reduction; break; - case vect_used_in_loop: + case vect_used_in_scope: relevant = vect_used_in_outer; break; @@ -468,18 +469,18 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) Here are the expected values of "relevant" for reduction phis/stmts: relevance: phi stmt - vect_unused_in_loop ok + vect_unused_in_scope ok vect_used_in_outer_by_reduction ok ok vect_used_in_outer ok ok vect_used_by_reduction ok - vect_used_in_loop */ + vect_used_in_scope */ if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def) { enum vect_relevant tmp_relevant = relevant; switch (tmp_relevant) { - case vect_unused_in_loop: + case vect_unused_in_scope: gcc_assert (gimple_code (stmt) != GIMPLE_PHI); relevant = vect_used_by_reduction; break; @@ -496,7 +497,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) if (gimple_code (stmt) == GIMPLE_PHI) break; /* fall through */ - case vect_used_in_loop: + case vect_used_in_scope: default: if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "unsupported use of reduction."); @@ -571,7 +572,7 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies, /* FORNOW: Assuming maximum 2 args per stmts. */ for (i = 0; i < 2; i++) { - if (dt[i] == vect_constant_def || dt[i] == vect_invariant_def) + if (dt[i] == vect_constant_def || dt[i] == vect_external_def) outside_cost += TARG_SCALAR_TO_VEC_COST; } @@ -619,7 +620,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, if (PURE_SLP_STMT (stmt_info)) return; - if (dt == vect_constant_def || dt == vect_invariant_def) + if (dt == vect_constant_def || dt == vect_external_def) outside_cost = TARG_SCALAR_TO_VEC_COST; /* Strided access? */ @@ -889,6 +890,9 @@ vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def) /* Case 1: operand is a constant. */ case vect_constant_def: { + vector_type = get_vectype_for_scalar_type (TREE_TYPE (op)); + gcc_assert (vector_type); + if (scalar_def) *scalar_def = op; @@ -900,12 +904,12 @@ vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def) { t = tree_cons (NULL_TREE, op, t); } - vec_cst = build_vector (vectype, t); - return vect_init_vector (stmt, vec_cst, vectype, NULL); + vec_cst = build_vector (vector_type, t); + return vect_init_vector (stmt, vec_cst, vector_type, NULL); } /* Case 2: operand is defined outside the loop - loop invariant. */ - case vect_invariant_def: + case vect_external_def: { vector_type = get_vectype_for_scalar_type (TREE_TYPE (def)); gcc_assert (vector_type); @@ -929,7 +933,7 @@ vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def) } /* Case 3: operand is defined inside the loop. */ - case vect_loop_def: + case vect_internal_def: { if (scalar_def) *scalar_def = NULL/* FIXME tuples: def_stmt*/; @@ -1042,7 +1046,7 @@ vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd) stmt_vec_info def_stmt_info; /* Do nothing; can reuse same def. */ - if (dt == vect_invariant_def || dt == vect_constant_def ) + if (dt == vect_external_def || dt == vect_constant_def ) return vec_oprnd; vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd); @@ -1190,7 +1194,7 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt) if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* FORNOW: SLP not supported. */ @@ -1508,7 +1512,7 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; if (!is_gimple_assign (stmt)) @@ -1771,7 +1775,7 @@ vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is vectorizable assignment? */ @@ -1885,7 +1889,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is STMT a vectorizable binary/unary operation? */ @@ -1947,7 +1951,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, shift_p = true; /* vector shifted by vector */ - if (dt[1] == vect_loop_def) + if (dt[1] == vect_internal_def) { optab = optab_for_tree_code (code, vectype, optab_vector); if (vect_print_dump_info (REPORT_DETAILS)) @@ -1956,7 +1960,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, /* See if the machine has a vector shifted by scalar insn and if not then see if it has a vector shifted by vector insn */ - else if (dt[1] == vect_constant_def || dt[1] == vect_invariant_def) + else if (dt[1] == vect_constant_def || dt[1] == vect_external_def) { optab = optab_for_tree_code (code, vectype, optab_scalar); if (optab @@ -1970,11 +1974,20 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, else { optab = optab_for_tree_code (code, vectype, optab_vector); - if (vect_print_dump_info (REPORT_DETAILS) - && optab + if (optab && (optab_handler (optab, TYPE_MODE (vectype))->insn_code != CODE_FOR_nothing)) - fprintf (vect_dump, "vector/vector shift/rotate found."); + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "vector/vector shift/rotate found."); + + /* Unlike the other binary operators, shifts/rotates have + the rhs being int, instead of the same type as the lhs, + so make sure the scalar is the right type if we are + dealing with vectors of short/char. */ + if (dt[1] == vect_constant_def) + op1 = fold_convert (TREE_TYPE (vectype), op1); + } } } @@ -2323,7 +2336,7 @@ vectorizable_type_demotion (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is STMT a vectorizable type-demotion operation? */ @@ -2590,7 +2603,7 @@ vectorizable_type_promotion (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is STMT a vectorizable type-promotion operation? */ @@ -2836,7 +2849,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is vectorizable store? */ @@ -3204,7 +3217,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* Is vectorizable load? */ @@ -3703,7 +3716,7 @@ vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi, if (!STMT_VINFO_RELEVANT_P (stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_loop_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def) return false; /* FORNOW: SLP not supported. */ @@ -3803,336 +3816,125 @@ vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi, } -/* Function vect_analyze_operations. - - Scan the loop stmts and make sure they are all vectorizable. */ +/* Make sure the statement is vectorizable. */ bool -vect_analyze_operations (loop_vec_info loop_vinfo) +vect_analyze_stmt (gimple stmt, bool *need_to_vectorize) { - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); - int nbbs = loop->num_nodes; - gimple_stmt_iterator si; - unsigned int vectorization_factor = 0; - int i; + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; - gimple phi; - stmt_vec_info stmt_info; - bool need_to_vectorize = false; - int min_profitable_iters; - int min_scalar_loop_bound; - unsigned int th; - bool only_slp_in_loop = true; if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "=== vect_analyze_operations ==="); - - gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo)); - vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - - for (i = 0; i < nbbs; i++) { - basic_block bb = bbs[i]; - - for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si)) - { - phi = gsi_stmt (si); - ok = true; - - stmt_info = vinfo_for_stmt (phi); - if (vect_print_dump_info (REPORT_DETAILS)) - { - fprintf (vect_dump, "examining phi: "); - print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM); - } - - if (! is_loop_header_bb_p (bb)) - { - /* inner-loop loop-closed exit phi in outer-loop vectorization - (i.e. a phi in the tail of the outer-loop). - FORNOW: we currently don't support the case that these phis - are not used in the outerloop, cause this case requires - to actually do something here. */ - if (!STMT_VINFO_RELEVANT_P (stmt_info) - || STMT_VINFO_LIVE_P (stmt_info)) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, - "Unsupported loop-closed phi in outer-loop."); - return false; - } - continue; - } - - gcc_assert (stmt_info); - - if (STMT_VINFO_LIVE_P (stmt_info)) - { - /* FORNOW: not yet supported. */ - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, "not vectorized: value used after loop."); - return false; - } - - if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_loop - && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def) - { - /* A scalar-dependence cycle that we don't support. */ - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, "not vectorized: scalar dependence cycle."); - return false; - } - - if (STMT_VINFO_RELEVANT_P (stmt_info)) - { - need_to_vectorize = true; - if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) - ok = vectorizable_induction (phi, NULL, NULL); - } - - if (!ok) - { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - { - fprintf (vect_dump, - "not vectorized: relevant phi not supported: "); - print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM); - } - return false; - } - } - - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) - { - gimple stmt = gsi_stmt (si); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); - - if (vect_print_dump_info (REPORT_DETAILS)) - { - fprintf (vect_dump, "==> examining statement: "); - print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); - } - - gcc_assert (stmt_info); - - /* skip stmts which do not need to be vectorized. - this is expected to include: - - the COND_EXPR which is the loop exit condition - - any LABEL_EXPRs in the loop - - computations that are used only for array indexing or loop - control */ - - if (!STMT_VINFO_RELEVANT_P (stmt_info) - && !STMT_VINFO_LIVE_P (stmt_info)) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "irrelevant."); - continue; - } - - switch (STMT_VINFO_DEF_TYPE (stmt_info)) - { - case vect_loop_def: - break; - - case vect_reduction_def: - gcc_assert (relevance == vect_used_in_outer - || relevance == vect_used_in_outer_by_reduction - || relevance == vect_unused_in_loop); - break; - - case vect_induction_def: - case vect_constant_def: - case vect_invariant_def: - case vect_unknown_def_type: - default: - gcc_unreachable (); - } - - if (STMT_VINFO_RELEVANT_P (stmt_info)) - { - gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt)))); - gcc_assert (STMT_VINFO_VECTYPE (stmt_info)); - need_to_vectorize = true; - } - - ok = true; - if (STMT_VINFO_RELEVANT_P (stmt_info) - || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def) - ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL) - || vectorizable_type_demotion (stmt, NULL, NULL, NULL) - || vectorizable_conversion (stmt, NULL, NULL, NULL) - || vectorizable_operation (stmt, NULL, NULL, NULL) - || vectorizable_assignment (stmt, NULL, NULL, NULL) - || vectorizable_load (stmt, NULL, NULL, NULL, NULL) - || vectorizable_call (stmt, NULL, NULL) - || vectorizable_store (stmt, NULL, NULL, NULL) - || vectorizable_condition (stmt, NULL, NULL) - || vectorizable_reduction (stmt, NULL, NULL)); - - if (!ok) - { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - { - fprintf (vect_dump, "not vectorized: relevant stmt not "); - fprintf (vect_dump, "supported: "); - print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); - } - return false; - } + fprintf (vect_dump, "==> examining statement: "); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } - /* Stmts that are (also) "live" (i.e. - that are used out of the loop) - need extra handling, except for vectorizable reductions. */ - if (STMT_VINFO_LIVE_P (stmt_info) - && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type) - ok = vectorizable_live_operation (stmt, NULL, NULL); + /* Skip stmts that do not need to be vectorized. In loops this is expected + to include: + - the COND_EXPR which is the loop exit condition + - any LABEL_EXPRs in the loop + - computations that are used only for array indexing or loop control. + In basic blocks we only analyze statements that are a part of some SLP + instance, therefore, all the statements are relevant. */ - if (!ok) - { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - { - fprintf (vect_dump, "not vectorized: live stmt not "); - fprintf (vect_dump, "supported: "); - print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); - } - return false; - } - - if (!PURE_SLP_STMT (stmt_info)) - { - /* STMT needs loop-based vectorization. */ - only_slp_in_loop = false; - - /* Groups of strided accesses whose size is not a power of 2 are - not vectorizable yet using loop-vectorization. Therefore, if - this stmt feeds non-SLP-able stmts (i.e., this stmt has to be - both SLPed and loop-based vectorized), the loop cannot be - vectorized. */ - if (STMT_VINFO_STRIDED_ACCESS (stmt_info) - && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt ( - DR_GROUP_FIRST_DR (stmt_info)))) == -1) - { - if (vect_print_dump_info (REPORT_DETAILS)) - { - fprintf (vect_dump, "not vectorized: the size of group " - "of strided accesses is not a power of 2"); - print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); - } - return false; - } - } - } /* stmts in bb */ - } /* bbs */ - - /* All operations in the loop are either irrelevant (deal with loop - control, or dead), or only used outside the loop and can be moved - out of the loop (e.g. invariants, inductions). The loop can be - optimized away by scalar optimizations. We're better off not - touching this loop. */ - if (!need_to_vectorize) + if (!STMT_VINFO_RELEVANT_P (stmt_info) + && !STMT_VINFO_LIVE_P (stmt_info)) { if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, - "All the computation can be taken out of the loop."); - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, - "not vectorized: redundant loop. no profit to vectorize."); - return false; - } + fprintf (vect_dump, "irrelevant."); - /* If all the stmts in the loop can be SLPed, we perform only SLP, and - vectorization factor of the loop is the unrolling factor required by the - SLP instances. If that unrolling factor is 1, we say, that we perform - pure SLP on loop - cross iteration parallelism is not exploited. */ - if (only_slp_in_loop) - vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo); - else - vectorization_factor = least_common_multiple (vectorization_factor, - LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo)); + return true; + } - LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor; + switch (STMT_VINFO_DEF_TYPE (stmt_info)) + { + case vect_internal_def: + break; - if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) - && vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, - "vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC, - vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo)); + case vect_reduction_def: + gcc_assert (relevance == vect_used_in_outer + || relevance == vect_used_in_outer_by_reduction + || relevance == vect_unused_in_scope); + break; + + case vect_induction_def: + case vect_constant_def: + case vect_external_def: + case vect_unknown_def_type: + default: + gcc_unreachable (); + } - if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) - && (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor)) + if (STMT_VINFO_RELEVANT_P (stmt_info)) { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, "not vectorized: iteration count too small."); - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump,"not vectorized: iteration count smaller than " - "vectorization factor."); - return false; + gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt)))); + gcc_assert (STMT_VINFO_VECTYPE (stmt_info)); + *need_to_vectorize = true; } - /* Analyze cost. Decide if worth while to vectorize. */ - - /* Once VF is set, SLP costs should be updated since the number of created - vector stmts depends on VF. */ - vect_update_slp_costs_according_to_vf (loop_vinfo); - - min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo); - LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters; - - if (min_profitable_iters < 0) + ok = true; + if (STMT_VINFO_RELEVANT_P (stmt_info) + || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def) + ok = (vectorizable_type_promotion (stmt, NULL, NULL, NULL) + || vectorizable_type_demotion (stmt, NULL, NULL, NULL) + || vectorizable_conversion (stmt, NULL, NULL, NULL) + || vectorizable_operation (stmt, NULL, NULL, NULL) + || vectorizable_assignment (stmt, NULL, NULL, NULL) + || vectorizable_load (stmt, NULL, NULL, NULL, NULL) + || vectorizable_call (stmt, NULL, NULL) + || vectorizable_store (stmt, NULL, NULL, NULL) + || vectorizable_condition (stmt, NULL, NULL) + || vectorizable_reduction (stmt, NULL, NULL)); + + if (!ok) { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, "not vectorized: vectorization not profitable."); - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "not vectorized: vector version will never be " - "profitable."); + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + { + fprintf (vect_dump, "not vectorized: relevant stmt not "); + fprintf (vect_dump, "supported: "); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } + return false; } - min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND) - * vectorization_factor) - 1); - - /* Use the cost model only if it is more conservative than user specified - threshold. */ - - th = (unsigned) min_scalar_loop_bound; - if (min_profitable_iters - && (!min_scalar_loop_bound - || min_profitable_iters > min_scalar_loop_bound)) - th = (unsigned) min_profitable_iters; + /* Stmts that are (also) "live" (i.e. - that are used out of the loop) + need extra handling, except for vectorizable reductions. */ + if (STMT_VINFO_LIVE_P (stmt_info) + && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type) + ok = vectorizable_live_operation (stmt, NULL, NULL); - if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) - && LOOP_VINFO_INT_NITERS (loop_vinfo) <= th) + if (!ok) { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, "not vectorized: vectorization not " - "profitable."); - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "not vectorized: iteration count smaller than " - "user specified loop bound parameter or minimum " - "profitable iterations (whichever is more conservative)."); - return false; + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + { + fprintf (vect_dump, "not vectorized: live stmt not "); + fprintf (vect_dump, "supported: "); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } + + return false; } - if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) - || LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0 - || LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo)) + if (!PURE_SLP_STMT (stmt_info)) { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "epilog loop required."); - if (!vect_can_advance_ivs_p (loop_vinfo)) + /* Groups of strided accesses whose size is not a power of 2 are not + vectorizable yet using loop-vectorization. Therefore, if this stmt + feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and + loop-based vectorized), the loop cannot be vectorized. */ + if (STMT_VINFO_STRIDED_ACCESS (stmt_info) + && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt ( + DR_GROUP_FIRST_DR (stmt_info)))) == -1) { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, - "not vectorized: can't create epilog loop 1."); - return false; - } - if (!slpeel_can_duplicate_loop_p (loop, single_exit (loop))) - { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS)) - fprintf (vect_dump, - "not vectorized: can't create epilog loop 2."); + if (vect_print_dump_info (REPORT_DETAILS)) + { + fprintf (vect_dump, "not vectorized: the size of group " + "of strided accesses is not a power of 2"); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } + return false; } } @@ -4343,7 +4145,7 @@ new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo) STMT_VINFO_TYPE (res) = undef_vec_info_type; STMT_VINFO_STMT (res) = stmt; STMT_VINFO_LOOP_VINFO (res) = loop_vinfo; - STMT_VINFO_RELEVANT (res) = vect_unused_in_loop; + STMT_VINFO_RELEVANT (res) = vect_unused_in_scope; STMT_VINFO_LIVE_P (res) = false; STMT_VINFO_VECTYPE (res) = NULL; STMT_VINFO_VEC_STMT (res) = NULL; @@ -4361,7 +4163,8 @@ new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo) && is_loop_header_bb_p (gimple_bb (stmt))) STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type; else - STMT_VINFO_DEF_TYPE (res) = vect_loop_def; + STMT_VINFO_DEF_TYPE (res) = vect_internal_def; + STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5); STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0; STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0; @@ -4499,7 +4302,7 @@ vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, gimple *def_stmt, if (is_gimple_min_invariant (operand)) { *def = operand; - *dt = vect_invariant_def; + *dt = vect_external_def; return true; } @@ -4530,18 +4333,18 @@ vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, gimple *def_stmt, print_gimple_stmt (vect_dump, *def_stmt, 0, TDF_SLIM); } - /* empty stmt is expected only in case of a function argument. + /* Empty stmt is expected only in case of a function argument. (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */ if (gimple_nop_p (*def_stmt)) { *def = operand; - *dt = vect_invariant_def; + *dt = vect_external_def; return true; } bb = gimple_bb (*def_stmt); if (!flow_bb_inside_loop_p (loop, bb)) - *dt = vect_invariant_def; + *dt = vect_external_def; else { stmt_vinfo = vinfo_for_stmt (*def_stmt); @@ -4618,7 +4421,7 @@ supportable_widening_operation (enum tree_code code, gimple stmt, tree vectype, struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info); bool ordered_p; enum machine_mode vec_mode; - enum insn_code icode1 = 0, icode2 = 0; + enum insn_code icode1, icode2; optab optab1, optab2; tree type = gimple_expr_type (stmt); tree wide_vectype = get_vectype_for_scalar_type (type); |