From patchwork Thu Sep 26 16:51:38 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Biener X-Patchwork-Id: 1168007 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=gcc.gnu.org (client-ip=209.132.180.131; helo=sourceware.org; envelope-from=gcc-patches-return-509657-incoming=patchwork.ozlabs.org@gcc.gnu.org; receiver=) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=suse.de Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=gcc.gnu.org header.i=@gcc.gnu.org header.b="BpCXYAWJ"; dkim-atps=neutral Received: from sourceware.org (server1.sourceware.org [209.132.180.131]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 46fLb341pBz9sNx for ; Fri, 27 Sep 2019 02:51:51 +1000 (AEST) DomainKey-Signature: a=rsa-sha1; c=nofws; d=gcc.gnu.org; h=list-id :list-unsubscribe:list-archive:list-post:list-help:sender:date :from:to:subject:message-id:mime-version:content-type; q=dns; s= default; b=VoVfbfDhF/UMUU/bzYeoIZeT2uvpggn5CKUcfifelldil52e76Fw9 FhV+Kx+K/pyxN4hZECTRuzuv6ptyPhk9fxUHNS6npmd5eNKrIBwA9KVKjvCib+J1 qLcDdR+lx/4g6EH/VZfuoIq9Grg5O3fYOdtRs9aVPiuHbEyNG8Hk5s= DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=gcc.gnu.org; h=list-id :list-unsubscribe:list-archive:list-post:list-help:sender:date :from:to:subject:message-id:mime-version:content-type; s= default; bh=3C8YyKsz93CqFQL76X4jLoeMp5A=; b=BpCXYAWJFAkAgyW8XYNa v12mP+63rFvd3HxDXcHZwIkWFo2m4TLnLPvjIciNOwhMki7jh5P1ZH4zuQpXBOro DzZ2OHGGd1lknLAGKBBlznTvJyhHs57zh+X7yxXYPgNp5npW81qN9hYNru8WwcNZ yufkia+fdNshSKN1Iy1Ls24= Received: (qmail 77149 invoked by alias); 26 Sep 2019 16:51:43 -0000 Mailing-List: contact gcc-patches-help@gcc.gnu.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Archive: List-Post: List-Help: Sender: gcc-patches-owner@gcc.gnu.org Delivered-To: mailing list gcc-patches@gcc.gnu.org Received: (qmail 77141 invoked by uid 89); 26 Sep 2019 16:51:43 -0000 Authentication-Results: sourceware.org; auth=none X-Spam-SWARE-Status: No, score=-12.8 required=5.0 tests=AWL, BAYES_00, GIT_PATCH_1, GIT_PATCH_2, GIT_PATCH_3, KAM_ASCII_DIVIDERS, SPF_PASS autolearn=ham version=3.3.1 spammy=Analyze X-HELO: mx1.suse.de Received: from mx2.suse.de (HELO mx1.suse.de) (195.135.220.15) by sourceware.org (qpsmtpd/0.93/v0.84-503-g423c35a) with ESMTP; Thu, 26 Sep 2019 16:51:41 +0000 Received: from relay2.suse.de (unknown [195.135.220.254]) by mx1.suse.de (Postfix) with ESMTP id C54D7AC28 for ; Thu, 26 Sep 2019 16:51:38 +0000 (UTC) Date: Thu, 26 Sep 2019 18:51:38 +0200 (CEST) From: Richard Biener To: gcc-patches@gcc.gnu.org Subject: [PATCH] Split out nested cycle loop-closed PHI vectorization Message-ID: User-Agent: Alpine 2.21 (LSU 202 2017-01-01) MIME-Version: 1.0 Bootstrapped / tested on x86_64-unknown-linux-gnu, applied. Richard. 2019-09-26 Richard Biener * tree-vect-loop.c (vect_analyze_loop_operations): Analyze loop-closed PHIs that are vect_internal_def. (vect_create_epilog_for_reduction): Exit early for nested cycles. Simplify. (vectorizable_lc_phi): New. * tree-vect-stmts.c (vect_analyze_stmt): Call vectorize_lc_phi. (vect_transform_stmt): Likewise. * tree-vectorizer.h (stmt_vec_info_type): Add lc_phi_info_type. (vectorizable_lc_phi): Declare. Index: gcc/tree-vect-loop.c =================================================================== --- gcc/tree-vect-loop.c (revision 276156) +++ gcc/tree-vect-loop.c (working copy) @@ -1519,12 +1519,16 @@ vect_analyze_loop_operations (loop_vec_i phi_op = PHI_ARG_DEF (phi, 0); stmt_vec_info op_def_info = loop_vinfo->lookup_def (phi_op); if (!op_def_info) - return opt_result::failure_at (phi, "unsupported phi"); + return opt_result::failure_at (phi, "unsupported phi\n"); if (STMT_VINFO_RELEVANT (op_def_info) != vect_used_in_outer && (STMT_VINFO_RELEVANT (op_def_info) != vect_used_in_outer_by_reduction)) - return opt_result::failure_at (phi, "unsupported phi"); + return opt_result::failure_at (phi, "unsupported phi\n"); + + if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def + && !vectorizable_lc_phi (stmt_info, NULL, NULL)) + return opt_result::failure_at (phi, "unsupported phi\n"); } continue; @@ -4396,6 +4400,10 @@ vect_create_epilog_for_reduction (vec # b1 = phi @@ -5313,7 +5311,6 @@ vect_create_epilog_for_reduction (vecinner; @@ -5473,7 +5470,7 @@ vect_create_epilog_for_reduction (vec (stmt_info->stmt) + || gimple_phi_num_args (stmt_info->stmt) != 1) + return false; + + /* To handle the nested_cycle_def for double-reductions we have to + refactor epilogue generation more. */ + if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def + /* && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def */) + return false; + + if (!vec_stmt) /* transformation not required. */ + { + STMT_VINFO_TYPE (stmt_info) = lc_phi_info_type; + return true; + } + + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree scalar_dest = gimple_phi_result (stmt_info->stmt); + basic_block bb = gimple_bb (stmt_info->stmt); + edge e = single_pred_edge (bb); + tree vec_dest = vect_create_destination_var (scalar_dest, vectype); + vec vec_oprnds = vNULL; + vect_get_vec_defs (gimple_phi_arg_def (stmt_info->stmt, 0), NULL_TREE, + stmt_info, &vec_oprnds, NULL, slp_node); + if (slp_node) + { + unsigned vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); + gcc_assert (vec_oprnds.length () == vec_num); + for (unsigned i = 0; i < vec_num; i++) + { + /* Create the vectorized LC PHI node. */ + gphi *new_phi = create_phi_node (vec_dest, bb); + add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION); + stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + } + } + else + { + unsigned ncopies = vect_get_num_copies (loop_vinfo, vectype); + stmt_vec_info prev_phi_info = NULL; + for (unsigned i = 0; i < ncopies; i++) + { + if (i != 0) + vect_get_vec_defs_for_stmt_copy (loop_vinfo, &vec_oprnds, NULL); + /* Create the vectorized LC PHI node. */ + gphi *new_phi = create_phi_node (vec_dest, bb); + add_phi_arg (new_phi, vec_oprnds[0], e, UNKNOWN_LOCATION); + stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); + if (i == 0) + STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi_info; + else + STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info; + prev_phi_info = new_phi_info; + } + } + vec_oprnds.release (); + + return true; +} + + /* Function vect_min_worthwhile_factor. For a loop where we could vectorize the operation indicated by CODE, @@ -8399,7 +8466,8 @@ vect_transform_loop (loop_vec_info loop_ if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def - || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle) + || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle + || STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def) && ! PURE_SLP_STMT (stmt_info)) { if (dump_enabled_p ()) Index: gcc/tree-vect-stmts.c =================================================================== --- gcc/tree-vect-stmts.c (revision 276156) +++ gcc/tree-vect-stmts.c (working copy) @@ -10671,7 +10671,8 @@ vect_analyze_stmt (stmt_vec_info stmt_in || vectorizable_condition (stmt_info, NULL, NULL, false, -1, node, cost_vec) || vectorizable_comparison (stmt_info, NULL, NULL, node, - cost_vec)); + cost_vec) + || vectorizable_lc_phi (stmt_info, NULL, node)); else { if (bb_vinfo) @@ -10820,6 +10821,11 @@ vect_transform_stmt (stmt_vec_info stmt_ gcc_assert (done); break; + case lc_phi_info_type: + done = vectorizable_lc_phi (stmt_info, &vec_stmt, slp_node); + gcc_assert (done); + break; + default: if (!STMT_VINFO_LIVE_P (stmt_info)) { Index: gcc/tree-vectorizer.h =================================================================== --- gcc/tree-vectorizer.h (revision 276156) +++ gcc/tree-vectorizer.h (working copy) @@ -694,6 +694,7 @@ enum stmt_vec_info_type { type_promotion_vec_info_type, type_demotion_vec_info_type, type_conversion_vec_info_type, + lc_phi_info_type, loop_exit_ctrl_vec_info_type }; @@ -1653,6 +1654,7 @@ extern bool vectorizable_reduction (stmt extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *, stmt_vec_info *, slp_tree, stmt_vector_for_cost *); +extern bool vectorizable_lc_phi (stmt_vec_info, stmt_vec_info *, slp_tree); extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, stmt_vector_for_cost *,