[06/11] Handle VMAT_INVARIANT separately

Invariant loads were handled as a variation on the code for contiguous
loads.  We detected whether they were invariant or not as a byproduct of
creating the vector pointer ivs: vect_create_data_ref_ptr passed back an
inv_p to say whether the pointer was invariant.

But vectorised invariant loads just keep the original scalar load,
so this meant that detecting invariant loads had the side-effect of
creating an unwanted vector pointer iv.  The placement of the code
also meant that we'd create a vector load and then not use the result.
In principle this is wrong code, since there's no guarantee that there's
a vector's worth of accessible data at that address, but we rely on DCE
to get rid of the load before any harm is done.

E.g., for an invariant load in an inner loop (which seems like the more
common use case for this code), we'd create:

   vectp_a.6_52 = &a + 4;

   # vectp_a.5_53 = PHI <vectp_a.5_54(9), vectp_a.6_52(2)>

   # vectp_a.5_55 = PHI <vectp_a.5_53(3), vectp_a.5_56(10)>

   vect_next_a_11.7_57 = MEM[(int *)vectp_a.5_55];
   next_a_11 = a[_1];
   vect_cst__58 = {next_a_11, next_a_11, next_a_11, next_a_11};

   vectp_a.5_56 = vectp_a.5_55 + 4;

   vectp_a.5_54 = vectp_a.5_53 + 0;

whereas all we want is:

   next_a_11 = a[_1];
   vect_cst__58 = {next_a_11, next_a_11, next_a_11, next_a_11};

This patch moves the handling to its own block and makes
vect_create_data_ref_ptr assert (when creating a full iv) that the
address isn't invariant.

The ncopies handling is unfortunate, but a preexisting issue.
Richi's suggestion of using a vector of vector statements would
let us reuse one statement for all copies.

2018-08-01  Richard Sandiford  <richard.sandiford@arm.com>

gcc/
	* tree-vectorizer.h (vect_create_data_ref_ptr): Remove inv_p
	parameter.
	* tree-vect-data-refs.c (vect_create_data_ref_ptr): Likewise.
	When creating an iv, assert that the step is not known to be zero.
	(vect_setup_realignment): Update call accordingly.
	* tree-vect-stmts.c (vectorizable_store): Likewise.
	(vectorizable_load): Likewise.  Handle VMAT_INVARIANT separately.

From-SVN: r263220

1 files changed, 11 insertions, 22 deletions

diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 1f09bd520cc..e20b502b719 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -4674,16 +4674,13 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
 
       Return the increment stmt that updates the pointer in PTR_INCR.
 
-   3. Set INV_P to true if the access pattern of the data reference in the
-      vectorized loop is invariant.  Set it to false otherwise.
-
-   4. Return the pointer.  */
+   3. Return the pointer.  */
 
 tree
 vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
 			  struct loop *at_loop, tree offset,
 			  tree *initial_address, gimple_stmt_iterator *gsi,
-			  gimple **ptr_incr, bool only_init, bool *inv_p,
+			  gimple **ptr_incr, bool only_init,
 			  tree byte_offset, tree iv_step)
 {
   const char *base_name;
@@ -4705,7 +4702,6 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
   bool insert_after;
   tree indx_before_incr, indx_after_incr;
   gimple *incr;
-  tree step;
   bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
 
   gcc_assert (iv_step != NULL_TREE
@@ -4726,14 +4722,6 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
       *ptr_incr = NULL;
     }
 
-  /* Check the step (evolution) of the load in LOOP, and record
-     whether it's invariant.  */
-  step = vect_dr_behavior (dr_info)->step;
-  if (integer_zerop (step))
-    *inv_p = true;
-  else
-    *inv_p = false;
-
   /* Create an expression for the first address accessed by this load
      in LOOP.  */
   base_name = get_name (DR_BASE_ADDRESS (dr));
@@ -4849,15 +4837,17 @@ vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
     aptr = aggr_ptr_init;
   else
     {
+      /* Accesses to invariant addresses should be handled specially
+	 by the caller.  */
+      tree step = vect_dr_behavior (dr_info)->step;
+      gcc_assert (!integer_zerop (step));
+
       if (iv_step == NULL_TREE)
 	{
-	  /* The step of the aggregate pointer is the type size.  */
+	  /* The step of the aggregate pointer is the type size,
+	     negated for downward accesses.  */
 	  iv_step = TYPE_SIZE_UNIT (aggr_type);
-	  /* One exception to the above is when the scalar step of the load in
-	     LOOP is zero. In this case the step here is also zero.  */
-	  if (*inv_p)
-	    iv_step = size_zero_node;
-	  else if (tree_int_cst_sgn (step) == -1)
+	  if (tree_int_cst_sgn (step) == -1)
 	    iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
 	}
 
@@ -5462,7 +5452,6 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
   gphi *phi_stmt;
   tree msq = NULL_TREE;
   gimple_seq stmts = NULL;
-  bool inv_p;
   bool compute_in_loop = false;
   bool nested_in_vect_loop = false;
   struct loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
@@ -5556,7 +5545,7 @@ vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
       vec_dest = vect_create_destination_var (scalar_dest, vectype);
       ptr = vect_create_data_ref_ptr (stmt_info, vectype,
 				      loop_for_initial_load, NULL_TREE,
-				      &init_addr, NULL, &inc, true, &inv_p);
+				      &init_addr, NULL, &inc, true);
       if (TREE_CODE (ptr) == SSA_NAME)
 	new_temp = copy_ssa_name (ptr);
       else