@@ -360,11 +360,11 @@
only ever used for the int64x1_t intrinsic, there is no scalar version. */
BUILTIN_VALLDI (UNOP, abs, 2)
- VAR1 (UNOP, vec_unpacks_hi_, 10, v4sf)
+ VAR2 (UNOP, vec_unpacks_hi_, 10, v4sf, v8hf)
VAR1 (BINOP, float_truncate_hi_, 0, v4sf)
VAR1 (BINOP, float_truncate_hi_, 0, v8hf)
- VAR1 (UNOP, float_extend_lo_, 0, v2df)
+ VAR2 (UNOP, float_extend_lo_, 0, v2df, v4sf)
BUILTIN_VDF (UNOP, float_truncate_lo_, 0)
/* Implemented by aarch64_ld1<VALL_F16:mode>. */
@@ -1677,36 +1677,57 @@
;; Float widening operations.
-(define_insn "vec_unpacks_lo_v4sf"
- [(set (match_operand:V2DF 0 "register_operand" "=w")
- (float_extend:V2DF
- (vec_select:V2SF
- (match_operand:V4SF 1 "register_operand" "w")
- (parallel [(const_int 0) (const_int 1)])
- )))]
+(define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
+ [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+ (float_extend:<VWIDE> (vec_select:<VHALF>
+ (match_operand:VQ_HSF 1 "register_operand" "w")
+ (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
+ )))]
"TARGET_SIMD"
- "fcvtl\\t%0.2d, %1.2s"
+ "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
[(set_attr "type" "neon_fp_cvt_widen_s")]
)
-(define_insn "aarch64_float_extend_lo_v2df"
- [(set (match_operand:V2DF 0 "register_operand" "=w")
- (float_extend:V2DF
- (match_operand:V2SF 1 "register_operand" "w")))]
+(define_expand "vec_unpacks_lo_<mode>"
+ [(match_operand:<VWIDE> 0 "register_operand" "")
+ (match_operand:VQ_HSF 1 "register_operand" "")]
"TARGET_SIMD"
- "fcvtl\\t%0.2d, %1.2s"
+ {
+ rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
+ emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
+ operands[1], p));
+ DONE;
+ }
+)
+
+(define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
+ [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+ (float_extend:<VWIDE> (vec_select:<VHALF>
+ (match_operand:VQ_HSF 1 "register_operand" "w")
+ (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
+ )))]
+ "TARGET_SIMD"
+ "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
[(set_attr "type" "neon_fp_cvt_widen_s")]
)
-(define_insn "vec_unpacks_hi_v4sf"
- [(set (match_operand:V2DF 0 "register_operand" "=w")
- (float_extend:V2DF
- (vec_select:V2SF
- (match_operand:V4SF 1 "register_operand" "w")
- (parallel [(const_int 2) (const_int 3)])
- )))]
+(define_expand "vec_unpacks_hi_<mode>"
+ [(match_operand:<VWIDE> 0 "register_operand" "")
+ (match_operand:VQ_HSF 1 "register_operand" "")]
+ "TARGET_SIMD"
+ {
+ rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
+ emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
+ operands[1], p));
+ DONE;
+ }
+)
+(define_insn "aarch64_float_extend_lo_<Vwide>"
+ [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+ (float_extend:<VWIDE>
+ (match_operand:VDF 1 "register_operand" "w")))]
"TARGET_SIMD"
- "fcvtl2\\t%0.2d, %1.4s"
+ "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
[(set_attr "type" "neon_fp_cvt_widen_s")]
)
@@ -6026,10 +6026,6 @@ vaddlvq_u32 (uint32x4_t a)
result; \
})
-/* vcvt_f32_f16 not supported */
-
-/* vcvt_high_f32_f16 not supported */
-
#define vcvt_n_f32_s32(a, b) \
__extension__ \
({ \
@@ -13420,6 +13416,12 @@ vcvt_high_f32_f64 (float32x2_t __a, float64x2_t __b)
/* vcvt (float -> double). */
+__extension__ static __inline float32x4_t __attribute__ ((__always_inline__))
+vcvt_f32_f16 (float16x4_t __a)
+{
+ return __builtin_aarch64_float_extend_lo_v4sf (__a);
+}
+
__extension__ static __inline float64x2_t __attribute__ ((__always_inline__))
vcvt_f64_f32 (float32x2_t __a)
{
@@ -13427,6 +13429,12 @@ vcvt_f64_f32 (float32x2_t __a)
return __builtin_aarch64_float_extend_lo_v2df (__a);
}
+__extension__ static __inline float32x4_t __attribute__ ((__always_inline__))
+vcvt_high_f32_f16 (float16x8_t __a)
+{
+ return __builtin_aarch64_vec_unpacks_hi_v8hf (__a);
+}
+
__extension__ static __inline float64x2_t __attribute__ ((__always_inline__))
vcvt_high_f64_f32 (float32x4_t __a)
{
@@ -91,6 +91,9 @@
;; Vector single Float modes.
(define_mode_iterator VDQSF [V2SF V4SF])
+;; Quad vector Float modes with half/single elements.
+(define_mode_iterator VQ_HSF [V8HF V4SF])
+
;; Modes suitable to use as the return type of a vcond expression.
(define_mode_iterator VDQF_COND [V2SF V2SI V4SF V4SI V2DF V2DI])
@@ -490,14 +493,18 @@
(V2SI "V2DI") (V16QI "V8HI")
(V8HI "V4SI") (V4SI "V2DI")
(HI "SI") (SI "DI")
+ (V8HF "V4SF") (V4SF "V2DF")
(V4HF "V4SF") (V2SF "V2DF")]
-
)
-;; Widened mode register suffixes for VD_BHSI/VQW.
+;; Widened modes of vector modes, lowercase
+(define_mode_attr Vwide [(V2SF "v2df") (V4HF "v4sf")])
+
+;; Widened mode register suffixes for VD_BHSI/VQW/VQ_HSF.
(define_mode_attr Vwtype [(V8QI "8h") (V4HI "4s")
(V2SI "2d") (V16QI "8h")
- (V8HI "4s") (V4SI "2d")])
+ (V8HI "4s") (V4SI "2d")
+ (V8HF "4s") (V4SF "2d")])
;; Widened mode register suffixes for VDW/VQW.
(define_mode_attr Vmwtype [(V8QI ".8h") (V4HI ".4s")
@@ -506,9 +513,10 @@
(V4HF ".4s") (V2SF ".2d")
(SI "") (HI "")])
-;; Lower part register suffixes for VQW.
+;; Lower part register suffixes for VQW/VQ_HSF.
(define_mode_attr Vhalftype [(V16QI "8b") (V8HI "4h")
- (V4SI "2s")])
+ (V4SI "2s") (V8HF "4h")
+ (V4SF "2s")])
;; Define corresponding core/FP element mode for each vector mode.
(define_mode_attr vw [(V8QI "w") (V16QI "w")
This adds the two remaining widening intrinsics, first adding patterns in aarch64-simd.md, then entries in aarch64-simd-builtins.def, and finally intrinsics in arm_neon.h . Note this changes the vector indices present in the RTL on bigendian for float vec_unpacks, to be the same as for integer vec_unpacks. This appears consistent with the usage of VEC_UNPACK_(FLOAT_)?EXPR in tree-vect-stmts.c, which uses a different EXPR for the same half of the vector depending on endianness. I was not able to construct a testcase where the RTL here mattered (i.e. where the RTL was constant-folded, but the tree had not been), but the correctness can be seen from a testcase: double d[4]; void bar (float *f) { for (int i = 0; i < 4; i++) d[i] = f[i]; } which used to produced as final RTL (-O3) (insn:TI 8 10 12 (set (reg:V2DF 33 v1 [orig:78 vect__9.19 ] [78]) (float_extend:V2DF (vec_select:V2SF (reg:V4SF 32 v0 [orig:77 MEM[(float *)f_6(D)] ] [77]) (parallel [ (const_int 2 [0x2]) (const_int 3 [0x3]) ])))) test.c:40 1274 {vec_unpacks_hi_v4sf} (expr_list:REG_EQUIV (mem/c:V2DF (reg/f:DI 0 x0 [79]) [2 MEM[(double *)&d]+0 S16 A64]) (nil))) (insn:TI 12 8 11 (set (reg:V2DF 32 v0 [orig:81 vect__9.19 ] [81]) (float_extend:V2DF (vec_select:V2SF (reg:V4SF 32 v0 [orig:77 MEM[(float *)f_6(D)] ] [77]) (parallel [ (const_int 0 [0]) (const_int 1 [0x1]) ])))) test.c:40 1272 {vec_unpacks_lo_v4sf} (expr_list:REG_EQUIV (mem/c:V2DF (plus:DI (reg/f:DI 0 x0 [79]) (const_int 16 [0x10])) [2 MEM[(double *)&d + 16B]+0 S16 A64]) (nil))) (insn:TI 11 12 15 (set (mem/c:V2DF (reg/f:DI 0 x0 [79]) [2 MEM[(double *)&d]+0 S16 A64]) (reg:V2DF 33 v1 [orig:78 vect__9.19 ] [78])) test.c:40 808 {*aarch64_simd_movv2df} (expr_list:REG_DEAD (reg:V2DF 33 v1 [orig:78 vect__9.19 ] [78]) (nil))) (insn:TI 15 11 22 (set (mem/c:V2DF (plus:DI (reg/f:DI 0 x0 [79]) (const_int 16 [0x10])) [2 MEM[(double *)&d + 16B]+0 S16 A64]) (reg:V2DF 32 v0 [orig:81 vect__9.19 ] [81])) test.c:40 808 {*aarch64_simd_movv2df} (expr_list:REG_DEAD (reg:V2DF 32 v0 [orig:81 vect__9.19 ] [81]) i.e. apparently storing vector elements 2 and 3 to the address of d, and elems 0+1 to address (d+16). Of course this was flipped back again to be correct at assembly time, but following this patch the RTL indices are also correct (elems 0+1 to address d, elems 2+3 to address d+16). gcc/ChangeLog: * config/aarch64/aarch64-simd.md (aarch64_simd_vec_unpacks_lo_<mode>, aarch64_simd_vec_unpacks_hi_<mode>): New insn. (vec_unpacks_lo_v4sf, vec_unpacks_hi_v4sf): Delete insn. (vec_unpacks_lo_<mode>, vec_unpacks_hi_<mode>): New expand. (aarch64_float_extend_lo_v2df): Rename to... (aarch64_float_extend_lo_<Vwide>): this, using VDF and so adding V4SF. * config/aarch64/aarch64-simd-builtins.def (vec_unpacks_hi): Add v8hf. (float_extend_lo): Add v4sf. * config/aarch64/arm_neon.h (vcvt_f32_f16, vcvt_high_f32_f16): New. * config/aarch64/iterators.md (VQ_HSF): New iterator. (VWIDE, Vwtype, Vhalftype): Add V8HF, V4SF. (Vwide): New mode_attr.