diff mbox series

[12/13] match-sat-alu.pd: Recognize complement masks for unsigned saturating truncation

Message ID 20260902145254.77832-14-ktkachov@nvidia.com
State New
Headers show
Series Saturating arithmetic matching improvements | expand

Commit Message

Kyrylo Tkachov Sept. 2, 2026, 2:52 p.m. UTC 
From: Kyrylo Tkachov <ktkachov@nvidia.com>

Unsigned saturating truncation can form its sign mask from ~X instead of -X.
This avoids negating the minimum signed value.

For negative X, ~X is nonnegative and its top-bit shift is zero.  For positive
X above the result range, ~X is negative and its top-bit shift is all ones.
Recognize this form.  Keep the unsigned range comparison because a signed
comparison does not clamp negative inputs.

With AArch64 -O2 -march=armv8-a -fopenmp-simd:

before:

complement_clip4:
	ldr	q31, [x1]
	movi	v30.4s, 0xff, msl 8
	not	v29.16b, v31.16b
	cmhi	v30.4s, v31.4s, v30.4s
	cmlt	v29.4s, v29.4s, #0
	bif	v29.16b, v31.16b, v30.16b
	xtn	v29.4h, v29.4s
	str	d29, [x0]
	ret

after:

complement_clip4:
	ldr	q31, [x1]
	movi	v30.4s, 0
	smax	v30.4s, v31.4s, v30.4s
	uqxtn	v30.4h, v30.4s
	str	d30, [x0]
	ret

Tests cover both widths, extrema, bounds, LTO, and a signed-comparison
counterexample.  The AArch64 test covers both valid forms.

Bootstrapped and tested on aarch64-none-linux-gnu.

Ok for trunk?

gcc/ChangeLog:

	* match-sat-alu.pd (unsigned_integer_narrow_clip): Add the complement
	form.

gcc/testsuite/ChangeLog:

	* gcc.dg/sat-trunc-signed-cmp-2.c: New test.
	* gcc.dg/vect/vect-sat-trunc-complement-1.c: Likewise.
	* gcc.target/aarch64/vect-sat-trunc-complement-1.c: Likewise.

Signed-off-by: Kyrylo Tkachov <ktkachov@nvidia.com>
---
 gcc/match-sat-alu.pd                          | 24 +++++++
 gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c | 42 +++++++++++
 .../gcc.dg/vect/vect-sat-trunc-complement-1.c | 70 +++++++++++++++++++
 .../aarch64/vect-sat-trunc-complement-1.c     | 30 ++++++++
 4 files changed, 166 insertions(+)
 create mode 100644 gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c
 create mode 100644 gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c
diff mbox series

Patch

diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index ec58d86486d..839f2ea52e8 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -334,6 +334,30 @@  along with GCC; see the file COPYING3.  If not see
 	 && cmp >= 0
 	 && expr_not_equal_to (@0, wi::min_value (itype_precision,
 						  SIGNED)))))))
+ (match (unsigned_integer_narrow_clip @0)
+  /* Use the sign of ~X for a source that avoids negating the minimum signed
+     value.  For a negative X, ~X is nonnegative and gives zero.  For a
+     positive X above the result range, ~X is negative and gives all ones.  */
+  (convert (cond^ (gt (nop_convert@3 @0) INTEGER_CST@1)
+	 (rshift:s (bit_not @0) INTEGER_CST@2)
+	 @0))
+  (if (!TYPE_UNSIGNED (TREE_TYPE (@0))
+       && TYPE_UNSIGNED (TREE_TYPE (@3)))
+   (with
+    {
+     unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0));
+     unsigned otype_precision = TYPE_PRECISION (type);
+     wide_int trunc_max = wi::mask (otype_precision, false, itype_precision);
+     wide_int int_cst_1 = wi::to_wide (@1, itype_precision);
+     wide_int int_cst_2 = wi::to_wide (@2, itype_precision);
+     wide_int shift_amount = wi::uhwi (itype_precision - 1,
+				  itype_precision);
+     int cmp = wi::cmp (int_cst_2, shift_amount,
+		       TYPE_SIGN (TREE_TYPE (@0)));
+    }
+    (if (otype_precision < itype_precision
+	 && wi::eq_p (trunc_max, int_cst_1)
+	 && cmp >= 0)))))
  (match (unsigned_integer_narrow_clip @0)
   /* SAT_US_TRUNC = (NT)MAX (MIN (X, NT_MAX), 0).  */
   (convert (integer_constant_clamp @0 @1 @2))
diff --git a/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c b/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c
new file mode 100644
index 00000000000..a06588ccedc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/sat-trunc-signed-cmp-2.c
@@ -0,0 +1,42 @@ 
+/* The complement spelling of the narrow-clip idiom is only a clamp when the
+   comparison that selects the saturated arm is unsigned.  With a signed
+   comparison a negative value takes the fall-through arm and keeps its low
+   bits, so it must not be turned into a saturating truncation.  */
+
+/* { dg-do run } */
+/* { dg-options "-O1 -ftree-vectorize -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+#define N 32
+
+static i32 a[N];
+static u16 r[N];
+
+__attribute__((noipa)) static void
+clip (u16 *__restrict d, const i32 *__restrict s, int n)
+{
+  for (int i = 0; i < n; i++)
+    {
+      i32 v = s[i];
+      d[i] = v > 65535 ? (~v) >> 31 : v;
+    }
+}
+
+int
+main (void)
+{
+  for (int i = 0; i < N; i++)
+    a[i] = -1;
+
+  clip (r, a, N);
+
+  for (int i = 0; i < N; i++)
+    if (r[i] != (u16) -1)
+      __builtin_abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c
new file mode 100644
index 00000000000..f7bb1656a5d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-complement-1.c
@@ -0,0 +1,70 @@ 
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */
+
+#include "tree-vect.h"
+
+typedef __UINT16_TYPE__ u16;
+typedef __UINT32_TYPE__ u32;
+typedef __INT32_TYPE__ i32;
+typedef __INT64_TYPE__ i64;
+
+#define N 96
+
+static i32 in32[N];
+static i64 in64[N];
+static u16 out16[N];
+static u32 out32[N];
+
+__attribute__((noipa)) static void
+clip_u16 (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = (x & ~65535) ? (~x) >> 31 : x;
+    }
+}
+
+__attribute__((noipa)) static void
+clip_u32 (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i64 x = in[i];
+      out[i] = (x & ~(i64) 0xffffffff) ? (~x) >> 63 : x;
+    }
+}
+
+int
+main (void)
+{
+  static const i32 values32[6] =
+    { (-2147483647 - 1), -1, 0, 65535, 65536, 2147483647 };
+  static const u16 expected16[6] =
+    { 0, 0, 0, 65535, 65535, 65535 };
+  static const i64 values64[6] =
+    { (-9223372036854775807LL - 1), -1, 0, 4294967295LL,
+      4294967296LL, 9223372036854775807LL };
+  static const u32 expected32[6] =
+    { 0, 0, 0, (u32) -1, (u32) -1, (u32) -1 };
+
+  check_vect ();
+
+  for (int i = 0; i < N; ++i)
+    {
+      in32[i] = values32[i % 6];
+      in64[i] = values64[i % 6];
+    }
+
+  clip_u16 (out16, in32, N);
+  clip_u32 (out32, in64, N);
+
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (out16[i] != expected16[i % 6]
+	|| out32[i] != expected32[i % 6])
+      __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c
new file mode 100644
index 00000000000..9c18ee063e0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-complement-1.c
@@ -0,0 +1,30 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __UINT32_TYPE__ u32;
+typedef __INT32_TYPE__ i32;
+typedef __INT64_TYPE__ i64;
+
+void
+clip_u16 (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = (x & ~65535) ? (~x) >> 31 : x;
+    }
+}
+
+void
+clip_u32 (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i64 x = in[i];
+      out[i] = (x & ~(i64) 4294967295LL) ? (~x) >> 63 : x;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 2 "vect" } } */