diff mbox series

[10/13] match-sat-alu.pd: Recognize unsigned constant-clamp truncation

Message ID 20260902145254.77832-12-ktkachov@nvidia.com
State New
Headers show
Series Saturating arithmetic matching improvements | expand

Commit Message

Kyrylo Tkachov Sept. 2, 2026, 2:52 p.m. UTC 
From: Kyrylo Tkachov <ktkachov@nvidia.com>

A signed value clamped to the range of a narrower unsigned type is unsigned
saturating truncation.  Recognize nested MIN and MAX operations in either
order.

Require exact zero and maximum bounds.  Require the outer clamp to have one
use so that the replacement removes it.  The inner clamp can have other uses.

With AArch64 -O2 -march=armv8-a -fopenmp-simd:

before:

clamp_u16_4:
	ldr	q0, [x1]
	movi	v30.4s, 0xff, msl 8
	movi	v31.4s, 0
	smin	v30.4s, v0.4s, v30.4s
	smax	v30.4s, v30.4s, v31.4s
	xtn	v30.4h, v30.4s
	str	d30, [x0]
	ret

after:

clamp_u16_4:
	ldr	q31, [x1]
	movi	v30.4s, 0
	smax	v30.4s, v31.4s, v30.4s
	uqxtn	v30.4h, v30.4s
	str	d30, [x0]
	ret

Runtime tests cover both clamp orders, two widths, and LTO.  Target tests
cover valid bounds, invalid bounds, and extra uses of each clamp.

Bootstrapped and tested on aarch64-none-linux-gnu.

Ok for trunk?

gcc/ChangeLog:

	* match-sat-alu.pd (integer_constant_clamp): New match.
	(unsigned_integer_narrow_clip): Add the constant-clamp form.

gcc/testsuite/ChangeLog:

	* gcc.dg/vect/vect-sat-trunc-clamp-1.c: New test.
	* gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c: Likewise.
	* gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c: Likewise.
	* gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c: Likewise.
	* gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c: Likewise.

Signed-off-by: Kyrylo Tkachov <ktkachov@nvidia.com>
---
 gcc/match-sat-alu.pd                          |  27 +++-
 .../gcc.dg/vect/vect-sat-trunc-clamp-1.c      | 121 ++++++++++++++++++
 .../vect-sat-trunc-clamp-inner-use-1.c        |  22 ++++
 .../vect-sat-trunc-clamp-multi-use-1.c        |  21 +++
 .../aarch64/vect-sat-trunc-clamp-u-1.c        |   7 +
 .../vect-sat-trunc-clamp-u-negative-1.c       |  30 +++++
 6 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
diff mbox series

Patch

diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index b141c87be03..ede949a34fe 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -23,6 +23,16 @@  along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
 
+/* Clamp @0 between the constant bounds @1 and @2.  The source can put the
+   MIN or the MAX first.  The outer operation must be single use so that
+   replacing its conversion also removes the complete clamp.  */
+(match (integer_constant_clamp @0 @1 @2)
+ (max (min @0 INTEGER_CST@2) INTEGER_CST@1)
+ (if (single_use (t))))
+(match (integer_constant_clamp @0 @1 @2)
+ (min (max @0 INTEGER_CST@1) INTEGER_CST@2)
+ (if (single_use (t))))
+
 /* Saturation add for unsigned integer.  */
 (if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
  (match (usadd_overflow_mask @0 @1)
@@ -323,7 +333,22 @@  along with GCC; see the file COPYING3.  If not see
 	 && wi::eq_p (trunc_max, int_cst_1)
 	 && cmp >= 0
 	 && expr_not_equal_to (@0, wi::min_value (itype_precision,
-						  SIGNED))))))))
+						  SIGNED)))))))
+ (match (unsigned_integer_narrow_clip @0)
+  /* SAT_US_TRUNC = (NT)MAX (MIN (X, NT_MAX), 0).  */
+  (convert (integer_constant_clamp @0 @1 @2))
+  (if (!TYPE_UNSIGNED (TREE_TYPE (@0)))
+   (with
+    {
+     unsigned itype_precision = TYPE_PRECISION (TREE_TYPE (@0));
+     unsigned otype_precision = TYPE_PRECISION (type);
+     wide_int trunc_max = wi::mask (otype_precision, false, itype_precision);
+     wide_int lo_cst = wi::to_wide (@1, itype_precision);
+     wide_int hi_cst = wi::to_wide (@2, itype_precision);
+    }
+    (if (otype_precision < itype_precision
+	 && wi::eq_p (lo_cst, 0)
+	 && wi::eq_p (hi_cst, trunc_max)))))))
 
 /* Saturation truncate for unsigned integer.  */
 (if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
new file mode 100644
index 00000000000..06a4428d0d2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-clamp-1.c
@@ -0,0 +1,121 @@ 
+/* { dg-do run } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */
+
+#include "tree-vect.h"
+
+typedef __UINT16_TYPE__ u16;
+typedef __UINT32_TYPE__ u32;
+typedef __UINT64_TYPE__ u64;
+typedef __INT32_TYPE__ i32;
+typedef __INT64_TYPE__ i64;
+
+#define N 259
+
+static i32 in32[N];
+static i64 in64[N];
+static u16 out16[N];
+static u32 out32[N];
+
+__attribute__((noipa)) static void
+clamp_u16_lo (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = x < 0 ? 0 : (x > 65535 ? 65535 : x);
+    }
+}
+
+__attribute__((noipa)) static void
+clamp_u16_hi (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = x > 65535 ? 65535 : (x < 0 ? 0 : x);
+    }
+}
+
+__attribute__((noipa)) static void
+clamp_u32_lo (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i64 x = in[i];
+      out[i] = x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x);
+    }
+}
+
+__attribute__((noipa)) static void
+clamp_u32_hi (u32 *__restrict out, const i64 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i64 x = in[i];
+      out[i] = x > 4294967295LL ? 4294967295LL : (x < 0 ? 0 : x);
+    }
+}
+
+static u16
+ref_u16 (i32 x)
+{
+  return x < 0 ? 0 : (x > 65535 ? 65535 : x);
+}
+
+static u32
+ref_u32 (i64 x)
+{
+  return x < 0 ? 0 : (x > 4294967295LL ? 4294967295LL : x);
+}
+
+static void
+check_u16 (void)
+{
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (out16[i] != ref_u16 (in32[i]))
+      __builtin_abort ();
+}
+
+static void
+check_u32 (void)
+{
+#pragma GCC novector
+  for (int i = 0; i < N; ++i)
+    if (out32[i] != ref_u32 (in64[i]))
+      __builtin_abort ();
+}
+
+int
+main (void)
+{
+  check_vect ();
+
+  for (int i = 0; i < N; ++i)
+    {
+      in32[i] = (i32) ((u32) i * 2654435761U + 1013904223U);
+      in64[i] = (i64) ((u64) i * 11400714819323198485ULL
+			     + 13787848793156543929ULL);
+    }
+
+  in32[0] = -1;
+  in32[1] = 0;
+  in32[2] = 65535;
+  in32[3] = 65536;
+  in64[0] = -1;
+  in64[1] = 0;
+  in64[2] = 4294967295LL;
+  in64[3] = 4294967296LL;
+
+  clamp_u16_lo (out16, in32, N);
+  check_u16 ();
+  clamp_u16_hi (out16, in32, N);
+  check_u16 ();
+  clamp_u32_lo (out32, in64, N);
+  check_u32 ();
+  clamp_u32_hi (out32, in64, N);
+  check_u32 ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
new file mode 100644
index 00000000000..b93bc56188e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-inner-use-1.c
@@ -0,0 +1,22 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+void
+clip (u16 *__restrict out, i32 *__restrict copy,
+      const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      i32 inner = x > 65535 ? 65535 : x;
+      i32 outer = inner < 0 ? 0 : inner;
+      out[i] = outer;
+      copy[i] = inner;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
new file mode 100644
index 00000000000..0b82d40ff2f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-multi-use-1.c
@@ -0,0 +1,21 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+void
+clip (u16 *__restrict out, i32 *__restrict copy,
+      const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      i32 clamped = x < 0 ? 0 : (x > 65535 ? 65535 : x);
+      out[i] = clamped;
+      copy[i] = clamped;
+    }
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
new file mode 100644
index 00000000000..f159a8f4dac
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-1.c
@@ -0,0 +1,7 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+#include "../../gcc.dg/vect/vect-sat-trunc-clamp-1.c"
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 4 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
new file mode 100644
index 00000000000..aa283f176ad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-clamp-u-negative-1.c
@@ -0,0 +1,30 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __UINT16_TYPE__ u16;
+typedef __INT32_TYPE__ i32;
+
+__attribute__((noipa))
+void
+bad_low (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = x < 1 ? 1 : (x > 65535 ? 65535 : x);
+    }
+}
+
+__attribute__((noipa))
+void
+bad_high (u16 *__restrict out, const i32 *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      i32 x = in[i];
+      out[i] = x > 65534 ? 65534 : (x < 0 ? 0 : x);
+    }
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */