@@ -6760,6 +6760,12 @@ aarch64_print_operand (FILE *f, rtx x, int code)
break;
case 'H':
+ if (x == const0_rtx)
+ {
+ asm_fprintf (f, "xzr");
+ break;
+ }
+
if (!REG_P (x) || !GP_REGNUM_P (REGNO (x) + 1))
{
output_operand_lossage ("invalid operand for '%%%c'", code);
@@ -17638,6 +17644,45 @@ aarch64_select_early_remat_modes (sbitmap modes)
}
}
+/* Override the default target speculation_safe_value. */
+static rtx
+aarch64_speculation_safe_value (machine_mode mode,
+ rtx result, rtx val, rtx failval)
+{
+ /* Maybe we should warn if falling back to hard barriers. They are
+ likely to be noticably more expensive than the alternative below. */
+ if (!aarch64_track_speculation)
+ return default_speculation_safe_value (mode, result, val, failval);
+
+ if (!REG_P (val))
+ val = copy_to_mode_reg (mode, val);
+
+ if (!aarch64_reg_or_zero (failval, mode))
+ failval = copy_to_mode_reg (mode, failval);
+
+ switch (mode)
+ {
+ case E_QImode:
+ emit_insn (gen_despeculate_copyqi (result, val, failval));
+ break;
+ case E_HImode:
+ emit_insn (gen_despeculate_copyhi (result, val, failval));
+ break;
+ case E_SImode:
+ emit_insn (gen_despeculate_copysi (result, val, failval));
+ break;
+ case E_DImode:
+ emit_insn (gen_despeculate_copydi (result, val, failval));
+ break;
+ case E_TImode:
+ emit_insn (gen_despeculate_copyti (result, val, failval));
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ return result;
+}
+
/* Target-specific selftests. */
#if CHECKING_P
@@ -18110,6 +18155,9 @@ aarch64_libgcc_floating_mode_supported_p
#undef TARGET_SELECT_EARLY_REMAT_MODES
#define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
+#undef TARGET_SPECULATION_SAFE_VALUE
+#define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
+
#if CHECKING_P
#undef TARGET_RUN_TARGET_SELFTESTS
#define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
@@ -6129,6 +6129,103 @@ (define_insn "speculation_barrier"
(set_attr "speculation_barrier" "true")]
)
+;; Support for __builtin_speculation_safe_value when we have speculation
+;; tracking enabled. Use the speculation tracker to decide whether to
+;; copy operand 1 to the target, or to copy the fail value (operand 2).
+(define_expand "despeculate_copy<ALLI_TI:mode>"
+ [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI_TI
+ [(match_operand:ALLI_TI 1 "register_operand" "r")
+ (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "
+ {
+ if (operands[2] == const0_rtx)
+ {
+ rtx tracker;
+ if (<MODE>mode == TImode)
+ tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ else
+ tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
+
+ emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
+ tracker));
+ DONE;
+ }
+ }
+ "
+)
+
+;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
+;; encoding for CSDB, but will work in older versions of the assembler.
+(define_insn "*despeculate_copy<ALLI:mode>_insn"
+ [(set (match_operand:ALLI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI
+ [(match_operand:ALLI 1 "register_operand" "r")
+ (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ {
+ operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
+ operands);
+ return "";
+ }
+ [(set_attr "length" "12")
+ (set_attr "type" "block")
+ (set_attr "speculation_barrier" "true")]
+)
+
+;; Pattern to match despeculate_copyti
+(define_insn "*despeculate_copyti_insn"
+ [(set (match_operand:TI 0 "register_operand" "=r")
+ (unspec_volatile:TI
+ [(match_operand:TI 1 "register_operand" "r")
+ (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
+ (use (reg:DI SPECULATION_TRACKER_REGNUM))
+ (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
+ ""
+ {
+ operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
+ output_asm_insn
+ ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
+ operands);
+ return "";
+ }
+ [(set_attr "length" "16")
+ (set_attr "type" "block")
+ (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simple<ALLI:mode>"
+ [(set (match_operand:ALLI 0 "register_operand" "=r")
+ (unspec_volatile:ALLI
+ [(match_operand:ALLI 1 "register_operand" "r")
+ (use (match_operand:ALLI 2 "register_operand" ""))]
+ UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
+ [(set_attr "type" "block")
+ (set_attr "length" "8")
+ (set_attr "speculation_barrier" "true")]
+)
+
+(define_insn "despeculate_simpleti"
+ [(set (match_operand:TI 0 "register_operand" "=r")
+ (unspec_volatile:TI
+ [(match_operand:TI 1 "register_operand" "r")
+ (use (match_operand:DI 2 "register_operand" ""))]
+ UNSPECV_SPECULATION_BARRIER))]
+ ""
+ "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
+ [(set_attr "type" "block")
+ (set_attr "length" "12")
+ (set_attr "speculation_barrier" "true")]
+)
+
;; AdvSIMD Stuff
(include "aarch64-simd.md")
@@ -35,6 +35,9 @@ (define_mode_iterator SHORT [QI HI])
;; Iterator for all integer modes (up to 64-bit)
(define_mode_iterator ALLI [QI HI SI DI])
+;; Iterator for all integer modes (up to 128-bit)
+(define_mode_iterator ALLI_TI [QI HI SI DI TI])
+
;; Iterator for all integer modes that can be extended (up to 64-bit)
(define_mode_iterator ALLX [QI HI SI])