@@ -1517,10 +1517,33 @@ emit_set_insn (rtx x, rtx y)
rtx
aarch64_gen_compare_reg (RTX_CODE code, rtx x, rtx y)
{
- machine_mode mode = SELECT_CC_MODE (code, x, y);
- rtx cc_reg = gen_rtx_REG (mode, CC_REGNUM);
+ machine_mode cmp_mode = GET_MODE (x);
+ machine_mode cc_mode;
+ rtx cc_reg;
- emit_set_insn (cc_reg, gen_rtx_COMPARE (mode, x, y));
+ if (cmp_mode == TImode)
+ {
+ gcc_assert (code == NE);
+
+ cc_mode = CCmode;
+ cc_reg = gen_rtx_REG (cc_mode, CC_REGNUM);
+
+ rtx x_lo = operand_subword (x, 0, 0, TImode);
+ rtx y_lo = operand_subword (y, 0, 0, TImode);
+ emit_set_insn (cc_reg, gen_rtx_COMPARE (cc_mode, x_lo, y_lo));
+
+ rtx x_hi = operand_subword (x, 1, 0, TImode);
+ rtx y_hi = operand_subword (y, 1, 0, TImode);
+ emit_insn (gen_ccmpdi (cc_reg, cc_reg, x_hi, y_hi,
+ gen_rtx_EQ (cc_mode, cc_reg, const0_rtx),
+ GEN_INT (AARCH64_EQ)));
+ }
+ else
+ {
+ cc_mode = SELECT_CC_MODE (code, x, y);
+ cc_reg = gen_rtx_REG (cc_mode, CC_REGNUM);
+ emit_set_insn (cc_reg, gen_rtx_COMPARE (cc_mode, x, y));
+ }
return cc_reg;
}
@@ -14145,40 +14168,54 @@ static void
aarch64_emit_load_exclusive (machine_mode mode, rtx rval,
rtx mem, rtx model_rtx)
{
- rtx (*gen) (rtx, rtx, rtx);
-
- switch (mode)
+ if (mode == TImode)
+ emit_insn (gen_aarch64_load_exclusive_pair (gen_lowpart (DImode, rval),
+ gen_highpart (DImode, rval),
+ mem, model_rtx));
+ else
{
- case E_QImode: gen = gen_aarch64_load_exclusiveqi; break;
- case E_HImode: gen = gen_aarch64_load_exclusivehi; break;
- case E_SImode: gen = gen_aarch64_load_exclusivesi; break;
- case E_DImode: gen = gen_aarch64_load_exclusivedi; break;
- default:
- gcc_unreachable ();
- }
+ rtx (*gen) (rtx, rtx, rtx);
+
+ switch (mode)
+ {
+ case E_QImode: gen = gen_aarch64_load_exclusiveqi; break;
+ case E_HImode: gen = gen_aarch64_load_exclusivehi; break;
+ case E_SImode: gen = gen_aarch64_load_exclusivesi; break;
+ case E_DImode: gen = gen_aarch64_load_exclusivedi; break;
+ default:
+ gcc_unreachable ();
+ }
- emit_insn (gen (rval, mem, model_rtx));
+ emit_insn (gen (rval, mem, model_rtx));
+ }
}
/* Emit store exclusive. */
static void
aarch64_emit_store_exclusive (machine_mode mode, rtx bval,
- rtx rval, rtx mem, rtx model_rtx)
+ rtx mem, rtx rval, rtx model_rtx)
{
- rtx (*gen) (rtx, rtx, rtx, rtx);
-
- switch (mode)
+ if (mode == TImode)
+ emit_insn (gen_aarch64_store_exclusive_pair
+ (bval, mem, operand_subword (rval, 0, 0, TImode),
+ operand_subword (rval, 1, 0, TImode), model_rtx));
+ else
{
- case E_QImode: gen = gen_aarch64_store_exclusiveqi; break;
- case E_HImode: gen = gen_aarch64_store_exclusivehi; break;
- case E_SImode: gen = gen_aarch64_store_exclusivesi; break;
- case E_DImode: gen = gen_aarch64_store_exclusivedi; break;
- default:
- gcc_unreachable ();
- }
+ rtx (*gen) (rtx, rtx, rtx, rtx);
+
+ switch (mode)
+ {
+ case E_QImode: gen = gen_aarch64_store_exclusiveqi; break;
+ case E_HImode: gen = gen_aarch64_store_exclusivehi; break;
+ case E_SImode: gen = gen_aarch64_store_exclusivesi; break;
+ case E_DImode: gen = gen_aarch64_store_exclusivedi; break;
+ default:
+ gcc_unreachable ();
+ }
- emit_insn (gen (bval, rval, mem, model_rtx));
+ emit_insn (gen (bval, mem, rval, model_rtx));
+ }
}
/* Mark the previous jump instruction as unlikely. */
@@ -14197,16 +14234,6 @@ aarch64_expand_compare_and_swap (rtx operands[])
{
rtx bval, rval, mem, oldval, newval, is_weak, mod_s, mod_f, x, cc_reg;
machine_mode mode, r_mode;
- typedef rtx (*gen_atomic_cas_fn) (rtx, rtx, rtx, rtx);
- int idx;
- gen_atomic_cas_fn atomic_gen;
- const gen_atomic_cas_fn atomic_cas[] =
- {
- gen_aarch64_compare_and_swapqi_lse,
- gen_aarch64_compare_and_swaphi_lse,
- gen_aarch64_compare_and_swapsi_lse,
- gen_aarch64_compare_and_swapdi_lse
- };
bval = operands[0];
rval = operands[1];
@@ -14232,18 +14259,29 @@ aarch64_expand_compare_and_swap (rtx operands[])
rval = gen_reg_rtx (r_mode);
}
- switch (mode)
- {
- case E_QImode: idx = 0; break;
- case E_HImode: idx = 1; break;
- case E_SImode: idx = 2; break;
- case E_DImode: idx = 3; break;
- default:
- gcc_unreachable ();
- }
if (TARGET_LSE)
{
- atomic_gen = atomic_cas[idx];
+ insn_code code;
+ switch (mode)
+ {
+ case E_QImode:
+ code = CODE_FOR_aarch64_compare_and_swapqi_lse;
+ break;
+ case E_HImode:
+ code = CODE_FOR_aarch64_compare_and_swaphi_lse;
+ break;
+ case E_SImode:
+ code = CODE_FOR_aarch64_compare_and_swapsi_lse;
+ break;
+ case E_DImode:
+ code = CODE_FOR_aarch64_compare_and_swapdi_lse;
+ break;
+ case E_TImode:
+ code = CODE_FOR_aarch64_compare_and_swapti_lse;
+ break;
+ default:
+ gcc_unreachable ();
+ }
/* The CAS insn requires oldval and rval overlap, but we need to
have a copy of oldval saved across the operation to tell if
the operation is successful. */
@@ -14252,7 +14290,7 @@ aarch64_expand_compare_and_swap (rtx operands[])
else
emit_move_insn (rval, gen_lowpart (r_mode, oldval));
- emit_insn (atomic_gen (rval, mem, newval, mod_s));
+ emit_insn (GEN_FCN (code) (rval, mem, newval, mod_s));
cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
}
@@ -14274,6 +14312,9 @@ aarch64_expand_compare_and_swap (rtx operands[])
case E_DImode:
code = CODE_FOR_aarch64_compare_and_swapdi;
break;
+ case E_TImode:
+ code = CODE_FOR_aarch64_compare_and_swapti;
+ break;
default:
gcc_unreachable ();
}
@@ -14345,7 +14386,7 @@ aarch64_split_compare_and_swap (rtx operands[])
CBNZ scratch, .label1
.label2:
CMP rval, 0. */
- bool strong_zero_p = !is_weak && oldval == const0_rtx;
+ bool strong_zero_p = !is_weak && oldval == const0_rtx && mode != TImode;
label1 = NULL;
if (!is_weak)
@@ -22,10 +22,10 @@
(define_expand "atomic_compare_and_swap<mode>"
[(match_operand:SI 0 "register_operand" "") ;; bool out
- (match_operand:ALLI 1 "register_operand" "") ;; val out
- (match_operand:ALLI 2 "aarch64_sync_memory_operand" "") ;; memory
- (match_operand:ALLI 3 "nonmemory_operand" "") ;; expected
- (match_operand:ALLI 4 "aarch64_reg_or_zero" "") ;; desired
+ (match_operand:ALLI_TI 1 "register_operand" "") ;; val out
+ (match_operand:ALLI_TI 2 "aarch64_sync_memory_operand" "") ;; memory
+ (match_operand:ALLI_TI 3 "nonmemory_operand" "") ;; expected
+ (match_operand:ALLI_TI 4 "aarch64_reg_or_zero" "") ;; desired
(match_operand:SI 5 "const_int_operand") ;; is_weak
(match_operand:SI 6 "const_int_operand") ;; mod_s
(match_operand:SI 7 "const_int_operand")] ;; mod_f
@@ -88,6 +88,30 @@
}
)
+(define_insn_and_split "aarch64_compare_and_swapti"
+ [(set (reg:CC CC_REGNUM) ;; bool out
+ (unspec_volatile:CC [(const_int 0)] UNSPECV_ATOMIC_CMPSW))
+ (set (match_operand:TI 0 "register_operand" "=&r") ;; val out
+ (match_operand:TI 1 "aarch64_sync_memory_operand" "+Q")) ;; memory
+ (set (match_dup 1)
+ (unspec_volatile:TI
+ [(match_operand:TI 2 "aarch64_reg_or_zero" "rZ") ;; expect
+ (match_operand:TI 3 "aarch64_reg_or_zero" "rZ") ;; desired
+ (match_operand:SI 4 "const_int_operand") ;; is_weak
+ (match_operand:SI 5 "const_int_operand") ;; mod_s
+ (match_operand:SI 6 "const_int_operand")] ;; mod_f
+ UNSPECV_ATOMIC_CMPSW))
+ (clobber (match_scratch:SI 7 "=&r"))]
+ ""
+ "#"
+ "&& reload_completed"
+ [(const_int 0)]
+ {
+ aarch64_split_compare_and_swap (operands);
+ DONE;
+ }
+)
+
(define_insn "aarch64_compare_and_swap<mode>_lse"
[(set (match_operand:SI 0 "register_operand" "+r") ;; val out
(zero_extend:SI
@@ -133,6 +157,28 @@
return "casal<atomic_sfx>\t%<w>0, %<w>2, %1";
})
+(define_insn "aarch64_compare_and_swapti_lse"
+ [(set (match_operand:TI 0 "register_operand" "+r") ;; val out
+ (match_operand:TI 1 "aarch64_sync_memory_operand" "+Q")) ;; memory
+ (set (match_dup 1)
+ (unspec_volatile:TI
+ [(match_dup 0) ;; expect
+ (match_operand:TI 2 "register_operand" "r") ;; desired
+ (match_operand:SI 3 "const_int_operand")] ;; mod_s
+ UNSPECV_ATOMIC_CMPSW))]
+ "TARGET_LSE"
+{
+ enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ if (is_mm_relaxed (model))
+ return "casp\t%0, %R0, %2, %R2, %1";
+ else if (is_mm_acquire (model) || is_mm_consume (model))
+ return "caspa\t%0, %R0, %2, %R2, %1";
+ else if (is_mm_release (model))
+ return "caspl\t%0, %R0, %2, %R2, %1";
+ else
+ return "caspal\t%0, %R0, %2, %R2, %1";
+})
+
(define_expand "atomic_exchange<mode>"
[(match_operand:ALLI 0 "register_operand" "")
(match_operand:ALLI 1 "aarch64_sync_memory_operand" "")
@@ -578,6 +624,24 @@
}
)
+(define_insn "aarch64_load_exclusive_pair"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (unspec_volatile:DI
+ [(match_operand:TI 2 "aarch64_sync_memory_operand" "Q")
+ (match_operand:SI 3 "const_int_operand")]
+ UNSPECV_LX))
+ (set (match_operand:DI 1 "register_operand" "=r")
+ (unspec_volatile:DI [(match_dup 2) (match_dup 3)] UNSPECV_LX))]
+ ""
+ {
+ enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ if (is_mm_relaxed (model) || is_mm_consume (model) || is_mm_release (model))
+ return "ldxp\t%0, %1, %2";
+ else
+ return "ldaxp\t%0, %1, %2";
+ }
+)
+
(define_insn "aarch64_store_exclusive<mode>"
[(set (match_operand:SI 0 "register_operand" "=&r")
(unspec_volatile:SI [(const_int 0)] UNSPECV_SX))
@@ -596,6 +660,25 @@
}
)
+(define_insn "aarch64_store_exclusive_pair"
+ [(set (match_operand:SI 0 "register_operand" "=&r")
+ (unspec_volatile:SI [(const_int 0)] UNSPECV_SX))
+ (set (match_operand:TI 1 "aarch64_sync_memory_operand" "=Q")
+ (unspec_volatile:TI
+ [(match_operand:DI 2 "aarch64_reg_or_zero" "rZ")
+ (match_operand:DI 3 "aarch64_reg_or_zero" "rZ")
+ (match_operand:SI 4 "const_int_operand")]
+ UNSPECV_SX))]
+ ""
+ {
+ enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ if (is_mm_relaxed (model) || is_mm_consume (model) || is_mm_acquire (model))
+ return "stxp\t%w0, %x2, %x3, %1";
+ else
+ return "stlxp\t%w0, %x2, %x3, %1";
+ }
+)
+
(define_expand "mem_thread_fence"
[(match_operand:SI 0 "const_int_operand" "")]
""
@@ -35,6 +35,9 @@
;; Iterator for all integer modes (up to 64-bit)
(define_mode_iterator ALLI [QI HI SI DI])
+;; Iterator for all integer modes (up to 128-bit)
+(define_mode_iterator ALLI_TI [QI HI SI DI TI])
+
;; Iterator for all integer modes that can be extended (up to 64-bit)
(define_mode_iterator ALLX [QI HI SI])