@@ -30,12 +30,15 @@ extern void hwasan_increment_tag ();
extern rtx hwasan_with_tag (rtx, poly_int64);
extern void hwasan_tag_init ();
extern rtx hwasan_create_untagged_base (rtx);
+extern rtx hwasan_extract_tag (rtx tagged_pointer);
extern rtx hwasan_base ();
extern void hwasan_emit_prologue (rtx *, rtx *, poly_int64 *, uint8_t *, size_t);
extern rtx_insn *hwasan_emit_uncolour_frame (rtx, rtx, rtx_insn *);
extern bool hwasan_expand_check_ifn (gimple_stmt_iterator *, bool);
+extern bool hwasan_expand_mark_ifn (gimple_stmt_iterator *);
extern bool memory_tagging_p (void);
extern bool gate_hwasan (void);
+extern bool hardware_memory_tagging_p (void);
extern rtx_insn *asan_emit_stack_protection (rtx, rtx, unsigned int,
HOST_WIDE_INT *, tree *, int);
extern rtx_insn *asan_emit_allocas_unpoison (rtx, rtx, rtx_insn *);
@@ -142,6 +145,13 @@ enum asan_mark_flags
#undef DEF
};
+enum hwasan_mark_flags
+{
+#define DEF(X) HWASAN_MARK_##X
+ IFN_ASAN_MARK_FLAGS
+#undef DEF
+};
+
/* Return true if STMT is ASAN_MARK with FLAG as first argument. */
extern bool asan_mark_p (gimple *stmt, enum asan_mark_flags flag);
@@ -3375,6 +3375,22 @@ asan_expand_mark_ifn (gimple_stmt_iterator *iter)
unsigned HOST_WIDE_INT size_in_bytes = tree_to_shwi (len);
gcc_assert (size_in_bytes);
+ if (memory_tagging_p ())
+ {
+ /* Here we swap the ASAN_MARK for HWASAN_MARK.
+ This is because we are using the (possibly temporary) approach to
+ always emit ASAN_MARK in TREE until here.
+ That approach means we don't yet have to duplicate all the special
+ cases for ASAN_MARK and ASAN_POISON with the exact same handling but
+ called HWASAN_MARK etc. */
+ gimple *hw_poison_call
+ = gimple_build_call_internal (IFN_HWASAN_MARK, 3,
+ gimple_call_arg (g, 0),
+ base, len);
+ gsi_replace (iter, hw_poison_call, false);
+ return false;
+ }
+
g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
NOP_EXPR, base);
gimple_set_location (g, loc);
@@ -3673,6 +3689,7 @@ asan_expand_poison_ifn (gimple_stmt_iterator *iter,
bool *need_commit_edge_insert,
hash_map<tree, tree> &shadow_vars_mapping)
{
+ gcc_assert (! memory_tagging_p ());
gimple *g = gsi_stmt (*iter);
tree poisoned_var = gimple_call_lhs (g);
if (!poisoned_var || has_zero_uses (poisoned_var))
@@ -3968,6 +3985,7 @@ hwasan_emit_prologue (rtx *bases,
}
else
{
+ gcc_assert (known_ge (end, start));
top = end;
bot = start;
}
@@ -4037,6 +4055,12 @@ hwasan_emit_uncolour_frame (rtx dynamic, rtx vars, rtx_insn *before)
do_pending_stack_adjust ();
rtx_insn *insns = get_insns ();
end_sequence ();
+
+ /* Clear the hash_map recording which variables are handled by HWASAN_MARK.
+ The only use in HWASAN is to decide which variables need to be coloured in
+ the prologue and which don't. */
+ delete asan_handled_variables;
+ asan_handled_variables = NULL;
return insns;
}
@@ -4052,6 +4076,19 @@ hwasan_create_untagged_base (rtx orig_base)
return untagged_base;
}
+rtx
+hwasan_extract_tag (rtx tagged_pointer)
+{
+ rtx tag = expand_simple_binop (Pmode,
+ LSHIFTRT,
+ tagged_pointer,
+ HWASAN_SHIFT_RTX,
+ NULL_RTX,
+ /* unsignedp = */0,
+ OPTAB_DIRECT);
+ return gen_lowpart (QImode, tag);
+}
+
/* Needs to be GTY(()), because cgraph_build_static_cdtor may
invoke ggc_collect. */
static GTY(()) tree hwasan_ctor_statements;
@@ -4166,10 +4203,25 @@ hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
}
bool
+hwasan_expand_mark_ifn (gimple_stmt_iterator *)
+{
+ /* HWASAN_MARK should only ever be available after the sanopt pass.
+ It might be nicer to have it everywhere in the future, so I"m leaving this
+ function and the declaration in asan.h around in case that's requested
+ upstream. */
+ gcc_unreachable ();
+}
+
+bool
gate_hwasan ()
{
return memory_tagging_p ();
}
+bool
+hardware_memory_tagging_p ()
+{
+ return memory_tagging_p () && HARDWARE_MEMORY_TAGGING;
+}
namespace {
@@ -739,6 +739,7 @@ dump_gimple_call_args (pretty_printer *buffer, gcall *gs, dump_flags_t flags)
limit = ARRAY_SIZE (reduction_args);
break;
+ case IFN_HWASAN_MARK:
case IFN_ASAN_MARK:
#define DEF(X) #X
static const char *const asan_mark_args[] = {IFN_ASAN_MARK_FLAGS};
@@ -1202,8 +1202,10 @@ asan_poison_variable (tree decl, bool poison, gimple_stmt_iterator *it,
/* It's necessary to have all stack variables aligned to ASAN granularity
bytes. */
- if (DECL_ALIGN_UNIT (decl) <= ASAN_SHADOW_GRANULARITY)
- SET_DECL_ALIGN (decl, BITS_PER_UNIT * ASAN_SHADOW_GRANULARITY);
+ unsigned shadow_granularity =
+ memory_tagging_p () ? HWASAN_TAG_GRANULE_SIZE : ASAN_SHADOW_GRANULARITY;
+ if (DECL_ALIGN_UNIT (decl) <= shadow_granularity)
+ SET_DECL_ALIGN (decl, BITS_PER_UNIT * shadow_granularity);
HOST_WIDE_INT flags = poison ? ASAN_MARK_POISON : ASAN_MARK_UNPOISON;
@@ -13816,7 +13818,7 @@ gimplify_function_tree (tree fndecl)
&& !needs_to_live_in_memory (ret))
DECL_GIMPLE_REG_P (ret) = 1;
- if (asan_sanitize_use_after_scope () && sanitize_flags_p (SANITIZE_ADDRESS))
+ if (asan_sanitize_use_after_scope ())
asan_poisoned_variables = new hash_set<tree> ();
bind = gimplify_body (fndecl, true);
if (asan_poisoned_variables)
@@ -471,11 +471,7 @@ expand_HWASAN_CHOOSE_COLOUR (internal_fn, gcall *gc)
machine_mode mode = GET_MODE (target);
gcc_assert (mode == QImode);
- rtx base_tag = expand_simple_binop (Pmode, LSHIFTRT, hwasan_base (),
- HWASAN_SHIFT_RTX,
- NULL_RTX, /* unsignedp = */0,
- OPTAB_DIRECT);
-
+ rtx base_tag = hwasan_extract_tag (hwasan_base ());
gcc_assert (base_tag);
rtx tag_offset = const_int_rtx[MAX_SAVED_CONST_INT + hwasan_current_tag ()];
rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
@@ -498,6 +494,39 @@ expand_HWASAN_CHOOSE_COLOUR (internal_fn, gcall *gc)
}
static void
+expand_HWASAN_MARK (internal_fn, gcall *gc)
+{
+ HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
+ bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
+
+ tree base = gimple_call_arg (gc, 1);
+ gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
+ rtx base_rtx = expand_normal (base);
+
+ rtx tag = is_poison ? const0_rtx : hwasan_extract_tag (base_rtx);
+ rtx address = hwasan_create_untagged_base (base_rtx);
+
+ tree len = gimple_call_arg (gc, 2);
+ gcc_assert (tree_fits_shwi_p (len));
+ unsigned HOST_WIDE_INT size_in_bytes = tree_to_shwi (len);
+ uint8_t tg_mask = HWASAN_TAG_GRANULE_SIZE - 1;
+ gcc_assert (size_in_bytes);
+ size_in_bytes = (size_in_bytes + tg_mask) & ~tg_mask;
+ rtx size = gen_int_mode (size_in_bytes, Pmode);
+
+ /* TODO Other options (i.e. inline options) */
+ rtx func = init_one_libfunc ("__hwasan_tag_memory");
+ emit_library_call (func,
+ LCT_NORMAL,
+ VOIDmode,
+ address, ptr_mode,
+ tag, QImode,
+ size, ptr_mode);
+}
+
+/* This should get expanded in the sanopt pass. */
+
+static void
expand_ASAN_CHECK (internal_fn, gcall *)
{
gcc_unreachable ();
@@ -290,6 +290,7 @@ DEF_INTERNAL_FN (ABNORMAL_DISPATCHER, ECF_NORETURN, NULL)
DEF_INTERNAL_FN (BUILTIN_EXPECT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (HWASAN_CHOOSE_COLOUR, ECF_LEAF | ECF_NOTHROW, ".")
DEF_INTERNAL_FN (HWASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
+DEF_INTERNAL_FN (HWASAN_MARK, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (ASAN_CHECK, ECF_TM_PURE | ECF_LEAF | ECF_NOTHROW, "..R..")
DEF_INTERNAL_FN (ASAN_MARK, ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (ASAN_POISON, ECF_LEAF | ECF_NOTHROW | ECF_NOVOPS, NULL)
@@ -1258,6 +1258,12 @@ sanitize_rewrite_addressable_params (function *fun)
unsigned int
pass_sanopt::execute (function *fun)
{
+ /*
+ n.b. ASAN_MARK is used for both HWASAN and ASAN.
+ asan_num_accesses is used to count either HWASAN_CHECK or ASAN_CHECK
+ stuff. This is fine because you can only have one of these active at a
+ time.
+ */
basic_block bb;
int asan_num_accesses = 0;
bool contains_asan_mark = false;
@@ -1345,6 +1351,9 @@ pass_sanopt::execute (function *fun)
&need_commit_edge_insert,
shadow_vars_mapping);
break;
+ case IFN_HWASAN_MARK:
+ no_next = hwasan_expand_mark_ifn (&gsi);
+ break;
default:
break;
}