@@ -1937,43 +1937,79 @@ make_pass_stv (gcc::context *ctxt)
return new pass_stv (ctxt);
}
-/* Inserting ENDBRANCH instructions. */
+/* Inserting ENDBR and pseudo patchable-area instructions. */
-static unsigned int
-rest_of_insert_endbranch (void)
+static void
+rest_of_insert_endbr_and_patchable_area (bool need_endbr,
+ bool need_patchable_area)
{
- timevar_push (TV_MACH_DEP);
-
- rtx cet_eb;
+ rtx endbr;
rtx_insn *insn;
+ rtx_insn *endbr_insn = NULL;
basic_block bb;
- /* Currently emit EB if it's a tracking function, i.e. 'nocf_check' is
- absent among function attributes. Later an optimization will be
- introduced to make analysis if an address of a static function is
- taken. A static function whose address is not taken will get a
- nocf_check attribute. This will allow to reduce the number of EB. */
+ if (need_endbr)
+ {
+ /* Currently emit EB if it's a tracking function, i.e. 'nocf_check'
+ is absent among function attributes. Later an optimization will
+ be introduced to make analysis if an address of a static function
+ is taken. A static function whose address is not taken will get
+ a nocf_check attribute. This will allow to reduce the number of
+ EB. */
+ if (!lookup_attribute ("nocf_check",
+ TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
+ && (!flag_manual_endbr
+ || lookup_attribute ("cf_check",
+ DECL_ATTRIBUTES (cfun->decl)))
+ && !cgraph_node::get (cfun->decl)->only_called_directly_p ())
+ {
+ if (crtl->profile && flag_fentry)
+ {
+ /* Queue ENDBR insertion to x86_function_profiler.
+ NB: Any patchable-area insn will be inserted after
+ ENDBR. */
+ cfun->machine->insn_queued_at_entrance = TYPE_ENDBR;
+ }
+ else
+ {
+ endbr = gen_nop_endbr ();
+ bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
+ rtx_insn *insn = BB_HEAD (bb);
+ endbr_insn = emit_insn_before (endbr, insn);
+ }
+ }
+ }
- if (!lookup_attribute ("nocf_check",
- TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
- && (!flag_manual_endbr
- || lookup_attribute ("cf_check",
- DECL_ATTRIBUTES (cfun->decl)))
- && !cgraph_node::get (cfun->decl)->only_called_directly_p ())
+ if (need_patchable_area)
{
- /* Queue ENDBR insertion to x86_function_profiler. */
if (crtl->profile && flag_fentry)
- cfun->machine->endbr_queued_at_entrance = true;
+ {
+ /* Queue patchable-area insertion to x86_function_profiler.
+ NB: If there is a queued ENDBR, x86_function_profiler
+ will also handle patchable-area. */
+ if (!cfun->machine->insn_queued_at_entrance)
+ cfun->machine->insn_queued_at_entrance = TYPE_PATCHABLE_AREA;
+ }
else
{
- cet_eb = gen_nop_endbr ();
-
- bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
- insn = BB_HEAD (bb);
- emit_insn_before (cet_eb, insn);
+ /* ix86_print_patchable_function_entry will provide actual
+ size. */
+ rtx patchable_area = gen_patchable_area (GEN_INT (0),
+ GEN_INT (0));
+ if (endbr_insn)
+ emit_insn_after (patchable_area, endbr_insn);
+ else
+ {
+ bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
+ insn = BB_HEAD (bb);
+ emit_insn_before (patchable_area, insn);
+ }
}
}
+ if (!need_endbr)
+ return;
+
bb = 0;
FOR_EACH_BB_FN (bb, cfun)
{
@@ -1982,7 +2018,6 @@ rest_of_insert_endbranch (void)
{
if (CALL_P (insn))
{
- bool need_endbr;
need_endbr = find_reg_note (insn, REG_SETJMP, NULL) != NULL;
if (!need_endbr && !SIBLING_CALL_P (insn))
{
@@ -2013,8 +2048,8 @@ rest_of_insert_endbranch (void)
/* Generate ENDBRANCH after CALL, which can return more than
twice, setjmp-like functions. */
- cet_eb = gen_nop_endbr ();
- emit_insn_after_setloc (cet_eb, insn, INSN_LOCATION (insn));
+ endbr = gen_nop_endbr ();
+ emit_insn_after_setloc (endbr, insn, INSN_LOCATION (insn));
continue;
}
@@ -2044,31 +2079,30 @@ rest_of_insert_endbranch (void)
dest_blk = e->dest;
insn = BB_HEAD (dest_blk);
gcc_assert (LABEL_P (insn));
- cet_eb = gen_nop_endbr ();
- emit_insn_after (cet_eb, insn);
+ endbr = gen_nop_endbr ();
+ emit_insn_after (endbr, insn);
}
continue;
}
if (LABEL_P (insn) && LABEL_PRESERVE_P (insn))
{
- cet_eb = gen_nop_endbr ();
- emit_insn_after (cet_eb, insn);
+ endbr = gen_nop_endbr ();
+ emit_insn_after (endbr, insn);
continue;
}
}
}
- timevar_pop (TV_MACH_DEP);
- return 0;
+ return;
}
namespace {
-const pass_data pass_data_insert_endbranch =
+const pass_data pass_data_insert_endbr_and_patchable_area =
{
RTL_PASS, /* type. */
- "cet", /* name. */
+ "endbr_and_patchable_area", /* name. */
OPTGROUP_NONE, /* optinfo_flags. */
TV_MACH_DEP, /* tv_id. */
0, /* properties_required. */
@@ -2078,32 +2112,44 @@ const pass_data pass_data_insert_endbranch =
0, /* todo_flags_finish. */
};
-class pass_insert_endbranch : public rtl_opt_pass
+class pass_insert_endbr_and_patchable_area : public rtl_opt_pass
{
public:
- pass_insert_endbranch (gcc::context *ctxt)
- : rtl_opt_pass (pass_data_insert_endbranch, ctxt)
+ pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_insert_endbr_and_patchable_area, ctxt)
{}
/* opt_pass methods: */
- virtual bool gate (function *)
- {
- return ((flag_cf_protection & CF_BRANCH));
+ virtual bool gate (function *fun)
+ {
+ need_endbr = (flag_cf_protection & CF_BRANCH) != 0;
+ need_patchable_area
+ = (function_entry_patch_area_size
+ || lookup_attribute ("patchable_function_entry",
+ DECL_ATTRIBUTES (fun->decl)));
+ return need_endbr || need_patchable_area;
}
virtual unsigned int execute (function *)
{
- return rest_of_insert_endbranch ();
+ timevar_push (TV_MACH_DEP);
+ rest_of_insert_endbr_and_patchable_area (need_endbr,
+ need_patchable_area);
+ timevar_pop (TV_MACH_DEP);
+ return 0;
}
-}; // class pass_insert_endbranch
+private:
+ bool need_endbr;
+ bool need_patchable_area;
+}; // class pass_insert_endbr_and_patchable_area
} // anon namespace
rtl_opt_pass *
-make_pass_insert_endbranch (gcc::context *ctxt)
+make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
{
- return new pass_insert_endbranch (ctxt);
+ return new pass_insert_endbr_and_patchable_area (ctxt);
}
/* At entry of the nearest common dominator for basic blocks with
@@ -30,6 +30,6 @@ along with GCC; see the file COPYING3. If not see
CONSTM1_RTX generated by the STV pass can be CSEed. */
INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
- INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbranch);
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
INSERT_PASS_AFTER (pass_combine, 1, pass_remove_partial_avx_dependency);
@@ -87,6 +87,8 @@ extern const char *output_fp_compare (rtx_insn *, rtx*, bool, bool);
extern const char *output_adjust_stack_and_probe (rtx);
extern const char *output_probe_stack_range (rtx, rtx);
+extern void ix86_output_patchable_area (unsigned int, bool);
+
extern void ix86_expand_clear (rtx);
extern void ix86_expand_move (machine_mode, rtx[]);
extern void ix86_expand_vector_move (machine_mode, rtx[]);
@@ -376,6 +378,7 @@ class rtl_opt_pass;
extern rtl_opt_pass *make_pass_insert_vzeroupper (gcc::context *);
extern rtl_opt_pass *make_pass_stv (gcc::context *);
-extern rtl_opt_pass *make_pass_insert_endbranch (gcc::context *);
+extern rtl_opt_pass *make_pass_insert_endbr_and_patchable_area
+ (gcc::context *);
extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
(gcc::context *);
@@ -1563,6 +1563,9 @@ ix86_asm_output_function_label (FILE *asm_out_file, const char *fname,
{
bool is_ms_hook = ix86_function_ms_hook_prologue (decl);
+ if (cfun)
+ cfun->machine->function_label_emitted = true;
+
if (is_ms_hook)
{
int i, filler_count = (TARGET_64BIT ? 32 : 16);
@@ -9118,6 +9121,80 @@ ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED)
}
}
+/* Implement TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY. */
+
+void
+ix86_print_patchable_function_entry (FILE *file,
+ unsigned HOST_WIDE_INT patch_area_size,
+ bool record_p)
+{
+ if (cfun->machine->function_label_emitted)
+ {
+ /* The insert_endbr_and_patchable_area pass inserted a dummy
+ UNSPECV_PATCHABLE_AREA with 0 patchable area size. If the
+ patchable area is placed after the function label, we replace
+ 0 patchable area size with the real one. Otherwise, the
+ dummy UNSPECV_PATCHABLE_AREA will be ignored. */
+ if (cfun->machine->insn_queued_at_entrance)
+ {
+ /* Record the patchable area. Both ENDBR and patchable area
+ will be inserted by x86_function_profiler later. */
+ cfun->machine->patch_area_size = patch_area_size;
+ cfun->machine->record_patch_area = record_p;
+ return;
+ }
+
+ /* We can have
+
+ UNSPECV_NOP_ENDBR
+ UNSPECV_PATCHABLE_AREA
+
+ or just
+
+ UNSPECV_PATCHABLE_AREA
+ */
+ rtx_insn *patchable_insn;
+ rtx_insn *insn = next_real_nondebug_insn (get_insns ());
+ if (insn
+ && INSN_P (insn)
+ && GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
+ && XINT (PATTERN (insn), 1) == UNSPECV_NOP_ENDBR)
+ patchable_insn = next_real_nondebug_insn (insn);
+ else
+ patchable_insn = insn;
+
+ if (patchable_insn && INSN_P (patchable_insn))
+ {
+ /* Replace the dummy patchable area size with the real one. */
+ rtx pattern = PATTERN (patchable_insn);
+ if (GET_CODE (pattern) == UNSPEC_VOLATILE
+ && XINT (pattern, 1) == UNSPECV_PATCHABLE_AREA)
+ {
+ XVECEXP (pattern, 0, 0) = GEN_INT (patch_area_size);
+ XVECEXP (pattern, 0, 1) = GEN_INT (record_p);
+ }
+ return;
+ }
+
+ gcc_unreachable ();
+ }
+
+ default_print_patchable_function_entry (file, patch_area_size,
+ record_p);
+}
+
+/* Output patchable area. NB: default_print_patchable_function_entry
+ isn't available in i386.md. */
+
+void
+ix86_output_patchable_area (unsigned int patch_area_size,
+ bool record_p)
+{
+ default_print_patchable_function_entry (asm_out_file,
+ patch_area_size,
+ record_p);
+}
+
/* Return a scratch register to use in the split stack prologue. The
split stack prologue is used for -fsplit-stack. It is the first
instructions in the function, even before the regular prologue.
@@ -20151,8 +20228,14 @@ current_fentry_section (const char **name)
void
x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
{
- if (cfun->machine->endbr_queued_at_entrance)
- fprintf (file, "\t%s\n", TARGET_64BIT ? "endbr64" : "endbr32");
+ if (cfun->machine->insn_queued_at_entrance)
+ {
+ if (cfun->machine->insn_queued_at_entrance == TYPE_ENDBR)
+ fprintf (file, "\t%s\n", TARGET_64BIT ? "endbr64" : "endbr32");
+ if (cfun->machine->patch_area_size)
+ ix86_output_patchable_area (cfun->machine->patch_area_size,
+ cfun->machine->record_patch_area);
+ }
const char *mcount_name = MCOUNT_NAME;
@@ -22744,6 +22827,10 @@ ix86_run_selftests (void)
#undef TARGET_ASM_FUNCTION_EPILOGUE
#define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
+#undef TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY
+#define TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY \
+ ix86_print_patchable_function_entry
+
#undef TARGET_ENCODE_SECTION_INFO
#ifndef SUBTARGET_ENCODE_SECTION_INFO
#define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
@@ -2751,6 +2751,13 @@ enum function_type
TYPE_EXCEPTION
};
+enum queued_insn_type
+{
+ TYPE_NONE = 0,
+ TYPE_ENDBR,
+ TYPE_PATCHABLE_AREA
+};
+
struct GTY(()) machine_function {
struct stack_local_entry *stack_locals;
int varargs_gpr_size;
@@ -2767,6 +2774,12 @@ struct GTY(()) machine_function {
structure. */
rtx split_stack_varargs_pointer;
+ /* The size of the patchable area at function entry. */
+ unsigned int patch_area_size;
+
+ /* If true, record patchable area at function entry. */
+ BOOL_BITFIELD record_patch_area : 1;
+
/* This value is used for amd64 targets and specifies the current abi
to be used. MS_ABI means ms abi. Otherwise SYSV_ABI means sysv abi. */
ENUM_BITFIELD(calling_abi) call_abi : 8;
@@ -2841,8 +2854,11 @@ struct GTY(()) machine_function {
/* Nonzero if the function places outgoing arguments on stack. */
BOOL_BITFIELD outgoing_args_on_stack : 1;
- /* If true, ENDBR is queued at function entrance. */
- BOOL_BITFIELD endbr_queued_at_entrance : 1;
+ /* If true, ENDBR or patchable area is queued at function entrance. */
+ ENUM_BITFIELD(queued_insn_type) insn_queued_at_entrance : 2;
+
+ /* If true, the function label has been emitted. */
+ BOOL_BITFIELD function_label_emitted : 1;
/* True if the function needs a stack frame. */
BOOL_BITFIELD stack_frame_required : 1;
@@ -300,6 +300,9 @@ (define_c_enum "unspecv" [
;; For ENQCMD and ENQCMDS support
UNSPECV_ENQCMD
UNSPECV_ENQCMDS
+
+ ;; For patchable area support
+ UNSPECV_PATCHABLE_AREA
])
;; Constants to represent rounding modes in the ROUND instruction
@@ -21280,6 +21283,18 @@ (define_insn "speculation_barrier"
[(set_attr "type" "other")
(set_attr "length" "3")])
+(define_insn "patchable_area"
+ [(unspec_volatile [(match_operand 0 "const_int_operand")
+ (match_operand 1 "const_int_operand")]
+ UNSPECV_PATCHABLE_AREA)]
+ ""
+{
+ if (INTVAL (operands[0]))
+ ix86_output_patchable_area (INTVAL (operands[0]),
+ INTVAL (operands[1]) != 0);
+ return "";
+})
+
(include "mmx.md")
(include "sse.md")
(include "sync.md")
new file mode 100644
@@ -0,0 +1,73 @@
+/* { dg-do "compile" } */
+/* { dg-options "-O1 -fcf-protection -mmanual-endbr" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/* Note: this test only checks the instructions in the function bodies,
+ not the placement of the patch label or nops before the function. */
+
+/*
+**f10_none:
+** nop
+** ret
+*/
+void
+__attribute__ ((nocf_check,patchable_function_entry (1, 0)))
+f10_none (void)
+{
+}
+
+/*
+**f10_endbr:
+** endbr(32|64)
+** nop
+** ret
+*/
+void
+__attribute__ ((cf_check,patchable_function_entry (1, 0)))
+f10_endbr (void)
+{
+}
+
+/*
+**f11_none:
+** ret
+*/
+void
+__attribute__ ((nocf_check,patchable_function_entry (1, 1)))
+f11_none (void)
+{
+}
+
+/*
+**f11_endbr:
+** endbr(32|64)
+** ret
+*/
+void
+__attribute__ ((cf_check,patchable_function_entry (1, 1)))
+f11_endbr (void)
+{
+}
+
+/*
+**f21_none:
+** nop
+** ret
+*/
+void
+__attribute__ ((nocf_check,patchable_function_entry (2, 1)))
+f21_none (void)
+{
+}
+
+/*
+**f21_endbr:
+** endbr(32|64)
+** nop
+** ret
+*/
+void
+__attribute__ ((cf_check,patchable_function_entry (2, 1)))
+f21_endbr (void)
+{
+}
new file mode 100644
@@ -0,0 +1,12 @@
+/* { dg-do "compile" } */
+/* { dg-options "-O1 -fcf-protection -mmanual-endbr -fasynchronous-unwind-tables" } */
+
+/* Test the placement of the .LPFE1 label. */
+
+void
+__attribute__ ((cf_check,patchable_function_entry (1, 0)))
+f10_endbr (void)
+{
+}
+
+/* { dg-final { scan-assembler "\t\.cfi_startproc\n\tendbr(32|64)\n.*\.LPFE1:\n\tnop\n\tret\n" } } */
new file mode 100644
@@ -0,0 +1,13 @@
+/* { dg-do "compile" } */
+/* { dg-require-effective-target mfentry } */
+/* { dg-options "-O1 -fcf-protection -mmanual-endbr -mfentry -pg -fasynchronous-unwind-tables" } */
+
+/* Test the placement of the .LPFE1 label. */
+
+void
+__attribute__ ((cf_check,patchable_function_entry (1, 0)))
+f10_endbr (void)
+{
+}
+
+/* { dg-final { scan-assembler "\t\.cfi_startproc\n\tendbr(32|64)\n.*\.LPFE1:\n\tnop\n1:\tcall\t__fentry__\n\tret\n" } } */
new file mode 100644
@@ -0,0 +1,11 @@
+/* { dg-do "compile" } */
+/* { dg-options "-O1 -fpatchable-function-entry=1 -fasynchronous-unwind-tables" } */
+
+/* Test the placement of the .LPFE1 label. */
+
+void
+foo (void)
+{
+}
+
+/* { dg-final { scan-assembler "\t\.cfi_startproc\n.*\.LPFE1:\n\tnop\n\tret\n" } } */
new file mode 100644
@@ -0,0 +1,12 @@
+/* { dg-do "compile" } */
+/* { dg-require-effective-target mfentry } */
+/* { dg-options "-O1 -fpatchable-function-entry=1 -mfentry -pg -fasynchronous-unwind-tables" } */
+
+/* Test the placement of the .LPFE1 label. */
+
+void
+foo (void)
+{
+}
+
+/* { dg-final { scan-assembler "\t\.cfi_startproc\n.*\.LPFE1:\n\tnop\n1:\tcall\t__fentry__\n\tret\n" } } */