@@ -27,6 +27,11 @@
#define MAP_32BIT 0x40 /* Only give out 32-bit addresses. */
#define MAP_ABOVE4G 0x80 /* Only map above 4GB. */
+#ifdef __USE_MISC
+/* Set up a restore token in the newly allocated shadow stack */
+# define SHADOW_STACK_SET_TOKEN 0x1
+#endif
+
#include <bits/mman-map-flags-generic.h>
/* Include generic Linux declarations. */
@@ -23,10 +23,15 @@
static inline int __attribute__ ((always_inline))
get_cet_status (void)
{
- unsigned long long cet_status[3];
- if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0)
- return cet_status[0];
- return 0;
+ unsigned long long kernel_feature;
+ unsigned int status = 0;
+ if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS,
+ &kernel_feature) == 0)
+ {
+ if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0)
+ status = GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+ }
+ return status;
}
# ifndef SHARED
@@ -21,12 +21,20 @@
static inline int __attribute__ ((always_inline))
dl_cet_disable_cet (unsigned int cet_feature)
{
- return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE,
- cet_feature);
+ if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
+ return -1;
+ long long int kernel_feature = ARCH_SHSTK_SHSTK;
+ return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE,
+ kernel_feature);
}
static inline int __attribute__ ((always_inline))
-dl_cet_lock_cet (void)
+dl_cet_lock_cet (unsigned int cet_feature)
{
- return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0);
+ if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
+ return -1;
+ /* Lock all SHSTK features. */
+ long long int kernel_feature = -1;
+ return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK,
+ kernel_feature);
}
@@ -4,24 +4,19 @@
#include_next <asm/prctl.h>
-#ifndef ARCH_CET_STATUS
-/* CET features:
- IBT: GNU_PROPERTY_X86_FEATURE_1_IBT
- SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK
- */
-/* Return CET features in unsigned long long *addr:
- features: addr[0].
- shadow stack base address: addr[1].
- shadow stack size: addr[2].
- */
-# define ARCH_CET_STATUS 0x3001
-/* Disable CET features in unsigned int features. */
-# define ARCH_CET_DISABLE 0x3002
-/* Lock all CET features. */
-# define ARCH_CET_LOCK 0x3003
-/* Allocate a new shadow stack with unsigned long long *addr:
- IN: requested shadow stack size: *addr.
- OUT: allocated shadow stack address: *addr.
- */
-# define ARCH_CET_ALLOC_SHSTK 0x3004
-#endif /* ARCH_CET_STATUS */
+#ifndef ARCH_SHSTK_ENABLE
+/* Enable SHSTK features in unsigned long int features. */
+# define ARCH_SHSTK_ENABLE 0x5001
+/* Disable SHSTK features in unsigned long int features. */
+# define ARCH_SHSTK_DISABLE 0x5002
+/* Lock SHSTK features in unsigned long int features. */
+# define ARCH_SHSTK_LOCK 0x5003
+/* Unlock SHSTK features in unsigned long int features. */
+# define ARCH_SHSTK_UNLOCK 0x5004
+/* Return SHSTK features in unsigned long int features. */
+# define ARCH_SHSTK_STATUS 0x5005
+
+/* ARCH_SHSTK_ features bits */
+# define ARCH_SHSTK_SHSTK 0x1
+# define ARCH_SHSTK_WRSS 0x2
+#endif
@@ -87,15 +87,14 @@ do_test (void)
ctx[4].uc_link = &ctx[0];
makecontext (&ctx[4], (void (*) (void)) f1, 0);
- /* NB: When shadow stack is enabled, makecontext calls arch_prctl
- with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which
- can be unmapped. The base address and size of the new shadow
- stack are returned in __ssp[1] and __ssp[2]. makecontext is
- called for CTX1, CTX3 and CTX4. But only CTX1 is used. New
- shadow stacks are allocated in the order of CTX3, CTX1, CTX4.
- It is very likely that CTX1's shadow stack is placed between
- CTX3 and CTX4. We munmap CTX3's and CTX4's shadow stacks to
- create gaps above and below CTX1's shadow stack. We check
+ /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack
+ to allocate a new shadow stack which can be unmapped. The base
+ address and size of the new shadow stack are returned in __ssp[1]
+ and __ssp[2]. makecontext is called for CTX1, CTX3 and CTX4. But
+ only CTX1 is used. New shadow stacks are allocated in the order
+ of CTX3, CTX1, CTX4. It is very likely that CTX1's shadow stack is
+ placed between CTX3 and CTX4. We munmap CTX3's and CTX4's shadow
+ stacks to create gaps above and below CTX1's shadow stack. We check
that setcontext CTX1 works correctly in this case. */
if (_get_ssp () != 0)
{
@@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl
endif
ifeq ($(subdir),stdlib)
-sysdep_routines += __start_context
+sysdep_routines += __start_context allocate-shadow-stack
endif
ifeq ($(subdir),csu)
@@ -24,20 +24,14 @@
/* Use CALL to push __start_context onto the new stack as well as the new
shadow stack. RDI points to ucontext:
Incoming:
- __ssp[0]: The original caller's shadow stack pointer.
- __ssp[1]: The size of the new shadow stack.
- __ssp[2]: The size of the new shadow stack.
- Outgoing:
__ssp[0]: The new shadow stack pointer.
__ssp[1]: The base address of the new shadow stack.
__ssp[2]: The size of the new shadow stack.
*/
ENTRY(__push___start_context)
- /* Save the pointer to ucontext. */
- movq %rdi, %r9
/* Get the original shadow stack pointer. */
- rdsspq %r8
+ rdsspq %rcx
/* Save the original stack pointer. */
movq %rsp, %rdx
/* Load the top of the new stack into RSI. */
@@ -45,24 +39,12 @@ ENTRY(__push___start_context)
/* Add 8 bytes to RSI since CALL will push the 8-byte return
address onto stack. */
leaq 8(%rsi), %rsp
- /* Allocate the new shadow stack. The size of the new shadow
- stack is passed in __ssp[1]. */
- lea (oSSP + 8)(%rdi), %RSI_LP
- movl $ARCH_CET_ALLOC_SHSTK, %edi
- movl $__NR_arch_prctl, %eax
- /* The new shadow stack base is returned in __ssp[1]. */
- syscall
- testq %rax, %rax
- jne L(hlt) /* This should never happen. */
-
- /* Get the size of the new shadow stack. */
- movq 8(%rsi), %rdi
-
- /* Get the base address of the new shadow stack. */
- movq (%rsi), %rsi
-
+ /* The size of the new shadow stack is stored in __ssp[2]. */
+ mov (oSSP + 16)(%rdi), %RSI_LP
+ /* The new shadow stack base is stored in __ssp[1]. */
+ mov (oSSP + 8)(%rdi), %RAX_LP
/* Use the restore stoken to restore the new shadow stack. */
- rstorssp -8(%rsi, %rdi)
+ rstorssp -8(%rax, %rsi)
/* Save the restore token on the original shadow stack. */
saveprevssp
@@ -73,18 +55,12 @@ ENTRY(__push___start_context)
jmp __start_context
1:
- /* Get the new shadow stack pointer. */
- rdsspq %rdi
-
/* Use the restore stoken to restore the original shadow stack. */
- rstorssp -8(%r8)
+ rstorssp -8(%rcx)
/* Save the restore token on the new shadow stack. */
saveprevssp
- /* Store the new shadow stack pointer in __ssp[0]. */
- movq %rdi, oSSP(%r9)
-
/* Restore the original stack. */
mov %rdx, %rsp
ret
new file mode 100644
@@ -0,0 +1,55 @@
+/* Helper function to allocate shadow stack.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <libc-pointer-arith.h>
+#include <allocate-shadow-stack.h>
+
+/* NB: This can be treated as a syscall by caller. */
+
+long int
+__allocate_shadow_stack (size_t stack_size,
+ shadow_stack_size_t *child_stack)
+{
+#ifdef __NR_map_shadow_stack
+ size_t shadow_stack_size
+ = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT;
+ /* Align shadow stack to 8 bytes. */
+ shadow_stack_size = ALIGN_UP (shadow_stack_size, 8);
+ /* Since sigaltstack shares shadow stack with the current context in
+ the thread, add extra 20 stack frames in shadow stack for signal
+ handlers. */
+ shadow_stack_size += 20 * 8;
+ void *shadow_stack = (void *)INLINE_SYSCALL_CALL
+ (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN);
+ /* Report the map_shadow_stack error. */
+ if (shadow_stack == MAP_FAILED)
+ return -errno;
+
+ /* Save the shadow stack base and size on child stack. */
+ child_stack[0] = (uintptr_t) shadow_stack;
+ child_stack[1] = shadow_stack_size;
+
+ return 0;
+#else
+ return -ENOSYS;
+#endif
+}
new file mode 100644
@@ -0,0 +1,24 @@
+/* Helper function to allocate shadow stack.
+ Copyright (C) 2023 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <ucontext.h>
+
+typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t;
+
+extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *)
+ attribute_hidden;
@@ -58,35 +58,15 @@ ENTRY(__getcontext)
testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET
jz L(no_shstk)
- /* Save RDI in RDX which won't be clobbered by syscall. */
- movq %rdi, %rdx
-
xorl %eax, %eax
cmpq %fs:SSP_BASE_OFFSET, %rax
jnz L(shadow_stack_bound_recorded)
- /* Get the base address and size of the default shadow stack
- which must be the current shadow stack since nothing has
- been recorded yet. */
- sub $24, %RSP_LP
- mov %RSP_LP, %RSI_LP
- movl $ARCH_CET_STATUS, %edi
- movl $__NR_arch_prctl, %eax
- syscall
- testq %rax, %rax
- jz L(continue_no_err)
-
- /* This should never happen. */
- hlt
-
-L(continue_no_err):
- /* Record the base of the current shadow stack. */
- movq 8(%rsp), %rax
+ /* When the shadow stack base is unset, the default shadow
+ stack is in use. Use the current shadow stack pointer
+ as the marker for the default shadow stack. */
+ rdsspq %rax
movq %rax, %fs:SSP_BASE_OFFSET
- add $24, %RSP_LP
-
- /* Restore RDI. */
- movq %rdx, %rdi
L(shadow_stack_bound_recorded):
/* Get the current shadow stack pointer. */
@@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded):
/* NB: Save the caller's shadow stack so that we can jump back
to the caller directly. */
addq $8, %rax
- movq %rax, oSSP(%rdx)
+ movq %rax, oSSP(%rdi)
/* Save the current shadow stack base in ucontext. */
movq %fs:SSP_BASE_OFFSET, %rax
@@ -24,6 +24,7 @@
# include <pthread.h>
# include <libc-pointer-arith.h>
# include <sys/prctl.h>
+# include <allocate-shadow-stack.h>
#endif
#include "ucontext_i.h"
@@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...)
if ((feature_1 & X86_FEATURE_1_SHSTK) != 0)
{
/* Shadow stack is enabled. We need to allocate a new shadow
- stack. */
- unsigned long ssp_size = (((uintptr_t) sp
- - (uintptr_t) ucp->uc_stack.ss_sp)
- >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT);
- /* Align shadow stack to 8 bytes. */
- ssp_size = ALIGN_UP (ssp_size, 8);
-
- ucp->__ssp[1] = ssp_size;
- ucp->__ssp[2] = ssp_size;
-
- /* Call __push___start_context to allocate a new shadow stack,
- push __start_context onto the new stack as well as the new
- shadow stack. NB: After __push___start_context returns,
+ stack. NB:
ucp->__ssp[0]: The new shadow stack pointer.
ucp->__ssp[1]: The base address of the new shadow stack.
ucp->__ssp[2]: The size of the new shadow stack.
*/
+ long int ret
+ = __allocate_shadow_stack (((uintptr_t) sp
+ - (uintptr_t) ucp->uc_stack.ss_sp),
+ &ucp->__ssp[1]);
+ if (ret != 0)
+ {
+ /* FIXME: What should we do? */
+ abort ();
+ }
+
+ ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8;
+ /* Call __push___start_context to push __start_context onto the new
+ stack as well as the new shadow stack. */
__push___start_context (ucp);
}
else
@@ -109,25 +109,11 @@ ENTRY(__swapcontext)
cmpq %fs:SSP_BASE_OFFSET, %rax
jnz L(shadow_stack_bound_recorded)
- /* Get the base address and size of the default shadow stack
- which must be the current shadow stack since nothing has
- been recorded yet. */
- sub $24, %RSP_LP
- mov %RSP_LP, %RSI_LP
- movl $ARCH_CET_STATUS, %edi
- movl $__NR_arch_prctl, %eax
- syscall
- testq %rax, %rax
- jz L(continue_no_err)
-
- /* This should never happen. */
- hlt
-
-L(continue_no_err):
- /* Record the base of the current shadow stack. */
- movq 8(%rsp), %rax
+ /* When the shadow stack base is unset, the default shadow
+ stack is in use. Use the current shadow stack pointer
+ as the marker for the default shadow stack. */
+ rdsspq %rax
movq %rax, %fs:SSP_BASE_OFFSET
- add $24, %RSP_LP
L(shadow_stack_bound_recorded):
/* If we unwind the stack, we can't undo stack unwinding. Just
@@ -1121,8 +1121,9 @@ no_cpuid:
# ifndef SHARED
/* Check if IBT and SHSTK are enabled by kernel. */
- if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT)
- || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK))
+ if ((cet_status
+ & (GNU_PROPERTY_X86_FEATURE_1_IBT
+ | GNU_PROPERTY_X86_FEATURE_1_SHSTK)))
{
/* Disable IBT and/or SHSTK if they are enabled by kernel, but
disabled by environment variable:
@@ -1131,9 +1132,11 @@ no_cpuid:
*/
unsigned int cet_feature = 0;
if (!CPU_FEATURE_USABLE (IBT))
- cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
+ cet_feature |= (cet_status
+ & GNU_PROPERTY_X86_FEATURE_1_IBT);
if (!CPU_FEATURE_USABLE (SHSTK))
- cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+ cet_feature |= (cet_status
+ & GNU_PROPERTY_X86_FEATURE_1_SHSTK);
if (cet_feature)
{
@@ -1148,7 +1151,9 @@ no_cpuid:
lock CET if IBT or SHSTK is enabled permissively. */
if (GL(dl_x86_feature_control).ibt != cet_permissive
&& GL(dl_x86_feature_control).shstk != cet_permissive)
- dl_cet_lock_cet ();
+ dl_cet_lock_cet (GL(dl_x86_feature_1)
+ & (GNU_PROPERTY_X86_FEATURE_1_IBT
+ | GNU_PROPERTY_X86_FEATURE_1_SHSTK));
}
# endif
}
@@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info)
feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
if (feature_1_lock != 0
- && dl_cet_lock_cet () != 0)
+ && dl_cet_lock_cet (feature_1_lock) != 0)
_dl_fatal_printf ("%s: can't lock CET\n", info->program);
}
@@ -60,7 +60,7 @@ typedef struct
void *__private_tm[4];
/* GCC split stack support. */
void *__private_ss;
- /* The lowest address of shadow stack, */
+ /* The marker for the current shadow stack. */
unsigned long long int ssp_base;
/* Must be kept even if it is no longer used by glibc since programs,
like AddressSanitizer, depend on the size of tcbhead_t. */