@@ -23,6 +23,7 @@
#include <linux/file_ref.h>
#include <net/sock.h>
#include <linux/init_task.h>
+#include <asm/syscall.h>
#include "internal.h"
@@ -643,6 +644,19 @@ static void fd_release(unsigned int fd)
/* Enough for SCM_MAX_FD, and a page of slots on 4K pages. */
#define FD_SLOTS_SPILL_MIN 256
+/* Make the syscall exit path call fd_slots_commit(). */
+#if defined(CONFIG_GENERIC_ENTRY)
+#define fd_slots_set_work() set_syscall_work(FD_SLOTS)
+#define fd_slots_clear_work() clear_syscall_work(FD_SLOTS)
+#elif defined(TIF_FD_SLOTS)
+#define fd_slots_set_work() set_thread_flag(TIF_FD_SLOTS)
+#define fd_slots_clear_work() clear_thread_flag(TIF_FD_SLOTS)
+#else
+/* Nothing commits until the architecture provides the flag. */
+#define fd_slots_set_work() do { } while (0)
+#define fd_slots_clear_work() do { } while (0)
+#endif
+
static struct fd_slot *fd_slot(struct fd_slots *slots, unsigned int idx)
{
if (idx < FD_SLOTS_INLINE)
@@ -690,6 +704,8 @@ static struct fd_slot *fd_slot_record(int fd)
}
ACCESS_PRIVATE(slot, fd) = fd;
ACCESS_PRIVATE(slot, file) = NULL;
+ if (!idx)
+ fd_slots_set_work();
slots->nr = idx + 1;
return slot;
}
@@ -809,6 +825,14 @@ static __always_inline void fd_slots_finish(struct fd_slots *slots, bool failed)
else
fd_slots_drop(slots);
slots->nr = 0;
+ fd_slots_clear_work();
+}
+
+/* Syscall exit hook, keyed on the return value the caller will see. */
+void fd_slots_commit(struct pt_regs *regs)
+{
+ fd_slots_finish(¤t->fd_slots,
+ syscall_get_error(current, regs) != 0);
}
/* Install or drop the prepared descriptors based on @ret. */
@@ -3,6 +3,7 @@
#define __LINUX_ENTRYCOMMON_H
#include <linux/audit.h>
+#include <linux/file.h>
#include <linux/irq-entry-common.h>
#include <linux/livepatch.h>
#include <linux/ptrace.h>
@@ -36,7 +37,8 @@
SYSCALL_WORK_SYSCALL_TRACE | \
SYSCALL_WORK_SYSCALL_AUDIT | \
SYSCALL_WORK_SYSCALL_USER_DISPATCH | \
- SYSCALL_WORK_SYSCALL_EXIT_TRAP)
+ SYSCALL_WORK_SYSCALL_EXIT_TRAP | \
+ SYSCALL_WORK_FD_SLOTS)
/**
* arch_ptrace_report_syscall_permit_entry - Architecture specific wrapper for
@@ -245,6 +247,10 @@ static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned lon
{
bool step;
+ /* Install or drop the descriptors the syscall prepared. */
+ if (work & SYSCALL_WORK_FD_SLOTS)
+ fd_slots_commit(regs);
+
/*
* If the syscall was rolled back due to syscall user dispatching,
* then the tracers below are not invoked for the same reason as
@@ -92,6 +92,8 @@ extern int __get_unused_fd_flags(unsigned flags, unsigned long nofile);
extern int get_unused_fd_flags(unsigned flags);
extern void put_unused_fd(unsigned int fd);
void __fd_slots_commit(long ret);
+struct pt_regs;
+void fd_slots_commit(struct pt_regs *regs);
void exit_fd_slots(void);
DEFINE_CLASS(get_unused_fd, int, if (_T >= 0) put_unused_fd(_T),
@@ -47,6 +47,7 @@ enum syscall_work_bit {
SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH,
SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP,
SYSCALL_WORK_BIT_SYSCALL_RSEQ_SLICE,
+ SYSCALL_WORK_BIT_FD_SLOTS,
};
#define SYSCALL_WORK_SECCOMP BIT(SYSCALL_WORK_BIT_SECCOMP)
@@ -57,6 +58,7 @@ enum syscall_work_bit {
#define SYSCALL_WORK_SYSCALL_USER_DISPATCH BIT(SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH)
#define SYSCALL_WORK_SYSCALL_EXIT_TRAP BIT(SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP)
#define SYSCALL_WORK_SYSCALL_RSEQ_SLICE BIT(SYSCALL_WORK_BIT_SYSCALL_RSEQ_SLICE)
+#define SYSCALL_WORK_FD_SLOTS BIT(SYSCALL_WORK_BIT_FD_SLOTS)
#endif
#include <asm/thread_info.h>
Reserving a descriptor with fd_prepare() marks the task with SYSCALL_WORK_FD_SLOTS. The syscall exit path already tests the syscall work bits so a syscall that deferred nothing pays nothing new and one that did takes the slow path. syscall_exit_work() commits the reservations before audit and ptrace. Architectures without the generic entry code grow TIF_FD_SLOTS bit in their syscall exit work in the following patches. Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org> --- fs/file.c | 24 ++++++++++++++++++++++++ include/linux/entry-common.h | 8 +++++++- include/linux/file.h | 2 ++ include/linux/thread_info.h | 2 ++ 4 files changed, 35 insertions(+), 1 deletion(-)