@@ -195,6 +195,7 @@ config LOCKDEP_SUPPORT
def_bool y
source "arch/riscv/Kconfig.socs"
+source "arch/riscv/Kconfig.erratas"
menu "Platform type"
new file mode 100644
@@ -0,0 +1,12 @@
+menu "CPU errata selection"
+
+config RISCV_ERRATA_ALTERNATIVE
+ bool "RISC-V alternative scheme"
+ default y
+ help
+ This Kconfig allows the kernel to automatically patch the
+ errata required by the execution platform at run time. The
+ code patching is performed once in the boot stages. It means
+ that the overhead from this mechanism is just taken once.
+
+endmenu
@@ -75,6 +75,7 @@ KBUILD_IMAGE := $(boot)/Image.gz
head-y := arch/riscv/kernel/head.o
core-y += arch/riscv/
+core-$(CONFIG_RISCV_ERRATA_ALTERNATIVE) += arch/riscv/errata/
libs-y += arch/riscv/lib/
libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
new file mode 100644
@@ -0,0 +1 @@
+obj-y += alternative.o
new file mode 100644
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * alternative runtime patching
+ * inspired by the ARM64 and x86 version
+ *
+ * Copyright (C) 2021 Sifive.
+ */
+
+#include <linux/init.h>
+#include <linux/cpu.h>
+#include <linux/uaccess.h>
+#include <asm/patch.h>
+#include <asm/alternative.h>
+#include <asm/sections.h>
+
+struct alt_region {
+ struct alt_entry *begin;
+ struct alt_entry *end;
+};
+
+static bool (*errata_checkfunc)(struct alt_entry *alt);
+typedef int (*patch_func_t)(void *addr, const void *insn, size_t size);
+
+static void __apply_alternatives(void *alt_region, void *alt_patch_func)
+{
+ struct alt_entry *alt;
+ struct alt_region *region = alt_region;
+
+ for (alt = region->begin; alt < region->end; alt++) {
+ if (!errata_checkfunc(alt))
+ continue;
+ ((patch_func_t)alt_patch_func)(alt->old_ptr, alt->alt_ptr, alt->alt_len);
+ }
+}
+
+static void __init init_alternative(void)
+{
+ struct errata_checkfunc_id *ptr;
+
+ for (ptr = (struct errata_checkfunc_id *)__alt_checkfunc_table;
+ ptr < (struct errata_checkfunc_id *)__alt_checkfunc_table_end;
+ ptr++) {
+ if (cpu_mfr_info.vendor_id == ptr->vendor_id)
+ errata_checkfunc = ptr->func;
+ }
+}
+
+/*
+ * This is called very early in the boot process (directly after we run
+ * a feature detect on the boot CPU). No need to worry about other CPUs
+ * here.
+ */
+void __init apply_boot_alternatives(void)
+{
+ struct alt_region region;
+
+ /* If called on non-boot cpu things could go wrong */
+ WARN_ON(smp_processor_id() != 0);
+
+ init_alternative();
+
+ if (!errata_checkfunc)
+ return;
+
+ region.begin = (struct alt_entry *)__alt_start;
+ region.end = (struct alt_entry *)__alt_end;
+ __apply_alternatives(®ion, patch_text_nosync);
+}
+
new file mode 100644
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_ALTERNATIVE_MACROS_H
+#define __ASM_ALTERNATIVE_MACROS_H
+
+#ifdef CONFIG_RISCV_ERRATA_ALTERNATIVE
+
+#ifndef __ASSEMBLY__
+
+#include <asm/asm.h>
+#include <linux/stringify.h>
+
+#define ALT_ENTRY(oldptr, altptr, vendor_id, errata_id, altlen) \
+ RISCV_PTR " " oldptr "\n" \
+ RISCV_PTR " " altptr "\n" \
+ REG_ASM " " vendor_id "\n" \
+ REG_ASM " " altlen "\n" \
+ ".word " errata_id "\n"
+
+#define __ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, enable) \
+ "886 :\n" \
+ oldinsn "\n" \
+ ".if " __stringify(enable) " == 1\n" \
+ "887 :\n" \
+ ".pushsection .alternative, \"a\"\n" \
+ ALT_ENTRY("886b", "888f", __stringify(vendor_id), __stringify(errata_id), "889f - 888f") \
+ ".popsection\n" \
+ ".subsection 1\n" \
+ "888 :\n" \
+ altinsn "\n" \
+ "889 :\n" \
+ ".previous\n" \
+ ".org . - (887b - 886b) + (889b - 888b)\n" \
+ ".org . - (889b - 888b) + (887b - 886b)\n" \
+ ".endif\n"
+
+#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
+ __ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, IS_ENABLED(CONFIG_k))
+
+#else /* __ASSEMBLY__ */
+
+.macro ALT_ENTRY oldptr altptr vendor_id errata_id alt_len
+ RISCV_PTR \oldptr
+ RISCV_PTR \altptr
+ REG_ASM \vendor_id
+ REG_ASM \alt_len
+ .word \errata_id
+.endm
+
+.macro __ALTERNATIVE_CFG insn1 insn2 vendor_id errata_id enable = 1
+886 :
+ \insn1
+ .if \enable
+887 :
+ .pushsection .alternative, "a"
+ ALT_ENTRY 886b, 888f, \vendor_id, \errata_id, 889f - 888f
+ .popsection
+ .subsection 1
+888 :
+ \insn2
+889 :
+ .previous
+ .org . - (889b - 888b) + (887b - 886b)
+ .org . - (887b - 886b) + (889b - 888b)
+ .endif
+.endm
+
+#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
+ __ALTERNATIVE_CFG oldinsn, altinsn, vendor_id, errata_id, IS_ENABLED(CONFIG_k)
+
+#endif /* !__ASSEMBLY__ */
+
+#else /* !CONFIG_RISCV_ERRATA_ALTERNATIVE*/
+#ifndef __ASSEMBLY__
+
+#define __ALTERNATIVE_CFG(oldinsn) \
+ oldinsn "\n"
+
+#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
+ __ALTERNATIVE_CFG(oldinsn)
+
+#else /* __ASSEMBLY__ */
+
+.macro __ALTERNATIVE_CFG insn1
+ \insn1
+.endm
+
+#define _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
+ __ALTERNATIVE_CFG oldinsn
+
+#endif /* !__ASSEMBLY__ */
+#endif /* CONFIG_RISCV_ERRATA_ALTERNATIVE */
+
+/*
+ * Usage:
+ * ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k)
+ * in the assembly code. Otherwise,
+ * asm(ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k));
+ *
+ * oldinsn: The old instruction which will be replaced.
+ * altinsn: The replacement instruction.
+ * vendor_id: The CPU vendor ID.
+ * errata_id: The errata ID.
+ * CONFIG_k: The Kconfig of this errata. The instructions replacement can
+ * be disabled by this Kconfig. When Kconfig is disabled, the
+ * oldinsn will always be executed.
+ */
+#define ALTERNATIVE(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k) \
+ _ALTERNATIVE_CFG(oldinsn, altinsn, vendor_id, errata_id, CONFIG_k)
+
+#endif
new file mode 100644
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2021 Sifive.
+ */
+
+#ifndef __ASM_ALTERNATIVE_H
+#define __ASM_ALTERNATIVE_H
+
+#define ERRATA_STRING_LENGTH_MAX 32
+
+#include <asm/alternative-macros.h>
+
+#ifndef __ASSEMBLY__
+
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/stddef.h>
+#include <asm/hwcap.h>
+
+void __init apply_boot_alternatives(void);
+
+struct alt_entry {
+ void *old_ptr; /* address of original instruciton or data */
+ void *alt_ptr; /* address of replacement instruction or data */
+ unsigned long vendor_id; /* cpu vendor id */
+ unsigned long alt_len; /* The replacement size */
+ unsigned int errata_id; /* The errata id */
+} __packed;
+
+struct errata_checkfunc_id {
+ unsigned long vendor_id;
+ bool (*func)(struct alt_entry *alt);
+};
+
+extern struct cpu_manufacturer_info_t cpu_mfr_info;
+
+#define REGISTER_ERRATA_CHECKFUNC(checkfunc, vendorid) \
+ static const struct errata_checkfunc_id _errata_check_##vendorid \
+ __used __section(".alt_checkfunc_table") \
+ __aligned(__alignof__(struct errata_checkfunc_id)) = \
+ { .vendor_id = vendorid, \
+ .func = checkfunc }
+#endif
+#endif
@@ -23,6 +23,7 @@
#define REG_L __REG_SEL(ld, lw)
#define REG_S __REG_SEL(sd, sw)
#define REG_SC __REG_SEL(sc.d, sc.w)
+#define REG_ASM __REG_SEL(.dword, .word)
#define SZREG __REG_SEL(8, 4)
#define LGREG __REG_SEL(3, 2)
new file mode 100644
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2021 Sifive.
+ */
+
+#ifdef CONFIG_SOC_SIFIVE
+#define ERRATA_NUMBER 0
+#endif
@@ -11,5 +11,7 @@ extern char _start[];
extern char _start_kernel[];
extern char __init_data_begin[], __init_data_end[];
extern char __init_text_begin[], __init_text_end[];
+extern char __alt_checkfunc_table[], __alt_checkfunc_table_end[];
+extern char __alt_start[], __alt_end[];
#endif /* __ASM_SECTIONS_H */
new file mode 100644
@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2021 SiFive
+ */
+
+#define SIFIVE_VENDOR_ID 0x489
@@ -31,6 +31,7 @@
#include <asm/sections.h>
#include <asm/sbi.h>
#include <asm/smp.h>
+#include <asm/alternative.h>
#include "head.h"
@@ -39,6 +40,9 @@ static DECLARE_COMPLETION(cpu_running);
void __init smp_prepare_boot_cpu(void)
{
init_cpu_topology();
+#ifdef CONFIG_RISCV_ERRATA_ALTERNATIVE
+ apply_boot_alternatives();
+#endif
}
void __init smp_prepare_cpus(unsigned int max_cpus)
@@ -90,6 +90,20 @@ SECTIONS
}
__init_data_end = .;
+
+ . = ALIGN(8);
+ .alt_checkfunc_table : {
+ __alt_checkfunc_table = .;
+ *(.alt_checkfunc_table)
+ __alt_checkfunc_table_end = .;
+ }
+
+ . = ALIGN(8);
+ .alternative : {
+ __alt_start = .;
+ *(.alternative)
+ __alt_end = .;
+ }
__init_end = .;
/* Start of data section */