summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristoph Muellner <christoph.muellner@theobroma-systems.com>2018-03-20 11:16:02 +0100
committerChristoph Muellner <christoph.muellner@theobroma-systems.com>2018-04-27 08:28:29 +0200
commit685aaecf4e90f0c1148222e95629ca562e8093db (patch)
tree0da2df49a7973f54a7cd0e1fb8fe48e725ce1621
parent5a0f312409eab96928a95598d440cf54b5cf7153 (diff)
arm64: Introduce retpoline for aarch64/arm64.linux-4.16.5-amp
This patch adds retpoline support for aarch64. This includes: * Kconfig flag CONFIG_RETPOLINE to enable it * testing for required compiler support * generation of external retpoline thunk functions * patches for the arm64 specific assembly code * Enable /sys/devices/system/cpu/vulnerabilities * arm64: retpoline: Use kernel's EXPORT_SYMBOL macro. * arm64: retpoline: Add thunks for x29 and x30. * arm64: retpoline: Add function signature for symbol versioning. Signed-off-by: Christoph Muellner <christoph.muellner@theobroma-systems.com>
-rw-r--r--arch/arm64/Kconfig15
-rw-r--r--arch/arm64/Makefile11
-rw-r--r--arch/arm64/include/asm/asm-prototypes.h41
-rw-r--r--arch/arm64/include/asm/assembler.h1
-rw-r--r--arch/arm64/include/asm/nospec-branch.h59
-rw-r--r--arch/arm64/kernel/cpu_errata.c64
-rw-r--r--arch/arm64/kernel/entry-ftrace.S2
-rw-r--r--arch/arm64/kernel/entry.S6
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S7
-rw-r--r--arch/arm64/kvm/hyp/hyp-entry.S2
-rw-r--r--arch/arm64/lib/Makefile4
-rw-r--r--arch/arm64/lib/retpoline.S61
-rw-r--r--drivers/firmware/efi/libstub/Makefile2
13 files changed, 267 insertions, 8 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 3551990a2726..0750abb45e51 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -68,6 +68,7 @@ config ARM64
select GENERIC_CLOCKEVENTS
select GENERIC_CLOCKEVENTS_BROADCAST
select GENERIC_CPU_AUTOPROBE
+ select GENERIC_CPU_VULNERABILITIES
select GENERIC_EARLY_IOREMAP
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IRQ_PROBE
@@ -305,6 +306,20 @@ endmenu
menu "Kernel Features"
+config RETPOLINE
+ bool "Avoid speculative indirect branches in kernel"
+ default y
+ select STACK_VALIDATION if HAVE_STACK_VALIDATION
+ help
+ Compile kernel with the retpoline compiler options to guard against
+ kernel-to-user data leaks by avoiding speculative indirect
+ branches. Requires a compiler with -mindirect-branch=thunk-extern
+ support for full protection. The kernel may run slower.
+
+ Without compiler support, at least indirect branches in assembler
+ code are eliminated. Since this includes the syscall entry path,
+ it is not entirely pointless.
+
menu "ARM errata workarounds via the alternatives framework"
config ARM64_ERRATUM_826319
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index a8f6aa92a144..8df25a60dbb9 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -88,6 +88,17 @@ ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)
KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds
endif
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern
+RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC))
+export RETPOLINE_CFLAGS
+
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+ifneq ($(RETPOLINE_CFLAGS),)
+ KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+endif
+endif
+
# Default value
head-y := arch/arm64/kernel/head.o
diff --git a/arch/arm64/include/asm/asm-prototypes.h b/arch/arm64/include/asm/asm-prototypes.h
new file mode 100644
index 000000000000..930391c5cef7
--- /dev/null
+++ b/arch/arm64/include/asm/asm-prototypes.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <asm-generic/asm-prototypes.h>
+
+#ifdef CONFIG_RETPOLINE
+#define INDIRECT_THUNK(reg) \
+extern asmlinkage void __aarch64_indirect_thunk_ ## reg(void);
+
+INDIRECT_THUNK(x0);
+INDIRECT_THUNK(x1);
+INDIRECT_THUNK(x2);
+INDIRECT_THUNK(x3);
+INDIRECT_THUNK(x4);
+INDIRECT_THUNK(x5);
+INDIRECT_THUNK(x6);
+INDIRECT_THUNK(x7);
+INDIRECT_THUNK(x8);
+INDIRECT_THUNK(x9);
+INDIRECT_THUNK(x10);
+INDIRECT_THUNK(x11);
+INDIRECT_THUNK(x12);
+INDIRECT_THUNK(x13);
+INDIRECT_THUNK(x14);
+INDIRECT_THUNK(x15);
+INDIRECT_THUNK(x16);
+INDIRECT_THUNK(x17);
+INDIRECT_THUNK(x18);
+INDIRECT_THUNK(x19);
+INDIRECT_THUNK(x20);
+INDIRECT_THUNK(x21);
+INDIRECT_THUNK(x22);
+INDIRECT_THUNK(x23);
+INDIRECT_THUNK(x24);
+INDIRECT_THUNK(x25);
+INDIRECT_THUNK(x26);
+INDIRECT_THUNK(x27);
+INDIRECT_THUNK(x28);
+INDIRECT_THUNK(x29);
+INDIRECT_THUNK(x30);
+
+#endif /* CONFIG_RETPOLINE */
+
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 3c78835bba94..73e8ecc8e8a5 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -30,6 +30,7 @@
#include <asm/pgtable-hwdef.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
+#include <asm/nospec-branch.h>
.macro save_and_disable_daif, flags
mrs \flags, daif
diff --git a/arch/arm64/include/asm/nospec-branch.h b/arch/arm64/include/asm/nospec-branch.h
new file mode 100644
index 000000000000..cc261d5f1afd
--- /dev/null
+++ b/arch/arm64/include/asm/nospec-branch.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_ARM64_NOSPEC_BRANCH_H_
+#define _ASM_ARM64_NOSPEC_BRANCH_H_
+
+#ifdef __ASSEMBLY__
+
+.macro retpoline
+ str x30, [sp, #-16]!
+ bl 101f
+100: //speculation trap
+ wfe
+ b 100b
+101: //do ROP
+ adrp x30, 102f
+ add x30, x30, :lo12:102f
+ ret
+102: //non-spec code
+ ldr x30, [sp], #16
+.endm
+
+.macro br_nospec reg
+#ifdef CONFIG_RETPOLINE
+ b __aarch64_indirect_thunk_\reg
+#else
+ br \reg
+#endif
+.endm
+
+.macro blr_nospec reg
+#ifdef CONFIG_RETPOLINE
+ bl __aarch64_indirect_thunk_\reg
+#else
+ blr \reg
+#endif
+.endm
+
+/*
+ * In case of "blr lr" we need to inline the retpoline
+ * as we cannot do a bl to the indirect_thunk, because
+ * it would destroy the contents of our link register.
+ */
+.macro blr_nospec_lr
+#ifdef CONFIG_RETPOLINE
+ retpoline
+ blr lr
+#else
+ blr lr
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+extern char __indirect_thunk_start[];
+extern char __indirect_thunk_end[];
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_ARM64_NOSPEC_BRANCH_H_ */
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index b5a28336c077..b6a20b767af6 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -462,3 +462,67 @@ void __init enable_errata_workarounds(void)
{
enable_cpu_capabilities(arm64_errata);
}
+
+static inline bool retp_compiler(void)
+{
+ return __is_defined(RETPOLINE);
+}
+
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+ SPECTRE_V2_NONE,
+ SPECTRE_V2_RETPOLINE_MINIMAL,
+ SPECTRE_V2_RETPOLINE_GENERIC,
+};
+
+static const char *spectre_v2_strings[] = {
+ [SPECTRE_V2_NONE] = "Vulnerable",
+ [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline",
+ [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
+};
+
+enum spectre_v2_mitigation get_spectre_v2_mitigation(void)
+{
+ enum spectre_v2_mitigation mode;
+
+#ifndef RETPOLINE
+ return SPECTRE_V2_NONE;
+#endif
+
+ mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
+ SPECTRE_V2_RETPOLINE_MINIMAL;
+
+ return mode;
+}
+
+#ifdef RETPOLINE
+static bool spectre_v2_bad_module;
+
+bool retpoline_module_ok(bool has_retpoline)
+{
+ if (has_retpoline)
+ return true;
+
+ pr_err("System may be vulnerable to spectre v2\n");
+ spectre_v2_bad_module = true;
+ return false;
+}
+
+static inline const char *spectre_v2_module_string(void)
+{
+ return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
+}
+#else
+static inline const char *spectre_v2_module_string(void) { return ""; }
+#endif
+
+#ifdef CONFIG_SYSFS
+ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ enum spectre_v2_mitigation mode;
+
+ mode = get_spectre_v2_mitigation();
+ return sprintf(buf, "%s%s\n", spectre_v2_strings[mode],
+ spectre_v2_module_string());
+}
+#endif
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 1175f5827ae1..27bb502aa5e6 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -106,7 +106,7 @@ ENTRY(_mcount)
mcount_get_pc x0 // function's pc
mcount_get_lr x1 // function's lr (= parent's pc)
- blr x2 // (*ftrace_trace_function)(pc, lr);
+ blr_nospec x2 // (*ftrace_trace_function)(pc, lr);
skip_ftrace_call: // }
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f409b7750114..7475dbdb816f 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -395,7 +395,7 @@ tsk .req x28 // current thread_info
ldr_l x1, handle_arch_irq
mov x0, sp
irq_stack_entry
- blr x1
+ blr_nospec x1
irq_stack_exit
.endm
@@ -976,7 +976,7 @@ el0_svc_naked: // compat entry point
b.hs ni_sys
mask_nospec64 xscno, xsc_nr, x19 // enforce bounds for syscall number
ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
- blr x16 // call sys_* routine
+ blr_nospec x16 // call sys_* routine
b ret_fast_syscall
ni_sys:
mov x0, sp
@@ -1006,7 +1006,7 @@ __sys_trace:
ldp x4, x5, [sp, #S_X4]
ldp x6, x7, [sp, #S_X6]
ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
- blr x16 // call sys_* routine
+ blr_nospec x16 // call sys_* routine
__sys_trace_return:
str x0, [sp, #S_X0] // save returned x0
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 0221aca6493d..7d534afe66b7 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -127,6 +127,13 @@ SECTIONS
TRAMP_TEXT
*(.fixup)
*(.gnu.warning)
+
+#ifdef CONFIG_RETPOLINE
+ __indirect_thunk_start = .;
+ *(.text.__aarch64.indirect_thunk)
+ __indirect_thunk_end = .;
+#endif /* CONFIG_RETPOLINE */
+
. = ALIGN(16);
*(.got) /* Global offset table */
}
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index f36464bd57c5..2e098c1248f7 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -38,7 +38,7 @@
mov x0, x1
mov x1, x2
mov x2, x3
- blr lr
+ blr_nospec_lr
ldr lr, [sp], #16
.endm
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 4e696f96451f..155d5080a674 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -2,8 +2,8 @@
lib-y := bitops.o clear_user.o delay.o copy_from_user.o \
copy_to_user.o copy_in_user.o copy_page.o \
clear_page.o memchr.o memcpy.o memmove.o memset.o \
- memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
- strchr.o strrchr.o tishift.o
+ memcmp.o retpoline.o strcmp.o strncmp.o strlen.o \
+ strnlen.o strchr.o strrchr.o tishift.o
# Tell the compiler to treat all general purpose registers (with the
# exception of the IP registers, which are already handled by the caller
diff --git a/arch/arm64/lib/retpoline.S b/arch/arm64/lib/retpoline.S
new file mode 100644
index 000000000000..c1a6f7c41b62
--- /dev/null
+++ b/arch/arm64/lib/retpoline.S
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/nospec-branch.h>
+#include <asm-generic/export.h>
+
+.macro THUNK reg
+ .section .text.__aarch64.indirect_thunk
+
+ENTRY(__aarch64_indirect_thunk_\reg)
+ retpoline
+ br \reg
+ENDPROC(__aarch64_indirect_thunk_\reg)
+.endm
+
+#define _ASM_ALIGN
+#define _ASM_PTR ".quad"
+
+# define _ASM_NOKPROBE(entry) \
+ .pushsection "_kprobe_blacklist","aw" ; \
+ _ASM_ALIGN \
+ _ASM_PTR (entry); \
+ .popsection
+
+#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
+#define EXPORT_THUNK(reg) __EXPORT_THUNK(__aarch64_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(x0)
+GENERATE_THUNK(x1)
+GENERATE_THUNK(x2)
+GENERATE_THUNK(x3)
+GENERATE_THUNK(x4)
+GENERATE_THUNK(x5)
+GENERATE_THUNK(x6)
+GENERATE_THUNK(x7)
+GENERATE_THUNK(x8)
+GENERATE_THUNK(x9)
+GENERATE_THUNK(x10)
+GENERATE_THUNK(x11)
+GENERATE_THUNK(x12)
+GENERATE_THUNK(x13)
+GENERATE_THUNK(x14)
+GENERATE_THUNK(x15)
+GENERATE_THUNK(x16)
+GENERATE_THUNK(x17)
+GENERATE_THUNK(x18)
+GENERATE_THUNK(x19)
+GENERATE_THUNK(x20)
+GENERATE_THUNK(x21)
+GENERATE_THUNK(x22)
+GENERATE_THUNK(x23)
+GENERATE_THUNK(x24)
+GENERATE_THUNK(x25)
+GENERATE_THUNK(x26)
+GENERATE_THUNK(x27)
+GENERATE_THUNK(x28)
+GENERATE_THUNK(x29)
+GENERATE_THUNK(x30)
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 7b3ba40f0745..6a150870a619 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -11,7 +11,7 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -O2 \
-fPIC -fno-strict-aliasing -mno-red-zone \
-mno-mmx -mno-sse
-cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie
+cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie -mindirect-branch=keep
cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \
-fno-builtin -fpic -mno-single-pic-base