From 685aaecf4e90f0c1148222e95629ca562e8093db Mon Sep 17 00:00:00 2001 From: Christoph Muellner Date: Tue, 20 Mar 2018 11:16:02 +0100 Subject: arm64: Introduce retpoline for aarch64/arm64. This patch adds retpoline support for aarch64. This includes: * Kconfig flag CONFIG_RETPOLINE to enable it * testing for required compiler support * generation of external retpoline thunk functions * patches for the arm64 specific assembly code * Enable /sys/devices/system/cpu/vulnerabilities * arm64: retpoline: Use kernel's EXPORT_SYMBOL macro. * arm64: retpoline: Add thunks for x29 and x30. * arm64: retpoline: Add function signature for symbol versioning. Signed-off-by: Christoph Muellner --- arch/arm64/Kconfig | 15 ++++++++ arch/arm64/Makefile | 11 ++++++ arch/arm64/include/asm/asm-prototypes.h | 41 +++++++++++++++++++++ arch/arm64/include/asm/assembler.h | 1 + arch/arm64/include/asm/nospec-branch.h | 59 ++++++++++++++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 64 +++++++++++++++++++++++++++++++++ arch/arm64/kernel/entry-ftrace.S | 2 +- arch/arm64/kernel/entry.S | 6 ++-- arch/arm64/kernel/vmlinux.lds.S | 7 ++++ arch/arm64/kvm/hyp/hyp-entry.S | 2 +- arch/arm64/lib/Makefile | 4 +-- arch/arm64/lib/retpoline.S | 61 +++++++++++++++++++++++++++++++ drivers/firmware/efi/libstub/Makefile | 2 +- 13 files changed, 267 insertions(+), 8 deletions(-) create mode 100644 arch/arm64/include/asm/asm-prototypes.h create mode 100644 arch/arm64/include/asm/nospec-branch.h create mode 100644 arch/arm64/lib/retpoline.S diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3551990a2726..0750abb45e51 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -68,6 +68,7 @@ config ARM64 select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_CPU_AUTOPROBE + select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP select GENERIC_IRQ_PROBE @@ -305,6 +306,20 @@ endmenu menu "Kernel Features" +config RETPOLINE + bool "Avoid speculative indirect branches in kernel" + default y + select STACK_VALIDATION if HAVE_STACK_VALIDATION + help + Compile kernel with the retpoline compiler options to guard against + kernel-to-user data leaks by avoiding speculative indirect + branches. Requires a compiler with -mindirect-branch=thunk-extern + support for full protection. The kernel may run slower. + + Without compiler support, at least indirect branches in assembler + code are eliminated. Since this includes the syscall entry path, + it is not entirely pointless. + menu "ARM errata workarounds via the alternatives framework" config ARM64_ERRATUM_826319 diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index a8f6aa92a144..8df25a60dbb9 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -88,6 +88,17 @@ ifeq ($(CONFIG_ARM64_MODULE_PLTS),y) KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds endif +RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern +RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC)) +export RETPOLINE_CFLAGS + +# Avoid indirect branches in kernel to deal with Spectre +ifdef CONFIG_RETPOLINE +ifneq ($(RETPOLINE_CFLAGS),) + KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE +endif +endif + # Default value head-y := arch/arm64/kernel/head.o diff --git a/arch/arm64/include/asm/asm-prototypes.h b/arch/arm64/include/asm/asm-prototypes.h new file mode 100644 index 000000000000..930391c5cef7 --- /dev/null +++ b/arch/arm64/include/asm/asm-prototypes.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include + +#ifdef CONFIG_RETPOLINE +#define INDIRECT_THUNK(reg) \ +extern asmlinkage void __aarch64_indirect_thunk_ ## reg(void); + +INDIRECT_THUNK(x0); +INDIRECT_THUNK(x1); +INDIRECT_THUNK(x2); +INDIRECT_THUNK(x3); +INDIRECT_THUNK(x4); +INDIRECT_THUNK(x5); +INDIRECT_THUNK(x6); +INDIRECT_THUNK(x7); +INDIRECT_THUNK(x8); +INDIRECT_THUNK(x9); +INDIRECT_THUNK(x10); +INDIRECT_THUNK(x11); +INDIRECT_THUNK(x12); +INDIRECT_THUNK(x13); +INDIRECT_THUNK(x14); +INDIRECT_THUNK(x15); +INDIRECT_THUNK(x16); +INDIRECT_THUNK(x17); +INDIRECT_THUNK(x18); +INDIRECT_THUNK(x19); +INDIRECT_THUNK(x20); +INDIRECT_THUNK(x21); +INDIRECT_THUNK(x22); +INDIRECT_THUNK(x23); +INDIRECT_THUNK(x24); +INDIRECT_THUNK(x25); +INDIRECT_THUNK(x26); +INDIRECT_THUNK(x27); +INDIRECT_THUNK(x28); +INDIRECT_THUNK(x29); +INDIRECT_THUNK(x30); + +#endif /* CONFIG_RETPOLINE */ + diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 3c78835bba94..73e8ecc8e8a5 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -30,6 +30,7 @@ #include #include #include +#include .macro save_and_disable_daif, flags mrs \flags, daif diff --git a/arch/arm64/include/asm/nospec-branch.h b/arch/arm64/include/asm/nospec-branch.h new file mode 100644 index 000000000000..cc261d5f1afd --- /dev/null +++ b/arch/arm64/include/asm/nospec-branch.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_ARM64_NOSPEC_BRANCH_H_ +#define _ASM_ARM64_NOSPEC_BRANCH_H_ + +#ifdef __ASSEMBLY__ + +.macro retpoline + str x30, [sp, #-16]! + bl 101f +100: //speculation trap + wfe + b 100b +101: //do ROP + adrp x30, 102f + add x30, x30, :lo12:102f + ret +102: //non-spec code + ldr x30, [sp], #16 +.endm + +.macro br_nospec reg +#ifdef CONFIG_RETPOLINE + b __aarch64_indirect_thunk_\reg +#else + br \reg +#endif +.endm + +.macro blr_nospec reg +#ifdef CONFIG_RETPOLINE + bl __aarch64_indirect_thunk_\reg +#else + blr \reg +#endif +.endm + +/* + * In case of "blr lr" we need to inline the retpoline + * as we cannot do a bl to the indirect_thunk, because + * it would destroy the contents of our link register. + */ +.macro blr_nospec_lr +#ifdef CONFIG_RETPOLINE + retpoline + blr lr +#else + blr lr +#endif +.endm + +#else /* __ASSEMBLY__ */ + +extern char __indirect_thunk_start[]; +extern char __indirect_thunk_end[]; + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ARM64_NOSPEC_BRANCH_H_ */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index b5a28336c077..b6a20b767af6 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -462,3 +462,67 @@ void __init enable_errata_workarounds(void) { enable_cpu_capabilities(arm64_errata); } + +static inline bool retp_compiler(void) +{ + return __is_defined(RETPOLINE); +} + +/* The Spectre V2 mitigation variants */ +enum spectre_v2_mitigation { + SPECTRE_V2_NONE, + SPECTRE_V2_RETPOLINE_MINIMAL, + SPECTRE_V2_RETPOLINE_GENERIC, +}; + +static const char *spectre_v2_strings[] = { + [SPECTRE_V2_NONE] = "Vulnerable", + [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline", + [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", +}; + +enum spectre_v2_mitigation get_spectre_v2_mitigation(void) +{ + enum spectre_v2_mitigation mode; + +#ifndef RETPOLINE + return SPECTRE_V2_NONE; +#endif + + mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : + SPECTRE_V2_RETPOLINE_MINIMAL; + + return mode; +} + +#ifdef RETPOLINE +static bool spectre_v2_bad_module; + +bool retpoline_module_ok(bool has_retpoline) +{ + if (has_retpoline) + return true; + + pr_err("System may be vulnerable to spectre v2\n"); + spectre_v2_bad_module = true; + return false; +} + +static inline const char *spectre_v2_module_string(void) +{ + return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; +} +#else +static inline const char *spectre_v2_module_string(void) { return ""; } +#endif + +#ifdef CONFIG_SYSFS +ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) +{ + enum spectre_v2_mitigation mode; + + mode = get_spectre_v2_mitigation(); + return sprintf(buf, "%s%s\n", spectre_v2_strings[mode], + spectre_v2_module_string()); +} +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 1175f5827ae1..27bb502aa5e6 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -106,7 +106,7 @@ ENTRY(_mcount) mcount_get_pc x0 // function's pc mcount_get_lr x1 // function's lr (= parent's pc) - blr x2 // (*ftrace_trace_function)(pc, lr); + blr_nospec x2 // (*ftrace_trace_function)(pc, lr); skip_ftrace_call: // } #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index f409b7750114..7475dbdb816f 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -395,7 +395,7 @@ tsk .req x28 // current thread_info ldr_l x1, handle_arch_irq mov x0, sp irq_stack_entry - blr x1 + blr_nospec x1 irq_stack_exit .endm @@ -976,7 +976,7 @@ el0_svc_naked: // compat entry point b.hs ni_sys mask_nospec64 xscno, xsc_nr, x19 // enforce bounds for syscall number ldr x16, [stbl, xscno, lsl #3] // address in the syscall table - blr x16 // call sys_* routine + blr_nospec x16 // call sys_* routine b ret_fast_syscall ni_sys: mov x0, sp @@ -1006,7 +1006,7 @@ __sys_trace: ldp x4, x5, [sp, #S_X4] ldp x6, x7, [sp, #S_X6] ldr x16, [stbl, xscno, lsl #3] // address in the syscall table - blr x16 // call sys_* routine + blr_nospec x16 // call sys_* routine __sys_trace_return: str x0, [sp, #S_X0] // save returned x0 diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 0221aca6493d..7d534afe66b7 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -127,6 +127,13 @@ SECTIONS TRAMP_TEXT *(.fixup) *(.gnu.warning) + +#ifdef CONFIG_RETPOLINE + __indirect_thunk_start = .; + *(.text.__aarch64.indirect_thunk) + __indirect_thunk_end = .; +#endif /* CONFIG_RETPOLINE */ + . = ALIGN(16); *(.got) /* Global offset table */ } diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index f36464bd57c5..2e098c1248f7 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -38,7 +38,7 @@ mov x0, x1 mov x1, x2 mov x2, x3 - blr lr + blr_nospec_lr ldr lr, [sp], #16 .endm diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 4e696f96451f..155d5080a674 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -2,8 +2,8 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_in_user.o copy_page.o \ clear_page.o memchr.o memcpy.o memmove.o memset.o \ - memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ - strchr.o strrchr.o tishift.o + memcmp.o retpoline.o strcmp.o strncmp.o strlen.o \ + strnlen.o strchr.o strrchr.o tishift.o # Tell the compiler to treat all general purpose registers (with the # exception of the IP registers, which are already handled by the caller diff --git a/arch/arm64/lib/retpoline.S b/arch/arm64/lib/retpoline.S new file mode 100644 index 000000000000..c1a6f7c41b62 --- /dev/null +++ b/arch/arm64/lib/retpoline.S @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include +#include +#include + +.macro THUNK reg + .section .text.__aarch64.indirect_thunk + +ENTRY(__aarch64_indirect_thunk_\reg) + retpoline + br \reg +ENDPROC(__aarch64_indirect_thunk_\reg) +.endm + +#define _ASM_ALIGN +#define _ASM_PTR ".quad" + +# define _ASM_NOKPROBE(entry) \ + .pushsection "_kprobe_blacklist","aw" ; \ + _ASM_ALIGN \ + _ASM_PTR (entry); \ + .popsection + +#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) +#define EXPORT_THUNK(reg) __EXPORT_THUNK(__aarch64_indirect_thunk_ ## reg) +#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg) + +GENERATE_THUNK(x0) +GENERATE_THUNK(x1) +GENERATE_THUNK(x2) +GENERATE_THUNK(x3) +GENERATE_THUNK(x4) +GENERATE_THUNK(x5) +GENERATE_THUNK(x6) +GENERATE_THUNK(x7) +GENERATE_THUNK(x8) +GENERATE_THUNK(x9) +GENERATE_THUNK(x10) +GENERATE_THUNK(x11) +GENERATE_THUNK(x12) +GENERATE_THUNK(x13) +GENERATE_THUNK(x14) +GENERATE_THUNK(x15) +GENERATE_THUNK(x16) +GENERATE_THUNK(x17) +GENERATE_THUNK(x18) +GENERATE_THUNK(x19) +GENERATE_THUNK(x20) +GENERATE_THUNK(x21) +GENERATE_THUNK(x22) +GENERATE_THUNK(x23) +GENERATE_THUNK(x24) +GENERATE_THUNK(x25) +GENERATE_THUNK(x26) +GENERATE_THUNK(x27) +GENERATE_THUNK(x28) +GENERATE_THUNK(x29) +GENERATE_THUNK(x30) diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 7b3ba40f0745..6a150870a619 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -11,7 +11,7 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -O2 \ -fPIC -fno-strict-aliasing -mno-red-zone \ -mno-mmx -mno-sse -cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie +cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie -mindirect-branch=keep cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \ -fno-builtin -fpic -mno-single-pic-base -- cgit v1.2.3