arm64: Add initial retpoline support.

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and
provide the corresponding thunks. Provide assembler macros for invoking the
thunks in the same way that GCC does, from native and inline assembler.
The integration is inspired by the x86 retpoline support.

Signed-off-by: Christoph Muellner <christoph.muellner@theobroma-systems.com>

9 files changed, 195 insertions, 3 deletions

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index be665760f2bd..c8c9f16e226f 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -305,6 +305,20 @@ endmenu
 
 menu "Kernel Features"
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	select STACK_VALIDATION if HAVE_STACK_VALIDATION
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 menu "ARM errata workarounds via the alternatives framework"
 
 config ARM64_ERRATUM_826319
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index b481b4a7c011..db3ab48e79cf 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -85,6 +85,21 @@ ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)
 KBUILD_LDFLAGS_MODULE	+= -T $(srctree)/arch/arm64/kernel/module.lds
 endif
 
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern
+RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC))
+export RETPOLINE_CFLAGS
+
+NO_RETPOLINE_CFLAGS_GCC := -mindirect-branch=keep
+NO_RETPOLINE_CFLAGS := $(call cc-option,$(NO_RETPOLINE_CFLAGS_GCC))
+export NO_RETPOLINE_CFLAGS
+
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+ifneq ($(RETPOLINE_CFLAGS),)
+  KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+endif
+endif
+
 # Default value
 head-y		:= arch/arm64/kernel/head.o
 
diff --git a/arch/arm64/include/asm/asm-prototypes.h b/arch/arm64/include/asm/asm-prototypes.h
new file mode 100644
index 000000000000..930391c5cef7
--- /dev/null
+++ b/arch/arm64/include/asm/asm-prototypes.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <asm-generic/asm-prototypes.h>
+
+#ifdef CONFIG_RETPOLINE
+#define INDIRECT_THUNK(reg) \
+extern asmlinkage void __aarch64_indirect_thunk_ ## reg(void);
+
+INDIRECT_THUNK(x0);
+INDIRECT_THUNK(x1);
+INDIRECT_THUNK(x2);
+INDIRECT_THUNK(x3);
+INDIRECT_THUNK(x4);
+INDIRECT_THUNK(x5);
+INDIRECT_THUNK(x6);
+INDIRECT_THUNK(x7);
+INDIRECT_THUNK(x8);
+INDIRECT_THUNK(x9);
+INDIRECT_THUNK(x10);
+INDIRECT_THUNK(x11);
+INDIRECT_THUNK(x12);
+INDIRECT_THUNK(x13);
+INDIRECT_THUNK(x14);
+INDIRECT_THUNK(x15);
+INDIRECT_THUNK(x16);
+INDIRECT_THUNK(x17);
+INDIRECT_THUNK(x18);
+INDIRECT_THUNK(x19);
+INDIRECT_THUNK(x20);
+INDIRECT_THUNK(x21);
+INDIRECT_THUNK(x22);
+INDIRECT_THUNK(x23);
+INDIRECT_THUNK(x24);
+INDIRECT_THUNK(x25);
+INDIRECT_THUNK(x26);
+INDIRECT_THUNK(x27);
+INDIRECT_THUNK(x28);
+INDIRECT_THUNK(x29);
+INDIRECT_THUNK(x30);
+
+#endif /* CONFIG_RETPOLINE */
+
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index a3ca19e68b73..c63b7d149bb0 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -31,6 +31,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
+#include <asm/nospec-branch.h>
 
 	.macro save_and_disable_daif, flags
 	mrs	\flags, daif
diff --git a/arch/arm64/include/asm/nospec-branch.h b/arch/arm64/include/asm/nospec-branch.h
new file mode 100644
index 000000000000..463ec183f4ec
--- /dev/null
+++ b/arch/arm64/include/asm/nospec-branch.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _ASM_ARM64_NOSPEC_BRANCH_H_
+#define _ASM_ARM64_NOSPEC_BRANCH_H_
+
+#ifdef __ASSEMBLY__
+
+.macro retpoline
+	str	x30, [sp, #-16]!
+	bl	101f
+100: //speculation trap
+	wfe
+	b	100b
+101: //do ROP
+	adr	x30, 102f
+	ret
+102: //non-spec code
+	ldr	x30, [sp], #16
+.endm
+
+.macro br_nospec reg
+#ifdef CONFIG_RETPOLINE
+	b	__aarch64_indirect_thunk_\reg
+#else
+	br	\reg
+#endif
+.endm
+
+.macro blr_nospec reg
+#ifdef CONFIG_RETPOLINE
+	bl	__aarch64_indirect_thunk_\reg
+#else
+	blr	\reg
+#endif
+.endm
+
+/*
+ * In case of "blr lr" we need to inline the retpoline
+ * as we cannot do a bl to the indirect_thunk, because
+ * it would destroy the contents of our link register.
+ */
+.macro blr_nospec_lr
+#ifdef CONFIG_RETPOLINE
+	retpoline
+#endif
+	blr	lr
+.endm
+
+#else /* __ASSEMBLY__ */
+
+extern char __indirect_thunk_start[];
+extern char __indirect_thunk_end[];
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_ARM64_NOSPEC_BRANCH_H_ */
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 0221aca6493d..7a4617bbed10 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -127,6 +127,13 @@ SECTIONS
 			TRAMP_TEXT
 			*(.fixup)
 			*(.gnu.warning)
+
+#ifdef CONFIG_RETPOLINE
+			__indirect_thunk_start = .;
+			*(.text.__aarch64.indirect_thunk)
+			__indirect_thunk_end = .;
+#endif /* CONFIG_RETPOLINE */
+
 		. = ALIGN(16);
 		*(.got)			/* Global offset table		*/
 	}
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 4e696f96451f..155d5080a674 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -2,8 +2,8 @@
 lib-y		:= bitops.o clear_user.o delay.o copy_from_user.o	\
 		   copy_to_user.o copy_in_user.o copy_page.o		\
 		   clear_page.o memchr.o memcpy.o memmove.o memset.o	\
-		   memcmp.o strcmp.o strncmp.o strlen.o strnlen.o	\
-		   strchr.o strrchr.o tishift.o
+		   memcmp.o retpoline.o strcmp.o strncmp.o strlen.o	\
+		   strnlen.o strchr.o strrchr.o tishift.o
 
 # Tell the compiler to treat all general purpose registers (with the
 # exception of the IP registers, which are already handled by the caller
diff --git a/arch/arm64/lib/retpoline.S b/arch/arm64/lib/retpoline.S
new file mode 100644
index 000000000000..39f72a001fa5
--- /dev/null
+++ b/arch/arm64/lib/retpoline.S
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/nospec-branch.h>
+#include <asm-generic/export.h>
+
+.macro THUNK reg
+	.section .text.__aarch64.indirect_thunk
+
+ENTRY(__aarch64_indirect_thunk_\reg)
+	retpoline
+	br	\reg
+ENDPROC(__aarch64_indirect_thunk_\reg)
+.endm
+
+#define _ASM_NOKPROBE(entry)					\
+	.pushsection "_kprobe_blacklist","aw" ;			\
+	.quad entry;						\
+	.popsection
+
+#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
+#define EXPORT_THUNK(reg) __EXPORT_THUNK(__aarch64_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(x0)
+GENERATE_THUNK(x1)
+GENERATE_THUNK(x2)
+GENERATE_THUNK(x3)
+GENERATE_THUNK(x4)
+GENERATE_THUNK(x5)
+GENERATE_THUNK(x6)
+GENERATE_THUNK(x7)
+GENERATE_THUNK(x8)
+GENERATE_THUNK(x9)
+GENERATE_THUNK(x10)
+GENERATE_THUNK(x11)
+GENERATE_THUNK(x12)
+GENERATE_THUNK(x13)
+GENERATE_THUNK(x14)
+GENERATE_THUNK(x15)
+GENERATE_THUNK(x16)
+GENERATE_THUNK(x17)
+GENERATE_THUNK(x18)
+GENERATE_THUNK(x19)
+GENERATE_THUNK(x20)
+GENERATE_THUNK(x21)
+GENERATE_THUNK(x22)
+GENERATE_THUNK(x23)
+GENERATE_THUNK(x24)
+GENERATE_THUNK(x25)
+GENERATE_THUNK(x26)
+GENERATE_THUNK(x27)
+GENERATE_THUNK(x28)
+GENERATE_THUNK(x29)
+GENERATE_THUNK(x30)
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 7b3ba40f0745..09f00169d1b3 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -11,7 +11,8 @@ cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ -O2 \
 				   -fPIC -fno-strict-aliasing -mno-red-zone \
 				   -mno-mmx -mno-sse
 
-cflags-$(CONFIG_ARM64)		:= $(subst -pg,,$(KBUILD_CFLAGS)) -fpie
+cflags-$(CONFIG_ARM64)		:= $(subst -pg,,$(KBUILD_CFLAGS)) -fpie \
+				   $(NO_RETPOLINE_CFLAGS)
 cflags-$(CONFIG_ARM)		:= $(subst -pg,,$(KBUILD_CFLAGS)) \
 				   -fno-builtin -fpic -mno-single-pic-base