Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJiri Kosina <jkosina@suse.cz>2018-02-19 22:13:08 +0100
committerJiri Kosina <jkosina@suse.cz>2018-02-19 22:13:08 +0100
commit518c19822943106a110f766b0b61f7691d9ec984 (patch)
tree4639e99dc1ccb02bcd4c4416ebaa5f5ab03e53c5
parent55556427367528baf0a25654b776433417906c26 (diff)
parentf6a66d06602b0a9fd56551a918deb4f510613ab8 (diff)
Merge remote-tracking branch 'origin/users/bpetkov/SLE11-SP4/for-next' into SLE11-SP4rpm-3.0.101-108.35
-rw-r--r--arch/x86/include/asm/alternative.h6
-rw-r--r--arch/x86/include/asm/asm-prototypes.h3
-rw-r--r--arch/x86/include/asm/asm.h5
-rw-r--r--arch/x86/include/asm/nospec-branch.h68
-rw-r--r--arch/x86/include/asm/system.h12
-rw-r--r--arch/x86/include/asm/thread_info.h4
-rw-r--r--arch/x86/lib/retpoline.S56
7 files changed, 67 insertions, 87 deletions
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index a8393463a6d4..35c332bc0d09 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -151,12 +151,6 @@ static inline int alternatives_text_reserved(void *start, void *end)
*/
#define ASM_OUTPUT2(a...) a
-/*
- * use this macro if you need clobbers but no inputs in
- * alternative_{input,io,call}()
- */
-#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
-
struct paravirt_patch_site;
#ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch_site *start,
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index a546be729f4f..1dcc152cd999 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -35,7 +35,4 @@ INDIRECT_THUNK(dx)
INDIRECT_THUNK(si)
INDIRECT_THUNK(di)
INDIRECT_THUNK(bp)
-asmlinkage void __fill_rsb(void);
-asmlinkage void __clear_rsb(void);
-
#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index ec3964e9e07d..60cdca3735a9 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -56,4 +56,9 @@
" .popsection\n"
#endif
+#ifndef __ASSEMBLY__
+register unsigned long current_stack_pointer asm(_ASM_SP);
+#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
+#endif
+
#endif /* _ASM_X86_ASM_H */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 8e9cbda72984..4461694af592 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -8,6 +8,48 @@
#include <asm/cpufeature.h>
#include <asm/nops.h>
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS 16 /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp) \
+ mov $(nr/2), reg; \
+771: \
+ call 772f; \
+773: /* speculation trap */ \
+ pause; \
+ jmp 773b; \
+772: \
+ call 774f; \
+775: /* speculation trap */ \
+ pause; \
+ jmp 775b; \
+774: \
+ dec reg; \
+ jnz 771b; \
+ add $(BITS_PER_LONG/8) * nr, sp;
+
#ifdef __ASSEMBLY__
/*
@@ -67,18 +109,20 @@
#endif
.endm
-/* This clobbers the BX register */
-.macro FILL_RETURN_BUFFER ftr:req
+ /*
+ * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+ * monstrosity above, manually.
+ */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
#ifdef CONFIG_RETPOLINE
- ALTERNATIVE ASM_NOP5, "call __clear_rsb", \ftr
+ ALTERNATIVE "jmp .Lskip_rsb_\@", ASM_NOP5, \ftr
+ __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
+.Lskip_rsb_\@:
#endif
.endm
#else /* __ASSEMBLY__ */
-#define FILL_RETURN_BUFFER(ftr) \
- ALTERNATIVE(ASM_NOP5, "call __clear_rsb", \ftr)
-
#if defined(CONFIG_X86_64) && defined(RETPOLINE)
/*
@@ -138,10 +182,14 @@ enum spectre_v2_mitigation {
static inline void vmexit_fill_RSB(void)
{
#ifdef CONFIG_RETPOLINE
- alternative_input(ASM_NOP5,
- "call __fill_rsb",
- X86_FEATURE_RETPOLINE,
- ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory"));
+ unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+ asm volatile (ALTERNATIVE("jmp 910f; " __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+ __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+ X86_FEATURE_RETPOLINE)
+ "910:"
+ : "=&r" (loops), "+r" (current_stack_pointer)
+ : "r" (loops) : "memory" );
#endif
}
#endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 89d83b0edf8a..7576481c608e 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -48,11 +48,9 @@ extern void show_regs_common(void);
* with userspace addresses. On CPUs where those concerns
* exist, overwrite the RSB with entries which capture
* speculative execution to prevent attack.
- *
- * Clobbers %ebx
*/
#ifdef CONFIG_RETPOLINE
-#define __switch_fill_rsb FILL_RETURN_BUFFER(X86_FEATURE_RSB_CTXSW)
+#define __switch_fill_rsb __stringify(__FILL_RETURN_BUFFER(%%ebx, RSB_CLEAR_LOOPS, %%esp))
#else
#define __switch_fill_rsb
#endif
@@ -79,7 +77,7 @@ do { \
"movl $1f,%[prev_ip]\n\t" /* save EIP */ \
"pushl %[next_ip]\n\t" /* restore EIP */ \
__switch_canary \
- __switch_fill_rsb \
+ __switch_fill_rsb "\n\t" \
"jmp __switch_to\n" /* regparm call */ \
"1:\t" \
"popl %%ebp\n\t" /* restore EBP */ \
@@ -149,11 +147,9 @@ do { \
* with userspace addresses. On CPUs where those concerns
* exist, overwrite the RSB with entries which capture
* speculative execution to prevent attack.
- *
- * Clobbers %rbx.
*/
#ifdef CONFIG_RETPOLINE
-#define __switch_fill_rsb FILL_RETURN_BUFFER(X86_FEATURE_RSB_CTXSW)
+#define __switch_fill_rsb __stringify(__FILL_RETURN_BUFFER(%%rbx, RSB_CLEAR_LOOPS, %%rsp))
#else
#define __switch_fill_rsb
#endif
@@ -163,7 +159,7 @@ do { \
asm volatile(SAVE_CONTEXT \
"movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
"movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
- __switch_fill_rsb \
+ __switch_fill_rsb "\n\t" \
"call __switch_to\n\t" \
THREAD_RETURN_SYM \
"movq "__percpu_arg([current_task])",%%rsi\n\t" \
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 7ccce89ed52a..a87773d28cbf 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -232,8 +232,6 @@ static inline struct thread_info *current_thread_info(void)
return ti;
}
-register unsigned long current_stack_pointer asm("rsp") __used;
-
#else /* !__ASSEMBLY__ */
/* how to get the thread information struct from ASM */
@@ -245,8 +243,6 @@ register unsigned long current_stack_pointer asm("rsp") __used;
#endif /* !X86_32 */
-#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
-
/*
* Thread-synchronous status.
*
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index 1743efb8065b..582fad52ec91 100644
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -7,7 +7,6 @@
#include <asm/alternative-asm.h>
#include <asm-generic/export.h>
#include <asm/nospec-branch.h>
-#include <asm/bitsperlong.h>
.macro THUNK reg
.section .text.__x86.indirect_thunk.\reg
@@ -46,58 +45,3 @@ GENERATE_THUNK(r13)
GENERATE_THUNK(r14)
GENERATE_THUNK(r15)
#endif
-
-/*
- * Fill the CPU return stack buffer.
- *
- * Each entry in the RSB, if used for a speculative 'ret', contains an
- * infinite 'pause; lfence; jmp' loop to capture speculative execution.
- *
- * This is required in various cases for retpoline and IBRS-based
- * mitigations for the Spectre variant 2 vulnerability. Sometimes to
- * eliminate potentially bogus entries from the RSB, and sometimes
- * purely to ensure that it doesn't get empty, which on some CPUs would
- * allow predictions from other (unwanted!) sources to be used.
- *
- * Google experimented with loop-unrolling and this turned out to be
- * the optimal version - two calls, each with their own speculation
- * trap should their return address end up getting used, in a loop.
- */
-.macro STUFF_RSB nr:req sp:req
- mov $(\nr / 2), %_ASM_BX
- .align 16
-771:
- call 772f
-773: /* speculation trap */
- pause
- lfence
- jmp 773b
- .align 16
-772:
- call 774f
-775: /* speculation trap */
- pause
- lfence
- jmp 775b
- .align 16
-774:
- dec %_ASM_BX
- jnz 771b
- add $((BITS_PER_LONG/8) * \nr), \sp
-.endm
-
-#define RSB_FILL_LOOPS 16 /* To avoid underflow */
-
-ENTRY(__fill_rsb)
- STUFF_RSB RSB_FILL_LOOPS, %_ASM_SP
- ret
-END(__fill_rsb)
-EXPORT_SYMBOL_GPL(__fill_rsb)
-
-#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
-
-ENTRY(__clear_rsb)
- STUFF_RSB RSB_CLEAR_LOOPS, %_ASM_SP
- ret
-END(__clear_rsb)
-EXPORT_SYMBOL_GPL(__clear_rsb)