Edit File: nospec-branch.h
/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NOSPEC_BRANCH_H__ #define __NOSPEC_BRANCH_H__ #include <asm/alternative.h> #include <asm/alternative-asm.h> #include <asm/cpufeatures.h> #include <asm/bitsperlong.h> #include <asm/percpu.h> #include <asm/nops.h> #include <asm/jump_label.h> /* * Fill the CPU return stack buffer. * * Each entry in the RSB, if used for a speculative 'ret', contains an * infinite 'pause; lfence; jmp' loop to capture speculative execution. * * This is required in various cases for retpoline and IBRS-based * mitigations for the Spectre variant 2 vulnerability. Sometimes to * eliminate potentially bogus entries from the RSB, and sometimes * purely to ensure that it doesn't get empty, which on some CPUs would * allow predictions from other (unwanted!) sources to be used. * * We define a CPP macro such that it can be used from both .S files and * inline assembly. It's possible to do a .macro and then include that * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. */ #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ #define RSB_FILL_LOOPS 16 /* To avoid underflow */ /* * Google experimented with loop-unrolling and this turned out to be * the optimal version — two calls, each with their own speculation * trap should their return address end up getting used, in a loop. */ #define __FILL_RETURN_BUFFER(reg, nr, sp) \ mov $(nr/2), reg; \ 771: \ call 772f; \ 773: /* speculation trap */ \ pause; \ lfence; \ jmp 773b; \ 772: \ call 774f; \ 775: /* speculation trap */ \ pause; \ lfence; \ jmp 775b; \ 774: \ add $(BITS_PER_LONG/8) * 2, sp; \ dec reg; \ jnz 771b; \ /* barrier for jnz misprediction */ \ lfence; #ifdef __ASSEMBLY__ /* * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP * monstrosity above, manually. */ .macro FILL_RETURN_BUFFER_CLOBBER reg:req ftr:req 661: jmp .Lskip_rsb_\@; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP4; 662: .pushsection .altinstr_replacement, "ax" 663: __FILL_RETURN_BUFFER(\reg, RSB_CLEAR_LOOPS, %_ASM_SP); 664: .popsection .pushsection .altinstructions, "a" altinstruction_entry 661b, 663b, \ftr, 662b-661b, 664b-663b .popsection .Lskip_rsb_\@: .endm .macro FILL_RETURN_BUFFER reg:req ftr:req push \reg FILL_RETURN_BUFFER_CLOBBER reg=\reg ftr=\ftr pop \reg .endm /* * These are the bare retpoline primitives for indirect jmp and call. * Do not use these directly; they only exist to make the ALTERNATIVE * invocation below less ugly. */ .macro RETPOLINE_JMP reg:req call .Ldo_rop_\@ .Lspec_trap_\@: pause lfence jmp .Lspec_trap_\@ .Ldo_rop_\@: mov \reg, (%_ASM_SP) ret .endm /* * This is a wrapper around RETPOLINE_JMP so the called function in reg * returns to the instruction after the macro. */ .macro RETPOLINE_CALL reg:req jmp .Ldo_call_\@ .Ldo_retpoline_jmp_\@: RETPOLINE_JMP \reg .Ldo_call_\@: call .Ldo_retpoline_jmp_\@ .endm .macro __JMP_NOSPEC reg:req 661: RETPOLINE_JMP \reg; 662: .pushsection .altinstr_replacement, "ax" 663: lfence; jmp *\reg; 664: .popsection .pushsection .altinstructions, "a" altinstruction_entry 661b, 663b, X86_FEATURE_RETPOLINE_AMD, 662b-661b, 664b-663b .popsection .endm .macro __CALL_NOSPEC reg:req 661: RETPOLINE_CALL \reg; 662: .pushsection .altinstr_replacement, "ax" 663: lfence; call *\reg; 664: .popsection .pushsection .altinstructions, "a" altinstruction_entry 661b, 663b, X86_FEATURE_RETPOLINE_AMD, 662b-661b, 664b-663b .popsection .endm /* * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple * indirect jmp/call which may be susceptible to the Spectre variant 2 * attack. */ .macro JMP_NOSPEC reg:req STATIC_JUMP .Lretp_\@, retp_enabled_key jmp *\reg .Lretp_\@: __JMP_NOSPEC \reg .endm .macro CALL_NOSPEC reg:req STATIC_JUMP .Lretp_\@, retp_enabled_key call *\reg jmp .Ldone_\@ .Lretp_\@: __CALL_NOSPEC \reg .Ldone_\@: .endm /* * MDS_USER_CLEAR_CPU_BUFFERS macro is the assembly equivalent of * mds_user_clear_cpu_buffers(). Like the C version, the __KERNEL_DS * is used for verw. * Note: The ZF flag will be clobbered after calling this macro. */ .macro MDS_USER_CLEAR_CPU_BUFFERS STATIC_JUMP .Lverw_\@, mds_user_clear jmp .Ldone_\@ .balign 2 .Lds_\@: .word __KERNEL_DS .Lverw_\@: verw .Lds_\@(%rip) .Ldone_\@: .endm /* * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the * return thunk isn't mapped into the userspace tables (then again, AMD * typically has NO_MELTDOWN). * * While zen_untrain_ret() doesn't clobber anything but requires stack, * entry_ibpb() will clobber AX, CX, DX. * * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point * where we have a stack but before any RET instruction. */ .macro UNTRAIN_RET #ifdef CONFIG_RETPOLINE 661: ASM_NOP5_ATOMIC; 662: .pushsection .altinstr_replacement, "ax" 6631: call zen_untrain_ret; 6641: 6632: call entry_ibpb; 6642: .popsection .pushsection .altinstructions, "a" altinstruction_entry 661b, 6631b, X86_FEATURE_UNRET, 662b-661b, 6641b-6631b altinstruction_entry 661b, 6632b, X86_FEATURE_ENTRY_IBPB, 662b-661b, 6642b-6632b .popsection #endif .endm #else /* __ASSEMBLY__ */ extern void __x86_return_thunk(void); extern void zen_untrain_ret(void); extern void entry_ibpb(void); #if defined(CONFIG_X86_64) && defined(RETPOLINE) /* * Since the inline asm uses the %V modifier which is only in newer GCC, * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. */ #define CALL_NOSPEC \ "call __x86_indirect_thunk_%V[thunk_target]\n" #define THUNK_TARGET(addr) [thunk_target] "r" (addr) #else /* No retpoline for C / inline asm */ # define CALL_NOSPEC "call *%[thunk_target]\n" # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #endif /* The Spectre V2 mitigation variants */ enum spectre_v2_mitigation { SPECTRE_V2_NONE, SPECTRE_V2_RETPOLINE_MINIMAL, SPECTRE_V2_RETPOLINE_NO_IBPB, SPECTRE_V2_RETPOLINE_UNSAFE_MODULE, SPECTRE_V2_RETPOLINE_AMD, SPECTRE_V2_RETPOLINE, SPECTRE_V2_RETPOLINE_IBRS_USER, SPECTRE_V2_IBRS, SPECTRE_V2_IBRS_ALWAYS, SPECTRE_V2_IBP_DISABLED, SPECTRE_V2_IBRS_ENHANCED, }; extern enum spectre_v2_mitigation spectre_v2_enabled; void __spectre_v2_select_mitigation(void); void spectre_v2_print_mitigation(void); static inline bool retp_compiler(void) { #ifdef RETPOLINE return true; #else return false; #endif } /* * The Intel specification for the SPEC_CTRL MSR requires that we * preserve any already set reserved bits at boot time (e.g. for * future additions that this kernel is not currently aware of). * We then set any additional mitigation bits that we want * ourselves and always use this as the base for SPEC_CTRL. * We also use this when handling guest entry/exit as below. */ extern u64 x86_spec_ctrl_base; /* The Speculative Store Bypass disable variants */ enum ssb_mitigation { SPEC_STORE_BYPASS_NONE, SPEC_STORE_BYPASS_DISABLE, SPEC_STORE_BYPASS_PRCTL, SPEC_STORE_BYPASS_SECCOMP, }; /* AMD specific Speculative Store Bypass MSR data */ extern u64 x86_amd_ls_cfg_base; extern u64 x86_amd_ls_cfg_ssbd_mask; /* * On VMEXIT we must ensure that no RSB predictions learned in the guest * can be followed in the host, by overwriting the RSB completely. Both * retpoline and IBRS mitigations for Spectre v2 need this; only on future * CPUs with IBRS_ATT *might* it be avoided. */ static inline void fill_RSB(void) { unsigned long loops; register unsigned long sp asm(_ASM_SP); asm volatile (__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)) : "=r" (loops), "+r" (sp) : : "memory" ); } /* * On VMEXIT we must ensure we untrain RET in order to mitigate * RETBleed for AMD/Hygon Zen uarch. */ static inline void untrain_ret(void) { asm volatile ( ALTERNATIVE_2(ASM_NOP5_ATOMIC, "call zen_untrain_ret", X86_FEATURE_UNRET, "call entry_ibpb", X86_FEATURE_ENTRY_IBPB) : : : "memory" ); } extern struct static_key mds_user_clear; extern struct static_key mds_idle_clear; extern struct static_key mmio_stale_data_clear; #include <asm/segment.h> /** * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability * * This uses the otherwise unused and obsolete VERW instruction in * combination with microcode which triggers a CPU buffer flush when the * instruction is executed. */ static inline void mds_clear_cpu_buffers(void) { static const u16 ds = __KERNEL_DS; /* * Has to be the memory-operand variant because only that * guarantees the CPU buffer flush functionality according to * documentation. The register-operand variant does not. * Works with any segment selector, but a valid writable * data segment is the fastest variant. * * "cc" clobber is required because VERW modifies ZF. */ asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); } /** * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability * * Clear CPU buffers if the corresponding static key is enabled * * RHEL7: This inline function from upstream isn't being used. The * equivalent MDS_USER_CLEAR_CPU_BUFFERS assembly macro is used in * the entry code to achieve the same effect. */ static inline void mds_user_clear_cpu_buffers(void) { if (static_key_false(&mds_user_clear)) mds_clear_cpu_buffers(); } /** * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability * * Clear CPU buffers if the corresponding static key is enabled */ static inline void mds_idle_clear_cpu_buffers(void) { if (static_key_false(&mds_idle_clear)) mds_clear_cpu_buffers(); } #endif /* __ASSEMBLY__ */ /* * Below is used in the eBPF JIT compiler and emits the byte sequence * for the following assembly: * * With retpolines configured: * * callq do_rop * spec_trap: * pause * lfence * jmp spec_trap * do_rop: * mov %rax,(%rsp) * retq * * Without retpolines configured: * * jmp *%rax */ #ifdef CONFIG_RETPOLINE # define RETPOLINE_RAX_BPF_JIT_SIZE 17 # define RETPOLINE_RAX_BPF_JIT() \ EMIT1_off32(0xE8, 7); /* callq do_rop */ \ /* spec_trap: */ \ EMIT2(0xF3, 0x90); /* pause */ \ EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ /* do_rop: */ \ EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ EMIT1(0xC3); /* retq */ #else # define RETPOLINE_RAX_BPF_JIT_SIZE 2 # define RETPOLINE_RAX_BPF_JIT() \ EMIT2(0xFF, 0xE0); /* jmp *%rax */ #endif #endif /* __NOSPEC_BRANCH_H__ */