8cb861e9e3
Processor MMIO Stale Data is a class of vulnerabilities that may
expose data after an MMIO operation. For details please refer to
Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst.
These vulnerabilities are broadly categorized as:
Device Register Partial Write (DRPW):
Some endpoint MMIO registers incorrectly handle writes that are
smaller than the register size. Instead of aborting the write or only
copying the correct subset of bytes (for example, 2 bytes for a 2-byte
write), more bytes than specified by the write transaction may be
written to the register. On some processors, this may expose stale
data from the fill buffers of the core that created the write
transaction.
Shared Buffers Data Sampling (SBDS):
After propagators may have moved data around the uncore and copied
stale data into client core fill buffers, processors affected by MFBDS
can leak data from the fill buffer.
Shared Buffers Data Read (SBDR):
It is similar to Shared Buffer Data Sampling (SBDS) except that the
data is directly read into the architectural software-visible state.
An attacker can use these vulnerabilities to extract data from CPU fill
buffers using MDS and TAA methods. Mitigate it by clearing the CPU fill
buffers using the VERW instruction before returning to a user or a
guest.
On CPUs not affected by MDS and TAA, user application cannot sample data
from CPU fill buffers using MDS or TAA. A guest with MMIO access can
still use DRPW or SBDR to extract data architecturally. Mitigate it with
VERW instruction to clear fill buffers before VMENTER for MMIO capable
guests.
Add a kernel parameter mmio_stale_data={off|full|full,nosmt} to control
the mitigation.
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
323 lines
8.6 KiB
C
323 lines
8.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
#ifndef _ASM_X86_NOSPEC_BRANCH_H_
|
|
#define _ASM_X86_NOSPEC_BRANCH_H_
|
|
|
|
#include <linux/static_key.h>
|
|
#include <linux/objtool.h>
|
|
#include <linux/linkage.h>
|
|
|
|
#include <asm/alternative.h>
|
|
#include <asm/cpufeatures.h>
|
|
#include <asm/msr-index.h>
|
|
#include <asm/unwind_hints.h>
|
|
|
|
#define RETPOLINE_THUNK_SIZE 32
|
|
|
|
/*
|
|
* Fill the CPU return stack buffer.
|
|
*
|
|
* Each entry in the RSB, if used for a speculative 'ret', contains an
|
|
* infinite 'pause; lfence; jmp' loop to capture speculative execution.
|
|
*
|
|
* This is required in various cases for retpoline and IBRS-based
|
|
* mitigations for the Spectre variant 2 vulnerability. Sometimes to
|
|
* eliminate potentially bogus entries from the RSB, and sometimes
|
|
* purely to ensure that it doesn't get empty, which on some CPUs would
|
|
* allow predictions from other (unwanted!) sources to be used.
|
|
*
|
|
* We define a CPP macro such that it can be used from both .S files and
|
|
* inline assembly. It's possible to do a .macro and then include that
|
|
* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
|
|
*/
|
|
|
|
#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
|
|
|
|
/*
|
|
* Google experimented with loop-unrolling and this turned out to be
|
|
* the optimal version - two calls, each with their own speculation
|
|
* trap should their return address end up getting used, in a loop.
|
|
*/
|
|
#define __FILL_RETURN_BUFFER(reg, nr, sp) \
|
|
mov $(nr/2), reg; \
|
|
771: \
|
|
ANNOTATE_INTRA_FUNCTION_CALL; \
|
|
call 772f; \
|
|
773: /* speculation trap */ \
|
|
UNWIND_HINT_EMPTY; \
|
|
pause; \
|
|
lfence; \
|
|
jmp 773b; \
|
|
772: \
|
|
ANNOTATE_INTRA_FUNCTION_CALL; \
|
|
call 774f; \
|
|
775: /* speculation trap */ \
|
|
UNWIND_HINT_EMPTY; \
|
|
pause; \
|
|
lfence; \
|
|
jmp 775b; \
|
|
774: \
|
|
add $(BITS_PER_LONG/8) * 2, sp; \
|
|
dec reg; \
|
|
jnz 771b;
|
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
/*
|
|
* This should be used immediately before an indirect jump/call. It tells
|
|
* objtool the subsequent indirect jump/call is vouched safe for retpoline
|
|
* builds.
|
|
*/
|
|
.macro ANNOTATE_RETPOLINE_SAFE
|
|
.Lannotate_\@:
|
|
.pushsection .discard.retpoline_safe
|
|
_ASM_PTR .Lannotate_\@
|
|
.popsection
|
|
.endm
|
|
|
|
/*
|
|
* JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
|
|
* indirect jmp/call which may be susceptible to the Spectre variant 2
|
|
* attack.
|
|
*/
|
|
.macro JMP_NOSPEC reg:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
|
|
__stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
|
|
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE
|
|
#else
|
|
jmp *%\reg
|
|
#endif
|
|
.endm
|
|
|
|
.macro CALL_NOSPEC reg:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \
|
|
__stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
|
|
__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE
|
|
#else
|
|
call *%\reg
|
|
#endif
|
|
.endm
|
|
|
|
/*
|
|
* A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
|
|
* monstrosity above, manually.
|
|
*/
|
|
.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
|
|
#ifdef CONFIG_RETPOLINE
|
|
ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
|
|
__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
|
|
.Lskip_rsb_\@:
|
|
#endif
|
|
.endm
|
|
|
|
#else /* __ASSEMBLY__ */
|
|
|
|
#define ANNOTATE_RETPOLINE_SAFE \
|
|
"999:\n\t" \
|
|
".pushsection .discard.retpoline_safe\n\t" \
|
|
_ASM_PTR " 999b\n\t" \
|
|
".popsection\n\t"
|
|
|
|
#ifdef CONFIG_RETPOLINE
|
|
|
|
typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
|
|
|
|
#define GEN(reg) \
|
|
extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;
|
|
#include <asm/GEN-for-each-reg.h>
|
|
#undef GEN
|
|
|
|
extern retpoline_thunk_t __x86_indirect_thunk_array[];
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
/*
|
|
* Inline asm uses the %V modifier which is only in newer GCC
|
|
* which is ensured when CONFIG_RETPOLINE is defined.
|
|
*/
|
|
# define CALL_NOSPEC \
|
|
ALTERNATIVE_2( \
|
|
ANNOTATE_RETPOLINE_SAFE \
|
|
"call *%[thunk_target]\n", \
|
|
"call __x86_indirect_thunk_%V[thunk_target]\n", \
|
|
X86_FEATURE_RETPOLINE, \
|
|
"lfence;\n" \
|
|
ANNOTATE_RETPOLINE_SAFE \
|
|
"call *%[thunk_target]\n", \
|
|
X86_FEATURE_RETPOLINE_LFENCE)
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
|
|
|
|
#else /* CONFIG_X86_32 */
|
|
/*
|
|
* For i386 we use the original ret-equivalent retpoline, because
|
|
* otherwise we'll run out of registers. We don't care about CET
|
|
* here, anyway.
|
|
*/
|
|
# define CALL_NOSPEC \
|
|
ALTERNATIVE_2( \
|
|
ANNOTATE_RETPOLINE_SAFE \
|
|
"call *%[thunk_target]\n", \
|
|
" jmp 904f;\n" \
|
|
" .align 16\n" \
|
|
"901: call 903f;\n" \
|
|
"902: pause;\n" \
|
|
" lfence;\n" \
|
|
" jmp 902b;\n" \
|
|
" .align 16\n" \
|
|
"903: lea 4(%%esp), %%esp;\n" \
|
|
" pushl %[thunk_target];\n" \
|
|
" ret;\n" \
|
|
" .align 16\n" \
|
|
"904: call 901b;\n", \
|
|
X86_FEATURE_RETPOLINE, \
|
|
"lfence;\n" \
|
|
ANNOTATE_RETPOLINE_SAFE \
|
|
"call *%[thunk_target]\n", \
|
|
X86_FEATURE_RETPOLINE_LFENCE)
|
|
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
|
#endif
|
|
#else /* No retpoline for C / inline asm */
|
|
# define CALL_NOSPEC "call *%[thunk_target]\n"
|
|
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
|
|
#endif
|
|
|
|
/* The Spectre V2 mitigation variants */
|
|
enum spectre_v2_mitigation {
|
|
SPECTRE_V2_NONE,
|
|
SPECTRE_V2_RETPOLINE,
|
|
SPECTRE_V2_LFENCE,
|
|
SPECTRE_V2_EIBRS,
|
|
SPECTRE_V2_EIBRS_RETPOLINE,
|
|
SPECTRE_V2_EIBRS_LFENCE,
|
|
};
|
|
|
|
/* The indirect branch speculation control variants */
|
|
enum spectre_v2_user_mitigation {
|
|
SPECTRE_V2_USER_NONE,
|
|
SPECTRE_V2_USER_STRICT,
|
|
SPECTRE_V2_USER_STRICT_PREFERRED,
|
|
SPECTRE_V2_USER_PRCTL,
|
|
SPECTRE_V2_USER_SECCOMP,
|
|
};
|
|
|
|
/* The Speculative Store Bypass disable variants */
|
|
enum ssb_mitigation {
|
|
SPEC_STORE_BYPASS_NONE,
|
|
SPEC_STORE_BYPASS_DISABLE,
|
|
SPEC_STORE_BYPASS_PRCTL,
|
|
SPEC_STORE_BYPASS_SECCOMP,
|
|
};
|
|
|
|
extern char __indirect_thunk_start[];
|
|
extern char __indirect_thunk_end[];
|
|
|
|
static __always_inline
|
|
void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
|
|
{
|
|
asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
|
|
: : "c" (msr),
|
|
"a" ((u32)val),
|
|
"d" ((u32)(val >> 32)),
|
|
[feature] "i" (feature)
|
|
: "memory");
|
|
}
|
|
|
|
static inline void indirect_branch_prediction_barrier(void)
|
|
{
|
|
u64 val = PRED_CMD_IBPB;
|
|
|
|
alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
|
|
}
|
|
|
|
/* The Intel SPEC CTRL MSR base value cache */
|
|
extern u64 x86_spec_ctrl_base;
|
|
|
|
/*
|
|
* With retpoline, we must use IBRS to restrict branch prediction
|
|
* before calling into firmware.
|
|
*
|
|
* (Implemented as CPP macros due to header hell.)
|
|
*/
|
|
#define firmware_restrict_branch_speculation_start() \
|
|
do { \
|
|
u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \
|
|
\
|
|
preempt_disable(); \
|
|
alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
|
|
X86_FEATURE_USE_IBRS_FW); \
|
|
} while (0)
|
|
|
|
#define firmware_restrict_branch_speculation_end() \
|
|
do { \
|
|
u64 val = x86_spec_ctrl_base; \
|
|
\
|
|
alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
|
|
X86_FEATURE_USE_IBRS_FW); \
|
|
preempt_enable(); \
|
|
} while (0)
|
|
|
|
DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
|
|
DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
|
|
DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
|
|
|
|
DECLARE_STATIC_KEY_FALSE(mds_user_clear);
|
|
DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
|
|
|
|
DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
|
|
|
|
DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
|
|
|
|
#include <asm/segment.h>
|
|
|
|
/**
|
|
* mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
|
|
*
|
|
* This uses the otherwise unused and obsolete VERW instruction in
|
|
* combination with microcode which triggers a CPU buffer flush when the
|
|
* instruction is executed.
|
|
*/
|
|
static __always_inline void mds_clear_cpu_buffers(void)
|
|
{
|
|
static const u16 ds = __KERNEL_DS;
|
|
|
|
/*
|
|
* Has to be the memory-operand variant because only that
|
|
* guarantees the CPU buffer flush functionality according to
|
|
* documentation. The register-operand variant does not.
|
|
* Works with any segment selector, but a valid writable
|
|
* data segment is the fastest variant.
|
|
*
|
|
* "cc" clobber is required because VERW modifies ZF.
|
|
*/
|
|
asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
|
|
}
|
|
|
|
/**
|
|
* mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
|
|
*
|
|
* Clear CPU buffers if the corresponding static key is enabled
|
|
*/
|
|
static __always_inline void mds_user_clear_cpu_buffers(void)
|
|
{
|
|
if (static_branch_likely(&mds_user_clear))
|
|
mds_clear_cpu_buffers();
|
|
}
|
|
|
|
/**
|
|
* mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
|
|
*
|
|
* Clear CPU buffers if the corresponding static key is enabled
|
|
*/
|
|
static inline void mds_idle_clear_cpu_buffers(void)
|
|
{
|
|
if (static_branch_likely(&mds_idle_clear))
|
|
mds_clear_cpu_buffers();
|
|
}
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
|