KVM/arm64 fixes for 6.14, take #2
- Large set of fixes for vector handling, specially in the interactions between host and guest state. This fixes a number of bugs affecting actual deployments, and greatly simplifies the FP/SIMD/SVE handling. Thanks to Mark Rutland for dealing with this thankless task. - Fix an ugly race between vcpu and vgic creation/init, resulting in unexpected behaviours. - Fix use of kernel VAs at EL2 when emulating timers with nVHE. - Small set of pKVM improvements and cleanups. -----BEGIN PGP SIGNATURE----- iQIzBAABCgAdFiEEn9UcU+C1Yxj9lZw9I9DQutE9ekMFAmevLKMACgkQI9DQutE9 ekP3hQ//db7pAPzLr++//PAyam0GP+ooKlgpB0ImZisQwkrTrTMP+IjNJG+NCJ46 y88anBErFijvWb3BINpeTM/dux7DmuoaolGx7lquFu+i0L8UfFFjYG7UU+NZscim KE4j0tJz8jm5ksN4iwaj3RIkGKc1zJtRyoPny3j1blOtm8aTtujRJB7/Gx2QefZR 1Z13RaIzk1tKdY0JxAmPpGkaRY99MQahx96iBsk2u4rlypcxmVr9aQ1Madp7Pc6Y pBcX9jZwLf75cj6CAK93YSjFF3j/x4QM8jSupLCu5tyin6YZ4sRaZa6sy52byk2v zes7i83l5g3+JEKv5oZVwjD5SFBu02UPbnMGSxKQitgz4Zej3qMIq5BxgII2kHZV jwXrNEx4trNegEcoqwFX5xA0FMUr1/g3Cr4+rZBoUramj80cBhzbBdUkhyWd3eey j2EOuAG3pgUD5Wv9SyojlbHBwmSAcBEtr3vqJpTjWQS6AyEmdKNvzh/8JCH1h7UM fBo4+LIEylzmZXbqDrZNwXh31tELoTCR9Ur3pTCEO3Yfg9npTLWmvKs+tAgO/282 IOjZE0N/ZtzPJ6Cgr+2efBGd+id81HXh+H8gWo35Dyx3EH2k44FHwQ3rW2NKOVzo 10eSbswYpjk3gi/6GxwC0lDqFi4Bk6ILvC6roqTghixBf7xThfY= =L5HS -----END PGP SIGNATURE----- Merge tag 'kvmarm-fixes-6.14-2' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD KVM/arm64 fixes for 6.14, take #2 - Large set of fixes for vector handling, specially in the interactions between host and guest state. This fixes a number of bugs affecting actual deployments, and greatly simplifies the FP/SIMD/SVE handling. Thanks to Mark Rutland for dealing with this thankless task. - Fix an ugly race between vcpu and vgic creation/init, resulting in unexpected behaviours. - Fix use of kernel VAs at EL2 when emulating timers with nVHE. - Small set of pKVM improvements and cleanups.
This commit is contained in:
Paolo Bonzini
commit
3bb7dcebd0
13 changed files with 286 additions and 374 deletions
|
|
@ -605,48 +605,6 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
|
|||
__cpacr_to_cptr_set(clr, set));\
|
||||
} while (0)
|
||||
|
||||
static __always_inline void kvm_write_cptr_el2(u64 val)
|
||||
{
|
||||
if (has_vhe() || has_hvhe())
|
||||
write_sysreg(val, cpacr_el1);
|
||||
else
|
||||
write_sysreg(val, cptr_el2);
|
||||
}
|
||||
|
||||
/* Resets the value of cptr_el2 when returning to the host. */
|
||||
static __always_inline void __kvm_reset_cptr_el2(struct kvm *kvm)
|
||||
{
|
||||
u64 val;
|
||||
|
||||
if (has_vhe()) {
|
||||
val = (CPACR_EL1_FPEN | CPACR_EL1_ZEN_EL1EN);
|
||||
if (cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPACR_EL1_SMEN_EL1EN;
|
||||
} else if (has_hvhe()) {
|
||||
val = CPACR_EL1_FPEN;
|
||||
|
||||
if (!kvm_has_sve(kvm) || !guest_owns_fp_regs())
|
||||
val |= CPACR_EL1_ZEN;
|
||||
if (cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPACR_EL1_SMEN;
|
||||
} else {
|
||||
val = CPTR_NVHE_EL2_RES1;
|
||||
|
||||
if (kvm_has_sve(kvm) && guest_owns_fp_regs())
|
||||
val |= CPTR_EL2_TZ;
|
||||
if (!cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPTR_EL2_TSM;
|
||||
}
|
||||
|
||||
kvm_write_cptr_el2(val);
|
||||
}
|
||||
|
||||
#ifdef __KVM_NVHE_HYPERVISOR__
|
||||
#define kvm_reset_cptr_el2(v) __kvm_reset_cptr_el2(kern_hyp_va((v)->kvm))
|
||||
#else
|
||||
#define kvm_reset_cptr_el2(v) __kvm_reset_cptr_el2((v)->kvm)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE
|
||||
* format if E2H isn't set.
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc,
|
|||
static inline void *pop_hyp_memcache(struct kvm_hyp_memcache *mc,
|
||||
void *(*to_va)(phys_addr_t phys))
|
||||
{
|
||||
phys_addr_t *p = to_va(mc->head);
|
||||
phys_addr_t *p = to_va(mc->head & PAGE_MASK);
|
||||
|
||||
if (!mc->nr_pages)
|
||||
return NULL;
|
||||
|
|
@ -615,8 +615,6 @@ struct cpu_sve_state {
|
|||
struct kvm_host_data {
|
||||
#define KVM_HOST_DATA_FLAG_HAS_SPE 0
|
||||
#define KVM_HOST_DATA_FLAG_HAS_TRBE 1
|
||||
#define KVM_HOST_DATA_FLAG_HOST_SVE_ENABLED 2
|
||||
#define KVM_HOST_DATA_FLAG_HOST_SME_ENABLED 3
|
||||
#define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4
|
||||
#define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5
|
||||
unsigned long flags;
|
||||
|
|
@ -624,23 +622,13 @@ struct kvm_host_data {
|
|||
struct kvm_cpu_context host_ctxt;
|
||||
|
||||
/*
|
||||
* All pointers in this union are hyp VA.
|
||||
* Hyp VA.
|
||||
* sve_state is only used in pKVM and if system_supports_sve().
|
||||
*/
|
||||
union {
|
||||
struct user_fpsimd_state *fpsimd_state;
|
||||
struct cpu_sve_state *sve_state;
|
||||
};
|
||||
struct cpu_sve_state *sve_state;
|
||||
|
||||
union {
|
||||
/* HYP VA pointer to the host storage for FPMR */
|
||||
u64 *fpmr_ptr;
|
||||
/*
|
||||
* Used by pKVM only, as it needs to provide storage
|
||||
* for the host
|
||||
*/
|
||||
u64 fpmr;
|
||||
};
|
||||
/* Used by pKVM only. */
|
||||
u64 fpmr;
|
||||
|
||||
/* Ownership of the FP regs */
|
||||
enum {
|
||||
|
|
|
|||
|
|
@ -1694,31 +1694,6 @@ void fpsimd_signal_preserve_current_state(void)
|
|||
sve_to_fpsimd(current);
|
||||
}
|
||||
|
||||
/*
|
||||
* Called by KVM when entering the guest.
|
||||
*/
|
||||
void fpsimd_kvm_prepare(void)
|
||||
{
|
||||
if (!system_supports_sve())
|
||||
return;
|
||||
|
||||
/*
|
||||
* KVM does not save host SVE state since we can only enter
|
||||
* the guest from a syscall so the ABI means that only the
|
||||
* non-saved SVE state needs to be saved. If we have left
|
||||
* SVE enabled for performance reasons then update the task
|
||||
* state to be FPSIMD only.
|
||||
*/
|
||||
get_cpu_fpsimd_context();
|
||||
|
||||
if (test_and_clear_thread_flag(TIF_SVE)) {
|
||||
sve_to_fpsimd(current);
|
||||
current->thread.fp_type = FP_STATE_FPSIMD;
|
||||
}
|
||||
|
||||
put_cpu_fpsimd_context();
|
||||
}
|
||||
|
||||
/*
|
||||
* Associate current's FPSIMD context with this cpu
|
||||
* The caller must have ownership of the cpu FPSIMD context before calling
|
||||
|
|
|
|||
|
|
@ -447,21 +447,19 @@ static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level)
|
|||
static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
|
||||
struct arch_timer_context *timer_ctx)
|
||||
{
|
||||
int ret;
|
||||
|
||||
kvm_timer_update_status(timer_ctx, new_level);
|
||||
|
||||
timer_ctx->irq.level = new_level;
|
||||
trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx),
|
||||
timer_ctx->irq.level);
|
||||
|
||||
if (!userspace_irqchip(vcpu->kvm)) {
|
||||
ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,
|
||||
timer_irq(timer_ctx),
|
||||
timer_ctx->irq.level,
|
||||
timer_ctx);
|
||||
WARN_ON(ret);
|
||||
}
|
||||
if (userspace_irqchip(vcpu->kvm))
|
||||
return;
|
||||
|
||||
kvm_vgic_inject_irq(vcpu->kvm, vcpu,
|
||||
timer_irq(timer_ctx),
|
||||
timer_ctx->irq.level,
|
||||
timer_ctx);
|
||||
}
|
||||
|
||||
/* Only called for a fully emulated timer */
|
||||
|
|
|
|||
|
|
@ -2481,14 +2481,6 @@ static void finalize_init_hyp_mode(void)
|
|||
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
|
||||
kern_hyp_va(sve_state);
|
||||
}
|
||||
} else {
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct user_fpsimd_state *fpsimd_state;
|
||||
|
||||
fpsimd_state = &per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->host_ctxt.fp_regs;
|
||||
per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->fpsimd_state =
|
||||
kern_hyp_va(fpsimd_state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -54,50 +54,18 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
|
|||
if (!system_supports_fpsimd())
|
||||
return;
|
||||
|
||||
fpsimd_kvm_prepare();
|
||||
|
||||
/*
|
||||
* We will check TIF_FOREIGN_FPSTATE just before entering the
|
||||
* guest in kvm_arch_vcpu_ctxflush_fp() and override this to
|
||||
* FP_STATE_FREE if the flag set.
|
||||
* Ensure that any host FPSIMD/SVE/SME state is saved and unbound such
|
||||
* that the host kernel is responsible for restoring this state upon
|
||||
* return to userspace, and the hyp code doesn't need to save anything.
|
||||
*
|
||||
* When the host may use SME, fpsimd_save_and_flush_cpu_state() ensures
|
||||
* that PSTATE.{SM,ZA} == {0,0}.
|
||||
*/
|
||||
*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
|
||||
*host_data_ptr(fpsimd_state) = kern_hyp_va(¤t->thread.uw.fpsimd_state);
|
||||
*host_data_ptr(fpmr_ptr) = kern_hyp_va(¤t->thread.uw.fpmr);
|
||||
fpsimd_save_and_flush_cpu_state();
|
||||
*host_data_ptr(fp_owner) = FP_STATE_FREE;
|
||||
|
||||
host_data_clear_flag(HOST_SVE_ENABLED);
|
||||
if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
|
||||
host_data_set_flag(HOST_SVE_ENABLED);
|
||||
|
||||
if (system_supports_sme()) {
|
||||
host_data_clear_flag(HOST_SME_ENABLED);
|
||||
if (read_sysreg(cpacr_el1) & CPACR_EL1_SMEN_EL0EN)
|
||||
host_data_set_flag(HOST_SME_ENABLED);
|
||||
|
||||
/*
|
||||
* If PSTATE.SM is enabled then save any pending FP
|
||||
* state and disable PSTATE.SM. If we leave PSTATE.SM
|
||||
* enabled and the guest does not enable SME via
|
||||
* CPACR_EL1.SMEN then operations that should be valid
|
||||
* may generate SME traps from EL1 to EL1 which we
|
||||
* can't intercept and which would confuse the guest.
|
||||
*
|
||||
* Do the same for PSTATE.ZA in the case where there
|
||||
* is state in the registers which has not already
|
||||
* been saved, this is very unlikely to happen.
|
||||
*/
|
||||
if (read_sysreg_s(SYS_SVCR) & (SVCR_SM_MASK | SVCR_ZA_MASK)) {
|
||||
*host_data_ptr(fp_owner) = FP_STATE_FREE;
|
||||
fpsimd_save_and_flush_cpu_state();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* If normal guests gain SME support, maintain this behavior for pKVM
|
||||
* guests, which don't support SME.
|
||||
*/
|
||||
WARN_ON(is_protected_kvm_enabled() && system_supports_sme() &&
|
||||
read_sysreg_s(SYS_SVCR));
|
||||
WARN_ON_ONCE(system_supports_sme() && read_sysreg_s(SYS_SVCR));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -162,52 +130,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
|
|||
|
||||
local_irq_save(flags);
|
||||
|
||||
/*
|
||||
* If we have VHE then the Hyp code will reset CPACR_EL1 to
|
||||
* the default value and we need to reenable SME.
|
||||
*/
|
||||
if (has_vhe() && system_supports_sme()) {
|
||||
/* Also restore EL0 state seen on entry */
|
||||
if (host_data_test_flag(HOST_SME_ENABLED))
|
||||
sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_SMEN);
|
||||
else
|
||||
sysreg_clear_set(CPACR_EL1,
|
||||
CPACR_EL1_SMEN_EL0EN,
|
||||
CPACR_EL1_SMEN_EL1EN);
|
||||
isb();
|
||||
}
|
||||
|
||||
if (guest_owns_fp_regs()) {
|
||||
if (vcpu_has_sve(vcpu)) {
|
||||
u64 zcr = read_sysreg_el1(SYS_ZCR);
|
||||
|
||||
/*
|
||||
* If the vCPU is in the hyp context then ZCR_EL1 is
|
||||
* loaded with its vEL2 counterpart.
|
||||
*/
|
||||
__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)) = zcr;
|
||||
|
||||
/*
|
||||
* Restore the VL that was saved when bound to the CPU,
|
||||
* which is the maximum VL for the guest. Because the
|
||||
* layout of the data when saving the sve state depends
|
||||
* on the VL, we need to use a consistent (i.e., the
|
||||
* maximum) VL.
|
||||
* Note that this means that at guest exit ZCR_EL1 is
|
||||
* not necessarily the same as on guest entry.
|
||||
*
|
||||
* ZCR_EL2 holds the guest hypervisor's VL when running
|
||||
* a nested guest, which could be smaller than the
|
||||
* max for the vCPU. Similar to above, we first need to
|
||||
* switch to a VL consistent with the layout of the
|
||||
* vCPU's SVE state. KVM support for NV implies VHE, so
|
||||
* using the ZCR_EL1 alias is safe.
|
||||
*/
|
||||
if (!has_vhe() || (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)))
|
||||
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
|
||||
SYS_ZCR_EL1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush (save and invalidate) the fpsimd/sve state so that if
|
||||
* the host tries to use fpsimd/sve, it's not using stale data
|
||||
|
|
@ -219,18 +142,6 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
|
|||
* when needed.
|
||||
*/
|
||||
fpsimd_save_and_flush_cpu_state();
|
||||
} else if (has_vhe() && system_supports_sve()) {
|
||||
/*
|
||||
* The FPSIMD/SVE state in the CPU has not been touched, and we
|
||||
* have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
|
||||
* reset by kvm_reset_cptr_el2() in the Hyp code, disabling SVE
|
||||
* for EL0. To avoid spurious traps, restore the trap state
|
||||
* seen by kvm_arch_vcpu_load_fp():
|
||||
*/
|
||||
if (host_data_test_flag(HOST_SVE_ENABLED))
|
||||
sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN);
|
||||
else
|
||||
sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
|
||||
}
|
||||
|
||||
local_irq_restore(flags);
|
||||
|
|
|
|||
|
|
@ -44,6 +44,11 @@ alternative_if ARM64_HAS_RAS_EXTN
|
|||
alternative_else_nop_endif
|
||||
mrs x1, isr_el1
|
||||
cbz x1, 1f
|
||||
|
||||
// Ensure that __guest_enter() always provides a context
|
||||
// synchronization event so that callers don't need ISBs for anything
|
||||
// that would usually be synchonized by the ERET.
|
||||
isb
|
||||
mov x0, #ARM_EXCEPTION_IRQ
|
||||
ret
|
||||
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
|
|||
return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
|
||||
arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2);
|
||||
|
|
@ -375,7 +375,87 @@ static inline void __hyp_sve_save_host(void)
|
|||
true);
|
||||
}
|
||||
|
||||
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu);
|
||||
static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 zcr_el1, zcr_el2;
|
||||
|
||||
if (!guest_owns_fp_regs())
|
||||
return;
|
||||
|
||||
if (vcpu_has_sve(vcpu)) {
|
||||
/* A guest hypervisor may restrict the effective max VL. */
|
||||
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
|
||||
zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
|
||||
else
|
||||
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
|
||||
|
||||
write_sysreg_el2(zcr_el2, SYS_ZCR);
|
||||
|
||||
zcr_el1 = __vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu));
|
||||
write_sysreg_el1(zcr_el1, SYS_ZCR);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void fpsimd_lazy_switch_to_host(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 zcr_el1, zcr_el2;
|
||||
|
||||
if (!guest_owns_fp_regs())
|
||||
return;
|
||||
|
||||
/*
|
||||
* When the guest owns the FP regs, we know that guest+hyp traps for
|
||||
* any FPSIMD/SVE/SME features exposed to the guest have been disabled
|
||||
* by either fpsimd_lazy_switch_to_guest() or kvm_hyp_handle_fpsimd()
|
||||
* prior to __guest_entry(). As __guest_entry() guarantees a context
|
||||
* synchronization event, we don't need an ISB here to avoid taking
|
||||
* traps for anything that was exposed to the guest.
|
||||
*/
|
||||
if (vcpu_has_sve(vcpu)) {
|
||||
zcr_el1 = read_sysreg_el1(SYS_ZCR);
|
||||
__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)) = zcr_el1;
|
||||
|
||||
/*
|
||||
* The guest's state is always saved using the guest's max VL.
|
||||
* Ensure that the host has the guest's max VL active such that
|
||||
* the host can save the guest's state lazily, but don't
|
||||
* artificially restrict the host to the guest's max VL.
|
||||
*/
|
||||
if (has_vhe()) {
|
||||
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
|
||||
write_sysreg_el2(zcr_el2, SYS_ZCR);
|
||||
} else {
|
||||
zcr_el2 = sve_vq_from_vl(kvm_host_sve_max_vl) - 1;
|
||||
write_sysreg_el2(zcr_el2, SYS_ZCR);
|
||||
|
||||
zcr_el1 = vcpu_sve_max_vq(vcpu) - 1;
|
||||
write_sysreg_el1(zcr_el1, SYS_ZCR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
/*
|
||||
* Non-protected kvm relies on the host restoring its sve state.
|
||||
* Protected kvm restores the host's sve state as not to reveal that
|
||||
* fpsimd was used by a guest nor leak upper sve bits.
|
||||
*/
|
||||
if (system_supports_sve()) {
|
||||
__hyp_sve_save_host();
|
||||
|
||||
/* Re-enable SVE traps if not supported for the guest vcpu. */
|
||||
if (!vcpu_has_sve(vcpu))
|
||||
cpacr_clear_set(CPACR_EL1_ZEN, 0);
|
||||
|
||||
} else {
|
||||
__fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs));
|
||||
}
|
||||
|
||||
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm)))
|
||||
*host_data_ptr(fpmr) = read_sysreg_s(SYS_FPMR);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* We trap the first access to the FP/SIMD to save the host context and
|
||||
|
|
@ -383,7 +463,7 @@ static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu);
|
|||
* If FP/SIMD is not implemented, handle the trap and inject an undefined
|
||||
* instruction exception to the guest. Similarly for trapped SVE accesses.
|
||||
*/
|
||||
static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
bool sve_guest;
|
||||
u8 esr_ec;
|
||||
|
|
@ -425,7 +505,7 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
isb();
|
||||
|
||||
/* Write out the host state if it's in the registers */
|
||||
if (host_owns_fp_regs())
|
||||
if (is_protected_kvm_enabled() && host_owns_fp_regs())
|
||||
kvm_hyp_save_fpsimd_host(vcpu);
|
||||
|
||||
/* Restore the guest state */
|
||||
|
|
@ -501,9 +581,22 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu)
|
|||
return true;
|
||||
}
|
||||
|
||||
/* Open-coded version of timer_get_offset() to allow for kern_hyp_va() */
|
||||
static inline u64 hyp_timer_get_offset(struct arch_timer_context *ctxt)
|
||||
{
|
||||
u64 offset = 0;
|
||||
|
||||
if (ctxt->offset.vm_offset)
|
||||
offset += *kern_hyp_va(ctxt->offset.vm_offset);
|
||||
if (ctxt->offset.vcpu_offset)
|
||||
offset += *kern_hyp_va(ctxt->offset.vcpu_offset);
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
static inline u64 compute_counter_value(struct arch_timer_context *ctxt)
|
||||
{
|
||||
return arch_timer_read_cntpct_el0() - timer_get_offset(ctxt);
|
||||
return arch_timer_read_cntpct_el0() - hyp_timer_get_offset(ctxt);
|
||||
}
|
||||
|
||||
static bool kvm_handle_cntxct(struct kvm_vcpu *vcpu)
|
||||
|
|
@ -587,7 +680,7 @@ static bool handle_ampere1_tcr(struct kvm_vcpu *vcpu)
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
|
||||
handle_tx2_tvm(vcpu))
|
||||
|
|
@ -607,7 +700,7 @@ static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
|
||||
__vgic_v3_perform_cpuif_access(vcpu) == 1)
|
||||
|
|
@ -616,19 +709,18 @@ static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_memory_fault(struct kvm_vcpu *vcpu,
|
||||
u64 *exit_code)
|
||||
{
|
||||
if (!__populate_fault_info(vcpu))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
static bool kvm_hyp_handle_iabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
__alias(kvm_hyp_handle_memory_fault);
|
||||
static bool kvm_hyp_handle_watchpt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
__alias(kvm_hyp_handle_memory_fault);
|
||||
#define kvm_hyp_handle_iabt_low kvm_hyp_handle_memory_fault
|
||||
#define kvm_hyp_handle_watchpt_low kvm_hyp_handle_memory_fault
|
||||
|
||||
static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
if (kvm_hyp_handle_memory_fault(vcpu, exit_code))
|
||||
return true;
|
||||
|
|
@ -658,23 +750,16 @@ static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
|
||||
typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *);
|
||||
|
||||
static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu);
|
||||
|
||||
static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code);
|
||||
|
||||
/*
|
||||
* Allow the hypervisor to handle the exit with an exit handler if it has one.
|
||||
*
|
||||
* Returns true if the hypervisor handled the exit, and control should go back
|
||||
* to the guest, or false if it hasn't.
|
||||
*/
|
||||
static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code,
|
||||
const exit_handler_fn *handlers)
|
||||
{
|
||||
const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu);
|
||||
exit_handler_fn fn;
|
||||
|
||||
fn = handlers[kvm_vcpu_trap_get_class(vcpu)];
|
||||
|
||||
exit_handler_fn fn = handlers[kvm_vcpu_trap_get_class(vcpu)];
|
||||
if (fn)
|
||||
return fn(vcpu, exit_code);
|
||||
|
||||
|
|
@ -704,20 +789,9 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu, u64 *exit_code
|
|||
* the guest, false when we should restore the host state and return to the
|
||||
* main run loop.
|
||||
*/
|
||||
static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool __fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code,
|
||||
const exit_handler_fn *handlers)
|
||||
{
|
||||
/*
|
||||
* Save PSTATE early so that we can evaluate the vcpu mode
|
||||
* early on.
|
||||
*/
|
||||
synchronize_vcpu_pstate(vcpu, exit_code);
|
||||
|
||||
/*
|
||||
* Check whether we want to repaint the state one way or
|
||||
* another.
|
||||
*/
|
||||
early_exit_filter(vcpu, exit_code);
|
||||
|
||||
if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)
|
||||
vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);
|
||||
|
||||
|
|
@ -747,7 +821,7 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
goto exit;
|
||||
|
||||
/* Check if there's an exit handler and allow it to handle the exit. */
|
||||
if (kvm_hyp_handle_exit(vcpu, exit_code))
|
||||
if (kvm_hyp_handle_exit(vcpu, exit_code, handlers))
|
||||
goto guest;
|
||||
exit:
|
||||
/* Return to the host kernel and handle the exit */
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
#include <hyp/adjust_pc.h>
|
||||
#include <hyp/switch.h>
|
||||
|
||||
#include <asm/pgtable-types.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
|
|
@ -83,7 +84,7 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
|
|||
if (system_supports_sve())
|
||||
__hyp_sve_restore_host();
|
||||
else
|
||||
__fpsimd_restore_state(*host_data_ptr(fpsimd_state));
|
||||
__fpsimd_restore_state(host_data_ptr(host_ctxt.fp_regs));
|
||||
|
||||
if (has_fpmr)
|
||||
write_sysreg_s(*host_data_ptr(fpmr), SYS_FPMR);
|
||||
|
|
@ -224,8 +225,12 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
|
|||
|
||||
sync_hyp_vcpu(hyp_vcpu);
|
||||
} else {
|
||||
struct kvm_vcpu *vcpu = kern_hyp_va(host_vcpu);
|
||||
|
||||
/* The host is fully trusted, run its vCPU directly. */
|
||||
ret = __kvm_vcpu_run(kern_hyp_va(host_vcpu));
|
||||
fpsimd_lazy_switch_to_guest(vcpu);
|
||||
ret = __kvm_vcpu_run(vcpu);
|
||||
fpsimd_lazy_switch_to_host(vcpu);
|
||||
}
|
||||
out:
|
||||
cpu_reg(host_ctxt, 1) = ret;
|
||||
|
|
@ -675,12 +680,6 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
|
|||
case ESR_ELx_EC_SMC64:
|
||||
handle_host_smc(host_ctxt);
|
||||
break;
|
||||
case ESR_ELx_EC_SVE:
|
||||
cpacr_clear_set(0, CPACR_EL1_ZEN);
|
||||
isb();
|
||||
sve_cond_update_zcr_vq(sve_vq_from_vl(kvm_host_sve_max_vl) - 1,
|
||||
SYS_ZCR_EL2);
|
||||
break;
|
||||
case ESR_ELx_EC_IABT_LOW:
|
||||
case ESR_ELx_EC_DABT_LOW:
|
||||
handle_host_mem_abort(host_ctxt);
|
||||
|
|
|
|||
|
|
@ -943,10 +943,10 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip
|
|||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (level != KVM_PGTABLE_LAST_LEVEL)
|
||||
return -E2BIG;
|
||||
if (!kvm_pte_valid(pte))
|
||||
return -ENOENT;
|
||||
if (level != KVM_PGTABLE_LAST_LEVEL)
|
||||
return -E2BIG;
|
||||
|
||||
state = guest_get_page_state(pte, ipa);
|
||||
if (state != PKVM_PAGE_SHARED_BORROWED)
|
||||
|
|
@ -998,44 +998,57 @@ unlock:
|
|||
return ret;
|
||||
}
|
||||
|
||||
int __pkvm_host_relax_perms_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu, enum kvm_pgtable_prot prot)
|
||||
static void assert_host_shared_guest(struct pkvm_hyp_vm *vm, u64 ipa)
|
||||
{
|
||||
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
|
||||
u64 ipa = hyp_pfn_to_phys(gfn);
|
||||
u64 phys;
|
||||
int ret;
|
||||
|
||||
if (prot & ~KVM_PGTABLE_PROT_RWX)
|
||||
return -EINVAL;
|
||||
if (!IS_ENABLED(CONFIG_NVHE_EL2_DEBUG))
|
||||
return;
|
||||
|
||||
host_lock_component();
|
||||
guest_lock_component(vm);
|
||||
|
||||
ret = __check_host_shared_guest(vm, &phys, ipa);
|
||||
if (!ret)
|
||||
ret = kvm_pgtable_stage2_relax_perms(&vm->pgt, ipa, prot, 0);
|
||||
|
||||
guest_unlock_component(vm);
|
||||
host_unlock_component();
|
||||
|
||||
WARN_ON(ret && ret != -ENOENT);
|
||||
}
|
||||
|
||||
int __pkvm_host_relax_perms_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu, enum kvm_pgtable_prot prot)
|
||||
{
|
||||
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
|
||||
u64 ipa = hyp_pfn_to_phys(gfn);
|
||||
int ret;
|
||||
|
||||
if (pkvm_hyp_vm_is_protected(vm))
|
||||
return -EPERM;
|
||||
|
||||
if (prot & ~KVM_PGTABLE_PROT_RWX)
|
||||
return -EINVAL;
|
||||
|
||||
assert_host_shared_guest(vm, ipa);
|
||||
guest_lock_component(vm);
|
||||
ret = kvm_pgtable_stage2_relax_perms(&vm->pgt, ipa, prot, 0);
|
||||
guest_unlock_component(vm);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __pkvm_host_wrprotect_guest(u64 gfn, struct pkvm_hyp_vm *vm)
|
||||
{
|
||||
u64 ipa = hyp_pfn_to_phys(gfn);
|
||||
u64 phys;
|
||||
int ret;
|
||||
|
||||
host_lock_component();
|
||||
if (pkvm_hyp_vm_is_protected(vm))
|
||||
return -EPERM;
|
||||
|
||||
assert_host_shared_guest(vm, ipa);
|
||||
guest_lock_component(vm);
|
||||
|
||||
ret = __check_host_shared_guest(vm, &phys, ipa);
|
||||
if (!ret)
|
||||
ret = kvm_pgtable_stage2_wrprotect(&vm->pgt, ipa, PAGE_SIZE);
|
||||
|
||||
ret = kvm_pgtable_stage2_wrprotect(&vm->pgt, ipa, PAGE_SIZE);
|
||||
guest_unlock_component(vm);
|
||||
host_unlock_component();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -1043,18 +1056,15 @@ int __pkvm_host_wrprotect_guest(u64 gfn, struct pkvm_hyp_vm *vm)
|
|||
int __pkvm_host_test_clear_young_guest(u64 gfn, bool mkold, struct pkvm_hyp_vm *vm)
|
||||
{
|
||||
u64 ipa = hyp_pfn_to_phys(gfn);
|
||||
u64 phys;
|
||||
int ret;
|
||||
|
||||
host_lock_component();
|
||||
if (pkvm_hyp_vm_is_protected(vm))
|
||||
return -EPERM;
|
||||
|
||||
assert_host_shared_guest(vm, ipa);
|
||||
guest_lock_component(vm);
|
||||
|
||||
ret = __check_host_shared_guest(vm, &phys, ipa);
|
||||
if (!ret)
|
||||
ret = kvm_pgtable_stage2_test_clear_young(&vm->pgt, ipa, PAGE_SIZE, mkold);
|
||||
|
||||
ret = kvm_pgtable_stage2_test_clear_young(&vm->pgt, ipa, PAGE_SIZE, mkold);
|
||||
guest_unlock_component(vm);
|
||||
host_unlock_component();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -1063,18 +1073,14 @@ int __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu)
|
|||
{
|
||||
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
|
||||
u64 ipa = hyp_pfn_to_phys(gfn);
|
||||
u64 phys;
|
||||
int ret;
|
||||
|
||||
host_lock_component();
|
||||
if (pkvm_hyp_vm_is_protected(vm))
|
||||
return -EPERM;
|
||||
|
||||
assert_host_shared_guest(vm, ipa);
|
||||
guest_lock_component(vm);
|
||||
|
||||
ret = __check_host_shared_guest(vm, &phys, ipa);
|
||||
if (!ret)
|
||||
kvm_pgtable_stage2_mkyoung(&vm->pgt, ipa, 0);
|
||||
|
||||
kvm_pgtable_stage2_mkyoung(&vm->pgt, ipa, 0);
|
||||
guest_unlock_component(vm);
|
||||
host_unlock_component();
|
||||
|
||||
return ret;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,9 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
|
|||
{
|
||||
u64 val = CPTR_EL2_TAM; /* Same bit irrespective of E2H */
|
||||
|
||||
if (!guest_owns_fp_regs())
|
||||
__activate_traps_fpsimd32(vcpu);
|
||||
|
||||
if (has_hvhe()) {
|
||||
val |= CPACR_EL1_TTA;
|
||||
|
||||
|
|
@ -47,6 +50,8 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
|
|||
if (vcpu_has_sve(vcpu))
|
||||
val |= CPACR_EL1_ZEN;
|
||||
}
|
||||
|
||||
write_sysreg(val, cpacr_el1);
|
||||
} else {
|
||||
val |= CPTR_EL2_TTA | CPTR_NVHE_EL2_RES1;
|
||||
|
||||
|
|
@ -61,12 +66,32 @@ static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
|
|||
|
||||
if (!guest_owns_fp_regs())
|
||||
val |= CPTR_EL2_TFP;
|
||||
|
||||
write_sysreg(val, cptr_el2);
|
||||
}
|
||||
}
|
||||
|
||||
if (!guest_owns_fp_regs())
|
||||
__activate_traps_fpsimd32(vcpu);
|
||||
static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (has_hvhe()) {
|
||||
u64 val = CPACR_EL1_FPEN;
|
||||
|
||||
kvm_write_cptr_el2(val);
|
||||
if (cpus_have_final_cap(ARM64_SVE))
|
||||
val |= CPACR_EL1_ZEN;
|
||||
if (cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPACR_EL1_SMEN;
|
||||
|
||||
write_sysreg(val, cpacr_el1);
|
||||
} else {
|
||||
u64 val = CPTR_NVHE_EL2_RES1;
|
||||
|
||||
if (!cpus_have_final_cap(ARM64_SVE))
|
||||
val |= CPTR_EL2_TZ;
|
||||
if (!cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPTR_EL2_TSM;
|
||||
|
||||
write_sysreg(val, cptr_el2);
|
||||
}
|
||||
}
|
||||
|
||||
static void __activate_traps(struct kvm_vcpu *vcpu)
|
||||
|
|
@ -119,7 +144,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
|
|||
|
||||
write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2);
|
||||
|
||||
kvm_reset_cptr_el2(vcpu);
|
||||
__deactivate_cptr_traps(vcpu);
|
||||
write_sysreg(__kvm_hyp_host_vector, vbar_el2);
|
||||
}
|
||||
|
||||
|
|
@ -192,34 +217,6 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
kvm_handle_pvm_sysreg(vcpu, exit_code));
|
||||
}
|
||||
|
||||
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
/*
|
||||
* Non-protected kvm relies on the host restoring its sve state.
|
||||
* Protected kvm restores the host's sve state as not to reveal that
|
||||
* fpsimd was used by a guest nor leak upper sve bits.
|
||||
*/
|
||||
if (unlikely(is_protected_kvm_enabled() && system_supports_sve())) {
|
||||
__hyp_sve_save_host();
|
||||
|
||||
/* Re-enable SVE traps if not supported for the guest vcpu. */
|
||||
if (!vcpu_has_sve(vcpu))
|
||||
cpacr_clear_set(CPACR_EL1_ZEN, 0);
|
||||
|
||||
} else {
|
||||
__fpsimd_save_state(*host_data_ptr(fpsimd_state));
|
||||
}
|
||||
|
||||
if (kvm_has_fpmr(kern_hyp_va(vcpu->kvm))) {
|
||||
u64 val = read_sysreg_s(SYS_FPMR);
|
||||
|
||||
if (unlikely(is_protected_kvm_enabled()))
|
||||
*host_data_ptr(fpmr) = val;
|
||||
else
|
||||
**host_data_ptr(fpmr_ptr) = val;
|
||||
}
|
||||
}
|
||||
|
||||
static const exit_handler_fn hyp_exit_handlers[] = {
|
||||
[0 ... ESR_ELx_EC_MAX] = NULL,
|
||||
[ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32,
|
||||
|
|
@ -251,19 +248,21 @@ static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
|
|||
return hyp_exit_handlers;
|
||||
}
|
||||
|
||||
/*
|
||||
* Some guests (e.g., protected VMs) are not be allowed to run in AArch32.
|
||||
* The ARMv8 architecture does not give the hypervisor a mechanism to prevent a
|
||||
* guest from dropping to AArch32 EL0 if implemented by the CPU. If the
|
||||
* hypervisor spots a guest in such a state ensure it is handled, and don't
|
||||
* trust the host to spot or fix it. The check below is based on the one in
|
||||
* kvm_arch_vcpu_ioctl_run().
|
||||
*
|
||||
* Returns false if the guest ran in AArch32 when it shouldn't have, and
|
||||
* thus should exit to the host, or true if a the guest run loop can continue.
|
||||
*/
|
||||
static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu);
|
||||
|
||||
synchronize_vcpu_pstate(vcpu, exit_code);
|
||||
|
||||
/*
|
||||
* Some guests (e.g., protected VMs) are not be allowed to run in
|
||||
* AArch32. The ARMv8 architecture does not give the hypervisor a
|
||||
* mechanism to prevent a guest from dropping to AArch32 EL0 if
|
||||
* implemented by the CPU. If the hypervisor spots a guest in such a
|
||||
* state ensure it is handled, and don't trust the host to spot or fix
|
||||
* it. The check below is based on the one in
|
||||
* kvm_arch_vcpu_ioctl_run().
|
||||
*/
|
||||
if (unlikely(vcpu_is_protected(vcpu) && vcpu_mode_is_32bit(vcpu))) {
|
||||
/*
|
||||
* As we have caught the guest red-handed, decide that it isn't
|
||||
|
|
@ -276,6 +275,8 @@ static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
*exit_code &= BIT(ARM_EXIT_WITH_SERROR_BIT);
|
||||
*exit_code |= ARM_EXCEPTION_IL;
|
||||
}
|
||||
|
||||
return __fixup_guest_exit(vcpu, exit_code, handlers);
|
||||
}
|
||||
|
||||
/* Switch to the guest for legacy non-VHE systems */
|
||||
|
|
|
|||
|
|
@ -136,6 +136,16 @@ write:
|
|||
write_sysreg(val, cpacr_el1);
|
||||
}
|
||||
|
||||
static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 val = CPACR_EL1_FPEN | CPACR_EL1_ZEN_EL1EN;
|
||||
|
||||
if (cpus_have_final_cap(ARM64_SME))
|
||||
val |= CPACR_EL1_SMEN_EL1EN;
|
||||
|
||||
write_sysreg(val, cpacr_el1);
|
||||
}
|
||||
|
||||
static void __activate_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 val;
|
||||
|
|
@ -207,7 +217,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
|
|||
*/
|
||||
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
|
||||
|
||||
kvm_reset_cptr_el2(vcpu);
|
||||
__deactivate_cptr_traps(vcpu);
|
||||
|
||||
if (!arm64_kernel_unmapped_at_el0())
|
||||
host_vectors = __this_cpu_read(this_cpu_vector);
|
||||
|
|
@ -413,14 +423,6 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
return true;
|
||||
}
|
||||
|
||||
static void kvm_hyp_save_fpsimd_host(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
__fpsimd_save_state(*host_data_ptr(fpsimd_state));
|
||||
|
||||
if (kvm_has_fpmr(vcpu->kvm))
|
||||
**host_data_ptr(fpmr_ptr) = read_sysreg_s(SYS_FPMR);
|
||||
}
|
||||
|
||||
static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
int ret = -EINVAL;
|
||||
|
|
@ -538,13 +540,10 @@ static const exit_handler_fn hyp_exit_handlers[] = {
|
|||
[ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops,
|
||||
};
|
||||
|
||||
static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu)
|
||||
static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
return hyp_exit_handlers;
|
||||
}
|
||||
synchronize_vcpu_pstate(vcpu, exit_code);
|
||||
|
||||
static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
|
||||
{
|
||||
/*
|
||||
* If we were in HYP context on entry, adjust the PSTATE view
|
||||
* so that the usual helpers work correctly.
|
||||
|
|
@ -564,6 +563,8 @@ static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code)
|
|||
*vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT);
|
||||
*vcpu_cpsr(vcpu) |= mode;
|
||||
}
|
||||
|
||||
return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);
|
||||
}
|
||||
|
||||
/* Switch to the guest for VHE systems running in EL2 */
|
||||
|
|
@ -578,6 +579,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
sysreg_save_host_state_vhe(host_ctxt);
|
||||
|
||||
fpsimd_lazy_switch_to_guest(vcpu);
|
||||
|
||||
/*
|
||||
* Note that ARM erratum 1165522 requires us to configure both stage 1
|
||||
* and stage 2 translation for the guest context before we clear
|
||||
|
|
@ -602,6 +605,8 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
|
|||
|
||||
__deactivate_traps(vcpu);
|
||||
|
||||
fpsimd_lazy_switch_to_host(vcpu);
|
||||
|
||||
sysreg_restore_host_state_vhe(host_ctxt);
|
||||
|
||||
if (guest_owns_fp_regs())
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@
|
|||
*
|
||||
* CPU Interface:
|
||||
*
|
||||
* - kvm_vgic_vcpu_init(): initialization of static data that
|
||||
* doesn't depend on any sizing information or emulation type. No
|
||||
* allocation is allowed there.
|
||||
* - kvm_vgic_vcpu_init(): initialization of static data that doesn't depend
|
||||
* on any sizing information. Private interrupts are allocated if not
|
||||
* already allocated at vgic-creation time.
|
||||
*/
|
||||
|
||||
/* EARLY INIT */
|
||||
|
|
@ -58,6 +58,8 @@ void kvm_vgic_early_init(struct kvm *kvm)
|
|||
|
||||
/* CREATION */
|
||||
|
||||
static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type);
|
||||
|
||||
/**
|
||||
* kvm_vgic_create: triggered by the instantiation of the VGIC device by
|
||||
* user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
|
||||
|
|
@ -112,6 +114,22 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
|
|||
goto out_unlock;
|
||||
}
|
||||
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
ret = vgic_allocate_private_irqs_locked(vcpu, type);
|
||||
if (ret)
|
||||
break;
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
kvm_for_each_vcpu(i, vcpu, kvm) {
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
kfree(vgic_cpu->private_irqs);
|
||||
vgic_cpu->private_irqs = NULL;
|
||||
}
|
||||
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
kvm->arch.vgic.in_kernel = true;
|
||||
kvm->arch.vgic.vgic_model = type;
|
||||
|
||||
|
|
@ -180,7 +198,7 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu)
|
||||
static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type)
|
||||
{
|
||||
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
|
||||
int i;
|
||||
|
|
@ -218,17 +236,28 @@ static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu)
|
|||
/* PPIs */
|
||||
irq->config = VGIC_CONFIG_LEVEL;
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case KVM_DEV_TYPE_ARM_VGIC_V3:
|
||||
irq->group = 1;
|
||||
irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
|
||||
break;
|
||||
case KVM_DEV_TYPE_ARM_VGIC_V2:
|
||||
irq->group = 0;
|
||||
irq->targets = BIT(vcpu->vcpu_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vgic_allocate_private_irqs(struct kvm_vcpu *vcpu)
|
||||
static int vgic_allocate_private_irqs(struct kvm_vcpu *vcpu, u32 type)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&vcpu->kvm->arch.config_lock);
|
||||
ret = vgic_allocate_private_irqs_locked(vcpu);
|
||||
ret = vgic_allocate_private_irqs_locked(vcpu, type);
|
||||
mutex_unlock(&vcpu->kvm->arch.config_lock);
|
||||
|
||||
return ret;
|
||||
|
|
@ -258,7 +287,7 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
|
|||
if (!irqchip_in_kernel(vcpu->kvm))
|
||||
return 0;
|
||||
|
||||
ret = vgic_allocate_private_irqs(vcpu);
|
||||
ret = vgic_allocate_private_irqs(vcpu, dist->vgic_model);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
|
|
@ -295,7 +324,7 @@ int vgic_init(struct kvm *kvm)
|
|||
{
|
||||
struct vgic_dist *dist = &kvm->arch.vgic;
|
||||
struct kvm_vcpu *vcpu;
|
||||
int ret = 0, i;
|
||||
int ret = 0;
|
||||
unsigned long idx;
|
||||
|
||||
lockdep_assert_held(&kvm->arch.config_lock);
|
||||
|
|
@ -315,35 +344,6 @@ int vgic_init(struct kvm *kvm)
|
|||
if (ret)
|
||||
goto out;
|
||||
|
||||
/* Initialize groups on CPUs created before the VGIC type was known */
|
||||
kvm_for_each_vcpu(idx, vcpu, kvm) {
|
||||
ret = vgic_allocate_private_irqs_locked(vcpu);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
|
||||
struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, i);
|
||||
|
||||
switch (dist->vgic_model) {
|
||||
case KVM_DEV_TYPE_ARM_VGIC_V3:
|
||||
irq->group = 1;
|
||||
irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
|
||||
break;
|
||||
case KVM_DEV_TYPE_ARM_VGIC_V2:
|
||||
irq->group = 0;
|
||||
irq->targets = 1U << idx;
|
||||
break;
|
||||
default:
|
||||
ret = -EINVAL;
|
||||
}
|
||||
|
||||
vgic_put_irq(kvm, irq);
|
||||
|
||||
if (ret)
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* If we have GICv4.1 enabled, unconditionally request enable the
|
||||
* v4 support so that we get HW-accelerated vSGIs. Otherwise, only
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue