Merge tag 'kvmarm-fixes-6.16-3' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 fixes for 6.16, take #3

- Fix another set of FP/SIMD/SVE bugs affecting NV, and plugging some
missing synchronisation

- A small fix for the irqbypass hook fixes, tightening the check and
ensuring that we only deal with MSI for both the old and the new
route entry

- Rework the way the shadow LRs are addressed in a nesting
configuration, plugging an embarrassing bug as well as simplifying
the whole process

- Add yet another fix for the dreaded arch_timer_edge_cases selftest

+215 -271
-62
arch/arm64/include/asm/kvm_emulate.h
··· 561 vcpu_set_flag((v), e); \ 562 } while (0) 563 564 - #define __build_check_all_or_none(r, bits) \ 565 - BUILD_BUG_ON(((r) & (bits)) && ((r) & (bits)) != (bits)) 566 - 567 - #define __cpacr_to_cptr_clr(clr, set) \ 568 - ({ \ 569 - u64 cptr = 0; \ 570 - \ 571 - if ((set) & CPACR_EL1_FPEN) \ 572 - cptr |= CPTR_EL2_TFP; \ 573 - if ((set) & CPACR_EL1_ZEN) \ 574 - cptr |= CPTR_EL2_TZ; \ 575 - if ((set) & CPACR_EL1_SMEN) \ 576 - cptr |= CPTR_EL2_TSM; \ 577 - if ((clr) & CPACR_EL1_TTA) \ 578 - cptr |= CPTR_EL2_TTA; \ 579 - if ((clr) & CPTR_EL2_TAM) \ 580 - cptr |= CPTR_EL2_TAM; \ 581 - if ((clr) & CPTR_EL2_TCPAC) \ 582 - cptr |= CPTR_EL2_TCPAC; \ 583 - \ 584 - cptr; \ 585 - }) 586 - 587 - #define __cpacr_to_cptr_set(clr, set) \ 588 - ({ \ 589 - u64 cptr = 0; \ 590 - \ 591 - if ((clr) & CPACR_EL1_FPEN) \ 592 - cptr |= CPTR_EL2_TFP; \ 593 - if ((clr) & CPACR_EL1_ZEN) \ 594 - cptr |= CPTR_EL2_TZ; \ 595 - if ((clr) & CPACR_EL1_SMEN) \ 596 - cptr |= CPTR_EL2_TSM; \ 597 - if ((set) & CPACR_EL1_TTA) \ 598 - cptr |= CPTR_EL2_TTA; \ 599 - if ((set) & CPTR_EL2_TAM) \ 600 - cptr |= CPTR_EL2_TAM; \ 601 - if ((set) & CPTR_EL2_TCPAC) \ 602 - cptr |= CPTR_EL2_TCPAC; \ 603 - \ 604 - cptr; \ 605 - }) 606 - 607 - #define cpacr_clear_set(clr, set) \ 608 - do { \ 609 - BUILD_BUG_ON((set) & CPTR_VHE_EL2_RES0); \ 610 - BUILD_BUG_ON((clr) & CPACR_EL1_E0POE); \ 611 - __build_check_all_or_none((clr), CPACR_EL1_FPEN); \ 612 - __build_check_all_or_none((set), CPACR_EL1_FPEN); \ 613 - __build_check_all_or_none((clr), CPACR_EL1_ZEN); \ 614 - __build_check_all_or_none((set), CPACR_EL1_ZEN); \ 615 - __build_check_all_or_none((clr), CPACR_EL1_SMEN); \ 616 - __build_check_all_or_none((set), CPACR_EL1_SMEN); \ 617 - \ 618 - if (has_vhe() || has_hvhe()) \ 619 - sysreg_clear_set(cpacr_el1, clr, set); \ 620 - else \ 621 - sysreg_clear_set(cptr_el2, \ 622 - __cpacr_to_cptr_clr(clr, set), \ 623 - __cpacr_to_cptr_set(clr, set));\ 624 - } while (0) 625 - 626 /* 627 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE 628 * format if E2H isn't set.
··· 561 vcpu_set_flag((v), e); \ 562 } while (0) 563 564 /* 565 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE 566 * format if E2H isn't set.
+2 -4
arch/arm64/include/asm/kvm_host.h
··· 1289 }) 1290 1291 /* 1292 - * The couple of isb() below are there to guarantee the same behaviour 1293 - * on VHE as on !VHE, where the eret to EL1 acts as a context 1294 - * synchronization event. 1295 */ 1296 #define kvm_call_hyp(f, ...) \ 1297 do { \ ··· 1308 \ 1309 if (has_vhe()) { \ 1310 ret = f(__VA_ARGS__); \ 1311 - isb(); \ 1312 } else { \ 1313 ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ 1314 } \
··· 1289 }) 1290 1291 /* 1292 + * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, 1293 + * where the eret to EL1 acts as a context synchronization event. 1294 */ 1295 #define kvm_call_hyp(f, ...) \ 1296 do { \ ··· 1309 \ 1310 if (has_vhe()) { \ 1311 ret = f(__VA_ARGS__); \ 1312 } else { \ 1313 ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ 1314 } \
+2 -1
arch/arm64/kvm/arm.c
··· 2764 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old, 2765 struct kvm_kernel_irq_routing_entry *new) 2766 { 2767 - if (new->type != KVM_IRQ_ROUTING_MSI) 2768 return true; 2769 2770 return memcmp(&old->msi, &new->msi, sizeof(new->msi));
··· 2764 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old, 2765 struct kvm_kernel_irq_routing_entry *new) 2766 { 2767 + if (old->type != KVM_IRQ_ROUTING_MSI || 2768 + new->type != KVM_IRQ_ROUTING_MSI) 2769 return true; 2770 2771 return memcmp(&old->msi, &new->msi, sizeof(new->msi));
+138 -9
arch/arm64/kvm/hyp/include/hyp/switch.h
··· 65 } 66 } 67 68 #define reg_to_fgt_masks(reg) \ 69 ({ \ 70 struct fgt_masks *m; \ ··· 616 */ 617 if (system_supports_sve()) { 618 __hyp_sve_save_host(); 619 - 620 - /* Re-enable SVE traps if not supported for the guest vcpu. */ 621 - if (!vcpu_has_sve(vcpu)) 622 - cpacr_clear_set(CPACR_EL1_ZEN, 0); 623 - 624 } else { 625 __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); 626 } ··· 666 /* Valid trap. Switch the context: */ 667 668 /* First disable enough traps to allow us to update the registers */ 669 - if (sve_guest || (is_protected_kvm_enabled() && system_supports_sve())) 670 - cpacr_clear_set(0, CPACR_EL1_FPEN | CPACR_EL1_ZEN); 671 - else 672 - cpacr_clear_set(0, CPACR_EL1_FPEN); 673 isb(); 674 675 /* Write out the host state if it's in the registers */ ··· 687 write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); 688 689 *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; 690 691 return true; 692 }
··· 65 } 66 } 67 68 + static inline void __activate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 69 + { 70 + u64 val = CPTR_NVHE_EL2_RES1 | CPTR_EL2_TAM | CPTR_EL2_TTA; 71 + 72 + /* 73 + * Always trap SME since it's not supported in KVM. 74 + * TSM is RES1 if SME isn't implemented. 75 + */ 76 + val |= CPTR_EL2_TSM; 77 + 78 + if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs()) 79 + val |= CPTR_EL2_TZ; 80 + 81 + if (!guest_owns_fp_regs()) 82 + val |= CPTR_EL2_TFP; 83 + 84 + write_sysreg(val, cptr_el2); 85 + } 86 + 87 + static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 88 + { 89 + /* 90 + * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 91 + * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 92 + * except for some missing controls, such as TAM. 93 + * In this case, CPTR_EL2.TAM has the same position with or without 94 + * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 95 + * shift value for trapping the AMU accesses. 96 + */ 97 + u64 val = CPTR_EL2_TAM | CPACR_EL1_TTA; 98 + u64 cptr; 99 + 100 + if (guest_owns_fp_regs()) { 101 + val |= CPACR_EL1_FPEN; 102 + if (vcpu_has_sve(vcpu)) 103 + val |= CPACR_EL1_ZEN; 104 + } 105 + 106 + if (!vcpu_has_nv(vcpu)) 107 + goto write; 108 + 109 + /* 110 + * The architecture is a bit crap (what a surprise): an EL2 guest 111 + * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, 112 + * as they are RES0 in the guest's view. To work around it, trap the 113 + * sucker using the very same bit it can't set... 114 + */ 115 + if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) 116 + val |= CPTR_EL2_TCPAC; 117 + 118 + /* 119 + * Layer the guest hypervisor's trap configuration on top of our own if 120 + * we're in a nested context. 121 + */ 122 + if (is_hyp_ctxt(vcpu)) 123 + goto write; 124 + 125 + cptr = vcpu_sanitised_cptr_el2(vcpu); 126 + 127 + /* 128 + * Pay attention, there's some interesting detail here. 129 + * 130 + * The CPTR_EL2.xEN fields are 2 bits wide, although there are only two 131 + * meaningful trap states when HCR_EL2.TGE = 0 (running a nested guest): 132 + * 133 + * - CPTR_EL2.xEN = x0, traps are enabled 134 + * - CPTR_EL2.xEN = x1, traps are disabled 135 + * 136 + * In other words, bit[0] determines if guest accesses trap or not. In 137 + * the interest of simplicity, clear the entire field if the guest 138 + * hypervisor has traps enabled to dispel any illusion of something more 139 + * complicated taking place. 140 + */ 141 + if (!(SYS_FIELD_GET(CPACR_EL1, FPEN, cptr) & BIT(0))) 142 + val &= ~CPACR_EL1_FPEN; 143 + if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0))) 144 + val &= ~CPACR_EL1_ZEN; 145 + 146 + if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 147 + val |= cptr & CPACR_EL1_E0POE; 148 + 149 + val |= cptr & CPTR_EL2_TCPAC; 150 + 151 + write: 152 + write_sysreg(val, cpacr_el1); 153 + } 154 + 155 + static inline void __activate_cptr_traps(struct kvm_vcpu *vcpu) 156 + { 157 + if (!guest_owns_fp_regs()) 158 + __activate_traps_fpsimd32(vcpu); 159 + 160 + if (has_vhe() || has_hvhe()) 161 + __activate_cptr_traps_vhe(vcpu); 162 + else 163 + __activate_cptr_traps_nvhe(vcpu); 164 + } 165 + 166 + static inline void __deactivate_cptr_traps_nvhe(struct kvm_vcpu *vcpu) 167 + { 168 + u64 val = CPTR_NVHE_EL2_RES1; 169 + 170 + if (!cpus_have_final_cap(ARM64_SVE)) 171 + val |= CPTR_EL2_TZ; 172 + if (!cpus_have_final_cap(ARM64_SME)) 173 + val |= CPTR_EL2_TSM; 174 + 175 + write_sysreg(val, cptr_el2); 176 + } 177 + 178 + static inline void __deactivate_cptr_traps_vhe(struct kvm_vcpu *vcpu) 179 + { 180 + u64 val = CPACR_EL1_FPEN; 181 + 182 + if (cpus_have_final_cap(ARM64_SVE)) 183 + val |= CPACR_EL1_ZEN; 184 + if (cpus_have_final_cap(ARM64_SME)) 185 + val |= CPACR_EL1_SMEN; 186 + 187 + write_sysreg(val, cpacr_el1); 188 + } 189 + 190 + static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) 191 + { 192 + if (has_vhe() || has_hvhe()) 193 + __deactivate_cptr_traps_vhe(vcpu); 194 + else 195 + __deactivate_cptr_traps_nvhe(vcpu); 196 + } 197 + 198 #define reg_to_fgt_masks(reg) \ 199 ({ \ 200 struct fgt_masks *m; \ ··· 486 */ 487 if (system_supports_sve()) { 488 __hyp_sve_save_host(); 489 } else { 490 __fpsimd_save_state(host_data_ptr(host_ctxt.fp_regs)); 491 } ··· 541 /* Valid trap. Switch the context: */ 542 543 /* First disable enough traps to allow us to update the registers */ 544 + __deactivate_cptr_traps(vcpu); 545 isb(); 546 547 /* Write out the host state if it's in the registers */ ··· 565 write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2); 566 567 *host_data_ptr(fp_owner) = FP_STATE_GUEST_OWNED; 568 + 569 + /* 570 + * Re-enable traps necessary for the current state of the guest, e.g. 571 + * those enabled by a guest hypervisor. The ERET to the guest will 572 + * provide the necessary context synchronization. 573 + */ 574 + __activate_cptr_traps(vcpu); 575 576 return true; 577 }
+4 -1
arch/arm64/kvm/hyp/nvhe/hyp-main.c
··· 69 if (!guest_owns_fp_regs()) 70 return; 71 72 - cpacr_clear_set(0, CPACR_EL1_FPEN | CPACR_EL1_ZEN); 73 isb(); 74 75 if (vcpu_has_sve(vcpu))
··· 69 if (!guest_owns_fp_regs()) 70 return; 71 72 + /* 73 + * Traps have been disabled by __deactivate_cptr_traps(), but there 74 + * hasn't necessarily been a context synchronization event yet. 75 + */ 76 isb(); 77 78 if (vcpu_has_sve(vcpu))
-59
arch/arm64/kvm/hyp/nvhe/switch.c
··· 47 48 extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); 49 50 - static void __activate_cptr_traps(struct kvm_vcpu *vcpu) 51 - { 52 - u64 val = CPTR_EL2_TAM; /* Same bit irrespective of E2H */ 53 - 54 - if (!guest_owns_fp_regs()) 55 - __activate_traps_fpsimd32(vcpu); 56 - 57 - if (has_hvhe()) { 58 - val |= CPACR_EL1_TTA; 59 - 60 - if (guest_owns_fp_regs()) { 61 - val |= CPACR_EL1_FPEN; 62 - if (vcpu_has_sve(vcpu)) 63 - val |= CPACR_EL1_ZEN; 64 - } 65 - 66 - write_sysreg(val, cpacr_el1); 67 - } else { 68 - val |= CPTR_EL2_TTA | CPTR_NVHE_EL2_RES1; 69 - 70 - /* 71 - * Always trap SME since it's not supported in KVM. 72 - * TSM is RES1 if SME isn't implemented. 73 - */ 74 - val |= CPTR_EL2_TSM; 75 - 76 - if (!vcpu_has_sve(vcpu) || !guest_owns_fp_regs()) 77 - val |= CPTR_EL2_TZ; 78 - 79 - if (!guest_owns_fp_regs()) 80 - val |= CPTR_EL2_TFP; 81 - 82 - write_sysreg(val, cptr_el2); 83 - } 84 - } 85 - 86 - static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) 87 - { 88 - if (has_hvhe()) { 89 - u64 val = CPACR_EL1_FPEN; 90 - 91 - if (cpus_have_final_cap(ARM64_SVE)) 92 - val |= CPACR_EL1_ZEN; 93 - if (cpus_have_final_cap(ARM64_SME)) 94 - val |= CPACR_EL1_SMEN; 95 - 96 - write_sysreg(val, cpacr_el1); 97 - } else { 98 - u64 val = CPTR_NVHE_EL2_RES1; 99 - 100 - if (!cpus_have_final_cap(ARM64_SVE)) 101 - val |= CPTR_EL2_TZ; 102 - if (!cpus_have_final_cap(ARM64_SME)) 103 - val |= CPTR_EL2_TSM; 104 - 105 - write_sysreg(val, cptr_el2); 106 - } 107 - } 108 - 109 static void __activate_traps(struct kvm_vcpu *vcpu) 110 { 111 ___activate_traps(vcpu, vcpu->arch.hcr_el2);
··· 47 48 extern void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); 49 50 static void __activate_traps(struct kvm_vcpu *vcpu) 51 { 52 ___activate_traps(vcpu, vcpu->arch.hcr_el2);
+14 -93
arch/arm64/kvm/hyp/vhe/switch.c
··· 90 return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE); 91 } 92 93 - static void __activate_cptr_traps(struct kvm_vcpu *vcpu) 94 - { 95 - u64 cptr; 96 - 97 - /* 98 - * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to 99 - * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, 100 - * except for some missing controls, such as TAM. 101 - * In this case, CPTR_EL2.TAM has the same position with or without 102 - * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM 103 - * shift value for trapping the AMU accesses. 104 - */ 105 - u64 val = CPACR_EL1_TTA | CPTR_EL2_TAM; 106 - 107 - if (guest_owns_fp_regs()) { 108 - val |= CPACR_EL1_FPEN; 109 - if (vcpu_has_sve(vcpu)) 110 - val |= CPACR_EL1_ZEN; 111 - } else { 112 - __activate_traps_fpsimd32(vcpu); 113 - } 114 - 115 - if (!vcpu_has_nv(vcpu)) 116 - goto write; 117 - 118 - /* 119 - * The architecture is a bit crap (what a surprise): an EL2 guest 120 - * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, 121 - * as they are RES0 in the guest's view. To work around it, trap the 122 - * sucker using the very same bit it can't set... 123 - */ 124 - if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) 125 - val |= CPTR_EL2_TCPAC; 126 - 127 - /* 128 - * Layer the guest hypervisor's trap configuration on top of our own if 129 - * we're in a nested context. 130 - */ 131 - if (is_hyp_ctxt(vcpu)) 132 - goto write; 133 - 134 - cptr = vcpu_sanitised_cptr_el2(vcpu); 135 - 136 - /* 137 - * Pay attention, there's some interesting detail here. 138 - * 139 - * The CPTR_EL2.xEN fields are 2 bits wide, although there are only two 140 - * meaningful trap states when HCR_EL2.TGE = 0 (running a nested guest): 141 - * 142 - * - CPTR_EL2.xEN = x0, traps are enabled 143 - * - CPTR_EL2.xEN = x1, traps are disabled 144 - * 145 - * In other words, bit[0] determines if guest accesses trap or not. In 146 - * the interest of simplicity, clear the entire field if the guest 147 - * hypervisor has traps enabled to dispel any illusion of something more 148 - * complicated taking place. 149 - */ 150 - if (!(SYS_FIELD_GET(CPACR_EL1, FPEN, cptr) & BIT(0))) 151 - val &= ~CPACR_EL1_FPEN; 152 - if (!(SYS_FIELD_GET(CPACR_EL1, ZEN, cptr) & BIT(0))) 153 - val &= ~CPACR_EL1_ZEN; 154 - 155 - if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 156 - val |= cptr & CPACR_EL1_E0POE; 157 - 158 - val |= cptr & CPTR_EL2_TCPAC; 159 - 160 - write: 161 - write_sysreg(val, cpacr_el1); 162 - } 163 - 164 - static void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) 165 - { 166 - u64 val = CPACR_EL1_FPEN | CPACR_EL1_ZEN_EL1EN; 167 - 168 - if (cpus_have_final_cap(ARM64_SME)) 169 - val |= CPACR_EL1_SMEN_EL1EN; 170 - 171 - write_sysreg(val, cpacr_el1); 172 - } 173 - 174 static void __activate_traps(struct kvm_vcpu *vcpu) 175 { 176 u64 val; ··· 558 host_ctxt = host_data_ptr(host_ctxt); 559 guest_ctxt = &vcpu->arch.ctxt; 560 561 - sysreg_save_host_state_vhe(host_ctxt); 562 - 563 fpsimd_lazy_switch_to_guest(vcpu); 564 565 /* 566 * Note that ARM erratum 1165522 requires us to configure both stage 1 ··· 586 587 __deactivate_traps(vcpu); 588 589 - fpsimd_lazy_switch_to_host(vcpu); 590 - 591 sysreg_restore_host_state_vhe(host_ctxt); 592 593 if (guest_owns_fp_regs()) 594 __fpsimd_save_fpexc32(vcpu); 595 - 596 - __debug_switch_to_host(vcpu); 597 598 return exit_code; 599 } ··· 632 */ 633 local_daif_restore(DAIF_PROCCTX_NOIRQ); 634 635 - /* 636 - * When we exit from the guest we change a number of CPU configuration 637 - * parameters, such as traps. We rely on the isb() in kvm_call_hyp*() 638 - * to make sure these changes take effect before running the host or 639 - * additional guests. 640 - */ 641 return ret; 642 } 643
··· 90 return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE); 91 } 92 93 static void __activate_traps(struct kvm_vcpu *vcpu) 94 { 95 u64 val; ··· 639 host_ctxt = host_data_ptr(host_ctxt); 640 guest_ctxt = &vcpu->arch.ctxt; 641 642 fpsimd_lazy_switch_to_guest(vcpu); 643 + 644 + sysreg_save_host_state_vhe(host_ctxt); 645 646 /* 647 * Note that ARM erratum 1165522 requires us to configure both stage 1 ··· 667 668 __deactivate_traps(vcpu); 669 670 sysreg_restore_host_state_vhe(host_ctxt); 671 + 672 + __debug_switch_to_host(vcpu); 673 + 674 + /* 675 + * Ensure that all system register writes above have taken effect 676 + * before returning to the host. In VHE mode, CPTR traps for 677 + * FPSIMD/SVE/SME also apply to EL2, so FPSIMD/SVE/SME state must be 678 + * manipulated after the ISB. 679 + */ 680 + isb(); 681 + 682 + fpsimd_lazy_switch_to_host(vcpu); 683 684 if (guest_owns_fp_regs()) 685 __fpsimd_save_fpexc32(vcpu); 686 687 return exit_code; 688 } ··· 705 */ 706 local_daif_restore(DAIF_PROCCTX_NOIRQ); 707 708 return ret; 709 } 710
+42 -39
arch/arm64/kvm/vgic/vgic-v3-nested.c
··· 36 37 static DEFINE_PER_CPU(struct shadow_if, shadow_if); 38 39 /* 40 * Nesting GICv3 support 41 * ··· 214 return reg; 215 } 216 217 /* 218 * For LRs which have HW bit set such as timer interrupts, we modify them to 219 * have the host hardware interrupt number instead of the virtual one programmed ··· 245 static void vgic_v3_create_shadow_lr(struct kvm_vcpu *vcpu, 246 struct vgic_v3_cpu_if *s_cpu_if) 247 { 248 - unsigned long lr_map = 0; 249 - int index = 0; 250 251 for (int i = 0; i < kvm_vgic_global_state.nr_lr; i++) { 252 u64 lr = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 253 - struct vgic_irq *irq; 254 255 if (!(lr & ICH_LR_STATE)) 256 - lr = 0; 257 258 - if (!(lr & ICH_LR_HW)) 259 - goto next; 260 261 - /* We have the HW bit set, check for validity of pINTID */ 262 - irq = vgic_get_vcpu_irq(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); 263 - if (!irq || !irq->hw || irq->intid > VGIC_MAX_SPI ) { 264 - /* There was no real mapping, so nuke the HW bit */ 265 - lr &= ~ICH_LR_HW; 266 - if (irq) 267 - vgic_put_irq(vcpu->kvm, irq); 268 - goto next; 269 - } 270 - 271 - /* Translate the virtual mapping to the real one */ 272 - lr &= ~ICH_LR_PHYS_ID_MASK; 273 - lr |= FIELD_PREP(ICH_LR_PHYS_ID_MASK, (u64)irq->hwintid); 274 - 275 - vgic_put_irq(vcpu->kvm, irq); 276 - 277 - next: 278 - s_cpu_if->vgic_lr[index] = lr; 279 - if (lr) { 280 - lr_map |= BIT(i); 281 - index++; 282 - } 283 } 284 285 - container_of(s_cpu_if, struct shadow_if, cpuif)->lr_map = lr_map; 286 - s_cpu_if->used_lrs = index; 287 } 288 289 void vgic_v3_sync_nested(struct kvm_vcpu *vcpu) 290 { 291 struct shadow_if *shadow_if = get_shadow_if(); 292 - int i, index = 0; 293 294 for_each_set_bit(i, &shadow_if->lr_map, kvm_vgic_global_state.nr_lr) { 295 u64 lr = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 296 struct vgic_irq *irq; 297 298 if (!(lr & ICH_LR_HW) || !(lr & ICH_LR_STATE)) 299 - goto next; 300 301 /* 302 * If we had a HW lr programmed by the guest hypervisor, we ··· 284 */ 285 irq = vgic_get_vcpu_irq(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); 286 if (WARN_ON(!irq)) /* Shouldn't happen as we check on load */ 287 - goto next; 288 289 - lr = __gic_v3_get_lr(index); 290 if (!(lr & ICH_LR_STATE)) 291 irq->active = false; 292 293 vgic_put_irq(vcpu->kvm, irq); 294 - next: 295 - index++; 296 } 297 } 298 ··· 373 val = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 374 375 val &= ~ICH_LR_STATE; 376 - val |= s_cpu_if->vgic_lr[i] & ICH_LR_STATE; 377 378 __vcpu_assign_sys_reg(vcpu, ICH_LRN(i), val); 379 - s_cpu_if->vgic_lr[i] = 0; 380 } 381 382 - shadow_if->lr_map = 0; 383 vcpu->arch.vgic_cpu.vgic_v3.used_lrs = 0; 384 } 385
··· 36 37 static DEFINE_PER_CPU(struct shadow_if, shadow_if); 38 39 + static int lr_map_idx_to_shadow_idx(struct shadow_if *shadow_if, int idx) 40 + { 41 + return hweight16(shadow_if->lr_map & (BIT(idx) - 1)); 42 + } 43 + 44 /* 45 * Nesting GICv3 support 46 * ··· 209 return reg; 210 } 211 212 + static u64 translate_lr_pintid(struct kvm_vcpu *vcpu, u64 lr) 213 + { 214 + struct vgic_irq *irq; 215 + 216 + if (!(lr & ICH_LR_HW)) 217 + return lr; 218 + 219 + /* We have the HW bit set, check for validity of pINTID */ 220 + irq = vgic_get_vcpu_irq(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); 221 + /* If there was no real mapping, nuke the HW bit */ 222 + if (!irq || !irq->hw || irq->intid > VGIC_MAX_SPI) 223 + lr &= ~ICH_LR_HW; 224 + 225 + /* Translate the virtual mapping to the real one, even if invalid */ 226 + if (irq) { 227 + lr &= ~ICH_LR_PHYS_ID_MASK; 228 + lr |= FIELD_PREP(ICH_LR_PHYS_ID_MASK, (u64)irq->hwintid); 229 + vgic_put_irq(vcpu->kvm, irq); 230 + } 231 + 232 + return lr; 233 + } 234 + 235 /* 236 * For LRs which have HW bit set such as timer interrupts, we modify them to 237 * have the host hardware interrupt number instead of the virtual one programmed ··· 217 static void vgic_v3_create_shadow_lr(struct kvm_vcpu *vcpu, 218 struct vgic_v3_cpu_if *s_cpu_if) 219 { 220 + struct shadow_if *shadow_if; 221 + 222 + shadow_if = container_of(s_cpu_if, struct shadow_if, cpuif); 223 + shadow_if->lr_map = 0; 224 225 for (int i = 0; i < kvm_vgic_global_state.nr_lr; i++) { 226 u64 lr = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 227 228 if (!(lr & ICH_LR_STATE)) 229 + continue; 230 231 + lr = translate_lr_pintid(vcpu, lr); 232 233 + s_cpu_if->vgic_lr[hweight16(shadow_if->lr_map)] = lr; 234 + shadow_if->lr_map |= BIT(i); 235 } 236 237 + s_cpu_if->used_lrs = hweight16(shadow_if->lr_map); 238 } 239 240 void vgic_v3_sync_nested(struct kvm_vcpu *vcpu) 241 { 242 struct shadow_if *shadow_if = get_shadow_if(); 243 + int i; 244 245 for_each_set_bit(i, &shadow_if->lr_map, kvm_vgic_global_state.nr_lr) { 246 u64 lr = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 247 struct vgic_irq *irq; 248 249 if (!(lr & ICH_LR_HW) || !(lr & ICH_LR_STATE)) 250 + continue; 251 252 /* 253 * If we had a HW lr programmed by the guest hypervisor, we ··· 277 */ 278 irq = vgic_get_vcpu_irq(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr)); 279 if (WARN_ON(!irq)) /* Shouldn't happen as we check on load */ 280 + continue; 281 282 + lr = __gic_v3_get_lr(lr_map_idx_to_shadow_idx(shadow_if, i)); 283 if (!(lr & ICH_LR_STATE)) 284 irq->active = false; 285 286 vgic_put_irq(vcpu->kvm, irq); 287 } 288 } 289 ··· 368 val = __vcpu_sys_reg(vcpu, ICH_LRN(i)); 369 370 val &= ~ICH_LR_STATE; 371 + val |= s_cpu_if->vgic_lr[lr_map_idx_to_shadow_idx(shadow_if, i)] & ICH_LR_STATE; 372 373 __vcpu_assign_sys_reg(vcpu, ICH_LRN(i), val); 374 } 375 376 vcpu->arch.vgic_cpu.vgic_v3.used_lrs = 0; 377 } 378
+13 -3
tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
··· 954 pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq); 955 } 956 957 static void test_vm_create(struct kvm_vm **vm, struct kvm_vcpu **vcpu, 958 enum arch_timer timer) 959 { ··· 970 vcpu_args_set(*vcpu, 1, timer); 971 972 test_init_timer_irq(*vm, *vcpu); 973 - vgic_v3_setup(*vm, 1, 64); 974 sync_global_to_guest(*vm, test_args); 975 sync_global_to_guest(*vm, CVAL_MAX); 976 sync_global_to_guest(*vm, DEF_CNT); 977 } 978 979 static void test_print_help(char *name) ··· 1070 if (test_args.test_virtual) { 1071 test_vm_create(&vm, &vcpu, VIRTUAL); 1072 test_run(vm, vcpu); 1073 - kvm_vm_free(vm); 1074 } 1075 1076 if (test_args.test_physical) { 1077 test_vm_create(&vm, &vcpu, PHYSICAL); 1078 test_run(vm, vcpu); 1079 - kvm_vm_free(vm); 1080 } 1081 1082 return 0;
··· 954 pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq); 955 } 956 957 + static int gic_fd; 958 + 959 static void test_vm_create(struct kvm_vm **vm, struct kvm_vcpu **vcpu, 960 enum arch_timer timer) 961 { ··· 968 vcpu_args_set(*vcpu, 1, timer); 969 970 test_init_timer_irq(*vm, *vcpu); 971 + gic_fd = vgic_v3_setup(*vm, 1, 64); 972 + __TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3"); 973 + 974 sync_global_to_guest(*vm, test_args); 975 sync_global_to_guest(*vm, CVAL_MAX); 976 sync_global_to_guest(*vm, DEF_CNT); 977 + } 978 + 979 + static void test_vm_cleanup(struct kvm_vm *vm) 980 + { 981 + close(gic_fd); 982 + kvm_vm_free(vm); 983 } 984 985 static void test_print_help(char *name) ··· 1060 if (test_args.test_virtual) { 1061 test_vm_create(&vm, &vcpu, VIRTUAL); 1062 test_run(vm, vcpu); 1063 + test_vm_cleanup(vm); 1064 } 1065 1066 if (test_args.test_physical) { 1067 test_vm_create(&vm, &vcpu, PHYSICAL); 1068 test_run(vm, vcpu); 1069 + test_vm_cleanup(vm); 1070 } 1071 1072 return 0;