Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513// SPDX-License-Identifier: GPL-2.0-only/* * KVM selftest powerpc library code - CPU-related functions (page tables...) */
#include <linux/sizes.h>
#include "processor.h"#include "kvm_util.h"#include "ucall_common.h"#include "guest_modes.h"#include "hcall.h"
#define RADIX_TREE_SIZE ((0x2UL << 61) | (0x5UL << 5)) /* 52-bits */#define RADIX_PGD_INDEX_SIZE 13
/* * A radix table must be aligned to its size, and the tables are larger than * the guest base page size for several level/page size combinations. */bool kvm_arch_needs_naturally_aligned_page_tables(void){ return true;}
static void set_proc_table(struct kvm_vm *vm, int pid, u64 dw0, u64 dw1){ u64 *proc_table;
proc_table = addr_gpa2hva(vm, vm->arch.prtb); proc_table[pid * 2 + 0] = cpu_to_be64(dw0); proc_table[pid * 2 + 1] = cpu_to_be64(dw1);}
static void set_radix_proc_table(struct kvm_vm *vm, int pid, gpa_t pgd){ set_proc_table(vm, pid, pgd | RADIX_TREE_SIZE | RADIX_PGD_INDEX_SIZE, 0);}
void virt_arch_pgd_alloc(struct kvm_vm *vm){ struct kvm_ppc_mmuv3_cfg mmu_cfg; gpa_t prtb, pgtb; size_t pgd_pages;
TEST_ASSERT((vm->mode == VM_MODE_P52V52_4K) || (vm->mode == VM_MODE_P52V52_64K), "Unsupported guest mode, mode: 0x%x", vm->mode);
if (vm->mmu.pgd_created) return;
prtb = vm_alloc_page_table(vm); vm->arch.prtb = prtb;
pgd_pages = (1UL << (RADIX_PGD_INDEX_SIZE + 3)) >> vm->page_shift; if (!pgd_pages) pgd_pages = 1; pgtb = vm_alloc_page_table_pages(vm, pgd_pages); vm->mmu.pgd = pgtb;
/* Set the base page directory in the proc table */ set_radix_proc_table(vm, 0, pgtb);
if (vm->mode == VM_MODE_P52V52_4K) mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x0; /* 4K size */ else /* vm->mode == VM_MODE_P52V52_64K */ mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x4; /* 64K size */ mmu_cfg.flags = KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE;
vm_ioctl(vm, KVM_PPC_CONFIGURE_V3_MMU, &mmu_cfg);
vm->mmu.pgd_created = true;}
static int pt_shift(struct kvm_vm *vm, int level){ switch (level) { case 1: return 13; case 2: case 3: return 9; case 4: if (vm->mode == VM_MODE_P52V52_4K) return 9; else /* vm->mode == VM_MODE_P52V52_64K */ return 5; default: TEST_ASSERT(false, "Invalid page table level %d\n", level); return 0; }}
static u64 pt_entry_coverage(struct kvm_vm *vm, int level){ u64 size = vm->page_size;
if (level == 4) return size; size <<= pt_shift(vm, 4); if (level == 3) return size; size <<= pt_shift(vm, 3); if (level == 2) return size; size <<= pt_shift(vm, 2); return size;}
static int pt_idx(struct kvm_vm *vm, gva_t gva, int level, u64 *nls){ switch (level) { case 1: if (nls) *nls = 0x9; return (gva >> 39) & 0x1fff; case 2: if (nls) *nls = 0x9; return (gva >> 30) & 0x1ff; case 3: if (vm->mode == VM_MODE_P52V52_4K) { if (nls) *nls = 0x9; } else { /* vm->mode == VM_MODE_P52V52_64K */ if (nls) *nls = 0x5; } return (gva >> 21) & 0x1ff; case 4: if (vm->mode == VM_MODE_P52V52_4K) return (gva >> 12) & 0x1ff; else /* vm->mode == VM_MODE_P52V52_64K */ return (gva >> 16) & 0x1f; default: TEST_ASSERT(false, "Invalid page table level %d\n", level); return 0; }}
static u64 *virt_get_pte(struct kvm_vm *vm, gpa_t pt, gva_t gva, int level, u64 *nls){ int idx = pt_idx(vm, gva, level, nls); u64 *ptep = addr_gpa2hva(vm, pt + idx * 8);
return ptep;}
#define PTE_VALID 0x8000000000000000ull#define PTE_LEAF 0x4000000000000000ull#define PTE_REFERENCED 0x0000000000000100ull#define PTE_CHANGED 0x0000000000000080ull#define PTE_PRIV 0x0000000000000008ull#define PTE_READ 0x0000000000000004ull#define PTE_RW 0x0000000000000002ull#define PTE_EXEC 0x0000000000000001ull#define PTE_PAGE_MASK 0x01fffffffffff000ull
#define PDE_VALID PTE_VALID#define PDE_NLS 0x0000000000000011ull#define PDE_PT_MASK 0x0fffffffffffff00ull
static gpa_t __vm_alloc_pt(struct kvm_vm *vm, u64 pt_shift){ gpa_t pt;
if (pt_shift >= vm->page_shift) { size_t pt_pages = 1ULL << (pt_shift - vm->page_shift);
pt = vm_alloc_page_table_pages(vm, pt_pages); } else { struct vm_pt_frag_cache *pt_frag_cache;
if (pt_shift == 8) { pt_frag_cache = &vm->arch.pt_frag_cache[0]; } else if (pt_shift == 12) { pt_frag_cache = &vm->arch.pt_frag_cache[1]; } else { TEST_ASSERT(0, "Invalid pt_shift:%lu\n", pt_shift); return 0; }
if (!pt_frag_cache->page) pt_frag_cache->page = vm_alloc_page_table(vm); pt = pt_frag_cache->page + pt_frag_cache->page_nr_used; pt_frag_cache->page_nr_used += (1 << pt_shift); if (pt_frag_cache->page_nr_used == vm->page_size) { pt_frag_cache->page = 0; pt_frag_cache->page_nr_used = 0; } }
return pt;}
void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa){ gpa_t pt = vm->mmu.pgd; u64 *ptep, pte; int level;
for (level = 1; level <= 3; level++) { u64 nls; u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); u64 pde = be64_to_cpu(*pdep);
if (pde) { TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), "Invalid PDE at level: %u gva: 0x%lx pde:0x%lx\n", level, gva, pde); pt = pde & PDE_PT_MASK; continue; }
pt = __vm_alloc_pt(vm, nls + 3); pde = PDE_VALID | nls | pt; *pdep = cpu_to_be64(pde); }
ptep = virt_get_pte(vm, pt, gva, level, NULL); pte = be64_to_cpu(*ptep);
TEST_ASSERT(!pte, "PTE already present at level: %u gva: 0x%lx pte:0x%lx\n", level, gva, pte);
pte = PTE_VALID | PTE_LEAF | PTE_REFERENCED | PTE_CHANGED | PTE_PRIV | PTE_READ | PTE_RW | PTE_EXEC | (gpa & PTE_PAGE_MASK); *ptep = cpu_to_be64(pte);}
gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva){ gpa_t pt = vm->mmu.pgd; u64 *ptep, pte; int level;
for (level = 1; level <= 3; level++) { u64 nls; u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); u64 pde = be64_to_cpu(*pdep);
TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), "PDE not present at level: %u gva: 0x%lx pde:0x%lx\n", level, gva, pde); pt = pde & PDE_PT_MASK; }
ptep = virt_get_pte(vm, pt, gva, level, NULL); pte = be64_to_cpu(*ptep);
TEST_ASSERT(pte, "PTE not present at level: %u gva: 0x%lx pte:0x%lx\n", level, gva, pte);
TEST_ASSERT((pte & PTE_VALID) && (pte & PTE_LEAF), "PTE not valid at level: %u gva: 0x%lx pte:0x%lx\n", level, gva, pte);
return (pte & PTE_PAGE_MASK) + (gva & (vm->page_size - 1));}
static void virt_dump_pt(FILE *stream, struct kvm_vm *vm, gpa_t pt, gva_t va, int level, u8 indent){ int size, idx;
size = 1U << (pt_shift(vm, level) + 3);
for (idx = 0; idx < size; idx += 8, va += pt_entry_coverage(vm, level)) { u64 *page_table = addr_gpa2hva(vm, pt + idx); u64 pte = be64_to_cpu(*page_table);
if (!(pte & PTE_VALID)) continue;
if (pte & PTE_LEAF) { fprintf(stream, "%*s PTE[%d] gVA:0x%016lx -> gRA:0x%016llx\n", indent, "", idx / 8, va, pte & PTE_PAGE_MASK); } else { fprintf(stream, "%*sPDE%d[%d] gVA:0x%016lx\n", indent, "", level, idx / 8, va); virt_dump_pt(stream, vm, pte & PDE_PT_MASK, va, level + 1, indent + 2); } }
}
void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent){ gpa_t pt = vm->mmu.pgd;
if (!vm->mmu.pgd_created) return;
virt_dump_pt(stream, vm, pt, 0, 1, indent);}
static unsigned long get_r2(void){ unsigned long r2;
asm("mr %0,%%r2" : "=r"(r2));
return r2;}
/* * Under ELFv1 a function pointer is the address of a descriptor holding the * entry point and the TOC, not the entry point itself. ELFv2 dropped * descriptors, i.e. the pointer is already the entry point. Dereference the * descriptor as needed so that the guest starts executing code and not data. */static uintptr_t function_entry(void *fn){#if defined(_CALL_ELF) && _CALL_ELF == 2 return (uintptr_t)fn;#else return ((const u64 *)fn)[0];#endif}
void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code){ uintptr_t entry = function_entry(guest_code); struct kvm_regs regs;
vcpu_regs_get(vcpu, ®s); regs.pc = entry; regs.gpr[12] = entry; vcpu_regs_set(vcpu, ®s);}
struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id){ const size_t stack_size = SZ_64K; gva_t stack_vaddr, ex_regs_vaddr; gpa_t ex_regs_paddr; struct ex_regs *ex_regs; struct kvm_regs regs; struct kvm_vcpu *vcpu; u64 lpcr;
stack_vaddr = __vm_alloc(vm, stack_size, DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
ex_regs_vaddr = __vm_alloc(vm, stack_size, DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA); ex_regs_paddr = addr_gva2gpa(vm, ex_regs_vaddr); ex_regs = addr_gpa2hva(vm, ex_regs_paddr); ex_regs->gva = ex_regs_vaddr;
vcpu = __vm_vcpu_add(vm, vcpu_id);
vcpu_enable_cap(vcpu, KVM_CAP_PPC_PAPR, 1);
/* Setup guest registers */ vcpu_regs_get(vcpu, ®s); lpcr = vcpu_get_reg(vcpu, KVM_REG_PPC_LPCR_64);
regs.gpr[1] = stack_vaddr + stack_size - 256; regs.gpr[2] = (uintptr_t)get_r2(); regs.gpr[13] = (uintptr_t)ex_regs_paddr;
regs.msr = MSR_SF | MSR_VEC | MSR_VSX | MSR_FP | MSR_ME | MSR_IR | MSR_DR | MSR_RI;
if (BYTE_ORDER == LITTLE_ENDIAN) { regs.msr |= MSR_LE; lpcr |= LPCR_ILE; } else { lpcr &= ~LPCR_ILE; }
vcpu_regs_set(vcpu, ®s); vcpu_set_reg(vcpu, KVM_REG_PPC_LPCR_64, lpcr);
return vcpu;}
void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...){ va_list ap; struct kvm_regs regs; int i;
TEST_ASSERT(num >= 1 && num <= 5, "Unsupported number of args: %u\n", num);
va_start(ap, num); vcpu_regs_get(vcpu, ®s);
for (i = 0; i < num; i++) regs.gpr[i + 3] = va_arg(ap, u64);
vcpu_regs_set(vcpu, ®s); va_end(ap);}
void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent){ struct kvm_regs regs;
vcpu_regs_get(vcpu, ®s);
fprintf(stream, "%*sNIA: 0x%016llx MSR: 0x%016llx\n", indent, "", regs.pc, regs.msr); fprintf(stream, "%*sLR: 0x%016llx CTR :0x%016llx\n", indent, "", regs.lr, regs.ctr); fprintf(stream, "%*sCR: 0x%08llx XER :0x%016llx\n", indent, "", regs.cr, regs.xer);}
void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus){ size_t excp_size = __interrupts_end - __interrupts_start; size_t excp_pages = (excp_size + vm->page_size - 1) / vm->page_size; gpa_t excp_paddr; void *mem;
/* * Guest interrupts are taken in real-mode, i.e. the vectors must live * at gPA 0. Use the innermost allocator to override the minimum GPA * that MEM_REGION_DATA would otherwise impose. */ excp_paddr = ____vm_phy_pages_alloc(vm, excp_pages, 0, vm->memslots[MEM_REGION_DATA], vm_arch_has_protected_memory(vm), false);
TEST_ASSERT(excp_paddr == 0, "Interrupt vectors not allocated at gPA address 0: (0x%lx)", excp_paddr);
mem = addr_gpa2hva(vm, excp_paddr); memcpy(mem, __interrupts_start, excp_size);}
void assert_on_unhandled_exception(struct kvm_vcpu *vcpu){ struct ucall uc;
if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { gpa_t ex_regs_paddr; struct ex_regs *ex_regs; struct kvm_regs regs;
vcpu_regs_get(vcpu, ®s); ex_regs_paddr = (gpa_t)regs.gpr[13]; ex_regs = addr_gpa2hva(vcpu->vm, ex_regs_paddr);
TEST_FAIL("Unexpected interrupt in guest NIA:0x%016lx MSR:0x%016lx TRAP:0x%04x", ex_regs->nia, ex_regs->msr, ex_regs->trap); }}
struct handler { void (*fn)(struct ex_regs *regs); int trap;};
#define NR_HANDLERS 10static struct handler handlers[NR_HANDLERS];
void route_interrupt(struct ex_regs *regs){ int i;
for (i = 0; i < NR_HANDLERS; i++) { if (handlers[i].trap == regs->trap) { handlers[i].fn(regs); return; } }
ucall(UCALL_UNHANDLED, 0);}
void vm_install_exception_handler(struct kvm_vm *vm, int trap, void (*fn)(struct ex_regs *)){ int i;
for (i = 0; i < NR_HANDLERS; i++) { if (!handlers[i].trap || handlers[i].trap == trap) { if (fn == NULL) trap = 0; /* Clear handler */ handlers[i].trap = trap; handlers[i].fn = fn; sync_global_to_guest(vm, handlers[i]); return; } }
TEST_FAIL("Out of exception handlers");}
void kvm_selftest_arch_init(void){ TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX));
/* * powerpc default mode is set by host page size and not static, * so start by computing that early. */ guest_modes_append_default();}