Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>#include <linux/linkage.h>#include <linux/types.h>#include <linux/kernel.h>#include <linux/pgtable.h>
#include <asm/init.h>#include <asm/sections.h>#include <asm/setup.h>#include <asm/sev.h>
extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];extern unsigned int next_early_pgt;
static inline bool check_la57_support(void){ /* * 5-level paging is detected and enabled at kernel decompression * stage. Only check if it has been enabled there. */ if (!(native_read_cr4() & X86_CR4_LA57)) return false;
__pgtable_l5_enabled = 1; pgdir_shift = 48; ptrs_per_p4d = 512;
return true;}
static unsigned long __init sme_postprocess_startup(struct boot_params *bp, pmdval_t *pmd, unsigned long p2v_offset){ unsigned long paddr, paddr_end; int i;
/* Encrypt the kernel and related (if SME is active) */ sme_encrypt_kernel(bp);
/* * Clear the memory encryption mask from the .bss..decrypted section. * The bss section will be memset to zero later in the initialization so * there is no need to zero it after changing the memory encryption * attribute. */ if (sme_get_me_mask()) { paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted); paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
for (; paddr < paddr_end; paddr += PMD_SIZE) { /* * On SNP, transition the page to shared in the RMP table so that * it is consistent with the page table attribute change. * * __start_bss_decrypted has a virtual address in the high range * mapping (kernel .text). PVALIDATE, by way of * early_snp_set_memory_shared(), requires a valid virtual * address but the kernel is currently running off of the identity * mapping so use the PA to get a *currently* valid virtual address. */ early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
i = pmd_index(paddr - p2v_offset); pmd[i] -= sme_get_me_mask(); } }
/* * Return the SME encryption mask (if SME is active) to be used as a * modifier for the initial pgdir entry programmed into CR3. */ return sme_get_me_mask();}
/* * This code is compiled using PIC codegen because it will execute from the * early 1:1 mapping of memory, which deviates from the mapping expected by the * linker. Due to this deviation, taking the address of a global variable will * produce an ambiguous result when using the plain & operator. Instead, * rip_rel_ptr() must be used, which will return the RIP-relative address in * the 1:1 mapping of memory. Kernel virtual addresses can be determined by * subtracting p2v_offset from the RIP-relative address. */unsigned long __init __startup_64(unsigned long p2v_offset, struct boot_params *bp){ pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts); unsigned long physaddr = (unsigned long)rip_rel_ptr(_text); unsigned long va_text, va_end; unsigned long pgtable_flags; unsigned long load_delta; pgdval_t *pgd; p4dval_t *p4d; pudval_t *pud; pmdval_t *pmd, pmd_entry; bool la57; int i;
la57 = check_la57_support();
/* Is the address too large? */ if (physaddr >> MAX_PHYSMEM_BITS) for (;;);
/* * Compute the delta between the address I am compiled to run at * and the address I am actually running at. */ phys_base = load_delta = __START_KERNEL_map + p2v_offset;
/* Is the address not 2M aligned? */ if (load_delta & ~PMD_MASK) for (;;);
va_text = physaddr - p2v_offset; va_end = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
/* Include the SME encryption mask in the fixup value */ load_delta += sme_get_me_mask();
/* Fixup the physical addresses in the page table */
pgd = rip_rel_ptr(early_top_pgt); pgd[pgd_index(__START_KERNEL_map)] += load_delta;
if (la57) { p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; }
level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta; level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) level2_fixmap_pgt[i].pmd += load_delta;
/* * Set up the identity mapping for the switchover. These * entries should *NOT* have the global bit set! This also * creates a bunch of nonsense entries but that is fine -- * it avoids problems around wraparound. */
pud = &early_pgts[0]->pmd; pmd = &early_pgts[1]->pmd; next_early_pgt = 2;
pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
if (la57) { p4d = &early_pgts[next_early_pgt++]->pmd;
i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
i = physaddr >> P4D_SHIFT; p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; } else { i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; pgd[i + 0] = (pgdval_t)pud + pgtable_flags; pgd[i + 1] = (pgdval_t)pud + pgtable_flags; }
i = physaddr >> PUD_SHIFT; pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags; pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; pmd_entry += sme_get_me_mask(); pmd_entry += physaddr;
for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) { int idx = i + (physaddr >> PMD_SHIFT);
pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE; }
/* * Fixup the kernel text+data virtual addresses. Note that * we might write invalid pmds, when the kernel is relocated * cleanup_highmap() fixes this up along with the mappings * beyond _end. * * Only the region occupied by the kernel image has so far * been checked against the table of usable memory regions * provided by the firmware, so invalidate pages outside that * region. A page table entry that maps to a reserved area of * memory would allow processor speculation into that area, * and on some hardware (particularly the UV platform) even * speculative access to some reserved areas is caught as an * error, causing the BIOS to halt the system. */
pmd = rip_rel_ptr(level2_kernel_pgt);
/* invalidate pages before the kernel image */ for (i = 0; i < pmd_index(va_text); i++) pmd[i] &= ~_PAGE_PRESENT;
/* fixup pages that are part of the kernel image */ for (; i <= pmd_index(va_end); i++) if (pmd[i] & _PAGE_PRESENT) pmd[i] += load_delta;
/* invalidate pages after the kernel image */ for (; i < PTRS_PER_PMD; i++) pmd[i] &= ~_PAGE_PRESENT;
return sme_postprocess_startup(bp, pmd, p2v_offset);}