Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#include <linux/linkage.h>
4#include <asm/segment.h>
5#include <asm/boot.h>
6#include <asm/msr.h>
7#include <asm/processor-flags.h>
8
9/*
10 * This is the 32-bit trampoline that will be copied over to low memory. It
11 * will be called using the ordinary 64-bit calling convention from code
12 * running in 64-bit mode.
13 *
14 * Return address is at the top of the stack (might be above 4G).
15 * The first argument (EDI) contains the address of the temporary PGD level
16 * page table in 32-bit addressable memory which will be programmed into
17 * register CR3.
18 */
19
20 .section ".rodata", "a", @progbits
21SYM_CODE_START(trampoline_32bit_src)
22 /*
23 * Preserve callee save 64-bit registers on the stack: this is
24 * necessary because the architecture does not guarantee that GPRs will
25 * retain their full 64-bit values across a 32-bit mode switch.
26 */
27 pushq %r15
28 pushq %r14
29 pushq %r13
30 pushq %r12
31 pushq %rbp
32 pushq %rbx
33
34 /* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
35 movq %rsp, %rbx
36 shrq $32, %rbx
37
38 /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
39 pushq $__KERNEL32_CS
40 leaq 0f(%rip), %rax
41 pushq %rax
42 lretq
43
44 /*
45 * The 32-bit code below will do a far jump back to long mode and end
46 * up here after reconfiguring the number of paging levels. First, the
47 * stack pointer needs to be restored to its full 64-bit value before
48 * the callee save register contents can be popped from the stack.
49 */
50.Lret:
51 shlq $32, %rbx
52 orq %rbx, %rsp
53
54 /* Restore the preserved 64-bit registers */
55 popq %rbx
56 popq %rbp
57 popq %r12
58 popq %r13
59 popq %r14
60 popq %r15
61 retq
62
63 .code32
640:
65 /* Disable paging */
66 movl %cr0, %eax
67 btrl $X86_CR0_PG_BIT, %eax
68 movl %eax, %cr0
69
70 /* Point CR3 to the trampoline's new top level page table */
71 movl %edi, %cr3
72
73 /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
74 movl $MSR_EFER, %ecx
75 rdmsr
76 btsl $_EFER_LME, %eax
77 /* Avoid writing EFER if no change was made (for TDX guest) */
78 jc 1f
79 wrmsr
801:
81 /* Toggle CR4.LA57 */
82 movl %cr4, %eax
83 btcl $X86_CR4_LA57_BIT, %eax
84 movl %eax, %cr4
85
86 /* Enable paging again. */
87 movl %cr0, %eax
88 btsl $X86_CR0_PG_BIT, %eax
89 movl %eax, %cr0
90
91 /*
92 * Return to the 64-bit calling code using LJMP rather than LRET, to
93 * avoid the need for a 32-bit addressable stack. The destination
94 * address will be adjusted after the template code is copied into a
95 * 32-bit addressable buffer.
96 */
97.Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampoline_32bit_src)
98SYM_CODE_END(trampoline_32bit_src)
99
100/*
101 * This symbol is placed right after trampoline_32bit_src() so its address can
102 * be used to infer the size of the trampoline code.
103 */
104SYM_DATA(trampoline_ljmp_imm_offset, .word .Ljmp + 1 - trampoline_32bit_src)
105
106 /*
107 * The trampoline code has a size limit.
108 * Make sure we fail to compile if the trampoline code grows
109 * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes.
110 */
111 .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE