tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Asm versions of Xen pv-ops, suitable for direct use.
4 *
5 * We only bother with direct forms (ie, vcpu in percpu data) of the
6 * operations here; the indirect forms are better handled in C.
7 */
8
9#include <asm/errno.h>
10#include <asm/asm-offsets.h>
11#include <asm/percpu.h>
12#include <asm/processor-flags.h>
13#include <asm/segment.h>
14#include <asm/thread_info.h>
15#include <asm/asm.h>
16#include <asm/frame.h>
17#include <asm/unwind_hints.h>
18
19#include <xen/interface/xen.h>
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <linux/objtool.h>
24#include <../entry/calling.h>
25
26.pushsection .noinstr.text, "ax"
27/*
28 * PV hypercall interface to the hypervisor.
29 *
30 * Called via inline asm(), so better preserve %rcx and %r11.
31 *
32 * Input:
33 * %eax: hypercall number
34 * %rdi, %rsi, %rdx, %r10, %r8: args 1..5 for the hypercall
35 * Output: %rax
36 */
37SYM_FUNC_START(xen_hypercall_pv)
38 ANNOTATE_NOENDBR
39 push %rcx
40 push %r11
41 UNWIND_HINT_SAVE
42 syscall
43 UNWIND_HINT_RESTORE
44 pop %r11
45 pop %rcx
46 RET
47SYM_FUNC_END(xen_hypercall_pv)
48
49/*
50 * Disabling events is simply a matter of making the event mask
51 * non-zero.
52 */
53SYM_FUNC_START(xen_irq_disable_direct)
54 movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
55 RET
56SYM_FUNC_END(xen_irq_disable_direct)
57
58/*
59 * Force an event check by making a hypercall, but preserve regs
60 * before making the call.
61 */
62SYM_FUNC_START(check_events)
63 FRAME_BEGIN
64 push %rax
65 push %rcx
66 push %rdx
67 push %rsi
68 push %rdi
69 push %r8
70 push %r9
71 push %r10
72 push %r11
73 call xen_force_evtchn_callback
74 pop %r11
75 pop %r10
76 pop %r9
77 pop %r8
78 pop %rdi
79 pop %rsi
80 pop %rdx
81 pop %rcx
82 pop %rax
83 FRAME_END
84 RET
85SYM_FUNC_END(check_events)
86
87/*
88 * Enable events. This clears the event mask and tests the pending
89 * event status with one and operation. If there are pending events,
90 * then enter the hypervisor to get them handled.
91 */
92SYM_FUNC_START(xen_irq_enable_direct)
93 FRAME_BEGIN
94 /* Unmask events */
95 movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
96
97 /*
98 * Preempt here doesn't matter because that will deal with any
99 * pending interrupts. The pending check may end up being run
100 * on the wrong CPU, but that doesn't hurt.
101 */
102
103 /* Test for pending */
104 testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_pending)
105 jz 1f
106
107 call check_events
1081:
109 FRAME_END
110 RET
111SYM_FUNC_END(xen_irq_enable_direct)
112
113/*
114 * (xen_)save_fl is used to get the current interrupt enable status.
115 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
116 * may be set in the return value. We take advantage of this by
117 * making sure that X86_EFLAGS_IF has the right value (and other bits
118 * in that byte are 0), but other bits in the return value are
119 * undefined. We need to toggle the state of the bit, because Xen and
120 * x86 use opposite senses (mask vs enable).
121 */
122SYM_FUNC_START(xen_save_fl_direct)
123 testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
124 setz %ah
125 addb %ah, %ah
126 RET
127SYM_FUNC_END(xen_save_fl_direct)
128
129SYM_FUNC_START(xen_read_cr2)
130 FRAME_BEGIN
131 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
132 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
133 FRAME_END
134 RET
135SYM_FUNC_END(xen_read_cr2);
136
137SYM_FUNC_START(xen_read_cr2_direct)
138 FRAME_BEGIN
139 _ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX
140 FRAME_END
141 RET
142SYM_FUNC_END(xen_read_cr2_direct);
143.popsection
144
145.macro xen_pv_trap name
146SYM_CODE_START(xen_\name)
147 UNWIND_HINT_ENTRY
148 ENDBR
149 pop %rcx
150 pop %r11
151 jmp \name
152SYM_CODE_END(xen_\name)
153_ASM_NOKPROBE(xen_\name)
154.endm
155
156xen_pv_trap asm_exc_divide_error
157xen_pv_trap asm_xenpv_exc_debug
158xen_pv_trap asm_exc_int3
159xen_pv_trap asm_xenpv_exc_nmi
160xen_pv_trap asm_exc_overflow
161xen_pv_trap asm_exc_bounds
162xen_pv_trap asm_exc_invalid_op
163xen_pv_trap asm_exc_device_not_available
164xen_pv_trap asm_xenpv_exc_double_fault
165xen_pv_trap asm_exc_coproc_segment_overrun
166xen_pv_trap asm_exc_invalid_tss
167xen_pv_trap asm_exc_segment_not_present
168xen_pv_trap asm_exc_stack_segment
169xen_pv_trap asm_exc_general_protection
170xen_pv_trap asm_exc_page_fault
171xen_pv_trap asm_exc_spurious_interrupt_bug
172xen_pv_trap asm_exc_coprocessor_error
173xen_pv_trap asm_exc_alignment_check
174#ifdef CONFIG_X86_CET
175xen_pv_trap asm_exc_control_protection
176#endif
177#ifdef CONFIG_X86_MCE
178xen_pv_trap asm_xenpv_exc_machine_check
179#endif /* CONFIG_X86_MCE */
180xen_pv_trap asm_exc_simd_coprocessor_error
181#ifdef CONFIG_IA32_EMULATION
182xen_pv_trap asm_int80_emulation
183#endif
184xen_pv_trap asm_exc_xen_unknown_trap
185xen_pv_trap asm_exc_xen_hypervisor_callback
186
187 __INIT
188SYM_CODE_START(xen_early_idt_handler_array)
189 i = 0
190 .rept NUM_EXCEPTION_VECTORS
191 UNWIND_HINT_UNDEFINED
192 ENDBR
193 pop %rcx
194 pop %r11
195 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
196 i = i + 1
197 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
198 .endr
199SYM_CODE_END(xen_early_idt_handler_array)
200 __FINIT
201
202/*
203 * Xen64 iret frame:
204 *
205 * ss
206 * rsp
207 * rflags
208 * cs
209 * rip <-- standard iret frame
210 *
211 * flags <-- xen_iret must push from here on
212 *
213 * rcx
214 * r11
215 * rsp->rax
216 */
217.macro xen_hypercall_iret
218 pushq $0 /* Flags */
219 push %rcx
220 push %r11
221 push %rax
222 mov $__HYPERVISOR_iret, %eax
223 syscall /* Do the IRET. */
224#ifdef CONFIG_MITIGATION_SLS
225 int3
226#endif
227.endm
228
229SYM_CODE_START(xen_iret)
230 UNWIND_HINT_UNDEFINED
231 ANNOTATE_NOENDBR
232 xen_hypercall_iret
233SYM_CODE_END(xen_iret)
234
235/*
236 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
237 * also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
238 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
239 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
240 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
241 * frame at the same address is useless.
242 */
243SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
244 UNWIND_HINT_REGS
245 POP_REGS
246
247 /* stackleak_erase() can work safely on the kernel stack. */
248 STACKLEAK_ERASE_NOCLOBBER
249
250 addq $8, %rsp /* skip regs->orig_ax */
251 jmp xen_iret
252SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
253
254/*
255 * Xen handles syscall callbacks much like ordinary exceptions, which
256 * means we have:
257 * - kernel gs
258 * - kernel rsp
259 * - an iret-like stack frame on the stack (including rcx and r11):
260 * ss
261 * rsp
262 * rflags
263 * cs
264 * rip
265 * r11
266 * rsp->rcx
267 */
268
269/* Normal 64-bit system call target */
270SYM_CODE_START(xen_entry_SYSCALL_64)
271 UNWIND_HINT_ENTRY
272 ENDBR
273 popq %rcx
274 popq %r11
275
276 /*
277 * Neither Xen nor the kernel really knows what the old SS and
278 * CS were. The kernel expects __USER_DS and __USER_CS, so
279 * report those values even though Xen will guess its own values.
280 */
281 movq $__USER_DS, 4*8(%rsp)
282 movq $__USER_CS, 1*8(%rsp)
283
284 jmp entry_SYSCALL_64_after_hwframe
285SYM_CODE_END(xen_entry_SYSCALL_64)
286
287#ifdef CONFIG_IA32_EMULATION
288
289/* 32-bit compat syscall target */
290SYM_CODE_START(xen_entry_SYSCALL_compat)
291 UNWIND_HINT_ENTRY
292 ENDBR
293 popq %rcx
294 popq %r11
295
296 /*
297 * Neither Xen nor the kernel really knows what the old SS and
298 * CS were. The kernel expects __USER_DS and __USER32_CS, so
299 * report those values even though Xen will guess its own values.
300 */
301 movq $__USER_DS, 4*8(%rsp)
302 movq $__USER32_CS, 1*8(%rsp)
303
304 jmp entry_SYSCALL_compat_after_hwframe
305SYM_CODE_END(xen_entry_SYSCALL_compat)
306
307/* 32-bit compat sysenter target */
308SYM_CODE_START(xen_entry_SYSENTER_compat)
309 UNWIND_HINT_ENTRY
310 ENDBR
311 /*
312 * NB: Xen is polite and clears TF from EFLAGS for us. This means
313 * that we don't need to guard against single step exceptions here.
314 */
315 popq %rcx
316 popq %r11
317
318 /*
319 * Neither Xen nor the kernel really knows what the old SS and
320 * CS were. The kernel expects __USER_DS and __USER32_CS, so
321 * report those values even though Xen will guess its own values.
322 */
323 movq $__USER_DS, 4*8(%rsp)
324 movq $__USER32_CS, 1*8(%rsp)
325
326 jmp entry_SYSENTER_compat_after_hwframe
327SYM_CODE_END(xen_entry_SYSENTER_compat)
328
329#else /* !CONFIG_IA32_EMULATION */
330
331SYM_CODE_START(xen_entry_SYSCALL_compat)
332SYM_CODE_START(xen_entry_SYSENTER_compat)
333 UNWIND_HINT_ENTRY
334 ENDBR
335 lea 16(%rsp), %rsp /* strip %rcx, %r11 */
336 mov $-ENOSYS, %rax
337 xen_hypercall_iret
338SYM_CODE_END(xen_entry_SYSENTER_compat)
339SYM_CODE_END(xen_entry_SYSCALL_compat)
340
341#endif /* CONFIG_IA32_EMULATION */