jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice, this
9 * list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <AK/Assertions.h>
28#include <AK/String.h>
29#include <AK/Types.h>
30#include <Kernel/Arch/i386/CPU.h>
31#include <Kernel/Arch/i386/ISRStubs.h>
32#include <Kernel/Interrupts/APIC.h>
33#include <Kernel/Interrupts/GenericInterruptHandler.h>
34#include <Kernel/Interrupts/IRQHandler.h>
35#include <Kernel/Interrupts/InterruptManagement.h>
36#include <Kernel/Interrupts/SharedIRQHandler.h>
37#include <Kernel/Interrupts/SpuriousInterruptHandler.h>
38#include <Kernel/Interrupts/UnhandledInterruptHandler.h>
39#include <Kernel/KSyms.h>
40#include <Kernel/Process.h>
41#include <Kernel/VM/MemoryManager.h>
42#include <LibBareMetal/IO.h>
43#include <LibC/mallocdefs.h>
44
45//#define PAGE_FAULT_DEBUG
46
47namespace Kernel {
48
49struct [[gnu::packed]] DescriptorTablePointer
50{
51 u16 limit;
52 void* address;
53};
54
55static DescriptorTablePointer s_idtr;
56static DescriptorTablePointer s_gdtr;
57static Descriptor s_idt[256];
58static Descriptor s_gdt[256];
59
60static GenericInterruptHandler* s_interrupt_handler[GENERIC_INTERRUPT_HANDLERS_COUNT];
61
62static Vector<u16>* s_gdt_freelist;
63
64static u16 s_gdt_length;
65
66u16 gdt_alloc_entry()
67{
68 ASSERT(s_gdt_freelist);
69 ASSERT(!s_gdt_freelist->is_empty());
70 return s_gdt_freelist->take_last();
71}
72
73void gdt_free_entry(u16 entry)
74{
75 s_gdt_freelist->append(entry);
76}
77
78extern "C" void handle_interrupt(RegisterState);
79
80#define EH_ENTRY(ec, title) \
81 extern "C" void title##_asm_entry(); \
82 extern "C" void title##_handler(RegisterState); \
83 asm( \
84 ".globl " #title "_asm_entry\n" \
85 "" #title "_asm_entry: \n" \
86 " pusha\n" \
87 " pushl %ds\n" \
88 " pushl %es\n" \
89 " pushl %fs\n" \
90 " pushl %gs\n" \
91 " pushl %ss\n" \
92 " mov $0x10, %ax\n" \
93 " mov %ax, %ds\n" \
94 " mov %ax, %es\n" \
95 " cld\n" \
96 " call " #title "_handler\n" \
97 " add $0x4, %esp \n" \
98 " popl %gs\n" \
99 " popl %fs\n" \
100 " popl %es\n" \
101 " popl %ds\n" \
102 " popa\n" \
103 " add $0x4, %esp\n" \
104 " iret\n");
105
106#define EH_ENTRY_NO_CODE(ec, title) \
107 extern "C" void title##_handler(RegisterState); \
108 extern "C" void title##_asm_entry(); \
109 asm( \
110 ".globl " #title "_asm_entry\n" \
111 "" #title "_asm_entry: \n" \
112 " pushl $0x0\n" \
113 " pusha\n" \
114 " pushl %ds\n" \
115 " pushl %es\n" \
116 " pushl %fs\n" \
117 " pushl %gs\n" \
118 " pushl %ss\n" \
119 " mov $0x10, %ax\n" \
120 " mov %ax, %ds\n" \
121 " mov %ax, %es\n" \
122 " cld\n" \
123 " call " #title "_handler\n" \
124 " add $0x4, %esp\n" \
125 " popl %gs\n" \
126 " popl %fs\n" \
127 " popl %es\n" \
128 " popl %ds\n" \
129 " popa\n" \
130 " add $0x4, %esp\n" \
131 " iret\n");
132
133static void dump(const RegisterState& regs)
134{
135 u16 ss;
136 u32 esp;
137 if (!Process::current || Process::current->is_ring0()) {
138 ss = regs.ss;
139 esp = regs.esp;
140 } else {
141 ss = regs.userspace_ss;
142 esp = regs.userspace_esp;
143 }
144
145 klog() << "exception code: " << String::format("%04x", regs.exception_code) << " (isr: " << String::format("%04x", regs.isr_number);
146 klog() << " pc=" << String::format("%04x", (u16)regs.cs) << ":" << String::format("%08x", regs.eip) << " flags=" << String::format("%04x", (u16)regs.eflags);
147 klog() << " stk=" << String::format("%04x", ss) << ":" << String::format("%08x", esp);
148 klog() << " ds=" << String::format("%04x", (u16)regs.ds) << " es=" << String::format("%04x", (u16)regs.es) << " fs=" << String::format("%04x", (u16)regs.fs) << " gs=" << String::format("%04x", (u16)regs.gs);
149 klog() << "eax=" << String::format("%08x", regs.eax) << " ebx=" << String::format("%08x", regs.ebx) << " ecx=" << String::format("%08x", regs.ecx) << " edx=" << String::format("%08x", regs.edx);
150 klog() << "ebp=" << String::format("%08x", regs.ebp) << " esp=" << String::format("%08x", regs.esp) << " esi=" << String::format("%08x", regs.esi) << " edi=" << String::format("%08x", regs.edi);
151 u32 cr0;
152 asm("movl %%cr0, %%eax"
153 : "=a"(cr0));
154 u32 cr2;
155 asm("movl %%cr2, %%eax"
156 : "=a"(cr2));
157 u32 cr3 = read_cr3();
158 u32 cr4;
159 asm("movl %%cr4, %%eax"
160 : "=a"(cr4));
161 klog() << "cr0=" << String::format("%08x", cr0) << " cr2=" << String::format("%08x", cr2) << " cr3=" << String::format("%08x", cr3) << " cr4=" << String::format("%08x", cr4);
162
163 if (Process::current && Process::current->validate_read((void*)regs.eip, 8)) {
164 SmapDisabler disabler;
165 u8* codeptr = (u8*)regs.eip;
166 klog() << "code: " << String::format("%02x", codeptr[0]) << " " << String::format("%02x", codeptr[1]) << " " << String::format("%02x", codeptr[2]) << " " << String::format("%02x", codeptr[3]) << " " << String::format("%02x", codeptr[4]) << " " << String::format("%02x", codeptr[5]) << " " << String::format("%02x", codeptr[6]) << " " << String::format("%02x", codeptr[7]);
167 }
168}
169
170void handle_crash(RegisterState& regs, const char* description, int signal)
171{
172 if (!Process::current) {
173 klog() << description << " with !current";
174 hang();
175 }
176
177 // If a process crashed while inspecting another process,
178 // make sure we switch back to the right page tables.
179 MM.enter_process_paging_scope(*Process::current);
180
181 klog() << "CRASH: " << description << ". Ring " << (Process::current->is_ring0() ? 0 : 3) << ".";
182 dump(regs);
183
184 if (Process::current->is_ring0()) {
185 klog() << "Oh shit, we've crashed in ring 0 :(";
186 dump_backtrace();
187 hang();
188 }
189
190 cli();
191 Process::current->crash(signal, regs.eip);
192}
193
194EH_ENTRY_NO_CODE(6, illegal_instruction);
195void illegal_instruction_handler(RegisterState regs)
196{
197 clac();
198 handle_crash(regs, "Illegal instruction", SIGILL);
199}
200
201EH_ENTRY_NO_CODE(0, divide_error);
202void divide_error_handler(RegisterState regs)
203{
204 clac();
205 handle_crash(regs, "Divide error", SIGFPE);
206}
207
208EH_ENTRY(13, general_protection_fault);
209void general_protection_fault_handler(RegisterState regs)
210{
211 clac();
212 handle_crash(regs, "General protection fault", SIGSEGV);
213}
214
215// 7: FPU not available exception
216EH_ENTRY_NO_CODE(7, fpu_exception);
217void fpu_exception_handler(RegisterState)
218{
219 // Just clear the TS flag. We've already restored the FPU state eagerly.
220 // FIXME: It would be nice if we didn't have to do this at all.
221 asm volatile("clts");
222}
223
224// 14: Page Fault
225EH_ENTRY(14, page_fault);
226void page_fault_handler(RegisterState regs)
227{
228 clac();
229
230 u32 fault_address;
231 asm("movl %%cr2, %%eax"
232 : "=a"(fault_address));
233
234#ifdef PAGE_FAULT_DEBUG
235 u32 fault_page_directory = read_cr3();
236 dbg() << "Ring " << (regs.cs & 3)
237 << " " << (regs.exception_code & 1 ? "PV" : "NP")
238 << " page fault in PD=" << String::format("%x", fault_page_directory) << ", "
239 << (regs.exception_code & 8 ? "reserved-bit " : "")
240 << (regs.exception_code & 2 ? "write" : "read")
241 << " " << VirtualAddress(fault_address);
242#endif
243
244#ifdef PAGE_FAULT_DEBUG
245 dump(regs);
246#endif
247
248 bool faulted_in_userspace = (regs.cs & 3) == 3;
249 if (faulted_in_userspace && !MM.validate_user_stack(*Process::current, VirtualAddress(regs.userspace_esp))) {
250 dbg() << "Invalid stack pointer: " << VirtualAddress(regs.userspace_esp);
251 handle_crash(regs, "Bad stack on page fault", SIGSTKFLT);
252 ASSERT_NOT_REACHED();
253 }
254
255 auto response = MM.handle_page_fault(PageFault(regs.exception_code, VirtualAddress(fault_address)));
256
257 if (response == PageFaultResponse::ShouldCrash) {
258 if (Thread::current->has_signal_handler(SIGSEGV)) {
259 Thread::current->send_urgent_signal_to_self(SIGSEGV);
260 return;
261 }
262
263 dbg() << "Unrecoverable page fault, "
264 << (regs.exception_code & PageFaultFlags::ReservedBitViolation ? "reserved bit violation / " : "")
265 << (regs.exception_code & PageFaultFlags::InstructionFetch ? "instruction fetch / " : "")
266 << (regs.exception_code & PageFaultFlags::Write ? "write to" : "read from")
267 << " address " << VirtualAddress(fault_address);
268 u32 malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE);
269 u32 free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE);
270 u32 kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE);
271 u32 kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE);
272 u32 slab_alloc_scrub_pattern = explode_byte(SLAB_ALLOC_SCRUB_BYTE);
273 u32 slab_dealloc_scrub_pattern = explode_byte(SLAB_DEALLOC_SCRUB_BYTE);
274 if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) {
275 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be uninitialized malloc() memory";
276 } else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) {
277 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be recently free()'d memory";
278 } else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) {
279 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be uninitialized kmalloc() memory";
280 } else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) {
281 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be recently kfree()'d memory";
282 } else if ((fault_address & 0xffff0000) == (slab_alloc_scrub_pattern & 0xffff0000)) {
283 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be uninitialized slab_alloc() memory";
284 } else if ((fault_address & 0xffff0000) == (slab_dealloc_scrub_pattern & 0xffff0000)) {
285 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like it may be recently slab_dealloc()'d memory";
286 } else if (fault_address < 4096) {
287 dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like a possible nullptr dereference";
288 }
289
290 handle_crash(regs, "Page Fault", SIGSEGV);
291 } else if (response == PageFaultResponse::Continue) {
292#ifdef PAGE_FAULT_DEBUG
293 dbg() << "Continuing after resolved page fault";
294#endif
295 } else {
296 ASSERT_NOT_REACHED();
297 }
298}
299
300#define EH(i, msg) \
301 static void _exception##i() \
302 { \
303 klog() << msg; \
304 u32 cr0, cr2, cr3, cr4; \
305 asm("movl %%cr0, %%eax" \
306 : "=a"(cr0)); \
307 asm("movl %%cr2, %%eax" \
308 : "=a"(cr2)); \
309 asm("movl %%cr3, %%eax" \
310 : "=a"(cr3)); \
311 asm("movl %%cr4, %%eax" \
312 : "=a"(cr4)); \
313 klog() << "CR0=" << String::format("%x", cr0) << " CR2=" << String::format("%x", cr2) << " CR3=" << String::format("%x", cr3) << " CR4=" << String::format("%x", cr4); \
314 hang(); \
315 }
316
317EH(1, "Debug exception")
318EH(2, "Unknown error")
319EH(3, "Breakpoint")
320EH(4, "Overflow")
321EH(5, "Bounds check")
322EH(8, "Double fault")
323EH(9, "Coprocessor segment overrun")
324EH(10, "Invalid TSS")
325EH(11, "Segment not present")
326EH(12, "Stack exception")
327EH(15, "Unknown error")
328EH(16, "Coprocessor error")
329
330static void write_raw_gdt_entry(u16 selector, u32 low, u32 high)
331{
332 u16 i = (selector & 0xfffc) >> 3;
333 s_gdt[i].low = low;
334 s_gdt[i].high = high;
335
336 if (i > s_gdt_length)
337 s_gdtr.limit = (s_gdt_length + 1) * 8 - 1;
338}
339
340void write_gdt_entry(u16 selector, Descriptor& descriptor)
341{
342 write_raw_gdt_entry(selector, descriptor.low, descriptor.high);
343}
344
345Descriptor& get_gdt_entry(u16 selector)
346{
347 u16 i = (selector & 0xfffc) >> 3;
348 return *(Descriptor*)(&s_gdt[i]);
349}
350
351void flush_gdt()
352{
353 s_gdtr.address = s_gdt;
354 s_gdtr.limit = (s_gdt_length * 8) - 1;
355 asm("lgdt %0" ::"m"(s_gdtr)
356 : "memory");
357}
358
359void gdt_init()
360{
361 s_gdt_length = 5;
362
363 s_gdt_freelist = new Vector<u16>();
364 s_gdt_freelist->ensure_capacity(256);
365 for (size_t i = s_gdt_length; i < 256; ++i)
366 s_gdt_freelist->append(i * 8);
367
368 s_gdt_length = 256;
369 s_gdtr.address = s_gdt;
370 s_gdtr.limit = (s_gdt_length * 8) - 1;
371
372 write_raw_gdt_entry(0x0000, 0x00000000, 0x00000000);
373 write_raw_gdt_entry(0x0008, 0x0000ffff, 0x00cf9a00);
374 write_raw_gdt_entry(0x0010, 0x0000ffff, 0x00cf9200);
375 write_raw_gdt_entry(0x0018, 0x0000ffff, 0x00cffa00);
376 write_raw_gdt_entry(0x0020, 0x0000ffff, 0x00cff200);
377
378 flush_gdt();
379
380 asm volatile(
381 "mov %%ax, %%ds\n"
382 "mov %%ax, %%es\n"
383 "mov %%ax, %%fs\n"
384 "mov %%ax, %%gs\n"
385 "mov %%ax, %%ss\n" ::"a"(0x10)
386 : "memory");
387
388 // Make sure CS points to the kernel code descriptor.
389 asm volatile(
390 "ljmpl $0x8, $sanity\n"
391 "sanity:\n");
392}
393
394static void unimp_trap()
395{
396 klog() << "Unhandled IRQ.";
397 hang();
398}
399
400GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number)
401{
402 ASSERT(s_interrupt_handler[interrupt_number] != nullptr);
403 return *s_interrupt_handler[interrupt_number];
404}
405
406static void revert_to_unused_handler(u8 interrupt_number)
407{
408 new UnhandledInterruptHandler(interrupt_number);
409}
410
411void register_generic_interrupt_handler(u8 interrupt_number, GenericInterruptHandler& handler)
412{
413 if (s_interrupt_handler[interrupt_number] != nullptr) {
414 if (s_interrupt_handler[interrupt_number]->type() == HandlerType::UnhandledInterruptHandler) {
415 s_interrupt_handler[interrupt_number] = &handler;
416 return;
417 }
418 if (s_interrupt_handler[interrupt_number]->is_shared_handler() && !s_interrupt_handler[interrupt_number]->is_sharing_with_others()) {
419 ASSERT(s_interrupt_handler[interrupt_number]->type() == HandlerType::SharedIRQHandler);
420 static_cast<SharedIRQHandler*>(s_interrupt_handler[interrupt_number])->register_handler(handler);
421 return;
422 }
423 if (!s_interrupt_handler[interrupt_number]->is_shared_handler()) {
424 ASSERT(s_interrupt_handler[interrupt_number]->type() == HandlerType::IRQHandler);
425 auto& previous_handler = *s_interrupt_handler[interrupt_number];
426 s_interrupt_handler[interrupt_number] = nullptr;
427 SharedIRQHandler::initialize(interrupt_number);
428 static_cast<SharedIRQHandler*>(s_interrupt_handler[interrupt_number])->register_handler(previous_handler);
429 static_cast<SharedIRQHandler*>(s_interrupt_handler[interrupt_number])->register_handler(handler);
430 return;
431 }
432 ASSERT_NOT_REACHED();
433 } else {
434 s_interrupt_handler[interrupt_number] = &handler;
435 }
436}
437
438void unregister_generic_interrupt_handler(u8 interrupt_number, GenericInterruptHandler& handler)
439{
440 ASSERT(s_interrupt_handler[interrupt_number] != nullptr);
441 if (s_interrupt_handler[interrupt_number]->type() == HandlerType::UnhandledInterruptHandler) {
442 dbg() << "Trying to unregister unused handler (?)";
443 return;
444 }
445 if (s_interrupt_handler[interrupt_number]->is_shared_handler() && !s_interrupt_handler[interrupt_number]->is_sharing_with_others()) {
446 ASSERT(s_interrupt_handler[interrupt_number]->type() == HandlerType::SharedIRQHandler);
447 static_cast<SharedIRQHandler*>(s_interrupt_handler[interrupt_number])->unregister_handler(handler);
448 if (!static_cast<SharedIRQHandler*>(s_interrupt_handler[interrupt_number])->sharing_devices_count()) {
449 revert_to_unused_handler(interrupt_number);
450 }
451 return;
452 }
453 if (!s_interrupt_handler[interrupt_number]->is_shared_handler()) {
454 ASSERT(s_interrupt_handler[interrupt_number]->type() == HandlerType::IRQHandler);
455 revert_to_unused_handler(interrupt_number);
456 return;
457 }
458 ASSERT_NOT_REACHED();
459}
460
461void register_interrupt_handler(u8 index, void (*f)())
462{
463 s_idt[index].low = 0x00080000 | LSW((f));
464 s_idt[index].high = ((u32)(f)&0xffff0000) | 0x8e00;
465}
466
467void register_user_callable_interrupt_handler(u8 index, void (*f)())
468{
469 s_idt[index].low = 0x00080000 | LSW((f));
470 s_idt[index].high = ((u32)(f)&0xffff0000) | 0xef00;
471}
472
473void flush_idt()
474{
475 asm("lidt %0" ::"m"(s_idtr));
476}
477
478void idt_init()
479{
480 s_idtr.address = s_idt;
481 s_idtr.limit = 0x100 * 8 - 1;
482
483 for (u8 i = 0xff; i > 0x10; --i)
484 register_interrupt_handler(i, unimp_trap);
485
486 register_interrupt_handler(0x00, divide_error_asm_entry);
487 register_interrupt_handler(0x01, _exception1);
488 register_interrupt_handler(0x02, _exception2);
489 register_interrupt_handler(0x03, _exception3);
490 register_interrupt_handler(0x04, _exception4);
491 register_interrupt_handler(0x05, _exception5);
492 register_interrupt_handler(0x06, illegal_instruction_asm_entry);
493 register_interrupt_handler(0x07, fpu_exception_asm_entry);
494 register_interrupt_handler(0x08, _exception8);
495 register_interrupt_handler(0x09, _exception9);
496 register_interrupt_handler(0x0a, _exception10);
497 register_interrupt_handler(0x0b, _exception11);
498 register_interrupt_handler(0x0c, _exception12);
499 register_interrupt_handler(0x0d, general_protection_fault_asm_entry);
500 register_interrupt_handler(0x0e, page_fault_asm_entry);
501 register_interrupt_handler(0x0f, _exception15);
502 register_interrupt_handler(0x10, _exception16);
503
504 register_interrupt_handler(0x50, interrupt_0_asm_entry);
505 register_interrupt_handler(0x51, interrupt_1_asm_entry);
506 register_interrupt_handler(0x52, interrupt_2_asm_entry);
507 register_interrupt_handler(0x53, interrupt_3_asm_entry);
508 register_interrupt_handler(0x54, interrupt_4_asm_entry);
509 register_interrupt_handler(0x55, interrupt_5_asm_entry);
510 register_interrupt_handler(0x56, interrupt_6_asm_entry);
511 register_interrupt_handler(0x57, interrupt_7_asm_entry);
512 register_interrupt_handler(0x58, interrupt_8_asm_entry);
513 register_interrupt_handler(0x59, interrupt_9_asm_entry);
514 register_interrupt_handler(0x5a, interrupt_10_asm_entry);
515 register_interrupt_handler(0x5b, interrupt_11_asm_entry);
516 register_interrupt_handler(0x5c, interrupt_12_asm_entry);
517 register_interrupt_handler(0x5d, interrupt_13_asm_entry);
518 register_interrupt_handler(0x5e, interrupt_14_asm_entry);
519 register_interrupt_handler(0x5f, interrupt_15_asm_entry);
520 register_interrupt_handler(0x60, interrupt_16_asm_entry);
521 register_interrupt_handler(0x61, interrupt_17_asm_entry);
522 register_interrupt_handler(0x62, interrupt_18_asm_entry);
523 register_interrupt_handler(0x63, interrupt_19_asm_entry);
524 register_interrupt_handler(0x64, interrupt_20_asm_entry);
525 register_interrupt_handler(0x65, interrupt_21_asm_entry);
526 register_interrupt_handler(0x66, interrupt_22_asm_entry);
527 register_interrupt_handler(0x67, interrupt_23_asm_entry);
528 register_interrupt_handler(0x68, interrupt_24_asm_entry);
529 register_interrupt_handler(0x69, interrupt_25_asm_entry);
530 register_interrupt_handler(0x6a, interrupt_26_asm_entry);
531 register_interrupt_handler(0x6b, interrupt_27_asm_entry);
532 register_interrupt_handler(0x6c, interrupt_28_asm_entry);
533 register_interrupt_handler(0x6d, interrupt_29_asm_entry);
534 register_interrupt_handler(0x6e, interrupt_30_asm_entry);
535 register_interrupt_handler(0x6f, interrupt_31_asm_entry);
536 register_interrupt_handler(0x70, interrupt_32_asm_entry);
537 register_interrupt_handler(0x71, interrupt_33_asm_entry);
538 register_interrupt_handler(0x72, interrupt_34_asm_entry);
539 register_interrupt_handler(0x73, interrupt_35_asm_entry);
540 register_interrupt_handler(0x74, interrupt_36_asm_entry);
541 register_interrupt_handler(0x75, interrupt_37_asm_entry);
542 register_interrupt_handler(0x76, interrupt_38_asm_entry);
543 register_interrupt_handler(0x77, interrupt_39_asm_entry);
544 register_interrupt_handler(0x78, interrupt_40_asm_entry);
545 register_interrupt_handler(0x79, interrupt_41_asm_entry);
546 register_interrupt_handler(0x7a, interrupt_42_asm_entry);
547 register_interrupt_handler(0x7b, interrupt_43_asm_entry);
548 register_interrupt_handler(0x7c, interrupt_44_asm_entry);
549 register_interrupt_handler(0x7d, interrupt_45_asm_entry);
550 register_interrupt_handler(0x7e, interrupt_46_asm_entry);
551 register_interrupt_handler(0x7f, interrupt_47_asm_entry);
552 register_interrupt_handler(0x80, interrupt_48_asm_entry);
553 register_interrupt_handler(0x81, interrupt_49_asm_entry);
554 register_interrupt_handler(0x82, interrupt_50_asm_entry);
555 register_interrupt_handler(0x83, interrupt_51_asm_entry);
556 register_interrupt_handler(0x84, interrupt_52_asm_entry);
557 register_interrupt_handler(0x85, interrupt_53_asm_entry);
558 register_interrupt_handler(0x86, interrupt_54_asm_entry);
559 register_interrupt_handler(0x87, interrupt_55_asm_entry);
560 register_interrupt_handler(0x88, interrupt_56_asm_entry);
561 register_interrupt_handler(0x89, interrupt_57_asm_entry);
562 register_interrupt_handler(0x8a, interrupt_58_asm_entry);
563 register_interrupt_handler(0x8b, interrupt_59_asm_entry);
564 register_interrupt_handler(0x8c, interrupt_60_asm_entry);
565 register_interrupt_handler(0x8d, interrupt_61_asm_entry);
566 register_interrupt_handler(0x8e, interrupt_62_asm_entry);
567 register_interrupt_handler(0x8f, interrupt_63_asm_entry);
568 register_interrupt_handler(0x90, interrupt_64_asm_entry);
569 register_interrupt_handler(0x91, interrupt_65_asm_entry);
570 register_interrupt_handler(0x92, interrupt_66_asm_entry);
571 register_interrupt_handler(0x93, interrupt_67_asm_entry);
572 register_interrupt_handler(0x94, interrupt_68_asm_entry);
573 register_interrupt_handler(0x95, interrupt_69_asm_entry);
574 register_interrupt_handler(0x96, interrupt_70_asm_entry);
575 register_interrupt_handler(0x97, interrupt_71_asm_entry);
576 register_interrupt_handler(0x98, interrupt_72_asm_entry);
577 register_interrupt_handler(0x99, interrupt_73_asm_entry);
578 register_interrupt_handler(0x9a, interrupt_74_asm_entry);
579 register_interrupt_handler(0x9b, interrupt_75_asm_entry);
580 register_interrupt_handler(0x9c, interrupt_76_asm_entry);
581 register_interrupt_handler(0x9d, interrupt_77_asm_entry);
582 register_interrupt_handler(0x9e, interrupt_78_asm_entry);
583 register_interrupt_handler(0x9f, interrupt_79_asm_entry);
584 register_interrupt_handler(0xa0, interrupt_80_asm_entry);
585 register_interrupt_handler(0xa1, interrupt_81_asm_entry);
586 register_interrupt_handler(0xa2, interrupt_82_asm_entry);
587 register_interrupt_handler(0xa3, interrupt_83_asm_entry);
588 register_interrupt_handler(0xa4, interrupt_84_asm_entry);
589 register_interrupt_handler(0xa5, interrupt_85_asm_entry);
590 register_interrupt_handler(0xa6, interrupt_86_asm_entry);
591 register_interrupt_handler(0xa7, interrupt_87_asm_entry);
592 register_interrupt_handler(0xa8, interrupt_88_asm_entry);
593 register_interrupt_handler(0xa9, interrupt_89_asm_entry);
594 register_interrupt_handler(0xaa, interrupt_90_asm_entry);
595 register_interrupt_handler(0xab, interrupt_91_asm_entry);
596 register_interrupt_handler(0xac, interrupt_92_asm_entry);
597 register_interrupt_handler(0xad, interrupt_93_asm_entry);
598 register_interrupt_handler(0xae, interrupt_94_asm_entry);
599 register_interrupt_handler(0xaf, interrupt_95_asm_entry);
600 register_interrupt_handler(0xb0, interrupt_96_asm_entry);
601 register_interrupt_handler(0xb1, interrupt_97_asm_entry);
602 register_interrupt_handler(0xb2, interrupt_98_asm_entry);
603 register_interrupt_handler(0xb3, interrupt_99_asm_entry);
604 register_interrupt_handler(0xb4, interrupt_100_asm_entry);
605 register_interrupt_handler(0xb5, interrupt_101_asm_entry);
606 register_interrupt_handler(0xb6, interrupt_102_asm_entry);
607 register_interrupt_handler(0xb7, interrupt_103_asm_entry);
608 register_interrupt_handler(0xb8, interrupt_104_asm_entry);
609 register_interrupt_handler(0xb9, interrupt_105_asm_entry);
610 register_interrupt_handler(0xba, interrupt_106_asm_entry);
611 register_interrupt_handler(0xbb, interrupt_107_asm_entry);
612 register_interrupt_handler(0xbc, interrupt_108_asm_entry);
613 register_interrupt_handler(0xbd, interrupt_109_asm_entry);
614 register_interrupt_handler(0xbe, interrupt_110_asm_entry);
615 register_interrupt_handler(0xbf, interrupt_111_asm_entry);
616 register_interrupt_handler(0xc0, interrupt_112_asm_entry);
617 register_interrupt_handler(0xc1, interrupt_113_asm_entry);
618 register_interrupt_handler(0xc2, interrupt_114_asm_entry);
619 register_interrupt_handler(0xc3, interrupt_115_asm_entry);
620 register_interrupt_handler(0xc4, interrupt_116_asm_entry);
621 register_interrupt_handler(0xc5, interrupt_117_asm_entry);
622 register_interrupt_handler(0xc6, interrupt_118_asm_entry);
623 register_interrupt_handler(0xc7, interrupt_119_asm_entry);
624 register_interrupt_handler(0xc8, interrupt_120_asm_entry);
625 register_interrupt_handler(0xc9, interrupt_121_asm_entry);
626 register_interrupt_handler(0xca, interrupt_122_asm_entry);
627 register_interrupt_handler(0xcb, interrupt_123_asm_entry);
628 register_interrupt_handler(0xcc, interrupt_124_asm_entry);
629 register_interrupt_handler(0xcd, interrupt_125_asm_entry);
630 register_interrupt_handler(0xce, interrupt_126_asm_entry);
631 register_interrupt_handler(0xcf, interrupt_127_asm_entry);
632
633 dbg() << "Installing Unhandled Handlers";
634
635 for (u8 i = 0; i < GENERIC_INTERRUPT_HANDLERS_COUNT; ++i) {
636 new UnhandledInterruptHandler(i);
637 }
638
639 flush_idt();
640}
641
642void load_task_register(u16 selector)
643{
644 asm("ltr %0" ::"r"(selector));
645}
646
647u32 g_in_irq;
648
649void handle_interrupt(RegisterState regs)
650{
651 clac();
652 ++g_in_irq;
653 ASSERT(regs.isr_number >= IRQ_VECTOR_BASE && regs.isr_number <= (IRQ_VECTOR_BASE + GENERIC_INTERRUPT_HANDLERS_COUNT));
654 u8 irq = (u8)(regs.isr_number - 0x50);
655 ASSERT(s_interrupt_handler[irq]);
656 s_interrupt_handler[irq]->handle_interrupt(regs);
657 s_interrupt_handler[irq]->increment_invoking_counter();
658 --g_in_irq;
659 s_interrupt_handler[irq]->eoi();
660}
661
662void sse_init()
663{
664 asm volatile(
665 "mov %cr0, %eax\n"
666 "andl $0xfffffffb, %eax\n"
667 "orl $0x2, %eax\n"
668 "mov %eax, %cr0\n"
669 "mov %cr4, %eax\n"
670 "orl $0x600, %eax\n"
671 "mov %eax, %cr4\n");
672}
673
674bool g_cpu_supports_nx;
675bool g_cpu_supports_pae;
676bool g_cpu_supports_pge;
677bool g_cpu_supports_rdrand;
678bool g_cpu_supports_smap;
679bool g_cpu_supports_smep;
680bool g_cpu_supports_sse;
681bool g_cpu_supports_tsc;
682bool g_cpu_supports_umip;
683
684void cpu_detect()
685{
686 CPUID processor_info(0x1);
687 g_cpu_supports_pae = (processor_info.edx() & (1 << 6));
688 g_cpu_supports_pge = (processor_info.edx() & (1 << 13));
689 g_cpu_supports_sse = (processor_info.edx() & (1 << 25));
690 g_cpu_supports_tsc = (processor_info.edx() & (1 << 4));
691 g_cpu_supports_rdrand = (processor_info.ecx() & (1 << 30));
692
693 CPUID extended_processor_info(0x80000001);
694 g_cpu_supports_nx = (extended_processor_info.edx() & (1 << 20));
695
696 CPUID extended_features(0x7);
697 g_cpu_supports_smap = (extended_features.ebx() & (1 << 20));
698 g_cpu_supports_smep = (extended_features.ebx() & (1 << 7));
699 g_cpu_supports_umip = (extended_features.ecx() & (1 << 2));
700}
701
702void stac()
703{
704 if (!g_cpu_supports_smap)
705 return;
706 asm volatile("stac" ::
707 : "cc");
708}
709
710void clac()
711{
712 if (!g_cpu_supports_smap)
713 return;
714 asm volatile("clac" ::
715 : "cc");
716}
717
718void cpu_setup()
719{
720 cpu_detect();
721
722 if (g_cpu_supports_sse) {
723 sse_init();
724 klog() << "x86: SSE support enabled";
725 }
726
727 asm volatile(
728 "movl %%cr0, %%eax\n"
729 "orl $0x00010000, %%eax\n"
730 "movl %%eax, %%cr0\n" ::
731 : "%eax", "memory");
732 klog() << "x86: WP support enabled";
733
734 if (g_cpu_supports_pge) {
735 // Turn on CR4.PGE so the CPU will respect the G bit in page tables.
736 asm volatile(
737 "mov %cr4, %eax\n"
738 "orl $0x80, %eax\n"
739 "mov %eax, %cr4\n");
740 klog() << "x86: PGE support enabled";
741 } else {
742 klog() << "x86: PGE support not detected";
743 }
744
745 if (g_cpu_supports_nx) {
746 // Turn on IA32_EFER.NXE
747 asm volatile(
748 "movl $0xc0000080, %ecx\n"
749 "rdmsr\n"
750 "orl $0x800, %eax\n"
751 "wrmsr\n");
752 klog() << "x86: NX support enabled";
753 } else {
754 klog() << "x86: NX support not detected";
755 }
756
757 if (g_cpu_supports_smep) {
758 // Turn on CR4.SMEP
759 asm volatile(
760 "mov %cr4, %eax\n"
761 "orl $0x100000, %eax\n"
762 "mov %eax, %cr4\n");
763 klog() << "x86: SMEP support enabled";
764 } else {
765 klog() << "x86: SMEP support not detected";
766 }
767
768 if (g_cpu_supports_smap) {
769 // Turn on CR4.SMAP
770 klog() << "x86: Enabling SMAP";
771 asm volatile(
772 "mov %cr4, %eax\n"
773 "orl $0x200000, %eax\n"
774 "mov %eax, %cr4\n");
775 klog() << "x86: SMAP support enabled";
776 } else {
777 klog() << "x86: SMAP support not detected";
778 }
779
780 if (g_cpu_supports_umip) {
781 asm volatile(
782 "mov %cr4, %eax\n"
783 "orl $0x800, %eax\n"
784 "mov %eax, %cr4\n");
785 klog() << "x86: UMIP support enabled";
786 }
787
788 if (g_cpu_supports_tsc) {
789 asm volatile(
790 "mov %cr4, %eax\n"
791 "orl $0x4, %eax\n"
792 "mov %eax, %cr4\n");
793 klog() << "x86: RDTSC support restricted";
794 }
795
796 if (g_cpu_supports_rdrand) {
797 klog() << "x86: Using RDRAND for good randomness";
798 } else {
799 klog() << "x86: No RDRAND support detected. Randomness will be shitty";
800 }
801}
802
803u32 read_cr3()
804{
805 u32 cr3;
806 asm("movl %%cr3, %%eax"
807 : "=a"(cr3));
808 return cr3;
809}
810
811void write_cr3(u32 cr3)
812{
813 asm volatile("movl %%eax, %%cr3" ::"a"(cr3)
814 : "memory");
815}
816
817}
818
819#ifdef DEBUG
820void __assertion_failed(const char* msg, const char* file, unsigned line, const char* func)
821{
822 asm volatile("cli");
823 klog() << "ASSERTION FAILED: " << msg << "\n"
824 << file << ":" << line << " in " << func;
825
826 // Switch back to the current process's page tables if there are any.
827 // Otherwise stack walking will be a disaster.
828 if (Process::current)
829 MM.enter_process_paging_scope(*Process::current);
830
831 Kernel::dump_backtrace();
832 asm volatile("hlt");
833 for (;;)
834 ;
835}
836#endif
837
838NonMaskableInterruptDisabler::NonMaskableInterruptDisabler()
839{
840 IO::out8(0x70, IO::in8(0x70) | 0x80);
841}
842
843NonMaskableInterruptDisabler::~NonMaskableInterruptDisabler()
844{
845 IO::out8(0x70, IO::in8(0x70) & 0x7F);
846}