LoongArch: Add ORC stack unwinder support

+2

arch/loongarch/Kconfig

··· 136 136 select HAVE_KVM 137 137 select HAVE_MOD_ARCH_SPECIFIC 138 138 select HAVE_NMI 139 + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS 139 140 select HAVE_PCI 140 141 select HAVE_PERF_EVENTS 141 142 select HAVE_PERF_REGS ··· 149 148 select HAVE_SAMPLE_FTRACE_DIRECT 150 149 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI 151 150 select HAVE_SETUP_PER_CPU_AREA if NUMA 151 + select HAVE_STACK_VALIDATION if HAVE_OBJTOOL 152 152 select HAVE_STACKPROTECTOR 153 153 select HAVE_SYSCALL_TRACEPOINTS 154 154 select HAVE_TIF_NOHZ

+11

arch/loongarch/Kconfig.debug

··· 26 26 Some of the addresses it reports may be incorrect (but better than the 27 27 Guess unwinder). 28 28 29 + config UNWINDER_ORC 30 + bool "ORC unwinder" 31 + select OBJTOOL 32 + help 33 + This option enables the ORC (Oops Rewind Capability) unwinder for 34 + unwinding kernel stack traces. It uses a custom data format which is 35 + a simplified version of the DWARF Call Frame Information standard. 36 + 37 + Enabling this option will increase the kernel's runtime memory usage 38 + by roughly 2-4MB, depending on your kernel config. 39 + 29 40 endchoice

+21 -2

arch/loongarch/Makefile

··· 26 26 32bit-emul = elf32loongarch 27 27 64bit-emul = elf64loongarch 28 28 29 + ifdef CONFIG_UNWINDER_ORC 30 + orc_hash_h := arch/$(SRCARCH)/include/generated/asm/orc_hash.h 31 + orc_hash_sh := $(srctree)/scripts/orc_hash.sh 32 + targets += $(orc_hash_h) 33 + quiet_cmd_orc_hash = GEN $@ 34 + cmd_orc_hash = mkdir -p $(dir $@); \ 35 + $(CONFIG_SHELL) $(orc_hash_sh) < $< > $@ 36 + $(orc_hash_h): $(srctree)/arch/loongarch/include/asm/orc_types.h $(orc_hash_sh) FORCE 37 + $(call if_changed,orc_hash) 38 + archprepare: $(orc_hash_h) 39 + endif 40 + 29 41 ifdef CONFIG_DYNAMIC_FTRACE 30 42 KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY 31 43 CC_FLAGS_FTRACE := -fpatchable-function-entry=2 ··· 84 72 KBUILD_CFLAGS_KERNEL += $(call cc-option,-fdirect-access-external-data) 85 73 KBUILD_AFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) 86 74 KBUILD_CFLAGS_MODULE += $(call cc-option,-fno-direct-access-external-data) 87 - KBUILD_AFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) 88 - KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) 89 75 else 90 76 cflags-y += $(call cc-option,-mno-explicit-relocs) 91 77 KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel 92 78 KBUILD_CFLAGS_KERNEL += -Wa,-mla-global-with-pcrel 93 79 KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs 94 80 KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs 81 + endif 82 + 83 + KBUILD_AFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) 84 + KBUILD_CFLAGS += $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax) 85 + KBUILD_AFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub) 86 + KBUILD_CFLAGS += $(call cc-option,-mthin-add-sub) $(call cc-option,-Wa$(comma)-mthin-add-sub) 87 + 88 + ifdef CONFIG_OBJTOOL 89 + KBUILD_CFLAGS += -fno-jump-tables 95 90 endif 96 91 97 92 KBUILD_RUSTFLAGS_MODULE += -Crelocation-model=pic

+2

arch/loongarch/include/asm/Kbuild

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 + generated-y += orc_hash.h 3 + 2 4 generic-y += dma-contiguous.h 3 5 generic-y += mcs_spinlock.h 4 6 generic-y += parport.h

+1

arch/loongarch/include/asm/bug.h

··· 44 44 do { \ 45 45 instrumentation_begin(); \ 46 46 __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \ 47 + annotate_reachable(); \ 47 48 instrumentation_end(); \ 48 49 } while (0) 49 50

+2

arch/loongarch/include/asm/exception.h

··· 6 6 #include <asm/ptrace.h> 7 7 #include <linux/kprobes.h> 8 8 9 + extern void *exception_table[]; 10 + 9 11 void show_registers(struct pt_regs *regs); 10 12 11 13 asmlinkage void cache_parity_error(void);

+7

arch/loongarch/include/asm/module.h

··· 6 6 #define _ASM_MODULE_H 7 7 8 8 #include <asm/inst.h> 9 + #include <asm/orc_types.h> 9 10 #include <asm-generic/module.h> 10 11 11 12 #define RELA_STACK_DEPTH 16 ··· 21 20 struct mod_section got; 22 21 struct mod_section plt; 23 22 struct mod_section plt_idx; 23 + 24 + #ifdef CONFIG_UNWINDER_ORC 25 + unsigned int num_orcs; 26 + int *orc_unwind_ip; 27 + struct orc_entry *orc_unwind; 28 + #endif 24 29 25 30 /* For CONFIG_DYNAMIC_FTRACE */ 26 31 struct plt_entry *ftrace_trampolines;

+18

arch/loongarch/include/asm/orc_header.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 + 3 + #ifndef _ORC_HEADER_H 4 + #define _ORC_HEADER_H 5 + 6 + #include <linux/types.h> 7 + #include <linux/compiler.h> 8 + #include <asm/orc_hash.h> 9 + 10 + /* 11 + * The header is currently a 20-byte hash of the ORC entry definition; see 12 + * scripts/orc_hash.sh. 13 + */ 14 + #define ORC_HEADER \ 15 + __used __section(".orc_header") __aligned(4) \ 16 + static const u8 orc_header[] = { ORC_HASH } 17 + 18 + #endif /* _ORC_HEADER_H */

+31

arch/loongarch/include/asm/orc_lookup.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 + #ifndef _ORC_LOOKUP_H 3 + #define _ORC_LOOKUP_H 4 + 5 + /* 6 + * This is a lookup table for speeding up access to the .orc_unwind table. 7 + * Given an input address offset, the corresponding lookup table entry 8 + * specifies a subset of the .orc_unwind table to search. 9 + * 10 + * Each block represents the end of the previous range and the start of the 11 + * next range. An extra block is added to give the last range an end. 12 + * 13 + * The block size should be a power of 2 to avoid a costly 'div' instruction. 14 + * 15 + * A block size of 256 was chosen because it roughly doubles unwinder 16 + * performance while only adding ~5% to the ORC data footprint. 17 + */ 18 + #define LOOKUP_BLOCK_ORDER 8 19 + #define LOOKUP_BLOCK_SIZE (1 << LOOKUP_BLOCK_ORDER) 20 + 21 + #ifndef LINKER_SCRIPT 22 + 23 + extern unsigned int orc_lookup[]; 24 + extern unsigned int orc_lookup_end[]; 25 + 26 + #define LOOKUP_START_IP (unsigned long)_stext 27 + #define LOOKUP_STOP_IP (unsigned long)_etext 28 + 29 + #endif /* LINKER_SCRIPT */ 30 + 31 + #endif /* _ORC_LOOKUP_H */

+58

arch/loongarch/include/asm/orc_types.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 + #ifndef _ORC_TYPES_H 3 + #define _ORC_TYPES_H 4 + 5 + #include <linux/types.h> 6 + 7 + /* 8 + * The ORC_REG_* registers are base registers which are used to find other 9 + * registers on the stack. 10 + * 11 + * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the 12 + * address of the previous frame: the caller's SP before it called the current 13 + * function. 14 + * 15 + * ORC_REG_UNDEFINED means the corresponding register's value didn't change in 16 + * the current frame. 17 + * 18 + * The most commonly used base registers are SP and FP -- which the previous SP 19 + * is usually based on -- and PREV_SP and UNDEFINED -- which the previous FP is 20 + * usually based on. 21 + * 22 + * The rest of the base registers are needed for special cases like entry code 23 + * and GCC realigned stacks. 24 + */ 25 + #define ORC_REG_UNDEFINED 0 26 + #define ORC_REG_PREV_SP 1 27 + #define ORC_REG_SP 2 28 + #define ORC_REG_FP 3 29 + #define ORC_REG_MAX 4 30 + 31 + #define ORC_TYPE_UNDEFINED 0 32 + #define ORC_TYPE_END_OF_STACK 1 33 + #define ORC_TYPE_CALL 2 34 + #define ORC_TYPE_REGS 3 35 + #define ORC_TYPE_REGS_PARTIAL 4 36 + 37 + #ifndef __ASSEMBLY__ 38 + /* 39 + * This struct is more or less a vastly simplified version of the DWARF Call 40 + * Frame Information standard. It contains only the necessary parts of DWARF 41 + * CFI, simplified for ease of access by the in-kernel unwinder. It tells the 42 + * unwinder how to find the previous SP and FP (and sometimes entry regs) on 43 + * the stack for a given code address. Each instance of the struct corresponds 44 + * to one or more code locations. 45 + */ 46 + struct orc_entry { 47 + s16 sp_offset; 48 + s16 fp_offset; 49 + s16 ra_offset; 50 + unsigned int sp_reg:4; 51 + unsigned int fp_reg:4; 52 + unsigned int ra_reg:4; 53 + unsigned int type:3; 54 + unsigned int signal:1; 55 + }; 56 + #endif /* __ASSEMBLY__ */ 57 + 58 + #endif /* _ORC_TYPES_H */

+3

arch/loongarch/include/asm/stackframe.h

··· 13 13 #include <asm/asm-offsets.h> 14 14 #include <asm/loongarch.h> 15 15 #include <asm/thread_info.h> 16 + #include <asm/unwind_hints.h> 16 17 17 18 /* Make the addition of cfi info a little easier. */ 18 19 .macro cfi_rel_offset reg offset=0 docfi=0 ··· 163 162 li.w t0, CSR_CRMD_WE 164 163 csrxchg t0, t0, LOONGARCH_CSR_CRMD 165 164 #endif 165 + UNWIND_HINT_REGS 166 166 .endm 167 167 168 168 .macro SAVE_ALL docfi=0 ··· 221 219 222 220 .macro RESTORE_SP_AND_RET docfi=0 223 221 cfi_ld sp, PT_R3, \docfi 222 + UNWIND_HINT_FUNC 224 223 ertn 225 224 .endm 226 225

+18 -2

arch/loongarch/include/asm/unwind.h

··· 16 16 enum unwinder_type { 17 17 UNWINDER_GUESS, 18 18 UNWINDER_PROLOGUE, 19 + UNWINDER_ORC, 19 20 }; 20 21 21 22 struct unwind_state { ··· 25 24 struct task_struct *task; 26 25 bool first, error, reset; 27 26 int graph_idx; 28 - unsigned long sp, pc, ra; 27 + unsigned long sp, fp, pc, ra; 29 28 }; 30 29 31 30 bool default_next_frame(struct unwind_state *state); ··· 62 61 state->sp = regs->regs[3]; 63 62 state->pc = regs->csr_era; 64 63 state->ra = regs->regs[1]; 64 + state->fp = regs->regs[22]; 65 65 } else if (task && task != current) { 66 66 state->sp = thread_saved_fp(task); 67 67 state->pc = thread_saved_ra(task); 68 68 state->ra = 0; 69 + state->fp = 0; 69 70 } else { 70 71 state->sp = (unsigned long)__builtin_frame_address(0); 71 72 state->pc = (unsigned long)__builtin_return_address(0); 72 73 state->ra = 0; 74 + state->fp = 0; 73 75 } 74 76 state->task = task; 75 77 get_stack_info(state->sp, state->task, &state->stack_info); ··· 81 77 82 78 static __always_inline unsigned long __unwind_get_return_address(struct unwind_state *state) 83 79 { 84 - return unwind_done(state) ? 0 : state->pc; 80 + if (unwind_done(state)) 81 + return 0; 82 + 83 + return __kernel_text_address(state->pc) ? state->pc : 0; 85 84 } 85 + 86 + #ifdef CONFIG_UNWINDER_ORC 87 + void unwind_init(void); 88 + void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size); 89 + #else 90 + static inline void unwind_init(void) {} 91 + static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} 92 + #endif 93 + 86 94 #endif /* _ASM_UNWIND_H */

+28

arch/loongarch/include/asm/unwind_hints.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef _ASM_LOONGARCH_UNWIND_HINTS_H 3 + #define _ASM_LOONGARCH_UNWIND_HINTS_H 4 + 5 + #include <linux/objtool.h> 6 + #include <asm/orc_types.h> 7 + 8 + #ifdef __ASSEMBLY__ 9 + 10 + .macro UNWIND_HINT_UNDEFINED 11 + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED 12 + .endm 13 + 14 + .macro UNWIND_HINT_END_OF_STACK 15 + UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK 16 + .endm 17 + 18 + .macro UNWIND_HINT_REGS 19 + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_REGS 20 + .endm 21 + 22 + .macro UNWIND_HINT_FUNC 23 + UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_CALL 24 + .endm 25 + 26 + #endif /* __ASSEMBLY__ */ 27 + 28 + #endif /* _ASM_LOONGARCH_UNWIND_HINTS_H */

+4

arch/loongarch/kernel/Makefile

··· 3 3 # Makefile for the Linux/LoongArch kernel. 4 4 # 5 5 6 + OBJECT_FILES_NON_STANDARD_head.o := y 7 + 6 8 extra-y := vmlinux.lds 7 9 8 10 obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \ ··· 23 21 24 22 CFLAGS_module.o += $(call cc-option,-Wno-override-init,) 25 23 CFLAGS_syscall.o += $(call cc-option,-Wno-override-init,) 24 + CFLAGS_traps.o += $(call cc-option,-Wno-override-init,) 26 25 CFLAGS_perf_event.o += $(call cc-option,-Wno-override-init,) 27 26 28 27 ifdef CONFIG_FUNCTION_TRACER ··· 65 62 66 63 obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o 67 64 obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o 65 + obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o 68 66 69 67 obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o 70 68 obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o

+5

arch/loongarch/kernel/entry.S

··· 14 14 #include <asm/regdef.h> 15 15 #include <asm/stackframe.h> 16 16 #include <asm/thread_info.h> 17 + #include <asm/unwind_hints.h> 17 18 18 19 .text 19 20 .cfi_sections .debug_frame 20 21 .align 5 21 22 SYM_CODE_START(handle_syscall) 23 + UNWIND_HINT_UNDEFINED 22 24 csrrd t0, PERCPU_BASE_KS 23 25 la.pcrel t1, kernelsp 24 26 add.d t1, t1, t0 ··· 59 57 cfi_st fp, PT_R22 60 58 61 59 SAVE_STATIC 60 + UNWIND_HINT_REGS 62 61 63 62 #ifdef CONFIG_KGDB 64 63 li.w t1, CSR_CRMD_WE ··· 78 75 _ASM_NOKPROBE(handle_syscall) 79 76 80 77 SYM_CODE_START(ret_from_fork) 78 + UNWIND_HINT_REGS 81 79 bl schedule_tail # a0 = struct task_struct *prev 82 80 move a0, sp 83 81 bl syscall_exit_to_user_mode ··· 88 84 SYM_CODE_END(ret_from_fork) 89 85 90 86 SYM_CODE_START(ret_from_kernel_thread) 87 + UNWIND_HINT_REGS 91 88 bl schedule_tail # a0 = struct task_struct *prev 92 89 move a0, s1 93 90 jirl ra, s0, 0

+7

arch/loongarch/kernel/fpu.S

··· 15 15 #include <asm/fpregdef.h> 16 16 #include <asm/loongarch.h> 17 17 #include <asm/regdef.h> 18 + #include <asm/unwind_hints.h> 18 19 19 20 #define FPU_REG_WIDTH 8 20 21 #define LSX_REG_WIDTH 16 ··· 527 526 .L_fpu_fault: 528 527 li.w a0, -EFAULT # failure 529 528 jr ra 529 + 530 + #ifdef CONFIG_CPU_HAS_LBT 531 + STACK_FRAME_NON_STANDARD _restore_fp 532 + STACK_FRAME_NON_STANDARD _restore_lsx 533 + STACK_FRAME_NON_STANDARD _restore_lasx 534 + #endif

+6

arch/loongarch/kernel/genex.S

··· 32 32 SYM_FUNC_END(__arch_cpu_idle) 33 33 34 34 SYM_CODE_START(handle_vint) 35 + UNWIND_HINT_UNDEFINED 35 36 BACKUP_T0T1 36 37 SAVE_ALL 37 38 la_abs t1, __arch_cpu_idle ··· 50 49 SYM_CODE_END(handle_vint) 51 50 52 51 SYM_CODE_START(except_vec_cex) 52 + UNWIND_HINT_UNDEFINED 53 53 b cache_parity_error 54 54 SYM_CODE_END(except_vec_cex) 55 55 ··· 69 67 .macro BUILD_HANDLER exception handler prep 70 68 .align 5 71 69 SYM_CODE_START(handle_\exception) 70 + UNWIND_HINT_UNDEFINED 72 71 666: 73 72 BACKUP_T0T1 74 73 SAVE_ALL ··· 80 77 668: 81 78 RESTORE_ALL_AND_RET 82 79 SYM_CODE_END(handle_\exception) 80 + .pushsection ".data", "aw", %progbits 83 81 SYM_DATA(unwind_hint_\exception, .word 668b - 666b) 82 + .popsection 84 83 .endm 85 84 86 85 BUILD_HANDLER ade ade badv ··· 99 94 BUILD_HANDLER reserved reserved none /* others */ 100 95 101 96 SYM_CODE_START(handle_sys) 97 + UNWIND_HINT_UNDEFINED 102 98 la_abs t0, handle_syscall 103 99 jr t0 104 100 SYM_CODE_END(handle_sys)

+3

arch/loongarch/kernel/lbt.S

··· 11 11 #include <asm/asm-offsets.h> 12 12 #include <asm/errno.h> 13 13 #include <asm/regdef.h> 14 + #include <asm/unwind_hints.h> 14 15 15 16 #define SCR_REG_WIDTH 8 16 17 ··· 154 153 .L_lbt_fault: 155 154 li.w a0, -EFAULT # failure 156 155 jr ra 156 + 157 + STACK_FRAME_NON_STANDARD _restore_ftop_context

+6

arch/loongarch/kernel/mcount_dyn.S

··· 73 73 SYM_FUNC_END(ftrace_stub) 74 74 75 75 SYM_CODE_START(ftrace_common) 76 + UNWIND_HINT_UNDEFINED 76 77 PTR_ADDI a0, ra, -8 /* arg0: ip */ 77 78 move a1, t0 /* arg1: parent_ip */ 78 79 la.pcrel t1, function_trace_op ··· 114 113 SYM_CODE_END(ftrace_common) 115 114 116 115 SYM_CODE_START(ftrace_caller) 116 + UNWIND_HINT_UNDEFINED 117 117 ftrace_regs_entry allregs=0 118 118 b ftrace_common 119 119 SYM_CODE_END(ftrace_caller) 120 120 121 121 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 122 122 SYM_CODE_START(ftrace_regs_caller) 123 + UNWIND_HINT_UNDEFINED 123 124 ftrace_regs_entry allregs=1 124 125 b ftrace_common 125 126 SYM_CODE_END(ftrace_regs_caller) ··· 129 126 130 127 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 131 128 SYM_CODE_START(ftrace_graph_caller) 129 + UNWIND_HINT_UNDEFINED 132 130 PTR_L a0, sp, PT_ERA 133 131 PTR_ADDI a0, a0, -8 /* arg0: self_addr */ 134 132 PTR_ADDI a1, sp, PT_R1 /* arg1: parent */ ··· 138 134 SYM_CODE_END(ftrace_graph_caller) 139 135 140 136 SYM_CODE_START(return_to_handler) 137 + UNWIND_HINT_UNDEFINED 141 138 /* Save return value regs */ 142 139 PTR_ADDI sp, sp, -FGRET_REGS_SIZE 143 140 PTR_S a0, sp, FGRET_REGS_A0 ··· 160 155 161 156 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 162 157 SYM_CODE_START(ftrace_stub_direct_tramp) 158 + UNWIND_HINT_UNDEFINED 163 159 jr t0 164 160 SYM_CODE_END(ftrace_stub_direct_tramp) 165 161 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */

+18 -4

arch/loongarch/kernel/module.c

··· 20 20 #include <linux/kernel.h> 21 21 #include <asm/alternative.h> 22 22 #include <asm/inst.h> 23 + #include <asm/unwind.h> 23 24 24 25 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 25 26 { ··· 516 515 int module_finalize(const Elf_Ehdr *hdr, 517 516 const Elf_Shdr *sechdrs, struct module *mod) 518 517 { 519 - const Elf_Shdr *s, *se; 520 518 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 519 + const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL; 521 520 522 - for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { 521 + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 523 522 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 524 - apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size); 523 + alt = s; 524 + if (!strcmp(".orc_unwind", secstrs + s->sh_name)) 525 + orc = s; 526 + if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name)) 527 + orc_ip = s; 525 528 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) 526 - module_init_ftrace_plt(hdr, s, mod); 529 + ftrace = s; 527 530 } 531 + 532 + if (alt) 533 + apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); 534 + 535 + if (orc && orc_ip) 536 + unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); 537 + 538 + if (ftrace) 539 + module_init_ftrace_plt(hdr, ftrace, mod); 528 540 529 541 return 0; 530 542 }

+4 -3

arch/loongarch/kernel/relocate_kernel.S

··· 15 15 #include <asm/addrspace.h> 16 16 17 17 SYM_CODE_START(relocate_new_kernel) 18 + UNWIND_HINT_UNDEFINED 18 19 /* 19 20 * a0: EFI boot flag for the new kernel 20 21 * a1: Command line pointer for the new kernel ··· 91 90 * then start at the entry point from LOONGARCH_IOCSR_MBUF0. 92 91 */ 93 92 SYM_CODE_START(kexec_smp_wait) 93 + UNWIND_HINT_UNDEFINED 94 94 1: li.w t0, 0x100 /* wait for init loop */ 95 95 2: addi.w t0, t0, -1 /* limit mailbox access */ 96 96 bnez t0, 2b ··· 108 106 109 107 relocate_new_kernel_end: 110 108 111 - SYM_DATA_START(relocate_new_kernel_size) 112 - PTR relocate_new_kernel_end - relocate_new_kernel 113 - SYM_DATA_END(relocate_new_kernel_size) 109 + .section ".data" 110 + SYM_DATA(relocate_new_kernel_size, .long relocate_new_kernel_end - relocate_new_kernel)

+1

arch/loongarch/kernel/rethook_trampoline.S

··· 76 76 .endm 77 77 78 78 SYM_CODE_START(arch_rethook_trampoline) 79 + UNWIND_HINT_UNDEFINED 79 80 addi.d sp, sp, -PT_SIZE 80 81 save_all_base_regs 81 82

+2

arch/loongarch/kernel/setup.c

··· 47 47 #include <asm/sections.h> 48 48 #include <asm/setup.h> 49 49 #include <asm/time.h> 50 + #include <asm/unwind.h> 50 51 51 52 #define SMBIOS_BIOSSIZE_OFFSET 0x09 52 53 #define SMBIOS_BIOSEXTERN_OFFSET 0x13 ··· 588 587 void __init setup_arch(char **cmdline_p) 589 588 { 590 589 cpu_probe(); 590 + unwind_init(); 591 591 592 592 init_environ(); 593 593 efi_init();

+1

arch/loongarch/kernel/stacktrace.c

··· 29 29 regs->csr_era = thread_saved_ra(task); 30 30 } 31 31 regs->regs[1] = 0; 32 + regs->regs[22] = 0; 32 33 } 33 34 34 35 for (unwind_start(&state, task, regs);

+29 -13

arch/loongarch/kernel/traps.c

··· 53 53 54 54 #include "access-helper.h" 55 55 56 + void *exception_table[EXCCODE_INT_START] = { 57 + [0 ... EXCCODE_INT_START - 1] = handle_reserved, 58 + 59 + [EXCCODE_TLBI] = handle_tlb_load, 60 + [EXCCODE_TLBL] = handle_tlb_load, 61 + [EXCCODE_TLBS] = handle_tlb_store, 62 + [EXCCODE_TLBM] = handle_tlb_modify, 63 + [EXCCODE_TLBNR] = handle_tlb_protect, 64 + [EXCCODE_TLBNX] = handle_tlb_protect, 65 + [EXCCODE_TLBPE] = handle_tlb_protect, 66 + [EXCCODE_ADE] = handle_ade, 67 + [EXCCODE_ALE] = handle_ale, 68 + [EXCCODE_BCE] = handle_bce, 69 + [EXCCODE_SYS] = handle_sys, 70 + [EXCCODE_BP] = handle_bp, 71 + [EXCCODE_INE] = handle_ri, 72 + [EXCCODE_IPE] = handle_ri, 73 + [EXCCODE_FPDIS] = handle_fpu, 74 + [EXCCODE_LSXDIS] = handle_lsx, 75 + [EXCCODE_LASXDIS] = handle_lasx, 76 + [EXCCODE_FPE] = handle_fpe, 77 + [EXCCODE_WATCH] = handle_watch, 78 + [EXCCODE_BTDIS] = handle_lbt, 79 + }; 80 + EXPORT_SYMBOL_GPL(exception_table); 81 + 56 82 static void show_backtrace(struct task_struct *task, const struct pt_regs *regs, 57 83 const char *loglvl, bool user) 58 84 { ··· 1176 1150 for (i = EXCCODE_INT_START; i <= EXCCODE_INT_END; i++) 1177 1151 set_handler(i * VECSIZE, handle_vint, VECSIZE); 1178 1152 1179 - set_handler(EXCCODE_ADE * VECSIZE, handle_ade, VECSIZE); 1180 - set_handler(EXCCODE_ALE * VECSIZE, handle_ale, VECSIZE); 1181 - set_handler(EXCCODE_BCE * VECSIZE, handle_bce, VECSIZE); 1182 - set_handler(EXCCODE_SYS * VECSIZE, handle_sys, VECSIZE); 1183 - set_handler(EXCCODE_BP * VECSIZE, handle_bp, VECSIZE); 1184 - set_handler(EXCCODE_INE * VECSIZE, handle_ri, VECSIZE); 1185 - set_handler(EXCCODE_IPE * VECSIZE, handle_ri, VECSIZE); 1186 - set_handler(EXCCODE_FPDIS * VECSIZE, handle_fpu, VECSIZE); 1187 - set_handler(EXCCODE_LSXDIS * VECSIZE, handle_lsx, VECSIZE); 1188 - set_handler(EXCCODE_LASXDIS * VECSIZE, handle_lasx, VECSIZE); 1189 - set_handler(EXCCODE_FPE * VECSIZE, handle_fpe, VECSIZE); 1190 - set_handler(EXCCODE_BTDIS * VECSIZE, handle_lbt, VECSIZE); 1191 - set_handler(EXCCODE_WATCH * VECSIZE, handle_watch, VECSIZE); 1153 + /* Set exception vector handler */ 1154 + for (i = EXCCODE_ADE; i <= EXCCODE_BTDIS; i++) 1155 + set_handler(i * VECSIZE, exception_table[i], VECSIZE); 1192 1156 1193 1157 cache_error_setup(); 1194 1158

+528

arch/loongarch/kernel/unwind_orc.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + #include <linux/objtool.h> 3 + #include <linux/module.h> 4 + #include <linux/sort.h> 5 + #include <asm/exception.h> 6 + #include <asm/orc_header.h> 7 + #include <asm/orc_lookup.h> 8 + #include <asm/orc_types.h> 9 + #include <asm/ptrace.h> 10 + #include <asm/setup.h> 11 + #include <asm/stacktrace.h> 12 + #include <asm/tlb.h> 13 + #include <asm/unwind.h> 14 + 15 + ORC_HEADER; 16 + 17 + #define orc_warn(fmt, ...) \ 18 + printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) 19 + 20 + extern int __start_orc_unwind_ip[]; 21 + extern int __stop_orc_unwind_ip[]; 22 + extern struct orc_entry __start_orc_unwind[]; 23 + extern struct orc_entry __stop_orc_unwind[]; 24 + 25 + static bool orc_init __ro_after_init; 26 + static unsigned int lookup_num_blocks __ro_after_init; 27 + 28 + /* Fake frame pointer entry -- used as a fallback for generated code */ 29 + static struct orc_entry orc_fp_entry = { 30 + .sp_reg = ORC_REG_FP, 31 + .sp_offset = 16, 32 + .fp_reg = ORC_REG_PREV_SP, 33 + .fp_offset = -16, 34 + .ra_reg = ORC_REG_PREV_SP, 35 + .ra_offset = -8, 36 + .type = ORC_TYPE_CALL 37 + }; 38 + 39 + /* 40 + * If we crash with IP==0, the last successfully executed instruction 41 + * was probably an indirect function call with a NULL function pointer, 42 + * and we don't have unwind information for NULL. 43 + * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function 44 + * pointer into its parent and then continue normally from there. 45 + */ 46 + static struct orc_entry orc_null_entry = { 47 + .sp_reg = ORC_REG_SP, 48 + .sp_offset = sizeof(long), 49 + .fp_reg = ORC_REG_UNDEFINED, 50 + .type = ORC_TYPE_CALL 51 + }; 52 + 53 + static inline unsigned long orc_ip(const int *ip) 54 + { 55 + return (unsigned long)ip + *ip; 56 + } 57 + 58 + static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, 59 + unsigned int num_entries, unsigned long ip) 60 + { 61 + int *first = ip_table; 62 + int *mid = first, *found = first; 63 + int *last = ip_table + num_entries - 1; 64 + 65 + if (!num_entries) 66 + return NULL; 67 + 68 + /* 69 + * Do a binary range search to find the rightmost duplicate of a given 70 + * starting address. Some entries are section terminators which are 71 + * "weak" entries for ensuring there are no gaps. They should be 72 + * ignored when they conflict with a real entry. 73 + */ 74 + while (first <= last) { 75 + mid = first + ((last - first) / 2); 76 + 77 + if (orc_ip(mid) <= ip) { 78 + found = mid; 79 + first = mid + 1; 80 + } else 81 + last = mid - 1; 82 + } 83 + 84 + return u_table + (found - ip_table); 85 + } 86 + 87 + #ifdef CONFIG_MODULES 88 + static struct orc_entry *orc_module_find(unsigned long ip) 89 + { 90 + struct module *mod; 91 + 92 + mod = __module_address(ip); 93 + if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) 94 + return NULL; 95 + 96 + return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, mod->arch.num_orcs, ip); 97 + } 98 + #else 99 + static struct orc_entry *orc_module_find(unsigned long ip) 100 + { 101 + return NULL; 102 + } 103 + #endif 104 + 105 + #ifdef CONFIG_DYNAMIC_FTRACE 106 + static struct orc_entry *orc_find(unsigned long ip); 107 + 108 + /* 109 + * Ftrace dynamic trampolines do not have orc entries of their own. 110 + * But they are copies of the ftrace entries that are static and 111 + * defined in ftrace_*.S, which do have orc entries. 112 + * 113 + * If the unwinder comes across a ftrace trampoline, then find the 114 + * ftrace function that was used to create it, and use that ftrace 115 + * function's orc entry, as the placement of the return code in 116 + * the stack will be identical. 117 + */ 118 + static struct orc_entry *orc_ftrace_find(unsigned long ip) 119 + { 120 + struct ftrace_ops *ops; 121 + unsigned long tramp_addr, offset; 122 + 123 + ops = ftrace_ops_trampoline(ip); 124 + if (!ops) 125 + return NULL; 126 + 127 + /* Set tramp_addr to the start of the code copied by the trampoline */ 128 + if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 129 + tramp_addr = (unsigned long)ftrace_regs_caller; 130 + else 131 + tramp_addr = (unsigned long)ftrace_caller; 132 + 133 + /* Now place tramp_addr to the location within the trampoline ip is at */ 134 + offset = ip - ops->trampoline; 135 + tramp_addr += offset; 136 + 137 + /* Prevent unlikely recursion */ 138 + if (ip == tramp_addr) 139 + return NULL; 140 + 141 + return orc_find(tramp_addr); 142 + } 143 + #else 144 + static struct orc_entry *orc_ftrace_find(unsigned long ip) 145 + { 146 + return NULL; 147 + } 148 + #endif 149 + 150 + static struct orc_entry *orc_find(unsigned long ip) 151 + { 152 + static struct orc_entry *orc; 153 + 154 + if (ip == 0) 155 + return &orc_null_entry; 156 + 157 + /* For non-init vmlinux addresses, use the fast lookup table: */ 158 + if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { 159 + unsigned int idx, start, stop; 160 + 161 + idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; 162 + 163 + if (unlikely((idx >= lookup_num_blocks-1))) { 164 + orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n", 165 + idx, lookup_num_blocks, (void *)ip); 166 + return NULL; 167 + } 168 + 169 + start = orc_lookup[idx]; 170 + stop = orc_lookup[idx + 1] + 1; 171 + 172 + if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || 173 + (__start_orc_unwind + stop > __stop_orc_unwind))) { 174 + orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n", 175 + idx, lookup_num_blocks, start, stop, (void *)ip); 176 + return NULL; 177 + } 178 + 179 + return __orc_find(__start_orc_unwind_ip + start, 180 + __start_orc_unwind + start, stop - start, ip); 181 + } 182 + 183 + /* vmlinux .init slow lookup: */ 184 + if (is_kernel_inittext(ip)) 185 + return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, 186 + __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); 187 + 188 + /* Module lookup: */ 189 + orc = orc_module_find(ip); 190 + if (orc) 191 + return orc; 192 + 193 + return orc_ftrace_find(ip); 194 + } 195 + 196 + #ifdef CONFIG_MODULES 197 + 198 + static DEFINE_MUTEX(sort_mutex); 199 + static int *cur_orc_ip_table = __start_orc_unwind_ip; 200 + static struct orc_entry *cur_orc_table = __start_orc_unwind; 201 + 202 + static void orc_sort_swap(void *_a, void *_b, int size) 203 + { 204 + int delta = _b - _a; 205 + int *a = _a, *b = _b, tmp; 206 + struct orc_entry *orc_a, *orc_b; 207 + 208 + /* Swap the .orc_unwind_ip entries: */ 209 + tmp = *a; 210 + *a = *b + delta; 211 + *b = tmp - delta; 212 + 213 + /* Swap the corresponding .orc_unwind entries: */ 214 + orc_a = cur_orc_table + (a - cur_orc_ip_table); 215 + orc_b = cur_orc_table + (b - cur_orc_ip_table); 216 + swap(*orc_a, *orc_b); 217 + } 218 + 219 + static int orc_sort_cmp(const void *_a, const void *_b) 220 + { 221 + const int *a = _a, *b = _b; 222 + unsigned long a_val = orc_ip(a); 223 + unsigned long b_val = orc_ip(b); 224 + struct orc_entry *orc_a; 225 + 226 + if (a_val > b_val) 227 + return 1; 228 + if (a_val < b_val) 229 + return -1; 230 + 231 + /* 232 + * The "weak" section terminator entries need to always be first 233 + * to ensure the lookup code skips them in favor of real entries. 234 + * These terminator entries exist to handle any gaps created by 235 + * whitelisted .o files which didn't get objtool generation. 236 + */ 237 + orc_a = cur_orc_table + (a - cur_orc_ip_table); 238 + 239 + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 240 + } 241 + 242 + void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, 243 + void *_orc, size_t orc_size) 244 + { 245 + int *orc_ip = _orc_ip; 246 + struct orc_entry *orc = _orc; 247 + unsigned int num_entries = orc_ip_size / sizeof(int); 248 + 249 + WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || 250 + orc_size % sizeof(*orc) != 0 || 251 + num_entries != orc_size / sizeof(*orc)); 252 + 253 + /* 254 + * The 'cur_orc_*' globals allow the orc_sort_swap() callback to 255 + * associate an .orc_unwind_ip table entry with its corresponding 256 + * .orc_unwind entry so they can both be swapped. 257 + */ 258 + mutex_lock(&sort_mutex); 259 + cur_orc_ip_table = orc_ip; 260 + cur_orc_table = orc; 261 + sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap); 262 + mutex_unlock(&sort_mutex); 263 + 264 + mod->arch.orc_unwind_ip = orc_ip; 265 + mod->arch.orc_unwind = orc; 266 + mod->arch.num_orcs = num_entries; 267 + } 268 + #endif 269 + 270 + void __init unwind_init(void) 271 + { 272 + int i; 273 + size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; 274 + size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; 275 + size_t num_entries = orc_ip_size / sizeof(int); 276 + struct orc_entry *orc; 277 + 278 + if (!num_entries || orc_ip_size % sizeof(int) != 0 || 279 + orc_size % sizeof(struct orc_entry) != 0 || 280 + num_entries != orc_size / sizeof(struct orc_entry)) { 281 + orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n"); 282 + return; 283 + } 284 + 285 + /* 286 + * Note, the orc_unwind and orc_unwind_ip tables were already 287 + * sorted at build time via the 'sorttable' tool. 288 + * It's ready for binary search straight away, no need to sort it. 289 + */ 290 + 291 + /* Initialize the fast lookup table: */ 292 + lookup_num_blocks = orc_lookup_end - orc_lookup; 293 + for (i = 0; i < lookup_num_blocks-1; i++) { 294 + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, 295 + num_entries, LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); 296 + if (!orc) { 297 + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); 298 + return; 299 + } 300 + 301 + orc_lookup[i] = orc - __start_orc_unwind; 302 + } 303 + 304 + /* Initialize the ending block: */ 305 + orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, LOOKUP_STOP_IP); 306 + if (!orc) { 307 + orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); 308 + return; 309 + } 310 + orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; 311 + 312 + orc_init = true; 313 + } 314 + 315 + static inline bool on_stack(struct stack_info *info, unsigned long addr, size_t len) 316 + { 317 + unsigned long begin = info->begin; 318 + unsigned long end = info->end; 319 + 320 + return (info->type != STACK_TYPE_UNKNOWN && 321 + addr >= begin && addr < end && addr + len > begin && addr + len <= end); 322 + } 323 + 324 + static bool stack_access_ok(struct unwind_state *state, unsigned long addr, size_t len) 325 + { 326 + struct stack_info *info = &state->stack_info; 327 + 328 + if (on_stack(info, addr, len)) 329 + return true; 330 + 331 + return !get_stack_info(addr, state->task, info) && on_stack(info, addr, len); 332 + } 333 + 334 + unsigned long unwind_get_return_address(struct unwind_state *state) 335 + { 336 + return __unwind_get_return_address(state); 337 + } 338 + EXPORT_SYMBOL_GPL(unwind_get_return_address); 339 + 340 + void unwind_start(struct unwind_state *state, struct task_struct *task, 341 + struct pt_regs *regs) 342 + { 343 + __unwind_start(state, task, regs); 344 + state->type = UNWINDER_ORC; 345 + if (!unwind_done(state) && !__kernel_text_address(state->pc)) 346 + unwind_next_frame(state); 347 + } 348 + EXPORT_SYMBOL_GPL(unwind_start); 349 + 350 + static bool is_entry_func(unsigned long addr) 351 + { 352 + extern u32 kernel_entry; 353 + extern u32 kernel_entry_end; 354 + 355 + return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end; 356 + } 357 + 358 + static inline unsigned long bt_address(unsigned long ra) 359 + { 360 + extern unsigned long eentry; 361 + 362 + if (__kernel_text_address(ra)) 363 + return ra; 364 + 365 + if (__module_text_address(ra)) 366 + return ra; 367 + 368 + if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { 369 + unsigned long func; 370 + unsigned long type = (ra - eentry) / VECSIZE; 371 + unsigned long offset = (ra - eentry) % VECSIZE; 372 + 373 + switch (type) { 374 + case 0 ... EXCCODE_INT_START - 1: 375 + func = (unsigned long)exception_table[type]; 376 + break; 377 + case EXCCODE_INT_START ... EXCCODE_INT_END: 378 + func = (unsigned long)handle_vint; 379 + break; 380 + default: 381 + func = (unsigned long)handle_reserved; 382 + break; 383 + } 384 + 385 + return func + offset; 386 + } 387 + 388 + return ra; 389 + } 390 + 391 + bool unwind_next_frame(struct unwind_state *state) 392 + { 393 + unsigned long *p, pc; 394 + struct pt_regs *regs; 395 + struct orc_entry *orc; 396 + struct stack_info *info = &state->stack_info; 397 + 398 + if (unwind_done(state)) 399 + return false; 400 + 401 + /* Don't let modules unload while we're reading their ORC data. */ 402 + preempt_disable(); 403 + 404 + if (is_entry_func(state->pc)) 405 + goto end; 406 + 407 + orc = orc_find(state->pc); 408 + if (!orc) { 409 + /* 410 + * As a fallback, try to assume this code uses a frame pointer. 411 + * This is useful for generated code, like BPF, which ORC 412 + * doesn't know about. This is just a guess, so the rest of 413 + * the unwind is no longer considered reliable. 414 + */ 415 + orc = &orc_fp_entry; 416 + state->error = true; 417 + } else { 418 + if (orc->type == ORC_TYPE_UNDEFINED) 419 + goto err; 420 + 421 + if (orc->type == ORC_TYPE_END_OF_STACK) 422 + goto end; 423 + } 424 + 425 + switch (orc->sp_reg) { 426 + case ORC_REG_SP: 427 + if (info->type == STACK_TYPE_IRQ && state->sp == info->end) 428 + orc->type = ORC_TYPE_REGS; 429 + else 430 + state->sp = state->sp + orc->sp_offset; 431 + break; 432 + case ORC_REG_FP: 433 + state->sp = state->fp; 434 + break; 435 + default: 436 + orc_warn("unknown SP base reg %d at %pB\n", orc->sp_reg, (void *)state->pc); 437 + goto err; 438 + } 439 + 440 + switch (orc->fp_reg) { 441 + case ORC_REG_PREV_SP: 442 + p = (unsigned long *)(state->sp + orc->fp_offset); 443 + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) 444 + goto err; 445 + 446 + state->fp = *p; 447 + break; 448 + case ORC_REG_UNDEFINED: 449 + /* Nothing. */ 450 + break; 451 + default: 452 + orc_warn("unknown FP base reg %d at %pB\n", orc->fp_reg, (void *)state->pc); 453 + goto err; 454 + } 455 + 456 + switch (orc->type) { 457 + case ORC_TYPE_CALL: 458 + if (orc->ra_reg == ORC_REG_PREV_SP) { 459 + p = (unsigned long *)(state->sp + orc->ra_offset); 460 + if (!stack_access_ok(state, (unsigned long)p, sizeof(unsigned long))) 461 + goto err; 462 + 463 + pc = unwind_graph_addr(state, *p, state->sp); 464 + pc -= LOONGARCH_INSN_SIZE; 465 + } else if (orc->ra_reg == ORC_REG_UNDEFINED) { 466 + if (!state->ra || state->ra == state->pc) 467 + goto err; 468 + 469 + pc = unwind_graph_addr(state, state->ra, state->sp); 470 + pc -= LOONGARCH_INSN_SIZE; 471 + state->ra = 0; 472 + } else { 473 + orc_warn("unknown ra base reg %d at %pB\n", orc->ra_reg, (void *)state->pc); 474 + goto err; 475 + } 476 + break; 477 + case ORC_TYPE_REGS: 478 + if (info->type == STACK_TYPE_IRQ && state->sp == info->end) 479 + regs = (struct pt_regs *)info->next_sp; 480 + else 481 + regs = (struct pt_regs *)state->sp; 482 + 483 + if (!stack_access_ok(state, (unsigned long)regs, sizeof(*regs))) 484 + goto err; 485 + 486 + if ((info->end == (unsigned long)regs + sizeof(*regs)) && 487 + !regs->regs[3] && !regs->regs[1]) 488 + goto end; 489 + 490 + if (user_mode(regs)) 491 + goto end; 492 + 493 + pc = regs->csr_era; 494 + if (!__kernel_text_address(pc)) 495 + goto err; 496 + 497 + state->sp = regs->regs[3]; 498 + state->ra = regs->regs[1]; 499 + state->fp = regs->regs[22]; 500 + get_stack_info(state->sp, state->task, info); 501 + 502 + break; 503 + default: 504 + orc_warn("unknown .orc_unwind entry type %d at %pB\n", orc->type, (void *)state->pc); 505 + goto err; 506 + } 507 + 508 + state->pc = bt_address(pc); 509 + if (!state->pc) { 510 + pr_err("cannot find unwind pc at %pK\n", (void *)pc); 511 + goto err; 512 + } 513 + 514 + if (!__kernel_text_address(state->pc)) 515 + goto err; 516 + 517 + preempt_enable(); 518 + return true; 519 + 520 + err: 521 + state->error = true; 522 + 523 + end: 524 + preempt_enable(); 525 + state->stack_info.type = STACK_TYPE_UNKNOWN; 526 + return false; 527 + } 528 + EXPORT_SYMBOL_GPL(unwind_next_frame);

+3

arch/loongarch/kernel/vmlinux.lds.S

··· 2 2 #include <linux/sizes.h> 3 3 #include <asm/asm-offsets.h> 4 4 #include <asm/thread_info.h> 5 + #include <asm/orc_lookup.h> 5 6 6 7 #define PAGE_SIZE _PAGE_SIZE 7 8 #define RO_EXCEPTION_TABLE_ALIGN 4 ··· 122 121 __la_abs_end = .; 123 122 } 124 123 #endif 124 + 125 + ORC_UNWIND_TABLE 125 126 126 127 .sdata : { 127 128 *(.sdata)

+8 -1

arch/loongarch/kvm/switch.S

··· 8 8 #include <asm/asmmacro.h> 9 9 #include <asm/loongarch.h> 10 10 #include <asm/regdef.h> 11 - #include <asm/stackframe.h> 11 + #include <asm/unwind_hints.h> 12 12 13 13 #define HGPR_OFFSET(x) (PT_R0 + 8*x) 14 14 #define GGPR_OFFSET(x) (KVM_ARCH_GGPR + 8*x) ··· 112 112 .text 113 113 .cfi_sections .debug_frame 114 114 SYM_CODE_START(kvm_exc_entry) 115 + UNWIND_HINT_UNDEFINED 115 116 csrwr a2, KVM_TEMP_KS 116 117 csrrd a2, KVM_VCPU_KS 117 118 addi.d a2, a2, KVM_VCPU_ARCH ··· 280 279 .section ".rodata" 281 280 SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) 282 281 SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) 282 + 283 + #ifdef CONFIG_CPU_HAS_LBT 284 + STACK_FRAME_NON_STANDARD kvm_restore_fpu 285 + STACK_FRAME_NON_STANDARD kvm_restore_lsx 286 + STACK_FRAME_NON_STANDARD kvm_restore_lasx 287 + #endif

+3

arch/loongarch/lib/clear_user.S

··· 10 10 #include <asm/asm-extable.h> 11 11 #include <asm/cpu.h> 12 12 #include <asm/regdef.h> 13 + #include <asm/unwind_hints.h> 13 14 14 15 SYM_FUNC_START(__clear_user) 15 16 /* ··· 205 204 _asm_extable 28b, .Lsmall_fixup 206 205 _asm_extable 29b, .Lexit 207 206 SYM_FUNC_END(__clear_user_fast) 207 + 208 + STACK_FRAME_NON_STANDARD __clear_user_fast

+3

arch/loongarch/lib/copy_user.S

··· 10 10 #include <asm/asm-extable.h> 11 11 #include <asm/cpu.h> 12 12 #include <asm/regdef.h> 13 + #include <asm/unwind_hints.h> 13 14 14 15 SYM_FUNC_START(__copy_user) 15 16 /* ··· 279 278 _asm_extable 58b, .Lexit 280 279 _asm_extable 59b, .Lexit 281 280 SYM_FUNC_END(__copy_user_fast) 281 + 282 + STACK_FRAME_NON_STANDARD __copy_user_fast

+3

arch/loongarch/lib/memcpy.S

··· 9 9 #include <asm/asmmacro.h> 10 10 #include <asm/cpu.h> 11 11 #include <asm/regdef.h> 12 + #include <asm/unwind_hints.h> 12 13 13 14 .section .noinstr.text, "ax" 14 15 ··· 198 197 jr ra 199 198 SYM_FUNC_END(__memcpy_fast) 200 199 _ASM_NOKPROBE(__memcpy_fast) 200 + 201 + STACK_FRAME_NON_STANDARD __memcpy_small

+3

arch/loongarch/lib/memset.S

··· 9 9 #include <asm/asmmacro.h> 10 10 #include <asm/cpu.h> 11 11 #include <asm/regdef.h> 12 + #include <asm/unwind_hints.h> 12 13 13 14 .macro fill_to_64 r0 14 15 bstrins.d \r0, \r0, 15, 8 ··· 167 166 jr ra 168 167 SYM_FUNC_END(__memset_fast) 169 168 _ASM_NOKPROBE(__memset_fast) 169 + 170 + STACK_FRAME_NON_STANDARD __memset_fast

+12 -15

arch/loongarch/mm/tlb.c

··· 9 9 #include <linux/hugetlb.h> 10 10 #include <linux/export.h> 11 11 12 - #include <asm/cpu.h> 13 12 #include <asm/bootinfo.h> 13 + #include <asm/cpu.h> 14 + #include <asm/exception.h> 14 15 #include <asm/mmu_context.h> 15 16 #include <asm/pgtable.h> 16 17 #include <asm/tlb.h> ··· 267 266 setup_ptwalker(); 268 267 local_flush_tlb_all(); 269 268 269 + if (cpu_has_ptw) { 270 + exception_table[EXCCODE_TLBI] = handle_tlb_load_ptw; 271 + exception_table[EXCCODE_TLBL] = handle_tlb_load_ptw; 272 + exception_table[EXCCODE_TLBS] = handle_tlb_store_ptw; 273 + exception_table[EXCCODE_TLBM] = handle_tlb_modify_ptw; 274 + } 275 + 270 276 /* The tlb handlers are generated only once */ 271 277 if (cpu == 0) { 272 278 memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); 273 279 local_flush_icache_range(tlbrentry, tlbrentry + 0x80); 274 - if (!cpu_has_ptw) { 275 - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); 276 - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); 277 - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); 278 - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); 279 - } else { 280 - set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE); 281 - set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE); 282 - set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE); 283 - set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE); 284 - } 285 - set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); 286 - set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); 287 - set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); 280 + 281 + for (int i = EXCCODE_TLBL; i <= EXCCODE_TLBPE; i++) 282 + set_handler(i * VECSIZE, exception_table[i], VECSIZE); 288 283 } else { 289 284 int vec_sz __maybe_unused; 290 285 void *addr __maybe_unused;

+9

arch/loongarch/mm/tlbex.S

··· 18 18 19 19 .macro tlb_do_page_fault, write 20 20 SYM_CODE_START(tlb_do_page_fault_\write) 21 + UNWIND_HINT_UNDEFINED 21 22 SAVE_ALL 22 23 csrrd a2, LOONGARCH_CSR_BADV 23 24 move a0, sp ··· 33 32 tlb_do_page_fault 1 34 33 35 34 SYM_CODE_START(handle_tlb_protect) 35 + UNWIND_HINT_UNDEFINED 36 36 BACKUP_T0T1 37 37 SAVE_ALL 38 38 move a0, sp ··· 46 44 SYM_CODE_END(handle_tlb_protect) 47 45 48 46 SYM_CODE_START(handle_tlb_load) 47 + UNWIND_HINT_UNDEFINED 49 48 csrwr t0, EXCEPTION_KS0 50 49 csrwr t1, EXCEPTION_KS1 51 50 csrwr ra, EXCEPTION_KS2 ··· 193 190 SYM_CODE_END(handle_tlb_load) 194 191 195 192 SYM_CODE_START(handle_tlb_load_ptw) 193 + UNWIND_HINT_UNDEFINED 196 194 csrwr t0, LOONGARCH_CSR_KS0 197 195 csrwr t1, LOONGARCH_CSR_KS1 198 196 la_abs t0, tlb_do_page_fault_0 ··· 201 197 SYM_CODE_END(handle_tlb_load_ptw) 202 198 203 199 SYM_CODE_START(handle_tlb_store) 200 + UNWIND_HINT_UNDEFINED 204 201 csrwr t0, EXCEPTION_KS0 205 202 csrwr t1, EXCEPTION_KS1 206 203 csrwr ra, EXCEPTION_KS2 ··· 351 346 SYM_CODE_END(handle_tlb_store) 352 347 353 348 SYM_CODE_START(handle_tlb_store_ptw) 349 + UNWIND_HINT_UNDEFINED 354 350 csrwr t0, LOONGARCH_CSR_KS0 355 351 csrwr t1, LOONGARCH_CSR_KS1 356 352 la_abs t0, tlb_do_page_fault_1 ··· 359 353 SYM_CODE_END(handle_tlb_store_ptw) 360 354 361 355 SYM_CODE_START(handle_tlb_modify) 356 + UNWIND_HINT_UNDEFINED 362 357 csrwr t0, EXCEPTION_KS0 363 358 csrwr t1, EXCEPTION_KS1 364 359 csrwr ra, EXCEPTION_KS2 ··· 507 500 SYM_CODE_END(handle_tlb_modify) 508 501 509 502 SYM_CODE_START(handle_tlb_modify_ptw) 503 + UNWIND_HINT_UNDEFINED 510 504 csrwr t0, LOONGARCH_CSR_KS0 511 505 csrwr t1, LOONGARCH_CSR_KS1 512 506 la_abs t0, tlb_do_page_fault_1 ··· 515 507 SYM_CODE_END(handle_tlb_modify_ptw) 516 508 517 509 SYM_CODE_START(handle_tlb_refill) 510 + UNWIND_HINT_UNDEFINED 518 511 csrwr t0, LOONGARCH_CSR_TLBRSAVE 519 512 csrrd t0, LOONGARCH_CSR_PGD 520 513 lddir t0, t0, 3

+1

arch/loongarch/vdso/Makefile

··· 4 4 KASAN_SANITIZE := n 5 5 UBSAN_SANITIZE := n 6 6 KCOV_INSTRUMENT := n 7 + OBJECT_FILES_NON_STANDARD := y 7 8 8 9 # Include the generic Makefile to check the built vdso. 9 10 include $(srctree)/lib/vdso/Makefile

+9

include/linux/compiler.h

··· 116 116 */ 117 117 #define __stringify_label(n) #n 118 118 119 + #define __annotate_reachable(c) ({ \ 120 + asm volatile(__stringify_label(c) ":\n\t" \ 121 + ".pushsection .discard.reachable\n\t" \ 122 + ".long " __stringify_label(c) "b - .\n\t" \ 123 + ".popsection\n\t"); \ 124 + }) 125 + #define annotate_reachable() __annotate_reachable(__COUNTER__) 126 + 119 127 #define __annotate_unreachable(c) ({ \ 120 128 asm volatile(__stringify_label(c) ":\n\t" \ 121 129 ".pushsection .discard.unreachable\n\t" \ ··· 136 128 #define __annotate_jump_table __section(".rodata..c_jump_table") 137 129 138 130 #else /* !CONFIG_OBJTOOL */ 131 + #define annotate_reachable() 139 132 #define annotate_unreachable() 140 133 #define __annotate_jump_table 141 134 #endif /* CONFIG_OBJTOOL */

+5 -2

scripts/Makefile

··· 31 31 32 32 ifdef CONFIG_UNWINDER_ORC 33 33 ifeq ($(ARCH),x86_64) 34 - ARCH := x86 34 + SRCARCH := x86 35 35 endif 36 - HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include 36 + ifeq ($(ARCH),loongarch) 37 + SRCARCH := loongarch 38 + endif 39 + HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/$(SRCARCH)/include 37 40 HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED 38 41 endif 39 42