Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

ARC: __switch_to: move ksp to thread_info from thread_struct

task's arch specific bits are carried in 2 places
- embedded thread_struct in task_struct
- associated thread_info (hoisted in task's stack page) and
syntactically: (thread_info *)(task_struct->stack)

ksp (dynamic kernel stack top) currently lives in thread_struct but
given its deep location in task struct likely to cache miss when
accessed from __switch_to(). Moving it to thread_info would be more
efficient given proximity to frequently accessed items such as
preempt_count thus very likely to be in cache, specially in schedular
code.

Note however that currently tsk.thread.ksp takes 1 memory access (off
of tsk pointer) while new code tsk->stack.ksp would take 2, but likely
to be in cache. Moreover if task is current the 2nd reference can be
elided and instead derived from SP as (SP & ~(THREAD_SIZE - 1))

All of this also makes __switch_to() code simpler and we can see the 2
ways of retirving ksp (descrobed above) in new code.

Signed-off-by: Vineet Gupta <vgupta@kernel.org>

+20 -23
+1 -2
arch/arc/include/asm/processor.h
··· 22 22 * struct thread_info 23 23 */ 24 24 struct thread_struct { 25 - unsigned long ksp; /* kernel mode stack pointer */ 26 25 unsigned long callee_reg; /* pointer to callee regs */ 27 26 unsigned long fault_address; /* dbls as brkpt holder as well */ 28 27 #ifdef CONFIG_ARC_DSP_SAVE_RESTORE_REGS ··· 53 54 * Where about of Task's sp, fp, blink when it was last seen in kernel mode. 54 55 * Look in process.c for details of kernel stack layout 55 56 */ 56 - #define TSK_K_ESP(tsk) (tsk->thread.ksp) 57 + #define TSK_K_ESP(tsk) (task_thread_info(tsk)->ksp) 57 58 58 59 #define TSK_K_REG(tsk, off) (*((unsigned long *)(TSK_K_ESP(tsk) + \ 59 60 sizeof(struct callee_regs) + off)))
+5 -5
arch/arc/include/asm/thread_info.h
··· 37 37 */ 38 38 struct thread_info { 39 39 unsigned long flags; /* low level flags */ 40 + unsigned long ksp; /* kernel mode stack top in __switch_to */ 40 41 int preempt_count; /* 0 => preemptable, <0 => BUG */ 41 - struct task_struct *task; /* main task structure */ 42 - __u32 cpu; /* current CPU */ 42 + int cpu; /* current CPU */ 43 43 unsigned long thr_ptr; /* TLS ptr */ 44 + struct task_struct *task; /* main task structure */ 44 45 }; 45 46 46 47 /* 47 - * macros/functions for gaining access to the thread information structure 48 - * 49 - * preempt_count needs to be 1 initially, until the scheduler is functional. 48 + * initilaize thread_info for any @tsk 49 + * - this is not related to init_task per se 50 50 */ 51 51 #define INIT_THREAD_INFO(tsk) \ 52 52 { \
+1 -1
arch/arc/kernel/asm-offsets.c
··· 20 20 21 21 BLANK(); 22 22 23 - DEFINE(THREAD_KSP, offsetof(struct thread_struct, ksp)); 24 23 DEFINE(THREAD_CALLEE_REG, offsetof(struct thread_struct, callee_reg)); 25 24 DEFINE(THREAD_FAULT_ADDR, 26 25 offsetof(struct thread_struct, fault_address)); 27 26 28 27 BLANK(); 29 28 29 + DEFINE(THREAD_INFO_KSP, offsetof(struct thread_info, ksp)); 30 30 DEFINE(THREAD_INFO_FLAGS, offsetof(struct thread_info, flags)); 31 31 DEFINE(THREAD_INFO_PREEMPT_COUNT, 32 32 offsetof(struct thread_info, preempt_count));
+10 -12
arch/arc/kernel/ctx_sw_asm.S
··· 11 11 #include <asm/entry.h> /* For the SAVE_* macros */ 12 12 #include <asm/asm-offsets.h> 13 13 14 - #define KSP_WORD_OFF ((TASK_THREAD + THREAD_KSP) / 4) 15 - 16 14 ; IN 17 15 ; - r0: prev task (also current) 18 16 ; - r1: next task ··· 35 37 /* kernel mode callee regs of @prev */ 36 38 SAVE_CALLEE_SAVED_KERNEL 37 39 38 - /* save final SP to @prev->thread.ksp */ 39 - #if KSP_WORD_OFF <= 255 40 - st.as sp, [r0, KSP_WORD_OFF] 41 - #else 42 - /* Workaround for NR_CPUS=4k as ST.as can only take s9 offset */ 43 - add2 r10, r0, KSP_WORD_OFF 44 - st sp, [r10] 45 - #endif 40 + /* 41 + * save final SP to @prev->thread_info.ksp 42 + * @prev is "current" so thread_info derived from SP 43 + */ 44 + GET_CURR_THR_INFO_FROM_SP r10 45 + st sp, [r10, THREAD_INFO_KSP] 46 + 46 47 /* update @next in _current_task[] and GP register caching it */ 47 48 SET_CURR_TASK_ON_CPU r1, r10 48 49 49 - /* load SP from @next->thread.ksp */ 50 - ld.as sp, [r1, KSP_WORD_OFF] 50 + /* load SP from @next->thread_info.ksp */ 51 + ld r10, [r1, TASK_THREAD_INFO] 52 + ld sp, [r10, THREAD_INFO_KSP] 51 53 52 54 /* restore callee regs, stack frame regs of @next */ 53 55 RESTORE_CALLEE_SAVED_KERNEL
+3 -3
arch/arc/kernel/process.c
··· 141 141 * | unused | 142 142 * | | 143 143 * ------------------ 144 - * | r25 | <==== top of Stack (thread.ksp) 144 + * | r25 | <==== top of Stack (thread_info.ksp) 145 145 * ~ ~ 146 146 * | --to-- | (CALLEE Regs of kernel mode) 147 147 * | r13 | ··· 181 181 c_callee = ((struct callee_regs *)childksp) - 1; 182 182 183 183 /* 184 - * __switch_to() uses thread.ksp to start unwinding stack 184 + * __switch_to() uses thread_info.ksp to start unwinding stack 185 185 * For kernel threads we don't need to create callee regs, the 186 186 * stack layout nevertheless needs to remain the same. 187 187 * Also, since __switch_to anyways unwinds callee regs, we use 188 188 * this to populate kernel thread entry-pt/args into callee regs, 189 189 * so that ret_from_kernel_thread() becomes simpler. 190 190 */ 191 - p->thread.ksp = (unsigned long)c_callee; /* THREAD_KSP */ 191 + task_thread_info(p)->ksp = (unsigned long)c_callee; /* THREAD_INFO_KSP */ 192 192 193 193 /* __switch_to expects FP(0), BLINK(return addr) at top */ 194 194 childksp[0] = 0; /* fp */