ARCv2: entry: rewrite to enable use of double load/stores LDD/STD

+164 -181

arch/arc/include/asm/entry-arcv2.h

··· 46 46 */ 47 47 48 48 /*------------------------------------------------------------------------*/ 49 - .macro INTERRUPT_PROLOGUE called_from 49 + .macro INTERRUPT_PROLOGUE 50 + 50 51 ; (A) Before jumping to Interrupt Vector, hardware micro-ops did following: 51 52 ; 1. SP auto-switched to kernel mode stack 52 53 ; 2. STATUS32.Z flag set if in U mode at time of interrupt (U:1,K:0) ··· 58 57 ; (B) Manually saved some regs: r12,r25,r30, sp,fp,gp, ACCL pair 59 58 60 59 #ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE 61 - .ifnc \called_from, exception 62 - st.as r9, [sp, -10] ; save r9 in it's final stack slot 63 - sub sp, sp, 12 ; skip JLI, LDI, EI 60 + ; carve pt_regs on stack (case #3), PC/STAT32 already on stack 61 + sub sp, sp, SZ_PT_REGS - 8 64 62 65 - PUSH lp_count 66 - PUSHAX lp_start 67 - PUSHAX lp_end 68 - PUSH blink 69 - 70 - PUSH r11 71 - PUSH r10 72 - 73 - sub sp, sp, 4 ; skip r9 74 - 75 - PUSH r8 76 - PUSH r7 77 - PUSH r6 78 - PUSH r5 79 - PUSH r4 80 - PUSH r3 81 - PUSH r2 82 - PUSH r1 83 - PUSH r0 84 - .endif 85 - #endif 86 - 87 - #ifdef CONFIG_ARC_HAS_ACCL_REGS 88 - PUSH r59 89 - PUSH r58 90 - #endif 91 - 92 - PUSH r30 93 - PUSH r12 94 - 95 - ; Saving pt_regs->sp correctly requires some extra work due to the way 96 - ; Auto stack switch works 97 - ; - U mode: retrieve it from AUX_USER_SP 98 - ; - K mode: add the offset from current SP where H/w starts auto push 99 - ; 100 - ; 1. Utilize the fact that Z bit is set if Intr taken in U mode 101 - ; 2. Upon entry SP is always saved (for any inspection, unwinding etc), 102 - ; but on return, restored only if U mode 103 - 104 - lr r9, [AUX_USER_SP] ; U mode SP 105 - 106 - mov.nz r9, sp 107 - add.nz r9, r9, SZ_PT_REGS - PT_sp - 4 ; K mode SP 108 - 109 - PUSH r9 ; SP (pt_regs->sp) 110 - 111 - PUSH fp 112 - PUSH gp 113 - 114 - #ifdef CONFIG_ARC_CURR_IN_REG 115 - PUSH r25 ; user_r25 116 - GET_CURR_TASK_ON_CPU r25 63 + __SAVE_REGFILE_HARD 117 64 #else 118 - sub sp, sp, 4 65 + ; carve pt_regs on stack (case #4), which grew partially already 66 + sub sp, sp, PT_r0 119 67 #endif 120 68 121 - .ifnc \called_from, exception 122 - sub sp, sp, 12 ; BTA/ECR/orig_r0 placeholder per pt_regs 123 - .endif 124 - 125 - .endm 126 - 127 - /*------------------------------------------------------------------------*/ 128 - .macro INTERRUPT_EPILOGUE called_from 129 - 130 - ; INPUT: r0 has STAT32 of calling context 131 - ; INPUT: Z flag set if returning to K mode 132 - .ifnc \called_from, exception 133 - add sp, sp, 12 ; skip BTA/ECR/orig_r0 placeholderss 134 - .endif 135 - 136 - #ifdef CONFIG_ARC_CURR_IN_REG 137 - POP r25 138 - #else 139 - add sp, sp, 4 140 - #endif 141 - 142 - POP gp 143 - POP fp 144 - 145 - ; Restore SP (into AUX_USER_SP) only if returning to U mode 146 - ; - for K mode, it will be implicitly restored as stack is unwound 147 - ; - Z flag set on K is inverse of what hardware does on interrupt entry 148 - ; but that doesn't really matter 149 - bz 1f 150 - 151 - POPAX AUX_USER_SP 152 - 1: 153 - POP r12 154 - POP r30 155 - 156 - #ifdef CONFIG_ARC_HAS_ACCL_REGS 157 - POP r58 158 - POP r59 159 - #endif 160 - 161 - #ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE 162 - .ifnc \called_from, exception 163 - POP r0 164 - POP r1 165 - POP r2 166 - POP r3 167 - POP r4 168 - POP r5 169 - POP r6 170 - POP r7 171 - POP r8 172 - POP r9 173 - POP r10 174 - POP r11 175 - 176 - POP blink 177 - POPAX lp_end 178 - POPAX lp_start 179 - 180 - POP r9 181 - mov lp_count, r9 182 - 183 - add sp, sp, 12 ; skip JLI, LDI, EI 184 - ld.as r9, [sp, -10] ; reload r9 which got clobbered 185 - .endif 186 - #endif 187 - 69 + __SAVE_REGFILE_SOFT 188 70 .endm 189 71 190 72 /*------------------------------------------------------------------------*/ ··· 79 195 ; 80 196 ; (B) Manually save the complete reg file below 81 197 82 - PUSH r9 ; freeup a register: slot of erstatus 198 + sub sp, sp, SZ_PT_REGS ; carve pt_regs 83 199 84 - PUSHAX eret 85 - sub sp, sp, 12 ; skip JLI, LDI, EI 86 - PUSH lp_count 87 - PUSHAX lp_start 88 - PUSHAX lp_end 89 - PUSH blink 200 + ; _HARD saves r10 clobbered by _SOFT as scratch hence comes first 90 201 91 - PUSH r11 92 - PUSH r10 202 + __SAVE_REGFILE_HARD 203 + __SAVE_REGFILE_SOFT 93 204 94 - ld.as r9, [sp, 10] ; load stashed r9 (status32 stack slot) 95 - lr r10, [erstatus] 96 - st.as r10, [sp, 10] ; save status32 at it's right stack slot 205 + st r0, [sp] ; orig_r0 97 206 98 - PUSH r9 99 - PUSH r8 100 - PUSH r7 101 - PUSH r6 102 - PUSH r5 103 - PUSH r4 104 - PUSH r3 105 - PUSH r2 106 - PUSH r1 107 - PUSH r0 207 + lr r10, [eret] 208 + lr r11, [erstatus] 209 + ST2 r10, r11, PT_ret 108 210 109 - ; -- for interrupts, regs above are auto-saved by h/w in that order -- 110 - ; Now do what ISR prologue does (manually save r12, sp, fp, gp, r25) 211 + lr r10, [ecr] 212 + lr r11, [erbta] 213 + ST2 r10, r11, PT_event 214 + mov r9, r10 111 215 112 - INTERRUPT_PROLOGUE exception 113 - 114 - PUSHAX erbta 115 - PUSHAX ecr ; r9 contains ECR, expected by EV_Trap 116 - 117 - PUSH r0 ; orig_r0 118 216 ; OUTPUT: r9 has ECR 217 + .endm 218 + 219 + /*------------------------------------------------------------------------ 220 + * This macro saves the registers manually which would normally be autosaved 221 + * by hardware on taken interrupts. It is used by 222 + * - exception handlers (which don't have autosave) 223 + * - interrupt autosave disabled due to CONFIG_ARC_IRQ_NO_AUTOSAVE 224 + */ 225 + .macro __SAVE_REGFILE_HARD 226 + 227 + ST2 r0, r1, PT_r0 228 + ST2 r2, r3, PT_r2 229 + ST2 r4, r5, PT_r4 230 + ST2 r6, r7, PT_r6 231 + ST2 r8, r9, PT_r8 232 + ST2 r10, r11, PT_r10 233 + 234 + st blink, [sp, PT_blink] 235 + 236 + lr r10, [lp_end] 237 + lr r11, [lp_start] 238 + ST2 r10, r11, PT_lpe 239 + 240 + st lp_count, [sp, PT_lpc] 241 + 242 + ; skip JLI, LDI, EI for now 243 + .endm 244 + 245 + /*------------------------------------------------------------------------ 246 + * This macros saves a bunch of other registers which can't be autosaved for 247 + * various reasons: 248 + * - r12: the last caller saved scratch reg since hardware saves in pairs so r0-r11 249 + * - r30: free reg, used by gcc as scratch 250 + * - ACCL/ACCH pair when they exist 251 + */ 252 + .macro __SAVE_REGFILE_SOFT 253 + 254 + ST2 gp, fp, PT_r26 ; gp (r26), fp (r27) 255 + 256 + st r12, [sp, PT_sp + 4] 257 + st r30, [sp, PT_sp + 8] 258 + 259 + ; Saving pt_regs->sp correctly requires some extra work due to the way 260 + ; Auto stack switch works 261 + ; - U mode: retrieve it from AUX_USER_SP 262 + ; - K mode: add the offset from current SP where H/w starts auto push 263 + ; 264 + ; 1. Utilize the fact that Z bit is set if Intr taken in U mode 265 + ; 2. Upon entry SP is always saved (for any inspection, unwinding etc), 266 + ; but on return, restored only if U mode 267 + 268 + lr r10, [AUX_USER_SP] ; U mode SP 269 + 270 + ; ISA requires ADD.nz to have same dest and src reg operands 271 + mov.nz r10, sp 272 + add.nz r10, r10, SZ_PT_REGS ; K mode SP 273 + 274 + st r10, [sp, PT_sp] ; SP (pt_regs->sp) 275 + 276 + #ifdef CONFIG_ARC_CURR_IN_REG 277 + st r25, [sp, PT_user_r25] 278 + GET_CURR_TASK_ON_CPU r25 279 + #endif 280 + 281 + #ifdef CONFIG_ARC_HAS_ACCL_REGS 282 + ST2 r58, r59, PT_sp + 12 283 + #endif 284 + 285 + .endm 286 + 287 + /*------------------------------------------------------------------------*/ 288 + .macro __RESTORE_REGFILE_SOFT 289 + 290 + LD2 gp, fp, PT_r26 ; gp (r26), fp (r27) 291 + 292 + ld r12, [sp, PT_sp + 4] 293 + ld r30, [sp, PT_sp + 8] 294 + 295 + ; Restore SP (into AUX_USER_SP) only if returning to U mode 296 + ; - for K mode, it will be implicitly restored as stack is unwound 297 + ; - Z flag set on K is inverse of what hardware does on interrupt entry 298 + ; but that doesn't really matter 299 + bz 1f 300 + 301 + ld r10, [sp, PT_sp] ; SP (pt_regs->sp) 302 + sr r10, [AUX_USER_SP] 303 + 1: 304 + 305 + #ifdef CONFIG_ARC_CURR_IN_REG 306 + ld r25, [sp, PT_user_r25] 307 + #endif 308 + 309 + #ifdef CONFIG_ARC_HAS_ACCL_REGS 310 + LD2 r58, r59, PT_sp + 12 311 + #endif 312 + .endm 313 + 314 + /*------------------------------------------------------------------------*/ 315 + .macro __RESTORE_REGFILE_HARD 316 + 317 + ld blink, [sp, PT_blink] 318 + 319 + LD2 r10, r11, PT_lpe 320 + sr r10, [lp_end] 321 + sr r11, [lp_start] 322 + 323 + ld r10, [sp, PT_lpc] ; lp_count can't be target of LD 324 + mov lp_count, r10 325 + 326 + LD2 r0, r1, PT_r0 327 + LD2 r2, r3, PT_r2 328 + LD2 r4, r5, PT_r4 329 + LD2 r6, r7, PT_r6 330 + LD2 r8, r9, PT_r8 331 + LD2 r10, r11, PT_r10 332 + .endm 333 + 334 + 335 + /*------------------------------------------------------------------------*/ 336 + .macro INTERRUPT_EPILOGUE 337 + 338 + ; INPUT: r0 has STAT32 of calling context 339 + ; INPUT: Z flag set if returning to K mode 340 + 341 + ; _SOFT clobbers r10 restored by _HARD hence the order 342 + 343 + __RESTORE_REGFILE_SOFT 344 + 345 + #ifdef CONFIG_ARC_IRQ_NO_AUTOSAVE 346 + __RESTORE_REGFILE_HARD 347 + add sp, sp, SZ_PT_REGS - 8 348 + #else 349 + add sp, sp, PT_r0 350 + #endif 351 + 119 352 .endm 120 353 121 354 /*------------------------------------------------------------------------*/ 122 355 .macro EXCEPTION_EPILOGUE 123 356 124 357 ; INPUT: r0 has STAT32 of calling context 125 - btst r0, STATUS_U_BIT ; Z flag set if K, used in INTERRUPT_EPILOGUE 126 358 127 - add sp, sp, 8 ; orig_r0/ECR don't need restoring 128 - POPAX erbta 359 + btst r0, STATUS_U_BIT ; Z flag set if K, used in restoring SP 129 360 130 - INTERRUPT_EPILOGUE exception 361 + ld r10, [sp, PT_event + 4] 362 + sr r10, [erbta] 131 363 132 - POP r0 133 - POP r1 134 - POP r2 135 - POP r3 136 - POP r4 137 - POP r5 138 - POP r6 139 - POP r7 140 - POP r8 141 - POP r9 142 - POP r10 143 - POP r11 364 + LD2 r10, r11, PT_ret 365 + sr r10, [eret] 366 + sr r11, [erstatus] 144 367 145 - POP blink 146 - POPAX lp_end 147 - POPAX lp_start 368 + __RESTORE_REGFILE_SOFT 369 + __RESTORE_REGFILE_HARD 148 370 149 - POP r9 150 - mov lp_count, r9 151 - 152 - add sp, sp, 12 ; skip JLI, LDI, EI 153 - POPAX eret 154 - POPAX erstatus 155 - 156 - ld.as r9, [sp, -12] ; reload r9 which got clobbered 371 + add sp, sp, SZ_PT_REGS 157 372 .endm 158 373 159 374 .macro FAKE_RET_FROM_EXCPN

+18

arch/arc/include/asm/linkage.h

··· 10 10 11 11 #ifdef __ASSEMBLY__ 12 12 13 + .macro ST2 e, o, off 14 + #ifdef CONFIG_ARC_HAS_LL64 15 + std \e, [sp, \off] 16 + #else 17 + st \e, [sp, \off] 18 + st \o, [sp, \off+4] 19 + #endif 20 + .endm 21 + 22 + .macro LD2 e, o, off 23 + #ifdef CONFIG_ARC_HAS_LL64 24 + ldd \e, [sp, \off] 25 + #else 26 + ld \e, [sp, \off] 27 + ld \o, [sp, \off+4] 28 + #endif 29 + .endm 30 + 13 31 #define ASM_NL ` /* use '`' to mark new line in macro */ 14 32 15 33 /* annotation for data we want in DCCM - if enabled in .config */

+7

arch/arc/kernel/asm-offsets.c

··· 55 55 DEFINE(PT_r5, offsetof(struct pt_regs, r5)); 56 56 DEFINE(PT_r6, offsetof(struct pt_regs, r6)); 57 57 DEFINE(PT_r7, offsetof(struct pt_regs, r7)); 58 + DEFINE(PT_r8, offsetof(struct pt_regs, r8)); 59 + DEFINE(PT_r10, offsetof(struct pt_regs, r10)); 60 + DEFINE(PT_r26, offsetof(struct pt_regs, r26)); 58 61 DEFINE(PT_ret, offsetof(struct pt_regs, ret)); 62 + DEFINE(PT_blink, offsetof(struct pt_regs, blink)); 63 + DEFINE(PT_lpe, offsetof(struct pt_regs, lp_end)); 64 + DEFINE(PT_lpc, offsetof(struct pt_regs, lp_count)); 65 + DEFINE(PT_user_r25, offsetof(struct pt_regs, user_r25)); 59 66 60 67 DEFINE(SZ_CALLEE_REGS, sizeof(struct callee_regs)); 61 68 DEFINE(SZ_PT_REGS, sizeof(struct pt_regs));

+2 -2

arch/arc/kernel/entry-arcv2.S

··· 67 67 68 68 ENTRY(handle_interrupt) 69 69 70 - INTERRUPT_PROLOGUE irq 70 + INTERRUPT_PROLOGUE 71 71 72 72 # irq control APIs local_irq_save/restore/disable/enable fiddle with 73 73 # global interrupt enable bits in STATUS32 (.IE for 1 prio, .E[] for 2 prio) ··· 223 223 bset.nz r11, r11, AUX_IRQ_ACT_BIT_U ; NZ means U 224 224 sr r11, [AUX_IRQ_ACT] 225 225 226 - INTERRUPT_EPILOGUE irq 226 + INTERRUPT_EPILOGUE 227 227 rtie 228 228 229 229 ;####### Return from Exception / pure kernel mode #######