tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * arch/arm/include/asm/assembler.h
4 *
5 * Copyright (C) 1996-2000 Russell King
6 *
7 * This file contains arm architecture specific defines
8 * for the different processors.
9 *
10 * Do not include any C declarations in this file - it is included by
11 * assembler source.
12 */
13#ifndef __ASM_ASSEMBLER_H__
14#define __ASM_ASSEMBLER_H__
15
16#ifndef __ASSEMBLY__
17#error "Only include this from assembly code"
18#endif
19
20#include <asm/ptrace.h>
21#include <asm/opcodes-virt.h>
22#include <asm/asm-offsets.h>
23#include <asm/page.h>
24#include <asm/thread_info.h>
25#include <asm/uaccess-asm.h>
26
27#define IOMEM(x) (x)
28
29/*
30 * Endian independent macros for shifting bytes within registers.
31 */
32#ifndef __ARMEB__
33#define lspull lsr
34#define lspush lsl
35#define get_byte_0 lsl #0
36#define get_byte_1 lsr #8
37#define get_byte_2 lsr #16
38#define get_byte_3 lsr #24
39#define put_byte_0 lsl #0
40#define put_byte_1 lsl #8
41#define put_byte_2 lsl #16
42#define put_byte_3 lsl #24
43#else
44#define lspull lsl
45#define lspush lsr
46#define get_byte_0 lsr #24
47#define get_byte_1 lsr #16
48#define get_byte_2 lsr #8
49#define get_byte_3 lsl #0
50#define put_byte_0 lsl #24
51#define put_byte_1 lsl #16
52#define put_byte_2 lsl #8
53#define put_byte_3 lsl #0
54#endif
55
56/* Select code for any configuration running in BE8 mode */
57#ifdef CONFIG_CPU_ENDIAN_BE8
58#define ARM_BE8(code...) code
59#else
60#define ARM_BE8(code...)
61#endif
62
63/*
64 * Data preload for architectures that support it
65 */
66#if __LINUX_ARM_ARCH__ >= 5
67#define PLD(code...) code
68#else
69#define PLD(code...)
70#endif
71
72/*
73 * This can be used to enable code to cacheline align the destination
74 * pointer when bulk writing to memory. Experiments on StrongARM and
75 * XScale didn't show this a worthwhile thing to do when the cache is not
76 * set to write-allocate (this would need further testing on XScale when WA
77 * is used).
78 *
79 * On Feroceon there is much to gain however, regardless of cache mode.
80 */
81#ifdef CONFIG_CPU_FEROCEON
82#define CALGN(code...) code
83#else
84#define CALGN(code...)
85#endif
86
87#define IMM12_MASK 0xfff
88
89/*
90 * Enable and disable interrupts
91 */
92#if __LINUX_ARM_ARCH__ >= 6
93 .macro disable_irq_notrace
94 cpsid i
95 .endm
96
97 .macro enable_irq_notrace
98 cpsie i
99 .endm
100#else
101 .macro disable_irq_notrace
102 msr cpsr_c, #PSR_I_BIT | SVC_MODE
103 .endm
104
105 .macro enable_irq_notrace
106 msr cpsr_c, #SVC_MODE
107 .endm
108#endif
109
110 .macro asm_trace_hardirqs_off, save=1
111#if defined(CONFIG_TRACE_IRQFLAGS)
112 .if \save
113 stmdb sp!, {r0-r3, ip, lr}
114 .endif
115 bl trace_hardirqs_off
116 .if \save
117 ldmia sp!, {r0-r3, ip, lr}
118 .endif
119#endif
120 .endm
121
122 .macro asm_trace_hardirqs_on, cond=al, save=1
123#if defined(CONFIG_TRACE_IRQFLAGS)
124 /*
125 * actually the registers should be pushed and pop'd conditionally, but
126 * after bl the flags are certainly clobbered
127 */
128 .if \save
129 stmdb sp!, {r0-r3, ip, lr}
130 .endif
131 bl\cond trace_hardirqs_on
132 .if \save
133 ldmia sp!, {r0-r3, ip, lr}
134 .endif
135#endif
136 .endm
137
138 .macro disable_irq, save=1
139 disable_irq_notrace
140 asm_trace_hardirqs_off \save
141 .endm
142
143 .macro enable_irq
144 asm_trace_hardirqs_on
145 enable_irq_notrace
146 .endm
147/*
148 * Save the current IRQ state and disable IRQs. Note that this macro
149 * assumes FIQs are enabled, and that the processor is in SVC mode.
150 */
151 .macro save_and_disable_irqs, oldcpsr
152#ifdef CONFIG_CPU_V7M
153 mrs \oldcpsr, primask
154#else
155 mrs \oldcpsr, cpsr
156#endif
157 disable_irq
158 .endm
159
160 .macro save_and_disable_irqs_notrace, oldcpsr
161#ifdef CONFIG_CPU_V7M
162 mrs \oldcpsr, primask
163#else
164 mrs \oldcpsr, cpsr
165#endif
166 disable_irq_notrace
167 .endm
168
169/*
170 * Restore interrupt state previously stored in a register. We don't
171 * guarantee that this will preserve the flags.
172 */
173 .macro restore_irqs_notrace, oldcpsr
174#ifdef CONFIG_CPU_V7M
175 msr primask, \oldcpsr
176#else
177 msr cpsr_c, \oldcpsr
178#endif
179 .endm
180
181 .macro restore_irqs, oldcpsr
182 tst \oldcpsr, #PSR_I_BIT
183 asm_trace_hardirqs_on cond=eq
184 restore_irqs_notrace \oldcpsr
185 .endm
186
187/*
188 * Assembly version of "adr rd, BSYM(sym)". This should only be used to
189 * reference local symbols in the same assembly file which are to be
190 * resolved by the assembler. Other usage is undefined.
191 */
192 .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
193 .macro badr\c, rd, sym
194#ifdef CONFIG_THUMB2_KERNEL
195 adr\c \rd, \sym + 1
196#else
197 adr\c \rd, \sym
198#endif
199 .endm
200 .endr
201
202 .macro get_current, rd
203#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
204 mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register
205#else
206 get_thread_info \rd
207 ldr \rd, [\rd, #TI_TASK]
208#endif
209 .endm
210
211 .macro set_current, rn
212#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
213 mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register
214#endif
215 .endm
216
217 .macro reload_current, t1:req, t2:req
218#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
219 adr_l \t1, __entry_task @ get __entry_task base address
220 mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset
221 ldr \t1, [\t1, \t2] @ load variable
222 mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
223#endif
224 .endm
225
226/*
227 * Get current thread_info.
228 */
229 .macro get_thread_info, rd
230#ifdef CONFIG_THREAD_INFO_IN_TASK
231 /* thread_info is the first member of struct task_struct */
232 get_current \rd
233#else
234 ARM( mov \rd, sp, lsr #THREAD_SIZE_ORDER + PAGE_SHIFT )
235 THUMB( mov \rd, sp )
236 THUMB( lsr \rd, \rd, #THREAD_SIZE_ORDER + PAGE_SHIFT )
237 mov \rd, \rd, lsl #THREAD_SIZE_ORDER + PAGE_SHIFT
238#endif
239 .endm
240
241/*
242 * Increment/decrement the preempt count.
243 */
244#ifdef CONFIG_PREEMPT_COUNT
245 .macro inc_preempt_count, ti, tmp
246 ldr \tmp, [\ti, #TI_PREEMPT] @ get preempt count
247 add \tmp, \tmp, #1 @ increment it
248 str \tmp, [\ti, #TI_PREEMPT]
249 .endm
250
251 .macro dec_preempt_count, ti, tmp
252 ldr \tmp, [\ti, #TI_PREEMPT] @ get preempt count
253 sub \tmp, \tmp, #1 @ decrement it
254 str \tmp, [\ti, #TI_PREEMPT]
255 .endm
256
257 .macro dec_preempt_count_ti, ti, tmp
258 get_thread_info \ti
259 dec_preempt_count \ti, \tmp
260 .endm
261#else
262 .macro inc_preempt_count, ti, tmp
263 .endm
264
265 .macro dec_preempt_count, ti, tmp
266 .endm
267
268 .macro dec_preempt_count_ti, ti, tmp
269 .endm
270#endif
271
272#define USERL(l, x...) \
2739999: x; \
274 .pushsection __ex_table,"a"; \
275 .align 3; \
276 .long 9999b,l; \
277 .popsection
278
279#define USER(x...) USERL(9001f, x)
280
281#ifdef CONFIG_SMP
282#define ALT_SMP(instr...) \
2839998: instr
284/*
285 * Note: if you get assembler errors from ALT_UP() when building with
286 * CONFIG_THUMB2_KERNEL, you almost certainly need to use
287 * ALT_SMP( W(instr) ... )
288 */
289#define ALT_UP(instr...) \
290 .pushsection ".alt.smp.init", "a" ;\
291 .align 2 ;\
292 .long 9998b - . ;\
2939997: instr ;\
294 .if . - 9997b == 2 ;\
295 nop ;\
296 .endif ;\
297 .if . - 9997b != 4 ;\
298 .error "ALT_UP() content must assemble to exactly 4 bytes";\
299 .endif ;\
300 .popsection
301#define ALT_UP_B(label) \
302 .pushsection ".alt.smp.init", "a" ;\
303 .align 2 ;\
304 .long 9998b - . ;\
305 W(b) . + (label - 9998b) ;\
306 .popsection
307#else
308#define ALT_SMP(instr...)
309#define ALT_UP(instr...) instr
310#define ALT_UP_B(label) b label
311#endif
312
313/*
314 * Instruction barrier
315 */
316 .macro instr_sync
317#if __LINUX_ARM_ARCH__ >= 7
318 isb
319#elif __LINUX_ARM_ARCH__ == 6
320 mcr p15, 0, r0, c7, c5, 4
321#endif
322 .endm
323
324/*
325 * SMP data memory barrier
326 */
327 .macro smp_dmb mode
328#ifdef CONFIG_SMP
329#if __LINUX_ARM_ARCH__ >= 7
330 .ifeqs "\mode","arm"
331 ALT_SMP(dmb ish)
332 .else
333 ALT_SMP(W(dmb) ish)
334 .endif
335#elif __LINUX_ARM_ARCH__ == 6
336 ALT_SMP(mcr p15, 0, r0, c7, c10, 5) @ dmb
337#else
338#error Incompatible SMP platform
339#endif
340 .ifeqs "\mode","arm"
341 ALT_UP(nop)
342 .else
343 ALT_UP(W(nop))
344 .endif
345#endif
346 .endm
347
348#if defined(CONFIG_CPU_V7M)
349 /*
350 * setmode is used to assert to be in svc mode during boot. For v7-M
351 * this is done in __v7m_setup, so setmode can be empty here.
352 */
353 .macro setmode, mode, reg
354 .endm
355#elif defined(CONFIG_THUMB2_KERNEL)
356 .macro setmode, mode, reg
357 mov \reg, #\mode
358 msr cpsr_c, \reg
359 .endm
360#else
361 .macro setmode, mode, reg
362 msr cpsr_c, #\mode
363 .endm
364#endif
365
366/*
367 * Helper macro to enter SVC mode cleanly and mask interrupts. reg is
368 * a scratch register for the macro to overwrite.
369 *
370 * This macro is intended for forcing the CPU into SVC mode at boot time.
371 * you cannot return to the original mode.
372 */
373.macro safe_svcmode_maskall reg:req
374#if __LINUX_ARM_ARCH__ >= 6 && !defined(CONFIG_CPU_V7M)
375 mrs \reg , cpsr
376 eor \reg, \reg, #HYP_MODE
377 tst \reg, #MODE_MASK
378 bic \reg , \reg , #MODE_MASK
379 orr \reg , \reg , #PSR_I_BIT | PSR_F_BIT | SVC_MODE
380THUMB( orr \reg , \reg , #PSR_T_BIT )
381 bne 1f
382 orr \reg, \reg, #PSR_A_BIT
383 badr lr, 2f
384 msr spsr_cxsf, \reg
385 __MSR_ELR_HYP(14)
386 __ERET
3871: msr cpsr_c, \reg
3882:
389#else
390/*
391 * workaround for possibly broken pre-v6 hardware
392 * (akita, Sharp Zaurus C-1000, PXA270-based)
393 */
394 setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, \reg
395#endif
396.endm
397
398/*
399 * STRT/LDRT access macros with ARM and Thumb-2 variants
400 */
401#ifdef CONFIG_THUMB2_KERNEL
402
403 .macro usraccoff, instr, reg, ptr, inc, off, cond, abort, t=TUSER()
4049999:
405 .if \inc == 1
406 \instr\()b\t\cond\().w \reg, [\ptr, #\off]
407 .elseif \inc == 4
408 \instr\t\cond\().w \reg, [\ptr, #\off]
409 .else
410 .error "Unsupported inc macro argument"
411 .endif
412
413 .pushsection __ex_table,"a"
414 .align 3
415 .long 9999b, \abort
416 .popsection
417 .endm
418
419 .macro usracc, instr, reg, ptr, inc, cond, rept, abort
420 @ explicit IT instruction needed because of the label
421 @ introduced by the USER macro
422 .ifnc \cond,al
423 .if \rept == 1
424 itt \cond
425 .elseif \rept == 2
426 ittt \cond
427 .else
428 .error "Unsupported rept macro argument"
429 .endif
430 .endif
431
432 @ Slightly optimised to avoid incrementing the pointer twice
433 usraccoff \instr, \reg, \ptr, \inc, 0, \cond, \abort
434 .if \rept == 2
435 usraccoff \instr, \reg, \ptr, \inc, \inc, \cond, \abort
436 .endif
437
438 add\cond \ptr, #\rept * \inc
439 .endm
440
441#else /* !CONFIG_THUMB2_KERNEL */
442
443 .macro usracc, instr, reg, ptr, inc, cond, rept, abort, t=TUSER()
444 .rept \rept
4459999:
446 .if \inc == 1
447 \instr\()b\t\cond \reg, [\ptr], #\inc
448 .elseif \inc == 4
449 \instr\t\cond \reg, [\ptr], #\inc
450 .else
451 .error "Unsupported inc macro argument"
452 .endif
453
454 .pushsection __ex_table,"a"
455 .align 3
456 .long 9999b, \abort
457 .popsection
458 .endr
459 .endm
460
461#endif /* CONFIG_THUMB2_KERNEL */
462
463 .macro strusr, reg, ptr, inc, cond=al, rept=1, abort=9001f
464 usracc str, \reg, \ptr, \inc, \cond, \rept, \abort
465 .endm
466
467 .macro ldrusr, reg, ptr, inc, cond=al, rept=1, abort=9001f
468 usracc ldr, \reg, \ptr, \inc, \cond, \rept, \abort
469 .endm
470
471/* Utility macro for declaring string literals */
472 .macro string name:req, string
473 .type \name , #object
474\name:
475 .asciz "\string"
476 .size \name , . - \name
477 .endm
478
479 .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo
480 .macro ret\c, reg
481#if __LINUX_ARM_ARCH__ < 6
482 mov\c pc, \reg
483#else
484 .ifeqs "\reg", "lr"
485 bx\c \reg
486 .else
487 mov\c pc, \reg
488 .endif
489#endif
490 .endm
491 .endr
492
493 .macro ret.w, reg
494 ret \reg
495#ifdef CONFIG_THUMB2_KERNEL
496 nop
497#endif
498 .endm
499
500 .macro bug, msg, line
501#ifdef CONFIG_THUMB2_KERNEL
5021: .inst 0xde02
503#else
5041: .inst 0xe7f001f2
505#endif
506#ifdef CONFIG_DEBUG_BUGVERBOSE
507 .pushsection .rodata.str, "aMS", %progbits, 1
5082: .asciz "\msg"
509 .popsection
510 .pushsection __bug_table, "aw"
511 .align 2
512 .word 1b, 2b
513 .hword \line
514 .popsection
515#endif
516 .endm
517
518#ifdef CONFIG_KPROBES
519#define _ASM_NOKPROBE(entry) \
520 .pushsection "_kprobe_blacklist", "aw" ; \
521 .balign 4 ; \
522 .long entry; \
523 .popsection
524#else
525#define _ASM_NOKPROBE(entry)
526#endif
527
528 .macro __adldst_l, op, reg, sym, tmp, c
529 .if __LINUX_ARM_ARCH__ < 7
530 ldr\c \tmp, .La\@
531 .subsection 1
532 .align 2
533.La\@: .long \sym - .Lpc\@
534 .previous
535 .else
536 .ifnb \c
537 THUMB( ittt \c )
538 .endif
539 movw\c \tmp, #:lower16:\sym - .Lpc\@
540 movt\c \tmp, #:upper16:\sym - .Lpc\@
541 .endif
542
543#ifndef CONFIG_THUMB2_KERNEL
544 .set .Lpc\@, . + 8 // PC bias
545 .ifc \op, add
546 add\c \reg, \tmp, pc
547 .else
548 \op\c \reg, [pc, \tmp]
549 .endif
550#else
551.Lb\@: add\c \tmp, \tmp, pc
552 /*
553 * In Thumb-2 builds, the PC bias depends on whether we are currently
554 * emitting into a .arm or a .thumb section. The size of the add opcode
555 * above will be 2 bytes when emitting in Thumb mode and 4 bytes when
556 * emitting in ARM mode, so let's use this to account for the bias.
557 */
558 .set .Lpc\@, . + (. - .Lb\@)
559
560 .ifnc \op, add
561 \op\c \reg, [\tmp]
562 .endif
563#endif
564 .endm
565
566 /*
567 * mov_l - move a constant value or [relocated] address into a register
568 */
569 .macro mov_l, dst:req, imm:req
570 .if __LINUX_ARM_ARCH__ < 7
571 ldr \dst, =\imm
572 .else
573 movw \dst, #:lower16:\imm
574 movt \dst, #:upper16:\imm
575 .endif
576 .endm
577
578 /*
579 * adr_l - adr pseudo-op with unlimited range
580 *
581 * @dst: destination register
582 * @sym: name of the symbol
583 * @cond: conditional opcode suffix
584 */
585 .macro adr_l, dst:req, sym:req, cond
586 __adldst_l add, \dst, \sym, \dst, \cond
587 .endm
588
589 /*
590 * ldr_l - ldr <literal> pseudo-op with unlimited range
591 *
592 * @dst: destination register
593 * @sym: name of the symbol
594 * @cond: conditional opcode suffix
595 */
596 .macro ldr_l, dst:req, sym:req, cond
597 __adldst_l ldr, \dst, \sym, \dst, \cond
598 .endm
599
600 /*
601 * str_l - str <literal> pseudo-op with unlimited range
602 *
603 * @src: source register
604 * @sym: name of the symbol
605 * @tmp: mandatory scratch register
606 * @cond: conditional opcode suffix
607 */
608 .macro str_l, src:req, sym:req, tmp:req, cond
609 __adldst_l str, \src, \sym, \tmp, \cond
610 .endm
611
612 /*
613 * rev_l - byte-swap a 32-bit value
614 *
615 * @val: source/destination register
616 * @tmp: scratch register
617 */
618 .macro rev_l, val:req, tmp:req
619 .if __LINUX_ARM_ARCH__ < 6
620 eor \tmp, \val, \val, ror #16
621 bic \tmp, \tmp, #0x00ff0000
622 mov \val, \val, ror #8
623 eor \val, \val, \tmp, lsr #8
624 .else
625 rev \val, \val
626 .endif
627 .endm
628
629#endif /* __ASM_ASSEMBLER_H__ */