Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
4 *
5 * Copyright (C) 1996-2000 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8#ifndef __ASSEMBLY__
9#error "Only include this from assembly code"
10#endif
11
12#ifndef __ASM_ASSEMBLER_H
13#define __ASM_ASSEMBLER_H
14
15#include <asm-generic/export.h>
16
17#include <asm/asm-offsets.h>
18#include <asm/cpufeature.h>
19#include <asm/cputype.h>
20#include <asm/debug-monitors.h>
21#include <asm/page.h>
22#include <asm/pgtable-hwdef.h>
23#include <asm/ptrace.h>
24#include <asm/thread_info.h>
25
26 .macro save_and_disable_daif, flags
27 mrs \flags, daif
28 msr daifset, #0xf
29 .endm
30
31 .macro disable_daif
32 msr daifset, #0xf
33 .endm
34
35 .macro enable_daif
36 msr daifclr, #0xf
37 .endm
38
39 .macro restore_daif, flags:req
40 msr daif, \flags
41 .endm
42
43 /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
44 .macro inherit_daif, pstate:req, tmp:req
45 and \tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
46 msr daif, \tmp
47 .endm
48
49 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */
50 .macro enable_da_f
51 msr daifclr, #(8 | 4 | 1)
52 .endm
53
54/*
55 * Save/restore interrupts.
56 */
57 .macro save_and_disable_irq, flags
58 mrs \flags, daif
59 msr daifset, #2
60 .endm
61
62 .macro restore_irq, flags
63 msr daif, \flags
64 .endm
65
66 .macro enable_dbg
67 msr daifclr, #8
68 .endm
69
70 .macro disable_step_tsk, flgs, tmp
71 tbz \flgs, #TIF_SINGLESTEP, 9990f
72 mrs \tmp, mdscr_el1
73 bic \tmp, \tmp, #DBG_MDSCR_SS
74 msr mdscr_el1, \tmp
75 isb // Synchronise with enable_dbg
769990:
77 .endm
78
79 /* call with daif masked */
80 .macro enable_step_tsk, flgs, tmp
81 tbz \flgs, #TIF_SINGLESTEP, 9990f
82 mrs \tmp, mdscr_el1
83 orr \tmp, \tmp, #DBG_MDSCR_SS
84 msr mdscr_el1, \tmp
859990:
86 .endm
87
88/*
89 * SMP data memory barrier
90 */
91 .macro smp_dmb, opt
92 dmb \opt
93 .endm
94
95/*
96 * RAS Error Synchronization barrier
97 */
98 .macro esb
99#ifdef CONFIG_ARM64_RAS_EXTN
100 hint #16
101#else
102 nop
103#endif
104 .endm
105
106/*
107 * Value prediction barrier
108 */
109 .macro csdb
110 hint #20
111 .endm
112
113/*
114 * Speculation barrier
115 */
116 .macro sb
117alternative_if_not ARM64_HAS_SB
118 dsb nsh
119 isb
120alternative_else
121 SB_BARRIER_INSN
122 nop
123alternative_endif
124 .endm
125
126/*
127 * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
128 * of bounds.
129 */
130 .macro mask_nospec64, idx, limit, tmp
131 sub \tmp, \idx, \limit
132 bic \tmp, \tmp, \idx
133 and \idx, \idx, \tmp, asr #63
134 csdb
135 .endm
136
137/*
138 * NOP sequence
139 */
140 .macro nops, num
141 .rept \num
142 nop
143 .endr
144 .endm
145
146/*
147 * Emit an entry into the exception table
148 */
149 .macro _asm_extable, from, to
150 .pushsection __ex_table, "a"
151 .align 3
152 .long (\from - .), (\to - .)
153 .popsection
154 .endm
155
156#define USER(l, x...) \
1579999: x; \
158 _asm_extable 9999b, l
159
160/*
161 * Register aliases.
162 */
163lr .req x30 // link register
164
165/*
166 * Vector entry
167 */
168 .macro ventry label
169 .align 7
170 b \label
171 .endm
172
173/*
174 * Select code when configured for BE.
175 */
176#ifdef CONFIG_CPU_BIG_ENDIAN
177#define CPU_BE(code...) code
178#else
179#define CPU_BE(code...)
180#endif
181
182/*
183 * Select code when configured for LE.
184 */
185#ifdef CONFIG_CPU_BIG_ENDIAN
186#define CPU_LE(code...)
187#else
188#define CPU_LE(code...) code
189#endif
190
191/*
192 * Define a macro that constructs a 64-bit value by concatenating two
193 * 32-bit registers. Note that on big endian systems the order of the
194 * registers is swapped.
195 */
196#ifndef CONFIG_CPU_BIG_ENDIAN
197 .macro regs_to_64, rd, lbits, hbits
198#else
199 .macro regs_to_64, rd, hbits, lbits
200#endif
201 orr \rd, \lbits, \hbits, lsl #32
202 .endm
203
204/*
205 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
206 * <symbol> is within the range +/- 4 GB of the PC.
207 */
208 /*
209 * @dst: destination register (64 bit wide)
210 * @sym: name of the symbol
211 */
212 .macro adr_l, dst, sym
213 adrp \dst, \sym
214 add \dst, \dst, :lo12:\sym
215 .endm
216
217 /*
218 * @dst: destination register (32 or 64 bit wide)
219 * @sym: name of the symbol
220 * @tmp: optional 64-bit scratch register to be used if <dst> is a
221 * 32-bit wide register, in which case it cannot be used to hold
222 * the address
223 */
224 .macro ldr_l, dst, sym, tmp=
225 .ifb \tmp
226 adrp \dst, \sym
227 ldr \dst, [\dst, :lo12:\sym]
228 .else
229 adrp \tmp, \sym
230 ldr \dst, [\tmp, :lo12:\sym]
231 .endif
232 .endm
233
234 /*
235 * @src: source register (32 or 64 bit wide)
236 * @sym: name of the symbol
237 * @tmp: mandatory 64-bit scratch register to calculate the address
238 * while <src> needs to be preserved.
239 */
240 .macro str_l, src, sym, tmp
241 adrp \tmp, \sym
242 str \src, [\tmp, :lo12:\sym]
243 .endm
244
245 /*
246 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)
247 * @sym: The name of the per-cpu variable
248 * @tmp: scratch register
249 */
250 .macro adr_this_cpu, dst, sym, tmp
251 adrp \tmp, \sym
252 add \dst, \tmp, #:lo12:\sym
253alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
254 mrs \tmp, tpidr_el1
255alternative_else
256 mrs \tmp, tpidr_el2
257alternative_endif
258 add \dst, \dst, \tmp
259 .endm
260
261 /*
262 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
263 * @sym: The name of the per-cpu variable
264 * @tmp: scratch register
265 */
266 .macro ldr_this_cpu dst, sym, tmp
267 adr_l \dst, \sym
268alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
269 mrs \tmp, tpidr_el1
270alternative_else
271 mrs \tmp, tpidr_el2
272alternative_endif
273 ldr \dst, [\dst, \tmp]
274 .endm
275
276/*
277 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
278 */
279 .macro vma_vm_mm, rd, rn
280 ldr \rd, [\rn, #VMA_VM_MM]
281 .endm
282
283/*
284 * mmid - get context id from mm pointer (mm->context.id)
285 */
286 .macro mmid, rd, rn
287 ldr \rd, [\rn, #MM_CONTEXT_ID]
288 .endm
289/*
290 * read_ctr - read CTR_EL0. If the system has mismatched register fields,
291 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
292 */
293 .macro read_ctr, reg
294alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
295 mrs \reg, ctr_el0 // read CTR
296 nop
297alternative_else
298 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
299alternative_endif
300 .endm
301
302
303/*
304 * raw_dcache_line_size - get the minimum D-cache line size on this CPU
305 * from the CTR register.
306 */
307 .macro raw_dcache_line_size, reg, tmp
308 mrs \tmp, ctr_el0 // read CTR
309 ubfm \tmp, \tmp, #16, #19 // cache line size encoding
310 mov \reg, #4 // bytes per word
311 lsl \reg, \reg, \tmp // actual cache line size
312 .endm
313
314/*
315 * dcache_line_size - get the safe D-cache line size across all CPUs
316 */
317 .macro dcache_line_size, reg, tmp
318 read_ctr \tmp
319 ubfm \tmp, \tmp, #16, #19 // cache line size encoding
320 mov \reg, #4 // bytes per word
321 lsl \reg, \reg, \tmp // actual cache line size
322 .endm
323
324/*
325 * raw_icache_line_size - get the minimum I-cache line size on this CPU
326 * from the CTR register.
327 */
328 .macro raw_icache_line_size, reg, tmp
329 mrs \tmp, ctr_el0 // read CTR
330 and \tmp, \tmp, #0xf // cache line size encoding
331 mov \reg, #4 // bytes per word
332 lsl \reg, \reg, \tmp // actual cache line size
333 .endm
334
335/*
336 * icache_line_size - get the safe I-cache line size across all CPUs
337 */
338 .macro icache_line_size, reg, tmp
339 read_ctr \tmp
340 and \tmp, \tmp, #0xf // cache line size encoding
341 mov \reg, #4 // bytes per word
342 lsl \reg, \reg, \tmp // actual cache line size
343 .endm
344
345/*
346 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
347 */
348 .macro tcr_set_t0sz, valreg, t0sz
349 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
350 .endm
351
352/*
353 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
354 * ID_AA64MMFR0_EL1.PARange value
355 *
356 * tcr: register with the TCR_ELx value to be updated
357 * pos: IPS or PS bitfield position
358 * tmp{0,1}: temporary registers
359 */
360 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1
361 mrs \tmp0, ID_AA64MMFR0_EL1
362 // Narrow PARange to fit the PS field in TCR_ELx
363 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
364 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX
365 cmp \tmp0, \tmp1
366 csel \tmp0, \tmp1, \tmp0, hi
367 bfi \tcr, \tmp0, \pos, #3
368 .endm
369
370/*
371 * Macro to perform a data cache maintenance for the interval
372 * [kaddr, kaddr + size)
373 *
374 * op: operation passed to dc instruction
375 * domain: domain used in dsb instruciton
376 * kaddr: starting virtual address of the region
377 * size: size of the region
378 * Corrupts: kaddr, size, tmp1, tmp2
379 */
380 .macro __dcache_op_workaround_clean_cache, op, kaddr
381alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
382 dc \op, \kaddr
383alternative_else
384 dc civac, \kaddr
385alternative_endif
386 .endm
387
388 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
389 dcache_line_size \tmp1, \tmp2
390 add \size, \kaddr, \size
391 sub \tmp2, \tmp1, #1
392 bic \kaddr, \kaddr, \tmp2
3939998:
394 .ifc \op, cvau
395 __dcache_op_workaround_clean_cache \op, \kaddr
396 .else
397 .ifc \op, cvac
398 __dcache_op_workaround_clean_cache \op, \kaddr
399 .else
400 .ifc \op, cvap
401 sys 3, c7, c12, 1, \kaddr // dc cvap
402 .else
403 .ifc \op, cvadp
404 sys 3, c7, c13, 1, \kaddr // dc cvadp
405 .else
406 dc \op, \kaddr
407 .endif
408 .endif
409 .endif
410 .endif
411 add \kaddr, \kaddr, \tmp1
412 cmp \kaddr, \size
413 b.lo 9998b
414 dsb \domain
415 .endm
416
417/*
418 * Macro to perform an instruction cache maintenance for the interval
419 * [start, end)
420 *
421 * start, end: virtual addresses describing the region
422 * label: A label to branch to on user fault.
423 * Corrupts: tmp1, tmp2
424 */
425 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
426 icache_line_size \tmp1, \tmp2
427 sub \tmp2, \tmp1, #1
428 bic \tmp2, \start, \tmp2
4299997:
430USER(\label, ic ivau, \tmp2) // invalidate I line PoU
431 add \tmp2, \tmp2, \tmp1
432 cmp \tmp2, \end
433 b.lo 9997b
434 dsb ish
435 isb
436 .endm
437
438/*
439 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
440 */
441 .macro reset_pmuserenr_el0, tmpreg
442 mrs \tmpreg, id_aa64dfr0_el1
443 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
444 cmp \tmpreg, #1 // Skip if no PMU present
445 b.lt 9000f
446 msr pmuserenr_el0, xzr // Disable PMU access from EL0
4479000:
448 .endm
449
450/*
451 * copy_page - copy src to dest using temp registers t1-t8
452 */
453 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
4549998: ldp \t1, \t2, [\src]
455 ldp \t3, \t4, [\src, #16]
456 ldp \t5, \t6, [\src, #32]
457 ldp \t7, \t8, [\src, #48]
458 add \src, \src, #64
459 stnp \t1, \t2, [\dest]
460 stnp \t3, \t4, [\dest, #16]
461 stnp \t5, \t6, [\dest, #32]
462 stnp \t7, \t8, [\dest, #48]
463 add \dest, \dest, #64
464 tst \src, #(PAGE_SIZE - 1)
465 b.ne 9998b
466 .endm
467
468/*
469 * Annotate a function as position independent, i.e., safe to be called before
470 * the kernel virtual mapping is activated.
471 */
472#define ENDPIPROC(x) \
473 .globl __pi_##x; \
474 .type __pi_##x, %function; \
475 .set __pi_##x, x; \
476 .size __pi_##x, . - x; \
477 ENDPROC(x)
478
479/*
480 * Annotate a function as being unsuitable for kprobes.
481 */
482#ifdef CONFIG_KPROBES
483#define NOKPROBE(x) \
484 .pushsection "_kprobe_blacklist", "aw"; \
485 .quad x; \
486 .popsection;
487#else
488#define NOKPROBE(x)
489#endif
490
491#ifdef CONFIG_KASAN
492#define EXPORT_SYMBOL_NOKASAN(name)
493#else
494#define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name)
495#endif
496
497 /*
498 * Emit a 64-bit absolute little endian symbol reference in a way that
499 * ensures that it will be resolved at build time, even when building a
500 * PIE binary. This requires cooperation from the linker script, which
501 * must emit the lo32/hi32 halves individually.
502 */
503 .macro le64sym, sym
504 .long \sym\()_lo32
505 .long \sym\()_hi32
506 .endm
507
508 /*
509 * mov_q - move an immediate constant into a 64-bit register using
510 * between 2 and 4 movz/movk instructions (depending on the
511 * magnitude and sign of the operand)
512 */
513 .macro mov_q, reg, val
514 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
515 movz \reg, :abs_g1_s:\val
516 .else
517 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
518 movz \reg, :abs_g2_s:\val
519 .else
520 movz \reg, :abs_g3:\val
521 movk \reg, :abs_g2_nc:\val
522 .endif
523 movk \reg, :abs_g1_nc:\val
524 .endif
525 movk \reg, :abs_g0_nc:\val
526 .endm
527
528/*
529 * Return the current task_struct.
530 */
531 .macro get_current_task, rd
532 mrs \rd, sp_el0
533 .endm
534
535/*
536 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
537 * orr is used as it can cover the immediate value (and is idempotent).
538 * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
539 * ttbr: Value of ttbr to set, modified.
540 */
541 .macro offset_ttbr1, ttbr
542#ifdef CONFIG_ARM64_USER_VA_BITS_52
543 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
544#endif
545 .endm
546
547/*
548 * Perform the reverse of offset_ttbr1.
549 * bic is used as it can cover the immediate value and, in future, won't need
550 * to be nop'ed out when dealing with 52-bit kernel VAs.
551 */
552 .macro restore_ttbr1, ttbr
553#ifdef CONFIG_ARM64_USER_VA_BITS_52
554 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
555#endif
556 .endm
557
558/*
559 * Arrange a physical address in a TTBR register, taking care of 52-bit
560 * addresses.
561 *
562 * phys: physical address, preserved
563 * ttbr: returns the TTBR value
564 */
565 .macro phys_to_ttbr, ttbr, phys
566#ifdef CONFIG_ARM64_PA_BITS_52
567 orr \ttbr, \phys, \phys, lsr #46
568 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52
569#else
570 mov \ttbr, \phys
571#endif
572 .endm
573
574 .macro phys_to_pte, pte, phys
575#ifdef CONFIG_ARM64_PA_BITS_52
576 /*
577 * We assume \phys is 64K aligned and this is guaranteed by only
578 * supporting this configuration with 64K pages.
579 */
580 orr \pte, \phys, \phys, lsr #36
581 and \pte, \pte, #PTE_ADDR_MASK
582#else
583 mov \pte, \phys
584#endif
585 .endm
586
587 .macro pte_to_phys, phys, pte
588#ifdef CONFIG_ARM64_PA_BITS_52
589 ubfiz \phys, \pte, #(48 - 16 - 12), #16
590 bfxil \phys, \pte, #16, #32
591 lsl \phys, \phys, #16
592#else
593 and \phys, \pte, #PTE_ADDR_MASK
594#endif
595 .endm
596
597/*
598 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
599 */
600 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2
601#ifdef CONFIG_FUJITSU_ERRATUM_010001
602 mrs \tmp1, midr_el1
603
604 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
605 and \tmp1, \tmp1, \tmp2
606 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001
607 cmp \tmp1, \tmp2
608 b.ne 10f
609
610 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
611 bic \tcr, \tcr, \tmp2
61210:
613#endif /* CONFIG_FUJITSU_ERRATUM_010001 */
614 .endm
615
616/**
617 * Errata workaround prior to disable MMU. Insert an ISB immediately prior
618 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
619 */
620 .macro pre_disable_mmu_workaround
621#ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
622 isb
623#endif
624 .endm
625
626 /*
627 * frame_push - Push @regcount callee saved registers to the stack,
628 * starting at x19, as well as x29/x30, and set x29 to
629 * the new value of sp. Add @extra bytes of stack space
630 * for locals.
631 */
632 .macro frame_push, regcount:req, extra
633 __frame st, \regcount, \extra
634 .endm
635
636 /*
637 * frame_pop - Pop the callee saved registers from the stack that were
638 * pushed in the most recent call to frame_push, as well
639 * as x29/x30 and any extra stack space that may have been
640 * allocated.
641 */
642 .macro frame_pop
643 __frame ld
644 .endm
645
646 .macro __frame_regs, reg1, reg2, op, num
647 .if .Lframe_regcount == \num
648 \op\()r \reg1, [sp, #(\num + 1) * 8]
649 .elseif .Lframe_regcount > \num
650 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8]
651 .endif
652 .endm
653
654 .macro __frame, op, regcount, extra=0
655 .ifc \op, st
656 .if (\regcount) < 0 || (\regcount) > 10
657 .error "regcount should be in the range [0 ... 10]"
658 .endif
659 .if ((\extra) % 16) != 0
660 .error "extra should be a multiple of 16 bytes"
661 .endif
662 .ifdef .Lframe_regcount
663 .if .Lframe_regcount != -1
664 .error "frame_push/frame_pop may not be nested"
665 .endif
666 .endif
667 .set .Lframe_regcount, \regcount
668 .set .Lframe_extra, \extra
669 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16
670 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
671 mov x29, sp
672 .endif
673
674 __frame_regs x19, x20, \op, 1
675 __frame_regs x21, x22, \op, 3
676 __frame_regs x23, x24, \op, 5
677 __frame_regs x25, x26, \op, 7
678 __frame_regs x27, x28, \op, 9
679
680 .ifc \op, ld
681 .if .Lframe_regcount == -1
682 .error "frame_push/frame_pop may not be nested"
683 .endif
684 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
685 .set .Lframe_regcount, -1
686 .endif
687 .endm
688
689/*
690 * Check whether to yield to another runnable task from kernel mode NEON code
691 * (which runs with preemption disabled).
692 *
693 * if_will_cond_yield_neon
694 * // pre-yield patchup code
695 * do_cond_yield_neon
696 * // post-yield patchup code
697 * endif_yield_neon <label>
698 *
699 * where <label> is optional, and marks the point where execution will resume
700 * after a yield has been performed. If omitted, execution resumes right after
701 * the endif_yield_neon invocation. Note that the entire sequence, including
702 * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT
703 * is not defined.
704 *
705 * As a convenience, in the case where no patchup code is required, the above
706 * sequence may be abbreviated to
707 *
708 * cond_yield_neon <label>
709 *
710 * Note that the patchup code does not support assembler directives that change
711 * the output section, any use of such directives is undefined.
712 *
713 * The yield itself consists of the following:
714 * - Check whether the preempt count is exactly 1 and a reschedule is also
715 * needed. If so, calling of preempt_enable() in kernel_neon_end() will
716 * trigger a reschedule. If it is not the case, yielding is pointless.
717 * - Disable and re-enable kernel mode NEON, and branch to the yield fixup
718 * code.
719 *
720 * This macro sequence may clobber all CPU state that is not guaranteed by the
721 * AAPCS to be preserved across an ordinary function call.
722 */
723
724 .macro cond_yield_neon, lbl
725 if_will_cond_yield_neon
726 do_cond_yield_neon
727 endif_yield_neon \lbl
728 .endm
729
730 .macro if_will_cond_yield_neon
731#ifdef CONFIG_PREEMPT
732 get_current_task x0
733 ldr x0, [x0, #TSK_TI_PREEMPT]
734 sub x0, x0, #PREEMPT_DISABLE_OFFSET
735 cbz x0, .Lyield_\@
736 /* fall through to endif_yield_neon */
737 .subsection 1
738.Lyield_\@ :
739#else
740 .section ".discard.cond_yield_neon", "ax"
741#endif
742 .endm
743
744 .macro do_cond_yield_neon
745 bl kernel_neon_end
746 bl kernel_neon_begin
747 .endm
748
749 .macro endif_yield_neon, lbl
750 .ifnb \lbl
751 b \lbl
752 .else
753 b .Lyield_out_\@
754 .endif
755 .previous
756.Lyield_out_\@ :
757 .endm
758
759#endif /* __ASM_ASSEMBLER_H */