Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-instrumented.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6/*
7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * To use this functionality an arch's atomic.h file needs to define all
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * this file at the end. This file provides atomic_read() that forwards to
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
16 */
17#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18#define _LINUX_ATOMIC_INSTRUMENTED_H
19
20#include <linux/build_bug.h>
21#include <linux/compiler.h>
22#include <linux/instrumented.h>
23
24static __always_inline int
25atomic_read(const atomic_t *v)
26{
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
29}
30
31static __always_inline int
32atomic_read_acquire(const atomic_t *v)
33{
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
36}
37
38static __always_inline void
39atomic_set(atomic_t *v, int i)
40{
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
43}
44
45static __always_inline void
46atomic_set_release(atomic_t *v, int i)
47{
48 kcsan_release();
49 instrument_atomic_write(v, sizeof(*v));
50 arch_atomic_set_release(v, i);
51}
52
53static __always_inline void
54atomic_add(int i, atomic_t *v)
55{
56 instrument_atomic_read_write(v, sizeof(*v));
57 arch_atomic_add(i, v);
58}
59
60static __always_inline int
61atomic_add_return(int i, atomic_t *v)
62{
63 kcsan_mb();
64 instrument_atomic_read_write(v, sizeof(*v));
65 return arch_atomic_add_return(i, v);
66}
67
68static __always_inline int
69atomic_add_return_acquire(int i, atomic_t *v)
70{
71 instrument_atomic_read_write(v, sizeof(*v));
72 return arch_atomic_add_return_acquire(i, v);
73}
74
75static __always_inline int
76atomic_add_return_release(int i, atomic_t *v)
77{
78 kcsan_release();
79 instrument_atomic_read_write(v, sizeof(*v));
80 return arch_atomic_add_return_release(i, v);
81}
82
83static __always_inline int
84atomic_add_return_relaxed(int i, atomic_t *v)
85{
86 instrument_atomic_read_write(v, sizeof(*v));
87 return arch_atomic_add_return_relaxed(i, v);
88}
89
90static __always_inline int
91atomic_fetch_add(int i, atomic_t *v)
92{
93 kcsan_mb();
94 instrument_atomic_read_write(v, sizeof(*v));
95 return arch_atomic_fetch_add(i, v);
96}
97
98static __always_inline int
99atomic_fetch_add_acquire(int i, atomic_t *v)
100{
101 instrument_atomic_read_write(v, sizeof(*v));
102 return arch_atomic_fetch_add_acquire(i, v);
103}
104
105static __always_inline int
106atomic_fetch_add_release(int i, atomic_t *v)
107{
108 kcsan_release();
109 instrument_atomic_read_write(v, sizeof(*v));
110 return arch_atomic_fetch_add_release(i, v);
111}
112
113static __always_inline int
114atomic_fetch_add_relaxed(int i, atomic_t *v)
115{
116 instrument_atomic_read_write(v, sizeof(*v));
117 return arch_atomic_fetch_add_relaxed(i, v);
118}
119
120static __always_inline void
121atomic_sub(int i, atomic_t *v)
122{
123 instrument_atomic_read_write(v, sizeof(*v));
124 arch_atomic_sub(i, v);
125}
126
127static __always_inline int
128atomic_sub_return(int i, atomic_t *v)
129{
130 kcsan_mb();
131 instrument_atomic_read_write(v, sizeof(*v));
132 return arch_atomic_sub_return(i, v);
133}
134
135static __always_inline int
136atomic_sub_return_acquire(int i, atomic_t *v)
137{
138 instrument_atomic_read_write(v, sizeof(*v));
139 return arch_atomic_sub_return_acquire(i, v);
140}
141
142static __always_inline int
143atomic_sub_return_release(int i, atomic_t *v)
144{
145 kcsan_release();
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_release(i, v);
148}
149
150static __always_inline int
151atomic_sub_return_relaxed(int i, atomic_t *v)
152{
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_sub_return_relaxed(i, v);
155}
156
157static __always_inline int
158atomic_fetch_sub(int i, atomic_t *v)
159{
160 kcsan_mb();
161 instrument_atomic_read_write(v, sizeof(*v));
162 return arch_atomic_fetch_sub(i, v);
163}
164
165static __always_inline int
166atomic_fetch_sub_acquire(int i, atomic_t *v)
167{
168 instrument_atomic_read_write(v, sizeof(*v));
169 return arch_atomic_fetch_sub_acquire(i, v);
170}
171
172static __always_inline int
173atomic_fetch_sub_release(int i, atomic_t *v)
174{
175 kcsan_release();
176 instrument_atomic_read_write(v, sizeof(*v));
177 return arch_atomic_fetch_sub_release(i, v);
178}
179
180static __always_inline int
181atomic_fetch_sub_relaxed(int i, atomic_t *v)
182{
183 instrument_atomic_read_write(v, sizeof(*v));
184 return arch_atomic_fetch_sub_relaxed(i, v);
185}
186
187static __always_inline void
188atomic_inc(atomic_t *v)
189{
190 instrument_atomic_read_write(v, sizeof(*v));
191 arch_atomic_inc(v);
192}
193
194static __always_inline int
195atomic_inc_return(atomic_t *v)
196{
197 kcsan_mb();
198 instrument_atomic_read_write(v, sizeof(*v));
199 return arch_atomic_inc_return(v);
200}
201
202static __always_inline int
203atomic_inc_return_acquire(atomic_t *v)
204{
205 instrument_atomic_read_write(v, sizeof(*v));
206 return arch_atomic_inc_return_acquire(v);
207}
208
209static __always_inline int
210atomic_inc_return_release(atomic_t *v)
211{
212 kcsan_release();
213 instrument_atomic_read_write(v, sizeof(*v));
214 return arch_atomic_inc_return_release(v);
215}
216
217static __always_inline int
218atomic_inc_return_relaxed(atomic_t *v)
219{
220 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_inc_return_relaxed(v);
222}
223
224static __always_inline int
225atomic_fetch_inc(atomic_t *v)
226{
227 kcsan_mb();
228 instrument_atomic_read_write(v, sizeof(*v));
229 return arch_atomic_fetch_inc(v);
230}
231
232static __always_inline int
233atomic_fetch_inc_acquire(atomic_t *v)
234{
235 instrument_atomic_read_write(v, sizeof(*v));
236 return arch_atomic_fetch_inc_acquire(v);
237}
238
239static __always_inline int
240atomic_fetch_inc_release(atomic_t *v)
241{
242 kcsan_release();
243 instrument_atomic_read_write(v, sizeof(*v));
244 return arch_atomic_fetch_inc_release(v);
245}
246
247static __always_inline int
248atomic_fetch_inc_relaxed(atomic_t *v)
249{
250 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_fetch_inc_relaxed(v);
252}
253
254static __always_inline void
255atomic_dec(atomic_t *v)
256{
257 instrument_atomic_read_write(v, sizeof(*v));
258 arch_atomic_dec(v);
259}
260
261static __always_inline int
262atomic_dec_return(atomic_t *v)
263{
264 kcsan_mb();
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return(v);
267}
268
269static __always_inline int
270atomic_dec_return_acquire(atomic_t *v)
271{
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_acquire(v);
274}
275
276static __always_inline int
277atomic_dec_return_release(atomic_t *v)
278{
279 kcsan_release();
280 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_dec_return_release(v);
282}
283
284static __always_inline int
285atomic_dec_return_relaxed(atomic_t *v)
286{
287 instrument_atomic_read_write(v, sizeof(*v));
288 return arch_atomic_dec_return_relaxed(v);
289}
290
291static __always_inline int
292atomic_fetch_dec(atomic_t *v)
293{
294 kcsan_mb();
295 instrument_atomic_read_write(v, sizeof(*v));
296 return arch_atomic_fetch_dec(v);
297}
298
299static __always_inline int
300atomic_fetch_dec_acquire(atomic_t *v)
301{
302 instrument_atomic_read_write(v, sizeof(*v));
303 return arch_atomic_fetch_dec_acquire(v);
304}
305
306static __always_inline int
307atomic_fetch_dec_release(atomic_t *v)
308{
309 kcsan_release();
310 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_dec_release(v);
312}
313
314static __always_inline int
315atomic_fetch_dec_relaxed(atomic_t *v)
316{
317 instrument_atomic_read_write(v, sizeof(*v));
318 return arch_atomic_fetch_dec_relaxed(v);
319}
320
321static __always_inline void
322atomic_and(int i, atomic_t *v)
323{
324 instrument_atomic_read_write(v, sizeof(*v));
325 arch_atomic_and(i, v);
326}
327
328static __always_inline int
329atomic_fetch_and(int i, atomic_t *v)
330{
331 kcsan_mb();
332 instrument_atomic_read_write(v, sizeof(*v));
333 return arch_atomic_fetch_and(i, v);
334}
335
336static __always_inline int
337atomic_fetch_and_acquire(int i, atomic_t *v)
338{
339 instrument_atomic_read_write(v, sizeof(*v));
340 return arch_atomic_fetch_and_acquire(i, v);
341}
342
343static __always_inline int
344atomic_fetch_and_release(int i, atomic_t *v)
345{
346 kcsan_release();
347 instrument_atomic_read_write(v, sizeof(*v));
348 return arch_atomic_fetch_and_release(i, v);
349}
350
351static __always_inline int
352atomic_fetch_and_relaxed(int i, atomic_t *v)
353{
354 instrument_atomic_read_write(v, sizeof(*v));
355 return arch_atomic_fetch_and_relaxed(i, v);
356}
357
358static __always_inline void
359atomic_andnot(int i, atomic_t *v)
360{
361 instrument_atomic_read_write(v, sizeof(*v));
362 arch_atomic_andnot(i, v);
363}
364
365static __always_inline int
366atomic_fetch_andnot(int i, atomic_t *v)
367{
368 kcsan_mb();
369 instrument_atomic_read_write(v, sizeof(*v));
370 return arch_atomic_fetch_andnot(i, v);
371}
372
373static __always_inline int
374atomic_fetch_andnot_acquire(int i, atomic_t *v)
375{
376 instrument_atomic_read_write(v, sizeof(*v));
377 return arch_atomic_fetch_andnot_acquire(i, v);
378}
379
380static __always_inline int
381atomic_fetch_andnot_release(int i, atomic_t *v)
382{
383 kcsan_release();
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_andnot_release(i, v);
386}
387
388static __always_inline int
389atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390{
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_andnot_relaxed(i, v);
393}
394
395static __always_inline void
396atomic_or(int i, atomic_t *v)
397{
398 instrument_atomic_read_write(v, sizeof(*v));
399 arch_atomic_or(i, v);
400}
401
402static __always_inline int
403atomic_fetch_or(int i, atomic_t *v)
404{
405 kcsan_mb();
406 instrument_atomic_read_write(v, sizeof(*v));
407 return arch_atomic_fetch_or(i, v);
408}
409
410static __always_inline int
411atomic_fetch_or_acquire(int i, atomic_t *v)
412{
413 instrument_atomic_read_write(v, sizeof(*v));
414 return arch_atomic_fetch_or_acquire(i, v);
415}
416
417static __always_inline int
418atomic_fetch_or_release(int i, atomic_t *v)
419{
420 kcsan_release();
421 instrument_atomic_read_write(v, sizeof(*v));
422 return arch_atomic_fetch_or_release(i, v);
423}
424
425static __always_inline int
426atomic_fetch_or_relaxed(int i, atomic_t *v)
427{
428 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_or_relaxed(i, v);
430}
431
432static __always_inline void
433atomic_xor(int i, atomic_t *v)
434{
435 instrument_atomic_read_write(v, sizeof(*v));
436 arch_atomic_xor(i, v);
437}
438
439static __always_inline int
440atomic_fetch_xor(int i, atomic_t *v)
441{
442 kcsan_mb();
443 instrument_atomic_read_write(v, sizeof(*v));
444 return arch_atomic_fetch_xor(i, v);
445}
446
447static __always_inline int
448atomic_fetch_xor_acquire(int i, atomic_t *v)
449{
450 instrument_atomic_read_write(v, sizeof(*v));
451 return arch_atomic_fetch_xor_acquire(i, v);
452}
453
454static __always_inline int
455atomic_fetch_xor_release(int i, atomic_t *v)
456{
457 kcsan_release();
458 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_xor_release(i, v);
460}
461
462static __always_inline int
463atomic_fetch_xor_relaxed(int i, atomic_t *v)
464{
465 instrument_atomic_read_write(v, sizeof(*v));
466 return arch_atomic_fetch_xor_relaxed(i, v);
467}
468
469static __always_inline int
470atomic_xchg(atomic_t *v, int i)
471{
472 kcsan_mb();
473 instrument_atomic_read_write(v, sizeof(*v));
474 return arch_atomic_xchg(v, i);
475}
476
477static __always_inline int
478atomic_xchg_acquire(atomic_t *v, int i)
479{
480 instrument_atomic_read_write(v, sizeof(*v));
481 return arch_atomic_xchg_acquire(v, i);
482}
483
484static __always_inline int
485atomic_xchg_release(atomic_t *v, int i)
486{
487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_xchg_release(v, i);
490}
491
492static __always_inline int
493atomic_xchg_relaxed(atomic_t *v, int i)
494{
495 instrument_atomic_read_write(v, sizeof(*v));
496 return arch_atomic_xchg_relaxed(v, i);
497}
498
499static __always_inline int
500atomic_cmpxchg(atomic_t *v, int old, int new)
501{
502 kcsan_mb();
503 instrument_atomic_read_write(v, sizeof(*v));
504 return arch_atomic_cmpxchg(v, old, new);
505}
506
507static __always_inline int
508atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509{
510 instrument_atomic_read_write(v, sizeof(*v));
511 return arch_atomic_cmpxchg_acquire(v, old, new);
512}
513
514static __always_inline int
515atomic_cmpxchg_release(atomic_t *v, int old, int new)
516{
517 kcsan_release();
518 instrument_atomic_read_write(v, sizeof(*v));
519 return arch_atomic_cmpxchg_release(v, old, new);
520}
521
522static __always_inline int
523atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524{
525 instrument_atomic_read_write(v, sizeof(*v));
526 return arch_atomic_cmpxchg_relaxed(v, old, new);
527}
528
529static __always_inline bool
530atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531{
532 kcsan_mb();
533 instrument_atomic_read_write(v, sizeof(*v));
534 instrument_atomic_read_write(old, sizeof(*old));
535 return arch_atomic_try_cmpxchg(v, old, new);
536}
537
538static __always_inline bool
539atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540{
541 instrument_atomic_read_write(v, sizeof(*v));
542 instrument_atomic_read_write(old, sizeof(*old));
543 return arch_atomic_try_cmpxchg_acquire(v, old, new);
544}
545
546static __always_inline bool
547atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548{
549 kcsan_release();
550 instrument_atomic_read_write(v, sizeof(*v));
551 instrument_atomic_read_write(old, sizeof(*old));
552 return arch_atomic_try_cmpxchg_release(v, old, new);
553}
554
555static __always_inline bool
556atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557{
558 instrument_atomic_read_write(v, sizeof(*v));
559 instrument_atomic_read_write(old, sizeof(*old));
560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561}
562
563static __always_inline bool
564atomic_sub_and_test(int i, atomic_t *v)
565{
566 kcsan_mb();
567 instrument_atomic_read_write(v, sizeof(*v));
568 return arch_atomic_sub_and_test(i, v);
569}
570
571static __always_inline bool
572atomic_dec_and_test(atomic_t *v)
573{
574 kcsan_mb();
575 instrument_atomic_read_write(v, sizeof(*v));
576 return arch_atomic_dec_and_test(v);
577}
578
579static __always_inline bool
580atomic_inc_and_test(atomic_t *v)
581{
582 kcsan_mb();
583 instrument_atomic_read_write(v, sizeof(*v));
584 return arch_atomic_inc_and_test(v);
585}
586
587static __always_inline bool
588atomic_add_negative(int i, atomic_t *v)
589{
590 kcsan_mb();
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_add_negative(i, v);
593}
594
595static __always_inline bool
596atomic_add_negative_acquire(int i, atomic_t *v)
597{
598 instrument_atomic_read_write(v, sizeof(*v));
599 return arch_atomic_add_negative_acquire(i, v);
600}
601
602static __always_inline bool
603atomic_add_negative_release(int i, atomic_t *v)
604{
605 kcsan_release();
606 instrument_atomic_read_write(v, sizeof(*v));
607 return arch_atomic_add_negative_release(i, v);
608}
609
610static __always_inline bool
611atomic_add_negative_relaxed(int i, atomic_t *v)
612{
613 instrument_atomic_read_write(v, sizeof(*v));
614 return arch_atomic_add_negative_relaxed(i, v);
615}
616
617static __always_inline int
618atomic_fetch_add_unless(atomic_t *v, int a, int u)
619{
620 kcsan_mb();
621 instrument_atomic_read_write(v, sizeof(*v));
622 return arch_atomic_fetch_add_unless(v, a, u);
623}
624
625static __always_inline bool
626atomic_add_unless(atomic_t *v, int a, int u)
627{
628 kcsan_mb();
629 instrument_atomic_read_write(v, sizeof(*v));
630 return arch_atomic_add_unless(v, a, u);
631}
632
633static __always_inline bool
634atomic_inc_not_zero(atomic_t *v)
635{
636 kcsan_mb();
637 instrument_atomic_read_write(v, sizeof(*v));
638 return arch_atomic_inc_not_zero(v);
639}
640
641static __always_inline bool
642atomic_inc_unless_negative(atomic_t *v)
643{
644 kcsan_mb();
645 instrument_atomic_read_write(v, sizeof(*v));
646 return arch_atomic_inc_unless_negative(v);
647}
648
649static __always_inline bool
650atomic_dec_unless_positive(atomic_t *v)
651{
652 kcsan_mb();
653 instrument_atomic_read_write(v, sizeof(*v));
654 return arch_atomic_dec_unless_positive(v);
655}
656
657static __always_inline int
658atomic_dec_if_positive(atomic_t *v)
659{
660 kcsan_mb();
661 instrument_atomic_read_write(v, sizeof(*v));
662 return arch_atomic_dec_if_positive(v);
663}
664
665static __always_inline s64
666atomic64_read(const atomic64_t *v)
667{
668 instrument_atomic_read(v, sizeof(*v));
669 return arch_atomic64_read(v);
670}
671
672static __always_inline s64
673atomic64_read_acquire(const atomic64_t *v)
674{
675 instrument_atomic_read(v, sizeof(*v));
676 return arch_atomic64_read_acquire(v);
677}
678
679static __always_inline void
680atomic64_set(atomic64_t *v, s64 i)
681{
682 instrument_atomic_write(v, sizeof(*v));
683 arch_atomic64_set(v, i);
684}
685
686static __always_inline void
687atomic64_set_release(atomic64_t *v, s64 i)
688{
689 kcsan_release();
690 instrument_atomic_write(v, sizeof(*v));
691 arch_atomic64_set_release(v, i);
692}
693
694static __always_inline void
695atomic64_add(s64 i, atomic64_t *v)
696{
697 instrument_atomic_read_write(v, sizeof(*v));
698 arch_atomic64_add(i, v);
699}
700
701static __always_inline s64
702atomic64_add_return(s64 i, atomic64_t *v)
703{
704 kcsan_mb();
705 instrument_atomic_read_write(v, sizeof(*v));
706 return arch_atomic64_add_return(i, v);
707}
708
709static __always_inline s64
710atomic64_add_return_acquire(s64 i, atomic64_t *v)
711{
712 instrument_atomic_read_write(v, sizeof(*v));
713 return arch_atomic64_add_return_acquire(i, v);
714}
715
716static __always_inline s64
717atomic64_add_return_release(s64 i, atomic64_t *v)
718{
719 kcsan_release();
720 instrument_atomic_read_write(v, sizeof(*v));
721 return arch_atomic64_add_return_release(i, v);
722}
723
724static __always_inline s64
725atomic64_add_return_relaxed(s64 i, atomic64_t *v)
726{
727 instrument_atomic_read_write(v, sizeof(*v));
728 return arch_atomic64_add_return_relaxed(i, v);
729}
730
731static __always_inline s64
732atomic64_fetch_add(s64 i, atomic64_t *v)
733{
734 kcsan_mb();
735 instrument_atomic_read_write(v, sizeof(*v));
736 return arch_atomic64_fetch_add(i, v);
737}
738
739static __always_inline s64
740atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
741{
742 instrument_atomic_read_write(v, sizeof(*v));
743 return arch_atomic64_fetch_add_acquire(i, v);
744}
745
746static __always_inline s64
747atomic64_fetch_add_release(s64 i, atomic64_t *v)
748{
749 kcsan_release();
750 instrument_atomic_read_write(v, sizeof(*v));
751 return arch_atomic64_fetch_add_release(i, v);
752}
753
754static __always_inline s64
755atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
756{
757 instrument_atomic_read_write(v, sizeof(*v));
758 return arch_atomic64_fetch_add_relaxed(i, v);
759}
760
761static __always_inline void
762atomic64_sub(s64 i, atomic64_t *v)
763{
764 instrument_atomic_read_write(v, sizeof(*v));
765 arch_atomic64_sub(i, v);
766}
767
768static __always_inline s64
769atomic64_sub_return(s64 i, atomic64_t *v)
770{
771 kcsan_mb();
772 instrument_atomic_read_write(v, sizeof(*v));
773 return arch_atomic64_sub_return(i, v);
774}
775
776static __always_inline s64
777atomic64_sub_return_acquire(s64 i, atomic64_t *v)
778{
779 instrument_atomic_read_write(v, sizeof(*v));
780 return arch_atomic64_sub_return_acquire(i, v);
781}
782
783static __always_inline s64
784atomic64_sub_return_release(s64 i, atomic64_t *v)
785{
786 kcsan_release();
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_sub_return_release(i, v);
789}
790
791static __always_inline s64
792atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
793{
794 instrument_atomic_read_write(v, sizeof(*v));
795 return arch_atomic64_sub_return_relaxed(i, v);
796}
797
798static __always_inline s64
799atomic64_fetch_sub(s64 i, atomic64_t *v)
800{
801 kcsan_mb();
802 instrument_atomic_read_write(v, sizeof(*v));
803 return arch_atomic64_fetch_sub(i, v);
804}
805
806static __always_inline s64
807atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
808{
809 instrument_atomic_read_write(v, sizeof(*v));
810 return arch_atomic64_fetch_sub_acquire(i, v);
811}
812
813static __always_inline s64
814atomic64_fetch_sub_release(s64 i, atomic64_t *v)
815{
816 kcsan_release();
817 instrument_atomic_read_write(v, sizeof(*v));
818 return arch_atomic64_fetch_sub_release(i, v);
819}
820
821static __always_inline s64
822atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
823{
824 instrument_atomic_read_write(v, sizeof(*v));
825 return arch_atomic64_fetch_sub_relaxed(i, v);
826}
827
828static __always_inline void
829atomic64_inc(atomic64_t *v)
830{
831 instrument_atomic_read_write(v, sizeof(*v));
832 arch_atomic64_inc(v);
833}
834
835static __always_inline s64
836atomic64_inc_return(atomic64_t *v)
837{
838 kcsan_mb();
839 instrument_atomic_read_write(v, sizeof(*v));
840 return arch_atomic64_inc_return(v);
841}
842
843static __always_inline s64
844atomic64_inc_return_acquire(atomic64_t *v)
845{
846 instrument_atomic_read_write(v, sizeof(*v));
847 return arch_atomic64_inc_return_acquire(v);
848}
849
850static __always_inline s64
851atomic64_inc_return_release(atomic64_t *v)
852{
853 kcsan_release();
854 instrument_atomic_read_write(v, sizeof(*v));
855 return arch_atomic64_inc_return_release(v);
856}
857
858static __always_inline s64
859atomic64_inc_return_relaxed(atomic64_t *v)
860{
861 instrument_atomic_read_write(v, sizeof(*v));
862 return arch_atomic64_inc_return_relaxed(v);
863}
864
865static __always_inline s64
866atomic64_fetch_inc(atomic64_t *v)
867{
868 kcsan_mb();
869 instrument_atomic_read_write(v, sizeof(*v));
870 return arch_atomic64_fetch_inc(v);
871}
872
873static __always_inline s64
874atomic64_fetch_inc_acquire(atomic64_t *v)
875{
876 instrument_atomic_read_write(v, sizeof(*v));
877 return arch_atomic64_fetch_inc_acquire(v);
878}
879
880static __always_inline s64
881atomic64_fetch_inc_release(atomic64_t *v)
882{
883 kcsan_release();
884 instrument_atomic_read_write(v, sizeof(*v));
885 return arch_atomic64_fetch_inc_release(v);
886}
887
888static __always_inline s64
889atomic64_fetch_inc_relaxed(atomic64_t *v)
890{
891 instrument_atomic_read_write(v, sizeof(*v));
892 return arch_atomic64_fetch_inc_relaxed(v);
893}
894
895static __always_inline void
896atomic64_dec(atomic64_t *v)
897{
898 instrument_atomic_read_write(v, sizeof(*v));
899 arch_atomic64_dec(v);
900}
901
902static __always_inline s64
903atomic64_dec_return(atomic64_t *v)
904{
905 kcsan_mb();
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_dec_return(v);
908}
909
910static __always_inline s64
911atomic64_dec_return_acquire(atomic64_t *v)
912{
913 instrument_atomic_read_write(v, sizeof(*v));
914 return arch_atomic64_dec_return_acquire(v);
915}
916
917static __always_inline s64
918atomic64_dec_return_release(atomic64_t *v)
919{
920 kcsan_release();
921 instrument_atomic_read_write(v, sizeof(*v));
922 return arch_atomic64_dec_return_release(v);
923}
924
925static __always_inline s64
926atomic64_dec_return_relaxed(atomic64_t *v)
927{
928 instrument_atomic_read_write(v, sizeof(*v));
929 return arch_atomic64_dec_return_relaxed(v);
930}
931
932static __always_inline s64
933atomic64_fetch_dec(atomic64_t *v)
934{
935 kcsan_mb();
936 instrument_atomic_read_write(v, sizeof(*v));
937 return arch_atomic64_fetch_dec(v);
938}
939
940static __always_inline s64
941atomic64_fetch_dec_acquire(atomic64_t *v)
942{
943 instrument_atomic_read_write(v, sizeof(*v));
944 return arch_atomic64_fetch_dec_acquire(v);
945}
946
947static __always_inline s64
948atomic64_fetch_dec_release(atomic64_t *v)
949{
950 kcsan_release();
951 instrument_atomic_read_write(v, sizeof(*v));
952 return arch_atomic64_fetch_dec_release(v);
953}
954
955static __always_inline s64
956atomic64_fetch_dec_relaxed(atomic64_t *v)
957{
958 instrument_atomic_read_write(v, sizeof(*v));
959 return arch_atomic64_fetch_dec_relaxed(v);
960}
961
962static __always_inline void
963atomic64_and(s64 i, atomic64_t *v)
964{
965 instrument_atomic_read_write(v, sizeof(*v));
966 arch_atomic64_and(i, v);
967}
968
969static __always_inline s64
970atomic64_fetch_and(s64 i, atomic64_t *v)
971{
972 kcsan_mb();
973 instrument_atomic_read_write(v, sizeof(*v));
974 return arch_atomic64_fetch_and(i, v);
975}
976
977static __always_inline s64
978atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
979{
980 instrument_atomic_read_write(v, sizeof(*v));
981 return arch_atomic64_fetch_and_acquire(i, v);
982}
983
984static __always_inline s64
985atomic64_fetch_and_release(s64 i, atomic64_t *v)
986{
987 kcsan_release();
988 instrument_atomic_read_write(v, sizeof(*v));
989 return arch_atomic64_fetch_and_release(i, v);
990}
991
992static __always_inline s64
993atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
994{
995 instrument_atomic_read_write(v, sizeof(*v));
996 return arch_atomic64_fetch_and_relaxed(i, v);
997}
998
999static __always_inline void
1000atomic64_andnot(s64 i, atomic64_t *v)
1001{
1002 instrument_atomic_read_write(v, sizeof(*v));
1003 arch_atomic64_andnot(i, v);
1004}
1005
1006static __always_inline s64
1007atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008{
1009 kcsan_mb();
1010 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_andnot(i, v);
1012}
1013
1014static __always_inline s64
1015atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016{
1017 instrument_atomic_read_write(v, sizeof(*v));
1018 return arch_atomic64_fetch_andnot_acquire(i, v);
1019}
1020
1021static __always_inline s64
1022atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023{
1024 kcsan_release();
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_fetch_andnot_release(i, v);
1027}
1028
1029static __always_inline s64
1030atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031{
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_fetch_andnot_relaxed(i, v);
1034}
1035
1036static __always_inline void
1037atomic64_or(s64 i, atomic64_t *v)
1038{
1039 instrument_atomic_read_write(v, sizeof(*v));
1040 arch_atomic64_or(i, v);
1041}
1042
1043static __always_inline s64
1044atomic64_fetch_or(s64 i, atomic64_t *v)
1045{
1046 kcsan_mb();
1047 instrument_atomic_read_write(v, sizeof(*v));
1048 return arch_atomic64_fetch_or(i, v);
1049}
1050
1051static __always_inline s64
1052atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053{
1054 instrument_atomic_read_write(v, sizeof(*v));
1055 return arch_atomic64_fetch_or_acquire(i, v);
1056}
1057
1058static __always_inline s64
1059atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060{
1061 kcsan_release();
1062 instrument_atomic_read_write(v, sizeof(*v));
1063 return arch_atomic64_fetch_or_release(i, v);
1064}
1065
1066static __always_inline s64
1067atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068{
1069 instrument_atomic_read_write(v, sizeof(*v));
1070 return arch_atomic64_fetch_or_relaxed(i, v);
1071}
1072
1073static __always_inline void
1074atomic64_xor(s64 i, atomic64_t *v)
1075{
1076 instrument_atomic_read_write(v, sizeof(*v));
1077 arch_atomic64_xor(i, v);
1078}
1079
1080static __always_inline s64
1081atomic64_fetch_xor(s64 i, atomic64_t *v)
1082{
1083 kcsan_mb();
1084 instrument_atomic_read_write(v, sizeof(*v));
1085 return arch_atomic64_fetch_xor(i, v);
1086}
1087
1088static __always_inline s64
1089atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090{
1091 instrument_atomic_read_write(v, sizeof(*v));
1092 return arch_atomic64_fetch_xor_acquire(i, v);
1093}
1094
1095static __always_inline s64
1096atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097{
1098 kcsan_release();
1099 instrument_atomic_read_write(v, sizeof(*v));
1100 return arch_atomic64_fetch_xor_release(i, v);
1101}
1102
1103static __always_inline s64
1104atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105{
1106 instrument_atomic_read_write(v, sizeof(*v));
1107 return arch_atomic64_fetch_xor_relaxed(i, v);
1108}
1109
1110static __always_inline s64
1111atomic64_xchg(atomic64_t *v, s64 i)
1112{
1113 kcsan_mb();
1114 instrument_atomic_read_write(v, sizeof(*v));
1115 return arch_atomic64_xchg(v, i);
1116}
1117
1118static __always_inline s64
1119atomic64_xchg_acquire(atomic64_t *v, s64 i)
1120{
1121 instrument_atomic_read_write(v, sizeof(*v));
1122 return arch_atomic64_xchg_acquire(v, i);
1123}
1124
1125static __always_inline s64
1126atomic64_xchg_release(atomic64_t *v, s64 i)
1127{
1128 kcsan_release();
1129 instrument_atomic_read_write(v, sizeof(*v));
1130 return arch_atomic64_xchg_release(v, i);
1131}
1132
1133static __always_inline s64
1134atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1135{
1136 instrument_atomic_read_write(v, sizeof(*v));
1137 return arch_atomic64_xchg_relaxed(v, i);
1138}
1139
1140static __always_inline s64
1141atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142{
1143 kcsan_mb();
1144 instrument_atomic_read_write(v, sizeof(*v));
1145 return arch_atomic64_cmpxchg(v, old, new);
1146}
1147
1148static __always_inline s64
1149atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150{
1151 instrument_atomic_read_write(v, sizeof(*v));
1152 return arch_atomic64_cmpxchg_acquire(v, old, new);
1153}
1154
1155static __always_inline s64
1156atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157{
1158 kcsan_release();
1159 instrument_atomic_read_write(v, sizeof(*v));
1160 return arch_atomic64_cmpxchg_release(v, old, new);
1161}
1162
1163static __always_inline s64
1164atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165{
1166 instrument_atomic_read_write(v, sizeof(*v));
1167 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1168}
1169
1170static __always_inline bool
1171atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1172{
1173 kcsan_mb();
1174 instrument_atomic_read_write(v, sizeof(*v));
1175 instrument_atomic_read_write(old, sizeof(*old));
1176 return arch_atomic64_try_cmpxchg(v, old, new);
1177}
1178
1179static __always_inline bool
1180atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1181{
1182 instrument_atomic_read_write(v, sizeof(*v));
1183 instrument_atomic_read_write(old, sizeof(*old));
1184 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1185}
1186
1187static __always_inline bool
1188atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1189{
1190 kcsan_release();
1191 instrument_atomic_read_write(v, sizeof(*v));
1192 instrument_atomic_read_write(old, sizeof(*old));
1193 return arch_atomic64_try_cmpxchg_release(v, old, new);
1194}
1195
1196static __always_inline bool
1197atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1198{
1199 instrument_atomic_read_write(v, sizeof(*v));
1200 instrument_atomic_read_write(old, sizeof(*old));
1201 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1202}
1203
1204static __always_inline bool
1205atomic64_sub_and_test(s64 i, atomic64_t *v)
1206{
1207 kcsan_mb();
1208 instrument_atomic_read_write(v, sizeof(*v));
1209 return arch_atomic64_sub_and_test(i, v);
1210}
1211
1212static __always_inline bool
1213atomic64_dec_and_test(atomic64_t *v)
1214{
1215 kcsan_mb();
1216 instrument_atomic_read_write(v, sizeof(*v));
1217 return arch_atomic64_dec_and_test(v);
1218}
1219
1220static __always_inline bool
1221atomic64_inc_and_test(atomic64_t *v)
1222{
1223 kcsan_mb();
1224 instrument_atomic_read_write(v, sizeof(*v));
1225 return arch_atomic64_inc_and_test(v);
1226}
1227
1228static __always_inline bool
1229atomic64_add_negative(s64 i, atomic64_t *v)
1230{
1231 kcsan_mb();
1232 instrument_atomic_read_write(v, sizeof(*v));
1233 return arch_atomic64_add_negative(i, v);
1234}
1235
1236static __always_inline bool
1237atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238{
1239 instrument_atomic_read_write(v, sizeof(*v));
1240 return arch_atomic64_add_negative_acquire(i, v);
1241}
1242
1243static __always_inline bool
1244atomic64_add_negative_release(s64 i, atomic64_t *v)
1245{
1246 kcsan_release();
1247 instrument_atomic_read_write(v, sizeof(*v));
1248 return arch_atomic64_add_negative_release(i, v);
1249}
1250
1251static __always_inline bool
1252atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253{
1254 instrument_atomic_read_write(v, sizeof(*v));
1255 return arch_atomic64_add_negative_relaxed(i, v);
1256}
1257
1258static __always_inline s64
1259atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260{
1261 kcsan_mb();
1262 instrument_atomic_read_write(v, sizeof(*v));
1263 return arch_atomic64_fetch_add_unless(v, a, u);
1264}
1265
1266static __always_inline bool
1267atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268{
1269 kcsan_mb();
1270 instrument_atomic_read_write(v, sizeof(*v));
1271 return arch_atomic64_add_unless(v, a, u);
1272}
1273
1274static __always_inline bool
1275atomic64_inc_not_zero(atomic64_t *v)
1276{
1277 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeof(*v));
1279 return arch_atomic64_inc_not_zero(v);
1280}
1281
1282static __always_inline bool
1283atomic64_inc_unless_negative(atomic64_t *v)
1284{
1285 kcsan_mb();
1286 instrument_atomic_read_write(v, sizeof(*v));
1287 return arch_atomic64_inc_unless_negative(v);
1288}
1289
1290static __always_inline bool
1291atomic64_dec_unless_positive(atomic64_t *v)
1292{
1293 kcsan_mb();
1294 instrument_atomic_read_write(v, sizeof(*v));
1295 return arch_atomic64_dec_unless_positive(v);
1296}
1297
1298static __always_inline s64
1299atomic64_dec_if_positive(atomic64_t *v)
1300{
1301 kcsan_mb();
1302 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic64_dec_if_positive(v);
1304}
1305
1306static __always_inline long
1307atomic_long_read(const atomic_long_t *v)
1308{
1309 instrument_atomic_read(v, sizeof(*v));
1310 return arch_atomic_long_read(v);
1311}
1312
1313static __always_inline long
1314atomic_long_read_acquire(const atomic_long_t *v)
1315{
1316 instrument_atomic_read(v, sizeof(*v));
1317 return arch_atomic_long_read_acquire(v);
1318}
1319
1320static __always_inline void
1321atomic_long_set(atomic_long_t *v, long i)
1322{
1323 instrument_atomic_write(v, sizeof(*v));
1324 arch_atomic_long_set(v, i);
1325}
1326
1327static __always_inline void
1328atomic_long_set_release(atomic_long_t *v, long i)
1329{
1330 kcsan_release();
1331 instrument_atomic_write(v, sizeof(*v));
1332 arch_atomic_long_set_release(v, i);
1333}
1334
1335static __always_inline void
1336atomic_long_add(long i, atomic_long_t *v)
1337{
1338 instrument_atomic_read_write(v, sizeof(*v));
1339 arch_atomic_long_add(i, v);
1340}
1341
1342static __always_inline long
1343atomic_long_add_return(long i, atomic_long_t *v)
1344{
1345 kcsan_mb();
1346 instrument_atomic_read_write(v, sizeof(*v));
1347 return arch_atomic_long_add_return(i, v);
1348}
1349
1350static __always_inline long
1351atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352{
1353 instrument_atomic_read_write(v, sizeof(*v));
1354 return arch_atomic_long_add_return_acquire(i, v);
1355}
1356
1357static __always_inline long
1358atomic_long_add_return_release(long i, atomic_long_t *v)
1359{
1360 kcsan_release();
1361 instrument_atomic_read_write(v, sizeof(*v));
1362 return arch_atomic_long_add_return_release(i, v);
1363}
1364
1365static __always_inline long
1366atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367{
1368 instrument_atomic_read_write(v, sizeof(*v));
1369 return arch_atomic_long_add_return_relaxed(i, v);
1370}
1371
1372static __always_inline long
1373atomic_long_fetch_add(long i, atomic_long_t *v)
1374{
1375 kcsan_mb();
1376 instrument_atomic_read_write(v, sizeof(*v));
1377 return arch_atomic_long_fetch_add(i, v);
1378}
1379
1380static __always_inline long
1381atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382{
1383 instrument_atomic_read_write(v, sizeof(*v));
1384 return arch_atomic_long_fetch_add_acquire(i, v);
1385}
1386
1387static __always_inline long
1388atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389{
1390 kcsan_release();
1391 instrument_atomic_read_write(v, sizeof(*v));
1392 return arch_atomic_long_fetch_add_release(i, v);
1393}
1394
1395static __always_inline long
1396atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397{
1398 instrument_atomic_read_write(v, sizeof(*v));
1399 return arch_atomic_long_fetch_add_relaxed(i, v);
1400}
1401
1402static __always_inline void
1403atomic_long_sub(long i, atomic_long_t *v)
1404{
1405 instrument_atomic_read_write(v, sizeof(*v));
1406 arch_atomic_long_sub(i, v);
1407}
1408
1409static __always_inline long
1410atomic_long_sub_return(long i, atomic_long_t *v)
1411{
1412 kcsan_mb();
1413 instrument_atomic_read_write(v, sizeof(*v));
1414 return arch_atomic_long_sub_return(i, v);
1415}
1416
1417static __always_inline long
1418atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419{
1420 instrument_atomic_read_write(v, sizeof(*v));
1421 return arch_atomic_long_sub_return_acquire(i, v);
1422}
1423
1424static __always_inline long
1425atomic_long_sub_return_release(long i, atomic_long_t *v)
1426{
1427 kcsan_release();
1428 instrument_atomic_read_write(v, sizeof(*v));
1429 return arch_atomic_long_sub_return_release(i, v);
1430}
1431
1432static __always_inline long
1433atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434{
1435 instrument_atomic_read_write(v, sizeof(*v));
1436 return arch_atomic_long_sub_return_relaxed(i, v);
1437}
1438
1439static __always_inline long
1440atomic_long_fetch_sub(long i, atomic_long_t *v)
1441{
1442 kcsan_mb();
1443 instrument_atomic_read_write(v, sizeof(*v));
1444 return arch_atomic_long_fetch_sub(i, v);
1445}
1446
1447static __always_inline long
1448atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449{
1450 instrument_atomic_read_write(v, sizeof(*v));
1451 return arch_atomic_long_fetch_sub_acquire(i, v);
1452}
1453
1454static __always_inline long
1455atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456{
1457 kcsan_release();
1458 instrument_atomic_read_write(v, sizeof(*v));
1459 return arch_atomic_long_fetch_sub_release(i, v);
1460}
1461
1462static __always_inline long
1463atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464{
1465 instrument_atomic_read_write(v, sizeof(*v));
1466 return arch_atomic_long_fetch_sub_relaxed(i, v);
1467}
1468
1469static __always_inline void
1470atomic_long_inc(atomic_long_t *v)
1471{
1472 instrument_atomic_read_write(v, sizeof(*v));
1473 arch_atomic_long_inc(v);
1474}
1475
1476static __always_inline long
1477atomic_long_inc_return(atomic_long_t *v)
1478{
1479 kcsan_mb();
1480 instrument_atomic_read_write(v, sizeof(*v));
1481 return arch_atomic_long_inc_return(v);
1482}
1483
1484static __always_inline long
1485atomic_long_inc_return_acquire(atomic_long_t *v)
1486{
1487 instrument_atomic_read_write(v, sizeof(*v));
1488 return arch_atomic_long_inc_return_acquire(v);
1489}
1490
1491static __always_inline long
1492atomic_long_inc_return_release(atomic_long_t *v)
1493{
1494 kcsan_release();
1495 instrument_atomic_read_write(v, sizeof(*v));
1496 return arch_atomic_long_inc_return_release(v);
1497}
1498
1499static __always_inline long
1500atomic_long_inc_return_relaxed(atomic_long_t *v)
1501{
1502 instrument_atomic_read_write(v, sizeof(*v));
1503 return arch_atomic_long_inc_return_relaxed(v);
1504}
1505
1506static __always_inline long
1507atomic_long_fetch_inc(atomic_long_t *v)
1508{
1509 kcsan_mb();
1510 instrument_atomic_read_write(v, sizeof(*v));
1511 return arch_atomic_long_fetch_inc(v);
1512}
1513
1514static __always_inline long
1515atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516{
1517 instrument_atomic_read_write(v, sizeof(*v));
1518 return arch_atomic_long_fetch_inc_acquire(v);
1519}
1520
1521static __always_inline long
1522atomic_long_fetch_inc_release(atomic_long_t *v)
1523{
1524 kcsan_release();
1525 instrument_atomic_read_write(v, sizeof(*v));
1526 return arch_atomic_long_fetch_inc_release(v);
1527}
1528
1529static __always_inline long
1530atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531{
1532 instrument_atomic_read_write(v, sizeof(*v));
1533 return arch_atomic_long_fetch_inc_relaxed(v);
1534}
1535
1536static __always_inline void
1537atomic_long_dec(atomic_long_t *v)
1538{
1539 instrument_atomic_read_write(v, sizeof(*v));
1540 arch_atomic_long_dec(v);
1541}
1542
1543static __always_inline long
1544atomic_long_dec_return(atomic_long_t *v)
1545{
1546 kcsan_mb();
1547 instrument_atomic_read_write(v, sizeof(*v));
1548 return arch_atomic_long_dec_return(v);
1549}
1550
1551static __always_inline long
1552atomic_long_dec_return_acquire(atomic_long_t *v)
1553{
1554 instrument_atomic_read_write(v, sizeof(*v));
1555 return arch_atomic_long_dec_return_acquire(v);
1556}
1557
1558static __always_inline long
1559atomic_long_dec_return_release(atomic_long_t *v)
1560{
1561 kcsan_release();
1562 instrument_atomic_read_write(v, sizeof(*v));
1563 return arch_atomic_long_dec_return_release(v);
1564}
1565
1566static __always_inline long
1567atomic_long_dec_return_relaxed(atomic_long_t *v)
1568{
1569 instrument_atomic_read_write(v, sizeof(*v));
1570 return arch_atomic_long_dec_return_relaxed(v);
1571}
1572
1573static __always_inline long
1574atomic_long_fetch_dec(atomic_long_t *v)
1575{
1576 kcsan_mb();
1577 instrument_atomic_read_write(v, sizeof(*v));
1578 return arch_atomic_long_fetch_dec(v);
1579}
1580
1581static __always_inline long
1582atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583{
1584 instrument_atomic_read_write(v, sizeof(*v));
1585 return arch_atomic_long_fetch_dec_acquire(v);
1586}
1587
1588static __always_inline long
1589atomic_long_fetch_dec_release(atomic_long_t *v)
1590{
1591 kcsan_release();
1592 instrument_atomic_read_write(v, sizeof(*v));
1593 return arch_atomic_long_fetch_dec_release(v);
1594}
1595
1596static __always_inline long
1597atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598{
1599 instrument_atomic_read_write(v, sizeof(*v));
1600 return arch_atomic_long_fetch_dec_relaxed(v);
1601}
1602
1603static __always_inline void
1604atomic_long_and(long i, atomic_long_t *v)
1605{
1606 instrument_atomic_read_write(v, sizeof(*v));
1607 arch_atomic_long_and(i, v);
1608}
1609
1610static __always_inline long
1611atomic_long_fetch_and(long i, atomic_long_t *v)
1612{
1613 kcsan_mb();
1614 instrument_atomic_read_write(v, sizeof(*v));
1615 return arch_atomic_long_fetch_and(i, v);
1616}
1617
1618static __always_inline long
1619atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620{
1621 instrument_atomic_read_write(v, sizeof(*v));
1622 return arch_atomic_long_fetch_and_acquire(i, v);
1623}
1624
1625static __always_inline long
1626atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627{
1628 kcsan_release();
1629 instrument_atomic_read_write(v, sizeof(*v));
1630 return arch_atomic_long_fetch_and_release(i, v);
1631}
1632
1633static __always_inline long
1634atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635{
1636 instrument_atomic_read_write(v, sizeof(*v));
1637 return arch_atomic_long_fetch_and_relaxed(i, v);
1638}
1639
1640static __always_inline void
1641atomic_long_andnot(long i, atomic_long_t *v)
1642{
1643 instrument_atomic_read_write(v, sizeof(*v));
1644 arch_atomic_long_andnot(i, v);
1645}
1646
1647static __always_inline long
1648atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649{
1650 kcsan_mb();
1651 instrument_atomic_read_write(v, sizeof(*v));
1652 return arch_atomic_long_fetch_andnot(i, v);
1653}
1654
1655static __always_inline long
1656atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657{
1658 instrument_atomic_read_write(v, sizeof(*v));
1659 return arch_atomic_long_fetch_andnot_acquire(i, v);
1660}
1661
1662static __always_inline long
1663atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664{
1665 kcsan_release();
1666 instrument_atomic_read_write(v, sizeof(*v));
1667 return arch_atomic_long_fetch_andnot_release(i, v);
1668}
1669
1670static __always_inline long
1671atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672{
1673 instrument_atomic_read_write(v, sizeof(*v));
1674 return arch_atomic_long_fetch_andnot_relaxed(i, v);
1675}
1676
1677static __always_inline void
1678atomic_long_or(long i, atomic_long_t *v)
1679{
1680 instrument_atomic_read_write(v, sizeof(*v));
1681 arch_atomic_long_or(i, v);
1682}
1683
1684static __always_inline long
1685atomic_long_fetch_or(long i, atomic_long_t *v)
1686{
1687 kcsan_mb();
1688 instrument_atomic_read_write(v, sizeof(*v));
1689 return arch_atomic_long_fetch_or(i, v);
1690}
1691
1692static __always_inline long
1693atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694{
1695 instrument_atomic_read_write(v, sizeof(*v));
1696 return arch_atomic_long_fetch_or_acquire(i, v);
1697}
1698
1699static __always_inline long
1700atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701{
1702 kcsan_release();
1703 instrument_atomic_read_write(v, sizeof(*v));
1704 return arch_atomic_long_fetch_or_release(i, v);
1705}
1706
1707static __always_inline long
1708atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709{
1710 instrument_atomic_read_write(v, sizeof(*v));
1711 return arch_atomic_long_fetch_or_relaxed(i, v);
1712}
1713
1714static __always_inline void
1715atomic_long_xor(long i, atomic_long_t *v)
1716{
1717 instrument_atomic_read_write(v, sizeof(*v));
1718 arch_atomic_long_xor(i, v);
1719}
1720
1721static __always_inline long
1722atomic_long_fetch_xor(long i, atomic_long_t *v)
1723{
1724 kcsan_mb();
1725 instrument_atomic_read_write(v, sizeof(*v));
1726 return arch_atomic_long_fetch_xor(i, v);
1727}
1728
1729static __always_inline long
1730atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731{
1732 instrument_atomic_read_write(v, sizeof(*v));
1733 return arch_atomic_long_fetch_xor_acquire(i, v);
1734}
1735
1736static __always_inline long
1737atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738{
1739 kcsan_release();
1740 instrument_atomic_read_write(v, sizeof(*v));
1741 return arch_atomic_long_fetch_xor_release(i, v);
1742}
1743
1744static __always_inline long
1745atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746{
1747 instrument_atomic_read_write(v, sizeof(*v));
1748 return arch_atomic_long_fetch_xor_relaxed(i, v);
1749}
1750
1751static __always_inline long
1752atomic_long_xchg(atomic_long_t *v, long i)
1753{
1754 kcsan_mb();
1755 instrument_atomic_read_write(v, sizeof(*v));
1756 return arch_atomic_long_xchg(v, i);
1757}
1758
1759static __always_inline long
1760atomic_long_xchg_acquire(atomic_long_t *v, long i)
1761{
1762 instrument_atomic_read_write(v, sizeof(*v));
1763 return arch_atomic_long_xchg_acquire(v, i);
1764}
1765
1766static __always_inline long
1767atomic_long_xchg_release(atomic_long_t *v, long i)
1768{
1769 kcsan_release();
1770 instrument_atomic_read_write(v, sizeof(*v));
1771 return arch_atomic_long_xchg_release(v, i);
1772}
1773
1774static __always_inline long
1775atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1776{
1777 instrument_atomic_read_write(v, sizeof(*v));
1778 return arch_atomic_long_xchg_relaxed(v, i);
1779}
1780
1781static __always_inline long
1782atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783{
1784 kcsan_mb();
1785 instrument_atomic_read_write(v, sizeof(*v));
1786 return arch_atomic_long_cmpxchg(v, old, new);
1787}
1788
1789static __always_inline long
1790atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791{
1792 instrument_atomic_read_write(v, sizeof(*v));
1793 return arch_atomic_long_cmpxchg_acquire(v, old, new);
1794}
1795
1796static __always_inline long
1797atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798{
1799 kcsan_release();
1800 instrument_atomic_read_write(v, sizeof(*v));
1801 return arch_atomic_long_cmpxchg_release(v, old, new);
1802}
1803
1804static __always_inline long
1805atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806{
1807 instrument_atomic_read_write(v, sizeof(*v));
1808 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1809}
1810
1811static __always_inline bool
1812atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1813{
1814 kcsan_mb();
1815 instrument_atomic_read_write(v, sizeof(*v));
1816 instrument_atomic_read_write(old, sizeof(*old));
1817 return arch_atomic_long_try_cmpxchg(v, old, new);
1818}
1819
1820static __always_inline bool
1821atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1822{
1823 instrument_atomic_read_write(v, sizeof(*v));
1824 instrument_atomic_read_write(old, sizeof(*old));
1825 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1826}
1827
1828static __always_inline bool
1829atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1830{
1831 kcsan_release();
1832 instrument_atomic_read_write(v, sizeof(*v));
1833 instrument_atomic_read_write(old, sizeof(*old));
1834 return arch_atomic_long_try_cmpxchg_release(v, old, new);
1835}
1836
1837static __always_inline bool
1838atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1839{
1840 instrument_atomic_read_write(v, sizeof(*v));
1841 instrument_atomic_read_write(old, sizeof(*old));
1842 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843}
1844
1845static __always_inline bool
1846atomic_long_sub_and_test(long i, atomic_long_t *v)
1847{
1848 kcsan_mb();
1849 instrument_atomic_read_write(v, sizeof(*v));
1850 return arch_atomic_long_sub_and_test(i, v);
1851}
1852
1853static __always_inline bool
1854atomic_long_dec_and_test(atomic_long_t *v)
1855{
1856 kcsan_mb();
1857 instrument_atomic_read_write(v, sizeof(*v));
1858 return arch_atomic_long_dec_and_test(v);
1859}
1860
1861static __always_inline bool
1862atomic_long_inc_and_test(atomic_long_t *v)
1863{
1864 kcsan_mb();
1865 instrument_atomic_read_write(v, sizeof(*v));
1866 return arch_atomic_long_inc_and_test(v);
1867}
1868
1869static __always_inline bool
1870atomic_long_add_negative(long i, atomic_long_t *v)
1871{
1872 kcsan_mb();
1873 instrument_atomic_read_write(v, sizeof(*v));
1874 return arch_atomic_long_add_negative(i, v);
1875}
1876
1877static __always_inline bool
1878atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879{
1880 instrument_atomic_read_write(v, sizeof(*v));
1881 return arch_atomic_long_add_negative_acquire(i, v);
1882}
1883
1884static __always_inline bool
1885atomic_long_add_negative_release(long i, atomic_long_t *v)
1886{
1887 kcsan_release();
1888 instrument_atomic_read_write(v, sizeof(*v));
1889 return arch_atomic_long_add_negative_release(i, v);
1890}
1891
1892static __always_inline bool
1893atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894{
1895 instrument_atomic_read_write(v, sizeof(*v));
1896 return arch_atomic_long_add_negative_relaxed(i, v);
1897}
1898
1899static __always_inline long
1900atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901{
1902 kcsan_mb();
1903 instrument_atomic_read_write(v, sizeof(*v));
1904 return arch_atomic_long_fetch_add_unless(v, a, u);
1905}
1906
1907static __always_inline bool
1908atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909{
1910 kcsan_mb();
1911 instrument_atomic_read_write(v, sizeof(*v));
1912 return arch_atomic_long_add_unless(v, a, u);
1913}
1914
1915static __always_inline bool
1916atomic_long_inc_not_zero(atomic_long_t *v)
1917{
1918 kcsan_mb();
1919 instrument_atomic_read_write(v, sizeof(*v));
1920 return arch_atomic_long_inc_not_zero(v);
1921}
1922
1923static __always_inline bool
1924atomic_long_inc_unless_negative(atomic_long_t *v)
1925{
1926 kcsan_mb();
1927 instrument_atomic_read_write(v, sizeof(*v));
1928 return arch_atomic_long_inc_unless_negative(v);
1929}
1930
1931static __always_inline bool
1932atomic_long_dec_unless_positive(atomic_long_t *v)
1933{
1934 kcsan_mb();
1935 instrument_atomic_read_write(v, sizeof(*v));
1936 return arch_atomic_long_dec_unless_positive(v);
1937}
1938
1939static __always_inline long
1940atomic_long_dec_if_positive(atomic_long_t *v)
1941{
1942 kcsan_mb();
1943 instrument_atomic_read_write(v, sizeof(*v));
1944 return arch_atomic_long_dec_if_positive(v);
1945}
1946
1947#define xchg(ptr, ...) \
1948({ \
1949 typeof(ptr) __ai_ptr = (ptr); \
1950 kcsan_mb(); \
1951 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1952 arch_xchg(__ai_ptr, __VA_ARGS__); \
1953})
1954
1955#define xchg_acquire(ptr, ...) \
1956({ \
1957 typeof(ptr) __ai_ptr = (ptr); \
1958 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1959 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1960})
1961
1962#define xchg_release(ptr, ...) \
1963({ \
1964 typeof(ptr) __ai_ptr = (ptr); \
1965 kcsan_release(); \
1966 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1967 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1968})
1969
1970#define xchg_relaxed(ptr, ...) \
1971({ \
1972 typeof(ptr) __ai_ptr = (ptr); \
1973 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1974 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1975})
1976
1977#define cmpxchg(ptr, ...) \
1978({ \
1979 typeof(ptr) __ai_ptr = (ptr); \
1980 kcsan_mb(); \
1981 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1982 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1983})
1984
1985#define cmpxchg_acquire(ptr, ...) \
1986({ \
1987 typeof(ptr) __ai_ptr = (ptr); \
1988 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1989 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1990})
1991
1992#define cmpxchg_release(ptr, ...) \
1993({ \
1994 typeof(ptr) __ai_ptr = (ptr); \
1995 kcsan_release(); \
1996 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1997 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1998})
1999
2000#define cmpxchg_relaxed(ptr, ...) \
2001({ \
2002 typeof(ptr) __ai_ptr = (ptr); \
2003 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2004 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
2005})
2006
2007#define cmpxchg64(ptr, ...) \
2008({ \
2009 typeof(ptr) __ai_ptr = (ptr); \
2010 kcsan_mb(); \
2011 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2012 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
2013})
2014
2015#define cmpxchg64_acquire(ptr, ...) \
2016({ \
2017 typeof(ptr) __ai_ptr = (ptr); \
2018 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2019 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
2020})
2021
2022#define cmpxchg64_release(ptr, ...) \
2023({ \
2024 typeof(ptr) __ai_ptr = (ptr); \
2025 kcsan_release(); \
2026 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2027 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
2028})
2029
2030#define cmpxchg64_relaxed(ptr, ...) \
2031({ \
2032 typeof(ptr) __ai_ptr = (ptr); \
2033 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
2035})
2036
2037#define try_cmpxchg(ptr, oldp, ...) \
2038({ \
2039 typeof(ptr) __ai_ptr = (ptr); \
2040 typeof(oldp) __ai_oldp = (oldp); \
2041 kcsan_mb(); \
2042 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045})
2046
2047#define try_cmpxchg_acquire(ptr, oldp, ...) \
2048({ \
2049 typeof(ptr) __ai_ptr = (ptr); \
2050 typeof(oldp) __ai_oldp = (oldp); \
2051 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2053 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2054})
2055
2056#define try_cmpxchg_release(ptr, oldp, ...) \
2057({ \
2058 typeof(ptr) __ai_ptr = (ptr); \
2059 typeof(oldp) __ai_oldp = (oldp); \
2060 kcsan_release(); \
2061 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2063 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2064})
2065
2066#define try_cmpxchg_relaxed(ptr, oldp, ...) \
2067({ \
2068 typeof(ptr) __ai_ptr = (ptr); \
2069 typeof(oldp) __ai_oldp = (oldp); \
2070 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2072 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2073})
2074
2075#define try_cmpxchg64(ptr, oldp, ...) \
2076({ \
2077 typeof(ptr) __ai_ptr = (ptr); \
2078 typeof(oldp) __ai_oldp = (oldp); \
2079 kcsan_mb(); \
2080 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2082 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2083})
2084
2085#define try_cmpxchg64_acquire(ptr, oldp, ...) \
2086({ \
2087 typeof(ptr) __ai_ptr = (ptr); \
2088 typeof(oldp) __ai_oldp = (oldp); \
2089 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2091 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2092})
2093
2094#define try_cmpxchg64_release(ptr, oldp, ...) \
2095({ \
2096 typeof(ptr) __ai_ptr = (ptr); \
2097 typeof(oldp) __ai_oldp = (oldp); \
2098 kcsan_release(); \
2099 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2101 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2102})
2103
2104#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2105({ \
2106 typeof(ptr) __ai_ptr = (ptr); \
2107 typeof(oldp) __ai_oldp = (oldp); \
2108 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2110 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2111})
2112
2113#define cmpxchg_local(ptr, ...) \
2114({ \
2115 typeof(ptr) __ai_ptr = (ptr); \
2116 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2117 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2118})
2119
2120#define cmpxchg64_local(ptr, ...) \
2121({ \
2122 typeof(ptr) __ai_ptr = (ptr); \
2123 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2124 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2125})
2126
2127#define sync_cmpxchg(ptr, ...) \
2128({ \
2129 typeof(ptr) __ai_ptr = (ptr); \
2130 kcsan_mb(); \
2131 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2132 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2133})
2134
2135#define try_cmpxchg_local(ptr, oldp, ...) \
2136({ \
2137 typeof(ptr) __ai_ptr = (ptr); \
2138 typeof(oldp) __ai_oldp = (oldp); \
2139 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2141 arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2142})
2143
2144#define try_cmpxchg64_local(ptr, oldp, ...) \
2145({ \
2146 typeof(ptr) __ai_ptr = (ptr); \
2147 typeof(oldp) __ai_oldp = (oldp); \
2148 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2150 arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2151})
2152
2153#define cmpxchg_double(ptr, ...) \
2154({ \
2155 typeof(ptr) __ai_ptr = (ptr); \
2156 kcsan_mb(); \
2157 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2158 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2159})
2160
2161
2162#define cmpxchg_double_local(ptr, ...) \
2163({ \
2164 typeof(ptr) __ai_ptr = (ptr); \
2165 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2166 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2167})
2168
2169#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e