Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * Atomic operations that C can't guarantee us. Useful for
3 * resource counting etc..
4 *
5 * But use these as seldom as possible since they are much more slower
6 * than regular operations.
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 *
12 * Copyright (C) 1996, 97, 99, 2000, 03, 04 by Ralf Baechle
13 */
14
15/*
16 * As workaround for the ATOMIC_DEC_AND_LOCK / atomic_dec_and_lock mess in
17 * <linux/spinlock.h> we have to include <linux/spinlock.h> outside the
18 * main big wrapper ...
19 */
20#include <linux/config.h>
21#include <linux/spinlock.h>
22
23#ifndef _ASM_ATOMIC_H
24#define _ASM_ATOMIC_H
25
26#include <asm/cpu-features.h>
27#include <asm/war.h>
28
29extern spinlock_t atomic_lock;
30
31typedef struct { volatile int counter; } atomic_t;
32
33#define ATOMIC_INIT(i) { (i) }
34
35/*
36 * atomic_read - read atomic variable
37 * @v: pointer of type atomic_t
38 *
39 * Atomically reads the value of @v.
40 */
41#define atomic_read(v) ((v)->counter)
42
43/*
44 * atomic_set - set atomic variable
45 * @v: pointer of type atomic_t
46 * @i: required value
47 *
48 * Atomically sets the value of @v to @i.
49 */
50#define atomic_set(v,i) ((v)->counter = (i))
51
52/*
53 * atomic_add - add integer to atomic variable
54 * @i: integer value to add
55 * @v: pointer of type atomic_t
56 *
57 * Atomically adds @i to @v.
58 */
59static __inline__ void atomic_add(int i, atomic_t * v)
60{
61 if (cpu_has_llsc && R10000_LLSC_WAR) {
62 unsigned long temp;
63
64 __asm__ __volatile__(
65 "1: ll %0, %1 # atomic_add \n"
66 " addu %0, %2 \n"
67 " sc %0, %1 \n"
68 " beqzl %0, 1b \n"
69 : "=&r" (temp), "=m" (v->counter)
70 : "Ir" (i), "m" (v->counter));
71 } else if (cpu_has_llsc) {
72 unsigned long temp;
73
74 __asm__ __volatile__(
75 "1: ll %0, %1 # atomic_add \n"
76 " addu %0, %2 \n"
77 " sc %0, %1 \n"
78 " beqz %0, 1b \n"
79 : "=&r" (temp), "=m" (v->counter)
80 : "Ir" (i), "m" (v->counter));
81 } else {
82 unsigned long flags;
83
84 spin_lock_irqsave(&atomic_lock, flags);
85 v->counter += i;
86 spin_unlock_irqrestore(&atomic_lock, flags);
87 }
88}
89
90/*
91 * atomic_sub - subtract the atomic variable
92 * @i: integer value to subtract
93 * @v: pointer of type atomic_t
94 *
95 * Atomically subtracts @i from @v.
96 */
97static __inline__ void atomic_sub(int i, atomic_t * v)
98{
99 if (cpu_has_llsc && R10000_LLSC_WAR) {
100 unsigned long temp;
101
102 __asm__ __volatile__(
103 "1: ll %0, %1 # atomic_sub \n"
104 " subu %0, %2 \n"
105 " sc %0, %1 \n"
106 " beqzl %0, 1b \n"
107 : "=&r" (temp), "=m" (v->counter)
108 : "Ir" (i), "m" (v->counter));
109 } else if (cpu_has_llsc) {
110 unsigned long temp;
111
112 __asm__ __volatile__(
113 "1: ll %0, %1 # atomic_sub \n"
114 " subu %0, %2 \n"
115 " sc %0, %1 \n"
116 " beqz %0, 1b \n"
117 : "=&r" (temp), "=m" (v->counter)
118 : "Ir" (i), "m" (v->counter));
119 } else {
120 unsigned long flags;
121
122 spin_lock_irqsave(&atomic_lock, flags);
123 v->counter -= i;
124 spin_unlock_irqrestore(&atomic_lock, flags);
125 }
126}
127
128/*
129 * Same as above, but return the result value
130 */
131static __inline__ int atomic_add_return(int i, atomic_t * v)
132{
133 unsigned long result;
134
135 if (cpu_has_llsc && R10000_LLSC_WAR) {
136 unsigned long temp;
137
138 __asm__ __volatile__(
139 "1: ll %1, %2 # atomic_add_return \n"
140 " addu %0, %1, %3 \n"
141 " sc %0, %2 \n"
142 " beqzl %0, 1b \n"
143 " addu %0, %1, %3 \n"
144 " sync \n"
145 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
146 : "Ir" (i), "m" (v->counter)
147 : "memory");
148 } else if (cpu_has_llsc) {
149 unsigned long temp;
150
151 __asm__ __volatile__(
152 "1: ll %1, %2 # atomic_add_return \n"
153 " addu %0, %1, %3 \n"
154 " sc %0, %2 \n"
155 " beqz %0, 1b \n"
156 " addu %0, %1, %3 \n"
157 " sync \n"
158 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
159 : "Ir" (i), "m" (v->counter)
160 : "memory");
161 } else {
162 unsigned long flags;
163
164 spin_lock_irqsave(&atomic_lock, flags);
165 result = v->counter;
166 result += i;
167 v->counter = result;
168 spin_unlock_irqrestore(&atomic_lock, flags);
169 }
170
171 return result;
172}
173
174static __inline__ int atomic_sub_return(int i, atomic_t * v)
175{
176 unsigned long result;
177
178 if (cpu_has_llsc && R10000_LLSC_WAR) {
179 unsigned long temp;
180
181 __asm__ __volatile__(
182 "1: ll %1, %2 # atomic_sub_return \n"
183 " subu %0, %1, %3 \n"
184 " sc %0, %2 \n"
185 " beqzl %0, 1b \n"
186 " subu %0, %1, %3 \n"
187 " sync \n"
188 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
189 : "Ir" (i), "m" (v->counter)
190 : "memory");
191 } else if (cpu_has_llsc) {
192 unsigned long temp;
193
194 __asm__ __volatile__(
195 "1: ll %1, %2 # atomic_sub_return \n"
196 " subu %0, %1, %3 \n"
197 " sc %0, %2 \n"
198 " beqz %0, 1b \n"
199 " subu %0, %1, %3 \n"
200 " sync \n"
201 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
202 : "Ir" (i), "m" (v->counter)
203 : "memory");
204 } else {
205 unsigned long flags;
206
207 spin_lock_irqsave(&atomic_lock, flags);
208 result = v->counter;
209 result -= i;
210 v->counter = result;
211 spin_unlock_irqrestore(&atomic_lock, flags);
212 }
213
214 return result;
215}
216
217/*
218 * atomic_sub_if_positive - add integer to atomic variable
219 * @v: pointer of type atomic_t
220 *
221 * Atomically test @v and decrement if it is greater than 0.
222 * The function returns the old value of @v minus 1.
223 */
224static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
225{
226 unsigned long result;
227
228 if (cpu_has_llsc && R10000_LLSC_WAR) {
229 unsigned long temp;
230
231 __asm__ __volatile__(
232 "1: ll %1, %2 # atomic_sub_if_positive\n"
233 " subu %0, %1, %3 \n"
234 " bltz %0, 1f \n"
235 " sc %0, %2 \n"
236 " beqzl %0, 1b \n"
237 " sync \n"
238 "1: \n"
239 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
240 : "Ir" (i), "m" (v->counter)
241 : "memory");
242 } else if (cpu_has_llsc) {
243 unsigned long temp;
244
245 __asm__ __volatile__(
246 "1: ll %1, %2 # atomic_sub_if_positive\n"
247 " subu %0, %1, %3 \n"
248 " bltz %0, 1f \n"
249 " sc %0, %2 \n"
250 " beqz %0, 1b \n"
251 " sync \n"
252 "1: \n"
253 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
254 : "Ir" (i), "m" (v->counter)
255 : "memory");
256 } else {
257 unsigned long flags;
258
259 spin_lock_irqsave(&atomic_lock, flags);
260 result = v->counter;
261 result -= i;
262 if (result >= 0)
263 v->counter = result;
264 spin_unlock_irqrestore(&atomic_lock, flags);
265 }
266
267 return result;
268}
269
270#define atomic_dec_return(v) atomic_sub_return(1,(v))
271#define atomic_inc_return(v) atomic_add_return(1,(v))
272
273/*
274 * atomic_sub_and_test - subtract value from variable and test result
275 * @i: integer value to subtract
276 * @v: pointer of type atomic_t
277 *
278 * Atomically subtracts @i from @v and returns
279 * true if the result is zero, or false for all
280 * other cases.
281 */
282#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
283
284/*
285 * atomic_inc_and_test - increment and test
286 * @v: pointer of type atomic_t
287 *
288 * Atomically increments @v by 1
289 * and returns true if the result is zero, or false for all
290 * other cases.
291 */
292#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
293
294/*
295 * atomic_dec_and_test - decrement by 1 and test
296 * @v: pointer of type atomic_t
297 *
298 * Atomically decrements @v by 1 and
299 * returns true if the result is 0, or false for all other
300 * cases.
301 */
302#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
303
304/*
305 * atomic_dec_if_positive - decrement by 1 if old value positive
306 * @v: pointer of type atomic_t
307 */
308#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v)
309
310/*
311 * atomic_inc - increment atomic variable
312 * @v: pointer of type atomic_t
313 *
314 * Atomically increments @v by 1.
315 */
316#define atomic_inc(v) atomic_add(1,(v))
317
318/*
319 * atomic_dec - decrement and test
320 * @v: pointer of type atomic_t
321 *
322 * Atomically decrements @v by 1.
323 */
324#define atomic_dec(v) atomic_sub(1,(v))
325
326/*
327 * atomic_add_negative - add and test if negative
328 * @v: pointer of type atomic_t
329 * @i: integer value to add
330 *
331 * Atomically adds @i to @v and returns true
332 * if the result is negative, or false when
333 * result is greater than or equal to zero.
334 */
335#define atomic_add_negative(i,v) (atomic_add_return(i, (v)) < 0)
336
337#ifdef CONFIG_64BIT
338
339typedef struct { volatile __s64 counter; } atomic64_t;
340
341#define ATOMIC64_INIT(i) { (i) }
342
343/*
344 * atomic64_read - read atomic variable
345 * @v: pointer of type atomic64_t
346 *
347 */
348#define atomic64_read(v) ((v)->counter)
349
350/*
351 * atomic64_set - set atomic variable
352 * @v: pointer of type atomic64_t
353 * @i: required value
354 */
355#define atomic64_set(v,i) ((v)->counter = (i))
356
357/*
358 * atomic64_add - add integer to atomic variable
359 * @i: integer value to add
360 * @v: pointer of type atomic64_t
361 *
362 * Atomically adds @i to @v.
363 */
364static __inline__ void atomic64_add(long i, atomic64_t * v)
365{
366 if (cpu_has_llsc && R10000_LLSC_WAR) {
367 unsigned long temp;
368
369 __asm__ __volatile__(
370 "1: lld %0, %1 # atomic64_add \n"
371 " addu %0, %2 \n"
372 " scd %0, %1 \n"
373 " beqzl %0, 1b \n"
374 : "=&r" (temp), "=m" (v->counter)
375 : "Ir" (i), "m" (v->counter));
376 } else if (cpu_has_llsc) {
377 unsigned long temp;
378
379 __asm__ __volatile__(
380 "1: lld %0, %1 # atomic64_add \n"
381 " addu %0, %2 \n"
382 " scd %0, %1 \n"
383 " beqz %0, 1b \n"
384 : "=&r" (temp), "=m" (v->counter)
385 : "Ir" (i), "m" (v->counter));
386 } else {
387 unsigned long flags;
388
389 spin_lock_irqsave(&atomic_lock, flags);
390 v->counter += i;
391 spin_unlock_irqrestore(&atomic_lock, flags);
392 }
393}
394
395/*
396 * atomic64_sub - subtract the atomic variable
397 * @i: integer value to subtract
398 * @v: pointer of type atomic64_t
399 *
400 * Atomically subtracts @i from @v.
401 */
402static __inline__ void atomic64_sub(long i, atomic64_t * v)
403{
404 if (cpu_has_llsc && R10000_LLSC_WAR) {
405 unsigned long temp;
406
407 __asm__ __volatile__(
408 "1: lld %0, %1 # atomic64_sub \n"
409 " subu %0, %2 \n"
410 " scd %0, %1 \n"
411 " beqzl %0, 1b \n"
412 : "=&r" (temp), "=m" (v->counter)
413 : "Ir" (i), "m" (v->counter));
414 } else if (cpu_has_llsc) {
415 unsigned long temp;
416
417 __asm__ __volatile__(
418 "1: lld %0, %1 # atomic64_sub \n"
419 " subu %0, %2 \n"
420 " scd %0, %1 \n"
421 " beqz %0, 1b \n"
422 : "=&r" (temp), "=m" (v->counter)
423 : "Ir" (i), "m" (v->counter));
424 } else {
425 unsigned long flags;
426
427 spin_lock_irqsave(&atomic_lock, flags);
428 v->counter -= i;
429 spin_unlock_irqrestore(&atomic_lock, flags);
430 }
431}
432
433/*
434 * Same as above, but return the result value
435 */
436static __inline__ long atomic64_add_return(long i, atomic64_t * v)
437{
438 unsigned long result;
439
440 if (cpu_has_llsc && R10000_LLSC_WAR) {
441 unsigned long temp;
442
443 __asm__ __volatile__(
444 "1: lld %1, %2 # atomic64_add_return \n"
445 " addu %0, %1, %3 \n"
446 " scd %0, %2 \n"
447 " beqzl %0, 1b \n"
448 " addu %0, %1, %3 \n"
449 " sync \n"
450 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
451 : "Ir" (i), "m" (v->counter)
452 : "memory");
453 } else if (cpu_has_llsc) {
454 unsigned long temp;
455
456 __asm__ __volatile__(
457 "1: lld %1, %2 # atomic64_add_return \n"
458 " addu %0, %1, %3 \n"
459 " scd %0, %2 \n"
460 " beqz %0, 1b \n"
461 " addu %0, %1, %3 \n"
462 " sync \n"
463 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
464 : "Ir" (i), "m" (v->counter)
465 : "memory");
466 } else {
467 unsigned long flags;
468
469 spin_lock_irqsave(&atomic_lock, flags);
470 result = v->counter;
471 result += i;
472 v->counter = result;
473 spin_unlock_irqrestore(&atomic_lock, flags);
474 }
475
476 return result;
477}
478
479static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
480{
481 unsigned long result;
482
483 if (cpu_has_llsc && R10000_LLSC_WAR) {
484 unsigned long temp;
485
486 __asm__ __volatile__(
487 "1: lld %1, %2 # atomic64_sub_return \n"
488 " subu %0, %1, %3 \n"
489 " scd %0, %2 \n"
490 " beqzl %0, 1b \n"
491 " subu %0, %1, %3 \n"
492 " sync \n"
493 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
494 : "Ir" (i), "m" (v->counter)
495 : "memory");
496 } else if (cpu_has_llsc) {
497 unsigned long temp;
498
499 __asm__ __volatile__(
500 "1: lld %1, %2 # atomic64_sub_return \n"
501 " subu %0, %1, %3 \n"
502 " scd %0, %2 \n"
503 " beqz %0, 1b \n"
504 " subu %0, %1, %3 \n"
505 " sync \n"
506 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
507 : "Ir" (i), "m" (v->counter)
508 : "memory");
509 } else {
510 unsigned long flags;
511
512 spin_lock_irqsave(&atomic_lock, flags);
513 result = v->counter;
514 result -= i;
515 v->counter = result;
516 spin_unlock_irqrestore(&atomic_lock, flags);
517 }
518
519 return result;
520}
521
522/*
523 * atomic64_sub_if_positive - add integer to atomic variable
524 * @v: pointer of type atomic64_t
525 *
526 * Atomically test @v and decrement if it is greater than 0.
527 * The function returns the old value of @v minus 1.
528 */
529static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
530{
531 unsigned long result;
532
533 if (cpu_has_llsc && R10000_LLSC_WAR) {
534 unsigned long temp;
535
536 __asm__ __volatile__(
537 "1: lld %1, %2 # atomic64_sub_if_positive\n"
538 " dsubu %0, %1, %3 \n"
539 " bltz %0, 1f \n"
540 " scd %0, %2 \n"
541 " beqzl %0, 1b \n"
542 " sync \n"
543 "1: \n"
544 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
545 : "Ir" (i), "m" (v->counter)
546 : "memory");
547 } else if (cpu_has_llsc) {
548 unsigned long temp;
549
550 __asm__ __volatile__(
551 "1: lld %1, %2 # atomic64_sub_if_positive\n"
552 " dsubu %0, %1, %3 \n"
553 " bltz %0, 1f \n"
554 " scd %0, %2 \n"
555 " beqz %0, 1b \n"
556 " sync \n"
557 "1: \n"
558 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
559 : "Ir" (i), "m" (v->counter)
560 : "memory");
561 } else {
562 unsigned long flags;
563
564 spin_lock_irqsave(&atomic_lock, flags);
565 result = v->counter;
566 result -= i;
567 if (result >= 0)
568 v->counter = result;
569 spin_unlock_irqrestore(&atomic_lock, flags);
570 }
571
572 return result;
573}
574
575#define atomic64_dec_return(v) atomic64_sub_return(1,(v))
576#define atomic64_inc_return(v) atomic64_add_return(1,(v))
577
578/*
579 * atomic64_sub_and_test - subtract value from variable and test result
580 * @i: integer value to subtract
581 * @v: pointer of type atomic64_t
582 *
583 * Atomically subtracts @i from @v and returns
584 * true if the result is zero, or false for all
585 * other cases.
586 */
587#define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0)
588
589/*
590 * atomic64_inc_and_test - increment and test
591 * @v: pointer of type atomic64_t
592 *
593 * Atomically increments @v by 1
594 * and returns true if the result is zero, or false for all
595 * other cases.
596 */
597#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
598
599/*
600 * atomic64_dec_and_test - decrement by 1 and test
601 * @v: pointer of type atomic64_t
602 *
603 * Atomically decrements @v by 1 and
604 * returns true if the result is 0, or false for all other
605 * cases.
606 */
607#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
608
609/*
610 * atomic64_dec_if_positive - decrement by 1 if old value positive
611 * @v: pointer of type atomic64_t
612 */
613#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v)
614
615/*
616 * atomic64_inc - increment atomic variable
617 * @v: pointer of type atomic64_t
618 *
619 * Atomically increments @v by 1.
620 */
621#define atomic64_inc(v) atomic64_add(1,(v))
622
623/*
624 * atomic64_dec - decrement and test
625 * @v: pointer of type atomic64_t
626 *
627 * Atomically decrements @v by 1.
628 */
629#define atomic64_dec(v) atomic64_sub(1,(v))
630
631/*
632 * atomic64_add_negative - add and test if negative
633 * @v: pointer of type atomic64_t
634 * @i: integer value to add
635 *
636 * Atomically adds @i to @v and returns true
637 * if the result is negative, or false when
638 * result is greater than or equal to zero.
639 */
640#define atomic64_add_negative(i,v) (atomic64_add_return(i, (v)) < 0)
641
642#endif /* CONFIG_64BIT */
643
644/*
645 * atomic*_return operations are serializing but not the non-*_return
646 * versions.
647 */
648#define smp_mb__before_atomic_dec() smp_mb()
649#define smp_mb__after_atomic_dec() smp_mb()
650#define smp_mb__before_atomic_inc() smp_mb()
651#define smp_mb__after_atomic_inc() smp_mb()
652
653#endif /* _ASM_ATOMIC_H */