Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 1999, 2000 Ralf Baechle (ralf@gnu.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#include <linux/compiler.h>
13#include <linux/types.h>
14#include <asm/bug.h>
15#include <asm/byteorder.h> /* sigh ... */
16#include <asm/cpu-features.h>
17
18#if (_MIPS_SZLONG == 32)
19#define SZLONG_LOG 5
20#define SZLONG_MASK 31UL
21#define __LL "ll "
22#define __SC "sc "
23#define cpu_to_lelongp(x) cpu_to_le32p((__u32 *) (x))
24#elif (_MIPS_SZLONG == 64)
25#define SZLONG_LOG 6
26#define SZLONG_MASK 63UL
27#define __LL "lld "
28#define __SC "scd "
29#define cpu_to_lelongp(x) cpu_to_le64p((__u64 *) (x))
30#endif
31
32#ifdef __KERNEL__
33
34#include <linux/irqflags.h>
35#include <asm/sgidefs.h>
36#include <asm/war.h>
37
38/*
39 * clear_bit() doesn't provide any barrier for the compiler.
40 */
41#define smp_mb__before_clear_bit() smp_mb()
42#define smp_mb__after_clear_bit() smp_mb()
43
44/*
45 * Only disable interrupt for kernel mode stuff to keep usermode stuff
46 * that dares to use kernel include files alive.
47 */
48
49#define __bi_flags unsigned long flags
50#define __bi_local_irq_save(x) local_irq_save(x)
51#define __bi_local_irq_restore(x) local_irq_restore(x)
52#else
53#define __bi_flags
54#define __bi_local_irq_save(x)
55#define __bi_local_irq_restore(x)
56#endif /* __KERNEL__ */
57
58/*
59 * set_bit - Atomically set a bit in memory
60 * @nr: the bit to set
61 * @addr: the address to start counting from
62 *
63 * This function is atomic and may not be reordered. See __set_bit()
64 * if you do not require the atomic guarantees.
65 * Note that @nr may be almost arbitrarily large; this function is not
66 * restricted to acting on a single-word quantity.
67 */
68static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
69{
70 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
71 unsigned long temp;
72
73 if (cpu_has_llsc && R10000_LLSC_WAR) {
74 __asm__ __volatile__(
75 " .set mips3 \n"
76 "1: " __LL "%0, %1 # set_bit \n"
77 " or %0, %2 \n"
78 " " __SC "%0, %1 \n"
79 " beqzl %0, 1b \n"
80 " .set mips0 \n"
81 : "=&r" (temp), "=m" (*m)
82 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
83 } else if (cpu_has_llsc) {
84 __asm__ __volatile__(
85 " .set mips3 \n"
86 "1: " __LL "%0, %1 # set_bit \n"
87 " or %0, %2 \n"
88 " " __SC "%0, %1 \n"
89 " beqz %0, 1b \n"
90 " .set mips0 \n"
91 : "=&r" (temp), "=m" (*m)
92 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
93 } else {
94 volatile unsigned long *a = addr;
95 unsigned long mask;
96 __bi_flags;
97
98 a += nr >> SZLONG_LOG;
99 mask = 1UL << (nr & SZLONG_MASK);
100 __bi_local_irq_save(flags);
101 *a |= mask;
102 __bi_local_irq_restore(flags);
103 }
104}
105
106/*
107 * clear_bit - Clears a bit in memory
108 * @nr: Bit to clear
109 * @addr: Address to start counting from
110 *
111 * clear_bit() is atomic and may not be reordered. However, it does
112 * not contain a memory barrier, so if it is used for locking purposes,
113 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
114 * in order to ensure changes are visible on other processors.
115 */
116static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
117{
118 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
119 unsigned long temp;
120
121 if (cpu_has_llsc && R10000_LLSC_WAR) {
122 __asm__ __volatile__(
123 " .set mips3 \n"
124 "1: " __LL "%0, %1 # clear_bit \n"
125 " and %0, %2 \n"
126 " " __SC "%0, %1 \n"
127 " beqzl %0, 1b \n"
128 " .set mips0 \n"
129 : "=&r" (temp), "=m" (*m)
130 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m));
131 } else if (cpu_has_llsc) {
132 __asm__ __volatile__(
133 " .set mips3 \n"
134 "1: " __LL "%0, %1 # clear_bit \n"
135 " and %0, %2 \n"
136 " " __SC "%0, %1 \n"
137 " beqz %0, 1b \n"
138 " .set mips0 \n"
139 : "=&r" (temp), "=m" (*m)
140 : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m));
141 } else {
142 volatile unsigned long *a = addr;
143 unsigned long mask;
144 __bi_flags;
145
146 a += nr >> SZLONG_LOG;
147 mask = 1UL << (nr & SZLONG_MASK);
148 __bi_local_irq_save(flags);
149 *a &= ~mask;
150 __bi_local_irq_restore(flags);
151 }
152}
153
154/*
155 * change_bit - Toggle a bit in memory
156 * @nr: Bit to change
157 * @addr: Address to start counting from
158 *
159 * change_bit() is atomic and may not be reordered.
160 * Note that @nr may be almost arbitrarily large; this function is not
161 * restricted to acting on a single-word quantity.
162 */
163static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
164{
165 if (cpu_has_llsc && R10000_LLSC_WAR) {
166 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
167 unsigned long temp;
168
169 __asm__ __volatile__(
170 " .set mips3 \n"
171 "1: " __LL "%0, %1 # change_bit \n"
172 " xor %0, %2 \n"
173 " " __SC "%0, %1 \n"
174 " beqzl %0, 1b \n"
175 " .set mips0 \n"
176 : "=&r" (temp), "=m" (*m)
177 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
178 } else if (cpu_has_llsc) {
179 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
180 unsigned long temp;
181
182 __asm__ __volatile__(
183 " .set mips3 \n"
184 "1: " __LL "%0, %1 # change_bit \n"
185 " xor %0, %2 \n"
186 " " __SC "%0, %1 \n"
187 " beqz %0, 1b \n"
188 " .set mips0 \n"
189 : "=&r" (temp), "=m" (*m)
190 : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m));
191 } else {
192 volatile unsigned long *a = addr;
193 unsigned long mask;
194 __bi_flags;
195
196 a += nr >> SZLONG_LOG;
197 mask = 1UL << (nr & SZLONG_MASK);
198 __bi_local_irq_save(flags);
199 *a ^= mask;
200 __bi_local_irq_restore(flags);
201 }
202}
203
204/*
205 * test_and_set_bit - Set a bit and return its old value
206 * @nr: Bit to set
207 * @addr: Address to count from
208 *
209 * This operation is atomic and cannot be reordered.
210 * It also implies a memory barrier.
211 */
212static inline int test_and_set_bit(unsigned long nr,
213 volatile unsigned long *addr)
214{
215 if (cpu_has_llsc && R10000_LLSC_WAR) {
216 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
217 unsigned long temp, res;
218
219 __asm__ __volatile__(
220 " .set mips3 \n"
221 "1: " __LL "%0, %1 # test_and_set_bit \n"
222 " or %2, %0, %3 \n"
223 " " __SC "%2, %1 \n"
224 " beqzl %2, 1b \n"
225 " and %2, %0, %3 \n"
226#ifdef CONFIG_SMP
227 " sync \n"
228#endif
229 " .set mips0 \n"
230 : "=&r" (temp), "=m" (*m), "=&r" (res)
231 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
232 : "memory");
233
234 return res != 0;
235 } else if (cpu_has_llsc) {
236 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
237 unsigned long temp, res;
238
239 __asm__ __volatile__(
240 " .set push \n"
241 " .set noreorder \n"
242 " .set mips3 \n"
243 "1: " __LL "%0, %1 # test_and_set_bit \n"
244 " or %2, %0, %3 \n"
245 " " __SC "%2, %1 \n"
246 " beqz %2, 1b \n"
247 " and %2, %0, %3 \n"
248#ifdef CONFIG_SMP
249 " sync \n"
250#endif
251 " .set pop \n"
252 : "=&r" (temp), "=m" (*m), "=&r" (res)
253 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
254 : "memory");
255
256 return res != 0;
257 } else {
258 volatile unsigned long *a = addr;
259 unsigned long mask;
260 int retval;
261 __bi_flags;
262
263 a += nr >> SZLONG_LOG;
264 mask = 1UL << (nr & SZLONG_MASK);
265 __bi_local_irq_save(flags);
266 retval = (mask & *a) != 0;
267 *a |= mask;
268 __bi_local_irq_restore(flags);
269
270 return retval;
271 }
272}
273
274/*
275 * test_and_clear_bit - Clear a bit and return its old value
276 * @nr: Bit to clear
277 * @addr: Address to count from
278 *
279 * This operation is atomic and cannot be reordered.
280 * It also implies a memory barrier.
281 */
282static inline int test_and_clear_bit(unsigned long nr,
283 volatile unsigned long *addr)
284{
285 if (cpu_has_llsc && R10000_LLSC_WAR) {
286 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
287 unsigned long temp, res;
288
289 __asm__ __volatile__(
290 " .set mips3 \n"
291 "1: " __LL "%0, %1 # test_and_clear_bit \n"
292 " or %2, %0, %3 \n"
293 " xor %2, %3 \n"
294 " " __SC "%2, %1 \n"
295 " beqzl %2, 1b \n"
296 " and %2, %0, %3 \n"
297#ifdef CONFIG_SMP
298 " sync \n"
299#endif
300 " .set mips0 \n"
301 : "=&r" (temp), "=m" (*m), "=&r" (res)
302 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
303 : "memory");
304
305 return res != 0;
306 } else if (cpu_has_llsc) {
307 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
308 unsigned long temp, res;
309
310 __asm__ __volatile__(
311 " .set push \n"
312 " .set noreorder \n"
313 " .set mips3 \n"
314 "1: " __LL "%0, %1 # test_and_clear_bit \n"
315 " or %2, %0, %3 \n"
316 " xor %2, %3 \n"
317 " " __SC "%2, %1 \n"
318 " beqz %2, 1b \n"
319 " and %2, %0, %3 \n"
320#ifdef CONFIG_SMP
321 " sync \n"
322#endif
323 " .set pop \n"
324 : "=&r" (temp), "=m" (*m), "=&r" (res)
325 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
326 : "memory");
327
328 return res != 0;
329 } else {
330 volatile unsigned long *a = addr;
331 unsigned long mask;
332 int retval;
333 __bi_flags;
334
335 a += nr >> SZLONG_LOG;
336 mask = 1UL << (nr & SZLONG_MASK);
337 __bi_local_irq_save(flags);
338 retval = (mask & *a) != 0;
339 *a &= ~mask;
340 __bi_local_irq_restore(flags);
341
342 return retval;
343 }
344}
345
346/*
347 * test_and_change_bit - Change a bit and return its old value
348 * @nr: Bit to change
349 * @addr: Address to count from
350 *
351 * This operation is atomic and cannot be reordered.
352 * It also implies a memory barrier.
353 */
354static inline int test_and_change_bit(unsigned long nr,
355 volatile unsigned long *addr)
356{
357 if (cpu_has_llsc && R10000_LLSC_WAR) {
358 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
359 unsigned long temp, res;
360
361 __asm__ __volatile__(
362 " .set mips3 \n"
363 "1: " __LL "%0, %1 # test_and_change_bit \n"
364 " xor %2, %0, %3 \n"
365 " " __SC "%2, %1 \n"
366 " beqzl %2, 1b \n"
367 " and %2, %0, %3 \n"
368#ifdef CONFIG_SMP
369 " sync \n"
370#endif
371 " .set mips0 \n"
372 : "=&r" (temp), "=m" (*m), "=&r" (res)
373 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
374 : "memory");
375
376 return res != 0;
377 } else if (cpu_has_llsc) {
378 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
379 unsigned long temp, res;
380
381 __asm__ __volatile__(
382 " .set push \n"
383 " .set noreorder \n"
384 " .set mips3 \n"
385 "1: " __LL "%0, %1 # test_and_change_bit \n"
386 " xor %2, %0, %3 \n"
387 " " __SC "\t%2, %1 \n"
388 " beqz %2, 1b \n"
389 " and %2, %0, %3 \n"
390#ifdef CONFIG_SMP
391 " sync \n"
392#endif
393 " .set pop \n"
394 : "=&r" (temp), "=m" (*m), "=&r" (res)
395 : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m)
396 : "memory");
397
398 return res != 0;
399 } else {
400 volatile unsigned long *a = addr;
401 unsigned long mask, retval;
402 __bi_flags;
403
404 a += nr >> SZLONG_LOG;
405 mask = 1UL << (nr & SZLONG_MASK);
406 __bi_local_irq_save(flags);
407 retval = (mask & *a) != 0;
408 *a ^= mask;
409 __bi_local_irq_restore(flags);
410
411 return retval;
412 }
413}
414
415#undef __bi_flags
416#undef __bi_local_irq_save
417#undef __bi_local_irq_restore
418
419#include <asm-generic/bitops/non-atomic.h>
420
421/*
422 * Return the bit position (0..63) of the most significant 1 bit in a word
423 * Returns -1 if no 1 bit exists
424 */
425static inline int __ilog2(unsigned long x)
426{
427 int lz;
428
429 if (sizeof(x) == 4) {
430 __asm__ (
431 " .set push \n"
432 " .set mips32 \n"
433 " clz %0, %1 \n"
434 " .set pop \n"
435 : "=r" (lz)
436 : "r" (x));
437
438 return 31 - lz;
439 }
440
441 BUG_ON(sizeof(x) != 8);
442
443 __asm__ (
444 " .set push \n"
445 " .set mips64 \n"
446 " dclz %0, %1 \n"
447 " .set pop \n"
448 : "=r" (lz)
449 : "r" (x));
450
451 return 63 - lz;
452}
453
454#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)
455
456/*
457 * __ffs - find first bit in word.
458 * @word: The word to search
459 *
460 * Returns 0..SZLONG-1
461 * Undefined if no bit exists, so code should check against 0 first.
462 */
463static inline unsigned long __ffs(unsigned long word)
464{
465 return __ilog2(word & -word);
466}
467
468/*
469 * fls - find last bit set.
470 * @word: The word to search
471 *
472 * This is defined the same way as ffs.
473 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
474 */
475static inline int fls(int word)
476{
477 __asm__ ("clz %0, %1" : "=r" (word) : "r" (word));
478
479 return 32 - word;
480}
481
482#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64)
483static inline int fls64(__u64 word)
484{
485 __asm__ ("dclz %0, %1" : "=r" (word) : "r" (word));
486
487 return 64 - word;
488}
489#else
490#include <asm-generic/bitops/fls64.h>
491#endif
492
493/*
494 * ffs - find first bit set.
495 * @word: The word to search
496 *
497 * This is defined the same way as
498 * the libc and compiler builtin ffs routines, therefore
499 * differs in spirit from the above ffz (man ffs).
500 */
501static inline int ffs(int word)
502{
503 if (!word)
504 return 0;
505
506 return fls(word & -word);
507}
508
509#else
510
511#include <asm-generic/bitops/__ffs.h>
512#include <asm-generic/bitops/ffs.h>
513#include <asm-generic/bitops/fls.h>
514#include <asm-generic/bitops/fls64.h>
515
516#endif /*defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) */
517
518#include <asm-generic/bitops/ffz.h>
519#include <asm-generic/bitops/find.h>
520
521#ifdef __KERNEL__
522
523#include <asm-generic/bitops/sched.h>
524#include <asm-generic/bitops/hweight.h>
525#include <asm-generic/bitops/ext2-non-atomic.h>
526#include <asm-generic/bitops/ext2-atomic.h>
527#include <asm-generic/bitops/minix.h>
528
529#endif /* __KERNEL__ */
530
531#endif /* _ASM_BITOPS_H */