arch/mips/include/asm/bitops.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (c) 1994 - 1997, 99, 2000, 06, 07  Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (c) 1999, 2000  Silicon Graphics, Inc.
   8  */
   9 #ifndef _ASM_BITOPS_H
  10 #define _ASM_BITOPS_H
  11
  12 #ifndef _LINUX_BITOPS_H
  13 #error only <linux/bitops.h> can be included directly
  14 #endif
  15
  16 #include <linux/compiler.h>
  17 #include <linux/types.h>
  18 #include <asm/barrier.h>
  19 #include <asm/byteorder.h>              /* sigh ... */
  20 #include <asm/compiler.h>
  21 #include <asm/cpu-features.h>
  22 #include <asm/llsc.h>
  23 #include <asm/sgidefs.h>
  24 #include <asm/war.h>
  25
  26 /*
  27  * These are the "slower" versions of the functions and are in bitops.c.
  28  * These functions call raw_local_irq_{save,restore}().
  29  */
  30 void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
  31 void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
  32 void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
  33 int __mips_test_and_set_bit(unsigned long nr,
  34                             volatile unsigned long *addr);
  35 int __mips_test_and_set_bit_lock(unsigned long nr,
  36                                  volatile unsigned long *addr);
  37 int __mips_test_and_clear_bit(unsigned long nr,
  38                               volatile unsigned long *addr);
  39 int __mips_test_and_change_bit(unsigned long nr,
  40                                volatile unsigned long *addr);
  41
  42
  43 /*
  44  * set_bit - Atomically set a bit in memory
  45  * @nr: the bit to set
  46  * @addr: the address to start counting from
  47  *
  48  * This function is atomic and may not be reordered.  See __set_bit()
  49  * if you do not require the atomic guarantees.
  50  * Note that @nr may be almost arbitrarily large; this function is not
  51  * restricted to acting on a single-word quantity.
  52  */
  53 static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
  54 {
  55         unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
  56         int bit = nr & SZLONG_MASK;
  57         unsigned long temp;
  58
  59         if (kernel_uses_llsc && R10000_LLSC_WAR) {
  60                 __asm__ __volatile__(
  61                 "       .set    push                                    \n"
  62                 "       .set    arch=r4000                              \n"
  63                 "1:     " __LL "%0, %1                  # set_bit       \n"
  64                 "       or      %0, %2                                  \n"
  65                 "       " __SC  "%0, %1                                 \n"
  66                 "       beqzl   %0, 1b                                  \n"
  67                 "       .set    pop                                     \n"
  68                 : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
  69                 : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
  70                 : __LLSC_CLOBBER);
  71 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
  72         } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
  73                 loongson_llsc_mb();
  74                 do {
  75                         __asm__ __volatile__(
  76                         "       " __LL "%0, %1          # set_bit       \n"
  77                         "       " __INS "%0, %3, %2, 1                  \n"
  78                         "       " __SC "%0, %1                          \n"
  79                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
  80                         : "ir" (bit), "r" (~0)
  81                         : __LLSC_CLOBBER);
  82                 } while (unlikely(!temp));
  83 #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
  84         } else if (kernel_uses_llsc) {
  85                 loongson_llsc_mb();
  86                 do {
  87                         __asm__ __volatile__(
  88                         "       .set    push                            \n"
  89                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
  90                         "       " __LL "%0, %1          # set_bit       \n"
  91                         "       or      %0, %2                          \n"
  92                         "       " __SC  "%0, %1                         \n"
  93                         "       .set    pop                             \n"
  94                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
  95                         : "ir" (1UL << bit)
  96                         : __LLSC_CLOBBER);
  97                 } while (unlikely(!temp));
  98         } else
  99                 __mips_set_bit(nr, addr);
 100 }
 101
 102 /*
 103  * clear_bit - Clears a bit in memory
 104  * @nr: Bit to clear
 105  * @addr: Address to start counting from
 106  *
 107  * clear_bit() is atomic and may not be reordered.  However, it does
 108  * not contain a memory barrier, so if it is used for locking purposes,
 109  * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
 110  * in order to ensure changes are visible on other processors.
 111  */
 112 static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
 113 {
 114         unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 115         int bit = nr & SZLONG_MASK;
 116         unsigned long temp;
 117
 118         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 119                 __asm__ __volatile__(
 120                 "       .set    push                                    \n"
 121                 "       .set    arch=r4000                              \n"
 122                 "1:     " __LL "%0, %1                  # clear_bit     \n"
 123                 "       and     %0, %2                                  \n"
 124                 "       " __SC "%0, %1                                  \n"
 125                 "       beqzl   %0, 1b                                  \n"
 126                 "       .set    pop                                     \n"
 127                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
 128                 : "ir" (~(1UL << bit))
 129                 : __LLSC_CLOBBER);
 130 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
 131         } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
 132                 loongson_llsc_mb();
 133                 do {
 134                         __asm__ __volatile__(
 135                         "       " __LL "%0, %1          # clear_bit     \n"
 136                         "       " __INS "%0, $0, %2, 1                  \n"
 137                         "       " __SC "%0, %1                          \n"
 138                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
 139                         : "ir" (bit)
 140                         : __LLSC_CLOBBER);
 141                 } while (unlikely(!temp));
 142 #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
 143         } else if (kernel_uses_llsc) {
 144                 loongson_llsc_mb();
 145                 do {
 146                         __asm__ __volatile__(
 147                         "       .set    push                            \n"
 148                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 149                         "       " __LL "%0, %1          # clear_bit     \n"
 150                         "       and     %0, %2                          \n"
 151                         "       " __SC "%0, %1                          \n"
 152                         "       .set    pop                             \n"
 153                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
 154                         : "ir" (~(1UL << bit))
 155                         : __LLSC_CLOBBER);
 156                 } while (unlikely(!temp));
 157         } else
 158                 __mips_clear_bit(nr, addr);
 159 }
 160
 161 /*
 162  * clear_bit_unlock - Clears a bit in memory
 163  * @nr: Bit to clear
 164  * @addr: Address to start counting from
 165  *
 166  * clear_bit() is atomic and implies release semantics before the memory
 167  * operation. It can be used for an unlock.
 168  */
 169 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
 170 {
 171         smp_mb__before_atomic();
 172         clear_bit(nr, addr);
 173 }
 174
 175 /*
 176  * change_bit - Toggle a bit in memory
 177  * @nr: Bit to change
 178  * @addr: Address to start counting from
 179  *
 180  * change_bit() is atomic and may not be reordered.
 181  * Note that @nr may be almost arbitrarily large; this function is not
 182  * restricted to acting on a single-word quantity.
 183  */
 184 static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
 185 {
 186         int bit = nr & SZLONG_MASK;
 187
 188         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 189                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 190                 unsigned long temp;
 191
 192                 __asm__ __volatile__(
 193                 "       .set    push                            \n"
 194                 "       .set    arch=r4000                      \n"
 195                 "1:     " __LL "%0, %1          # change_bit    \n"
 196                 "       xor     %0, %2                          \n"
 197                 "       " __SC  "%0, %1                         \n"
 198                 "       beqzl   %0, 1b                          \n"
 199                 "       .set    pop                             \n"
 200                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
 201                 : "ir" (1UL << bit)
 202                 : __LLSC_CLOBBER);
 203         } else if (kernel_uses_llsc) {
 204                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 205                 unsigned long temp;
 206
 207                 loongson_llsc_mb();
 208                 do {
 209                         __asm__ __volatile__(
 210                         "       .set    push                            \n"
 211                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 212                         "       " __LL "%0, %1          # change_bit    \n"
 213                         "       xor     %0, %2                          \n"
 214                         "       " __SC  "%0, %1                         \n"
 215                         "       .set    pop                             \n"
 216                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
 217                         : "ir" (1UL << bit)
 218                         : __LLSC_CLOBBER);
 219                 } while (unlikely(!temp));
 220         } else
 221                 __mips_change_bit(nr, addr);
 222 }
 223
 224 /*
 225  * test_and_set_bit - Set a bit and return its old value
 226  * @nr: Bit to set
 227  * @addr: Address to count from
 228  *
 229  * This operation is atomic and cannot be reordered.
 230  * It also implies a memory barrier.
 231  */
 232 static inline int test_and_set_bit(unsigned long nr,
 233         volatile unsigned long *addr)
 234 {
 235         int bit = nr & SZLONG_MASK;
 236         unsigned long res;
 237
 238         smp_mb__before_llsc();
 239
 240         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 241                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 242                 unsigned long temp;
 243
 244                 __asm__ __volatile__(
 245                 "       .set    push                                    \n"
 246                 "       .set    arch=r4000                              \n"
 247                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 248                 "       or      %2, %0, %3                              \n"
 249                 "       " __SC  "%2, %1                                 \n"
 250                 "       beqzl   %2, 1b                                  \n"
 251                 "       and     %2, %0, %3                              \n"
 252                 "       .set    pop                                     \n"
 253                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 254                 : "r" (1UL << bit)
 255                 : __LLSC_CLOBBER);
 256         } else if (kernel_uses_llsc) {
 257                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 258                 unsigned long temp;
 259
 260                 loongson_llsc_mb();
 261                 do {
 262                         __asm__ __volatile__(
 263                         "       .set    push                            \n"
 264                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 265                         "       " __LL "%0, %1  # test_and_set_bit      \n"
 266                         "       or      %2, %0, %3                      \n"
 267                         "       " __SC  "%2, %1                         \n"
 268                         "       .set    pop                             \n"
 269                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 270                         : "r" (1UL << bit)
 271                         : __LLSC_CLOBBER);
 272                 } while (unlikely(!res));
 273
 274                 res = temp & (1UL << bit);
 275         } else
 276                 res = __mips_test_and_set_bit(nr, addr);
 277
 278         smp_llsc_mb();
 279
 280         return res != 0;
 281 }
 282
 283 /*
 284  * test_and_set_bit_lock - Set a bit and return its old value
 285  * @nr: Bit to set
 286  * @addr: Address to count from
 287  *
 288  * This operation is atomic and implies acquire ordering semantics
 289  * after the memory operation.
 290  */
 291 static inline int test_and_set_bit_lock(unsigned long nr,
 292         volatile unsigned long *addr)
 293 {
 294         int bit = nr & SZLONG_MASK;
 295         unsigned long res;
 296
 297         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 298                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 299                 unsigned long temp;
 300
 301                 __asm__ __volatile__(
 302                 "       .set    push                                    \n"
 303                 "       .set    arch=r4000                              \n"
 304                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 305                 "       or      %2, %0, %3                              \n"
 306                 "       " __SC  "%2, %1                                 \n"
 307                 "       beqzl   %2, 1b                                  \n"
 308                 "       and     %2, %0, %3                              \n"
 309                 "       .set    pop                                     \n"
 310                 : "=&r" (temp), "+m" (*m), "=&r" (res)
 311                 : "r" (1UL << bit)
 312                 : __LLSC_CLOBBER);
 313         } else if (kernel_uses_llsc) {
 314                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 315                 unsigned long temp;
 316
 317                 loongson_llsc_mb();
 318                 do {
 319                         __asm__ __volatile__(
 320                         "       .set    push                            \n"
 321                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 322                         "       " __LL "%0, %1  # test_and_set_bit      \n"
 323                         "       or      %2, %0, %3                      \n"
 324                         "       " __SC  "%2, %1                         \n"
 325                         "       .set    pop                             \n"
 326                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 327                         : "r" (1UL << bit)
 328                         : __LLSC_CLOBBER);
 329                 } while (unlikely(!res));
 330
 331                 res = temp & (1UL << bit);
 332         } else
 333                 res = __mips_test_and_set_bit_lock(nr, addr);
 334
 335         smp_llsc_mb();
 336
 337         return res != 0;
 338 }
 339 /*
 340  * test_and_clear_bit - Clear a bit and return its old value
 341  * @nr: Bit to clear
 342  * @addr: Address to count from
 343  *
 344  * This operation is atomic and cannot be reordered.
 345  * It also implies a memory barrier.
 346  */
 347 static inline int test_and_clear_bit(unsigned long nr,
 348         volatile unsigned long *addr)
 349 {
 350         int bit = nr & SZLONG_MASK;
 351         unsigned long res;
 352
 353         smp_mb__before_llsc();
 354
 355         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 356                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 357                 unsigned long temp;
 358
 359                 __asm__ __volatile__(
 360                 "       .set    push                                    \n"
 361                 "       .set    arch=r4000                              \n"
 362                 "1:     " __LL  "%0, %1         # test_and_clear_bit    \n"
 363                 "       or      %2, %0, %3                              \n"
 364                 "       xor     %2, %3                                  \n"
 365                 "       " __SC  "%2, %1                                 \n"
 366                 "       beqzl   %2, 1b                                  \n"
 367                 "       and     %2, %0, %3                              \n"
 368                 "       .set    pop                                     \n"
 369                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 370                 : "r" (1UL << bit)
 371                 : __LLSC_CLOBBER);
 372 #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
 373         } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
 374                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 375                 unsigned long temp;
 376
 377                 loongson_llsc_mb();
 378                 do {
 379                         __asm__ __volatile__(
 380                         "       " __LL  "%0, %1 # test_and_clear_bit    \n"
 381                         "       " __EXT "%2, %0, %3, 1                  \n"
 382                         "       " __INS "%0, $0, %3, 1                  \n"
 383                         "       " __SC  "%0, %1                         \n"
 384                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 385                         : "ir" (bit)
 386                         : __LLSC_CLOBBER);
 387                 } while (unlikely(!temp));
 388 #endif
 389         } else if (kernel_uses_llsc) {
 390                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 391                 unsigned long temp;
 392
 393                 loongson_llsc_mb();
 394                 do {
 395                         __asm__ __volatile__(
 396                         "       .set    push                            \n"
 397                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 398                         "       " __LL  "%0, %1 # test_and_clear_bit    \n"
 399                         "       or      %2, %0, %3                      \n"
 400                         "       xor     %2, %3                          \n"
 401                         "       " __SC  "%2, %1                         \n"
 402                         "       .set    pop                             \n"
 403                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 404                         : "r" (1UL << bit)
 405                         : __LLSC_CLOBBER);
 406                 } while (unlikely(!res));
 407
 408                 res = temp & (1UL << bit);
 409         } else
 410                 res = __mips_test_and_clear_bit(nr, addr);
 411
 412         smp_llsc_mb();
 413
 414         return res != 0;
 415 }
 416
 417 /*
 418  * test_and_change_bit - Change a bit and return its old value
 419  * @nr: Bit to change
 420  * @addr: Address to count from
 421  *
 422  * This operation is atomic and cannot be reordered.
 423  * It also implies a memory barrier.
 424  */
 425 static inline int test_and_change_bit(unsigned long nr,
 426         volatile unsigned long *addr)
 427 {
 428         int bit = nr & SZLONG_MASK;
 429         unsigned long res;
 430
 431         smp_mb__before_llsc();
 432
 433         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 434                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 435                 unsigned long temp;
 436
 437                 __asm__ __volatile__(
 438                 "       .set    push                                    \n"
 439                 "       .set    arch=r4000                              \n"
 440                 "1:     " __LL  "%0, %1         # test_and_change_bit   \n"
 441                 "       xor     %2, %0, %3                              \n"
 442                 "       " __SC  "%2, %1                                 \n"
 443                 "       beqzl   %2, 1b                                  \n"
 444                 "       and     %2, %0, %3                              \n"
 445                 "       .set    pop                                     \n"
 446                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 447                 : "r" (1UL << bit)
 448                 : __LLSC_CLOBBER);
 449         } else if (kernel_uses_llsc) {
 450                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 451                 unsigned long temp;
 452
 453                 loongson_llsc_mb();
 454                 do {
 455                         __asm__ __volatile__(
 456                         "       .set    push                            \n"
 457                         "       .set    "MIPS_ISA_ARCH_LEVEL"           \n"
 458                         "       " __LL  "%0, %1 # test_and_change_bit   \n"
 459                         "       xor     %2, %0, %3                      \n"
 460                         "       " __SC  "\t%2, %1                       \n"
 461                         "       .set    pop                             \n"
 462                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
 463                         : "r" (1UL << bit)
 464                         : __LLSC_CLOBBER);
 465                 } while (unlikely(!res));
 466
 467                 res = temp & (1UL << bit);
 468         } else
 469                 res = __mips_test_and_change_bit(nr, addr);
 470
 471         smp_llsc_mb();
 472
 473         return res != 0;
 474 }
 475
 476 #include <asm-generic/bitops/non-atomic.h>
 477
 478 /*
 479  * __clear_bit_unlock - Clears a bit in memory
 480  * @nr: Bit to clear
 481  * @addr: Address to start counting from
 482  *
 483  * __clear_bit() is non-atomic and implies release semantics before the memory
 484  * operation. It can be used for an unlock if no other CPUs can concurrently
 485  * modify other bits in the word.
 486  */
 487 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
 488 {
 489         smp_mb__before_llsc();
 490         __clear_bit(nr, addr);
 491         nudge_writes();
 492 }
 493
 494 /*
 495  * Return the bit position (0..63) of the most significant 1 bit in a word
 496  * Returns -1 if no 1 bit exists
 497  */
 498 static __always_inline unsigned long __fls(unsigned long word)
 499 {
 500         int num;
 501
 502         if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
 503             __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
 504                 __asm__(
 505                 "       .set    push                                    \n"
 506                 "       .set    "MIPS_ISA_LEVEL"                        \n"
 507                 "       clz     %0, %1                                  \n"
 508                 "       .set    pop                                     \n"
 509                 : "=r" (num)
 510                 : "r" (word));
 511
 512                 return 31 - num;
 513         }
 514
 515         if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
 516             __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
 517                 __asm__(
 518                 "       .set    push                                    \n"
 519                 "       .set    "MIPS_ISA_LEVEL"                        \n"
 520                 "       dclz    %0, %1                                  \n"
 521                 "       .set    pop                                     \n"
 522                 : "=r" (num)
 523                 : "r" (word));
 524
 525                 return 63 - num;
 526         }
 527
 528         num = BITS_PER_LONG - 1;
 529
 530 #if BITS_PER_LONG == 64
 531         if (!(word & (~0ul << 32))) {
 532                 num -= 32;
 533                 word <<= 32;
 534         }
 535 #endif
 536         if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
 537                 num -= 16;
 538                 word <<= 16;
 539         }
 540         if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
 541                 num -= 8;
 542                 word <<= 8;
 543         }
 544         if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
 545                 num -= 4;
 546                 word <<= 4;
 547         }
 548         if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
 549                 num -= 2;
 550                 word <<= 2;
 551         }
 552         if (!(word & (~0ul << (BITS_PER_LONG-1))))
 553                 num -= 1;
 554         return num;
 555 }
 556
 557 /*
 558  * __ffs - find first bit in word.
 559  * @word: The word to search
 560  *
 561  * Returns 0..SZLONG-1
 562  * Undefined if no bit exists, so code should check against 0 first.
 563  */
 564 static __always_inline unsigned long __ffs(unsigned long word)
 565 {
 566         return __fls(word & -word);
 567 }
 568
 569 /*
 570  * fls - find last bit set.
 571  * @word: The word to search
 572  *
 573  * This is defined the same way as ffs.
 574  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 575  */
 576 static inline int fls(unsigned int x)
 577 {
 578         int r;
 579
 580         if (!__builtin_constant_p(x) &&
 581             __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
 582                 __asm__(
 583                 "       .set    push                                    \n"
 584                 "       .set    "MIPS_ISA_LEVEL"                        \n"
 585                 "       clz     %0, %1                                  \n"
 586                 "       .set    pop                                     \n"
 587                 : "=r" (x)
 588                 : "r" (x));
 589
 590                 return 32 - x;
 591         }
 592
 593         r = 32;
 594         if (!x)
 595                 return 0;
 596         if (!(x & 0xffff0000u)) {
 597                 x <<= 16;
 598                 r -= 16;
 599         }
 600         if (!(x & 0xff000000u)) {
 601                 x <<= 8;
 602                 r -= 8;
 603         }
 604         if (!(x & 0xf0000000u)) {
 605                 x <<= 4;
 606                 r -= 4;
 607         }
 608         if (!(x & 0xc0000000u)) {
 609                 x <<= 2;
 610                 r -= 2;
 611         }
 612         if (!(x & 0x80000000u)) {
 613                 x <<= 1;
 614                 r -= 1;
 615         }
 616         return r;
 617 }
 618
 619 #include <asm-generic/bitops/fls64.h>
 620
 621 /*
 622  * ffs - find first bit set.
 623  * @word: The word to search
 624  *
 625  * This is defined the same way as
 626  * the libc and compiler builtin ffs routines, therefore
 627  * differs in spirit from the above ffz (man ffs).
 628  */
 629 static inline int ffs(int word)
 630 {
 631         if (!word)
 632                 return 0;
 633
 634         return fls(word & -word);
 635 }
 636
 637 #include <asm-generic/bitops/ffz.h>
 638 #include <asm-generic/bitops/find.h>
 639
 640 #ifdef __KERNEL__
 641
 642 #include <asm-generic/bitops/sched.h>
 643
 644 #include <asm/arch_hweight.h>
 645 #include <asm-generic/bitops/const_hweight.h>
 646
 647 #include <asm-generic/bitops/le.h>
 648 #include <asm-generic/bitops/ext2-atomic.h>
 649
 650 #endif /* __KERNEL__ */
 651
 652 #endif /* _ASM_BITOPS_H */