diff options
Diffstat (limited to 'src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h')
| -rw-r--r-- | src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h | 541 |
1 files changed, 541 insertions, 0 deletions
diff --git a/src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h b/src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h new file mode 100644 index 0000000000..7f521a67e2 --- /dev/null +++ b/src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h @@ -0,0 +1,541 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ + +/** + * @file + * + * This file provides atomic operations + * + * @addtogroup hal + * @{ + */ + +/** + * Atomically adds a signed value to a 32 bit (aligned) memory location. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. (This should NOT be used for reference counting - + * use the standard version instead.) + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + */ +static inline void bdk_atomic_add32_nosync(int32_t *ptr, int32_t incr) +{ + /* Atomic add with no ordering */ + asm volatile ("ldadd %w[i], wzr, [%[b]]" + : [r] "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); +} + +/** + * Atomically adds a signed value to a 32 bit (aligned) memory location. + * + * Memory access ordering is enforced before/after the atomic operation, + * so no additional 'sync' instructions are required. + * + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + */ +static inline void bdk_atomic_add32(int32_t *ptr, int32_t incr) +{ + /* Atomic add with acquire and release */ + asm volatile ("ldaddal %w[i], wzr, [%[b]]" + : "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); +} + +/** + * Atomically sets a 32 bit (aligned) memory location to a value + * + * @param ptr address of memory to set + * @param value value to set memory location to. + */ +static inline void bdk_atomic_set32(int32_t *ptr, int32_t value) +{ + /* Implies a release */ + asm volatile ("stlr %w[v], [%[b]]" + : "+m" (*ptr) + : [v] "r" (value), [b] "r" (ptr) + : "memory"); +} + +/** + * Returns the current value of a 32 bit (aligned) memory + * location. + * + * @param ptr Address of memory to get + * @return Value of the memory + */ +static inline int32_t bdk_atomic_get32(int32_t *ptr) +{ + return *(volatile int32_t *)ptr; +} + +/** + * Atomically adds a signed value to a 64 bit (aligned) memory location. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. (This should NOT be used for reference counting - + * use the standard version instead.) + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + */ +static inline void bdk_atomic_add64_nosync(int64_t *ptr, int64_t incr) +{ + /* Atomic add with no ordering */ + asm volatile ("ldadd %x[i], xzr, [%[b]]" + : [r] "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); +} + +/** + * Atomically adds a signed value to a 64 bit (aligned) memory location. + * + * Memory access ordering is enforced before/after the atomic operation, + * so no additional 'sync' instructions are required. + * + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + */ +static inline void bdk_atomic_add64(int64_t *ptr, int64_t incr) +{ + /* Atomic add with acquire and release */ + asm volatile ("ldaddal %x[i], xzr, [%[b]]" + : [r] "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); +} + +/** + * Atomically sets a 64 bit (aligned) memory location to a value + * + * @param ptr address of memory to set + * @param value value to set memory location to. + */ +static inline void bdk_atomic_set64(int64_t *ptr, int64_t value) +{ + /* Implies a release */ + asm volatile ("stlr %x[v], [%[b]]" + : "+m" (*ptr) + : [v] "r" (value), [b] "r" (ptr) + : "memory"); +} + +/** + * Returns the current value of a 64 bit (aligned) memory + * location. + * + * @param ptr Address of memory to get + * @return Value of the memory + */ +static inline int64_t bdk_atomic_get64(int64_t *ptr) +{ + return *(volatile int64_t *)ptr; +} + +/** + * Atomically compares the old value with the value at ptr, and if they match, + * stores new_val to ptr. + * If *ptr and old don't match, function returns failure immediately. + * If *ptr and old match, function spins until *ptr updated to new atomically, or + * until *ptr and old no longer match + * + * Does no memory synchronization. + * + * @return 1 on success (match and store) + * 0 on no match + */ +static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline)); +static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val) +{ + uint32_t val = old_val; + + /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever + Don't use compare and swap on these chips */ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X)) + { + asm volatile ("1: ldxr %w[v], [%[b]] \n" + " cmp %w[v], %w[o] \n" + " b.ne 2f \n" + " stxr %w[v], %w[n], [%[b]]\n" /* Returns zero on success */ + " cbnz %w[v], 1b \n" + " mov %w[v], %w[o] \n" + "2: \n" + : [mem] "+m" (*ptr), [v] "=&r" (val) + : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val) + : ); + } + else + { + asm volatile ("cas %w[o], %w[n], [%[b]]" + : [mem] "+m" (*ptr), [o] "+r" (val) + : [b] "r" (ptr), [n] "r" (new_val) + : ); + } + return old_val == val; +} + +/** + * Atomically compares the old value with the value at ptr, and if they match, + * stores new_val to ptr. + * If *ptr and old don't match, function returns failure immediately. + * If *ptr and old match, function spins until *ptr updated to new atomically, or + * until *ptr and old no longer match + * + * Does memory synchronization that is required to use this as a locking primitive. + * + * @return 1 on success (match and store) + * 0 on no match + */ +static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline)); +static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val) +{ + uint32_t val = old_val; + + /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever + Don't use compare and swap on these chips */ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X)) + { + asm volatile ("1: ldaxr %w[v], [%[b]] \n" + " cmp %w[v], %w[o] \n" + " b.ne 2f \n" + " stlxr %w[v], %w[n], [%[b]]\n" /* Returns zero on success */ + " cbnz %w[v], 1b \n" + " mov %w[v], %w[o] \n" + "2: \n" + : [mem] "+m" (*ptr), [v] "=&r" (val) + : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val) + : ); + } + else + { + asm volatile ("casal %w[o], %w[n], [%[b]]" + : [mem] "+m" (*ptr), [o] "+r" (val) + : [b] "r" (ptr), [n] "r" (new_val) + : ); + } + return old_val == val; +} + +/** + * Atomically compares the old value with the value at ptr, and if they match, + * stores new_val to ptr. + * If *ptr and old don't match, function returns failure immediately. + * If *ptr and old match, function spins until *ptr updated to new atomically, or + * until *ptr and old no longer match + * + * Does no memory synchronization. + * + * @return 1 on success (match and store) + * 0 on no match + */ +static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline)); +static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val) +{ + uint32_t val = old_val; + + /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever + Don't use compare and swap on these chips */ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X)) + { + asm volatile ("1: ldxr %x[v], [%[b]] \n" + " cmp %x[v], %x[o] \n" + " b.ne 2f \n" + " stxr %x[v], %x[n], [%[b]]\n" /* Returns zero on success */ + " cbnz %x[v], 1b \n" + " mov %x[v], %x[o] \n" + "2: \n" + : [mem] "+m" (*ptr), [v] "=&r" (val) + : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val) + : ); + } + else + { + asm volatile ("cas %x[o], %x[n], [%[b]]" + : [mem] "+m" (*ptr), [o] "+r" (val) + : [b] "r" (ptr), [n] "r" (new_val) + : ); + } + return old_val == val; +} + +/** + * Atomically compares the old value with the value at ptr, and if they match, + * stores new_val to ptr. + * If *ptr and old don't match, function returns failure immediately. + * If *ptr and old match, function spins until *ptr updated to new atomically, or + * until *ptr and old no longer match + * + * Does memory synchronization that is required to use this as a locking primitive. + * + * @return 1 on success (match and store) + * 0 on no match + */ +static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline)); +static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val) +{ + uint32_t val = old_val; + + /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever + Don't use compare and swap on these chips */ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X)) + { + asm volatile ("1: ldaxr %x[v], [%[b]] \n" + " cmp %x[v], %x[o] \n" + " b.ne 2f \n" + " stlxr %x[v], %x[n], [%[b]]\n" /* Returns zero on success */ + " cbnz %x[v], 1b \n" + " mov %x[v], %x[o] \n" + "2: \n" + : [mem] "+m" (*ptr), [v] "=&r" (val) + : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val) + : ); + } + else + { + asm volatile ("casal %x[o], %x[n], [%[b]]" + : [mem] "+m" (*ptr), [o] "+r" (val) + : [b] "r" (ptr), [n] "r" (new_val) + : ); + } + return old_val == val; +} + +/** + * Atomically adds a signed value to a 64 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. (This should NOT be used for reference counting - + * use the standard version instead.) + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + * + * @return Value of memory location before increment + */ +static inline int64_t bdk_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr) +{ + int64_t result; + /* Atomic add with no ordering */ + asm volatile ("ldadd %x[i], %x[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically adds a signed value to a 64 bit (aligned) memory location, + * and returns previous value. + * + * Memory access ordering is enforced before/after the atomic operation, + * so no additional 'sync' instructions are required. + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + * + * @return Value of memory location before increment + */ +static inline int64_t bdk_atomic_fetch_and_add64(int64_t *ptr, int64_t incr) +{ + int64_t result; + /* Atomic add with acquire/release */ + asm volatile ("ldaddal %x[i], %x[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically adds a signed value to a 32 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. (This should NOT be used for reference counting - + * use the standard version instead.) + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + * + * @return Value of memory location before increment + */ +static inline int32_t bdk_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr) +{ + int32_t result; + /* Atomic add with no ordering */ + asm volatile ("ldadd %w[i], %w[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically adds a signed value to a 32 bit (aligned) memory location, + * and returns previous value. + * + * Memory access ordering is enforced before/after the atomic operation, + * so no additional 'sync' instructions are required. + * + * @param ptr address in memory to add incr to + * @param incr amount to increment memory location by (signed) + * + * @return Value of memory location before increment + */ +static inline int32_t bdk_atomic_fetch_and_add32(int32_t *ptr, int32_t incr) +{ + int32_t result; + /* Atomic add with acquire/release */ + asm volatile ("ldaddal %w[i], %w[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (incr), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically set bits in a 64 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. + * + * @param ptr address in memory + * @param mask mask of bits to set + * + * @return Value of memory location before setting bits + */ +static inline uint64_t bdk_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask) +{ + uint64_t result; + /* Atomic or with no ordering */ + asm volatile ("ldset %x[i], %x[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (mask), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically set bits in a 32 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. + * + * @param ptr address in memory + * @param mask mask of bits to set + * + * @return Value of memory location before setting bits + */ +static inline uint32_t bdk_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask) +{ + uint32_t result; + /* Atomic or with no ordering */ + asm volatile ("ldset %w[i], %w[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (mask), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically clear bits in a 64 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. + * + * @param ptr address in memory + * @param mask mask of bits to clear + * + * @return Value of memory location before clearing bits + */ +static inline uint64_t bdk_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask) +{ + uint64_t result; + /* Atomic and with no ordering */ + asm volatile ("ldclr %x[i], %x[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (mask), [b] "r" (ptr) + : "memory"); + return result; +} + +/** + * Atomically clear bits in a 32 bit (aligned) memory location, + * and returns previous value. + * + * This version does not perform 'sync' operations to enforce memory + * operations. This should only be used when there are no memory operation + * ordering constraints. + * + * @param ptr address in memory + * @param mask mask of bits to clear + * + * @return Value of memory location before clearing bits + */ +static inline uint32_t bdk_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask) +{ + uint32_t result; + /* Atomic and with no ordering */ + asm volatile ("ldclr %w[i], %w[r], [%[b]]" + : [r] "=r" (result), "+m" (*ptr) + : [i] "r" (mask), [b] "r" (ptr) + : "memory"); + return result; +} + +/** @} */ |