path: root/src/vendorcode/cavium/include/bdk/libbdk-hal/bdk-atomic.h

/***********************license start***********************************
* Copyright (c) 2003-2017  Cavium Inc. (support@cavium.com). All rights
* reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
*   * Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer.
*
*   * Redistributions in binary form must reproduce the above
*     copyright notice, this list of conditions and the following
*     disclaimer in the documentation and/or other materials provided
*     with the distribution.
*
*   * Neither the name of Cavium Inc. nor the names of
*     its contributors may be used to endorse or promote products
*     derived from this software without specific prior written
*     permission.
*
* This Software, including technical data, may be subject to U.S. export
* control laws, including the U.S. Export Administration Act and its
* associated regulations, and may be subject to export or import
* regulations in other countries.
*
* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK
* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
***********************license end**************************************/

/**
 * @file
 *
 * This file provides atomic operations
 *
 * @addtogroup hal
 * @{
 */

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void bdk_atomic_add32_nosync(int32_t *ptr, int32_t incr)
{
    /* Atomic add with no ordering */
    asm volatile ("ldadd %w[i], wzr, [%[b]]"
                  : [r] "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void bdk_atomic_add32(int32_t *ptr, int32_t incr)
{
    /* Atomic add with acquire and release */
    asm volatile ("ldaddal %w[i], wzr, [%[b]]"
                  : "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
}

/**
 * Atomically sets a 32 bit (aligned) memory location to a value
 *
 * @param ptr    address of memory to set
 * @param value  value to set memory location to.
 */
static inline void bdk_atomic_set32(int32_t *ptr, int32_t value)
{
    /* Implies a release */
    asm volatile ("stlr %w[v], [%[b]]"
                  : "+m" (*ptr)
                  : [v] "r" (value), [b] "r" (ptr)
                  : "memory");
}

/**
 * Returns the current value of a 32 bit (aligned) memory
 * location.
 *
 * @param ptr    Address of memory to get
 * @return Value of the memory
 */
static inline int32_t bdk_atomic_get32(int32_t *ptr)
{
    return *(volatile int32_t *)ptr;
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void bdk_atomic_add64_nosync(int64_t *ptr, int64_t incr)
{
    /* Atomic add with no ordering */
    asm volatile ("ldadd %x[i], xzr, [%[b]]"
                  : [r] "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 */
static inline void bdk_atomic_add64(int64_t *ptr, int64_t incr)
{
    /* Atomic add with acquire and release */
    asm volatile ("ldaddal %x[i], xzr, [%[b]]"
                  : [r] "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
}

/**
 * Atomically sets a 64 bit (aligned) memory location to a value
 *
 * @param ptr    address of memory to set
 * @param value  value to set memory location to.
 */
static inline void bdk_atomic_set64(int64_t *ptr, int64_t value)
{
    /* Implies a release */
    asm volatile ("stlr %x[v], [%[b]]"
                  : "+m" (*ptr)
                  : [v] "r" (value), [b] "r" (ptr)
                  : "memory");
}

/**
 * Returns the current value of a 64 bit (aligned) memory
 * location.
 *
 * @param ptr    Address of memory to get
 * @return Value of the memory
 */
static inline int64_t bdk_atomic_get64(int64_t *ptr)
{
    return *(volatile int64_t *)ptr;
}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does no memory synchronization.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline));
static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
{
    uint32_t val = old_val;

    /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
       Don't use compare and swap on these chips */
    if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
    {
        asm volatile ("1:   ldxr    %w[v], [%[b]]       \n"
                      "     cmp     %w[v], %w[o]        \n"
                      "     b.ne    2f                  \n"
                      "     stxr    %w[v], %w[n], [%[b]]\n" /* Returns zero on success */
                      "     cbnz    %w[v], 1b           \n"
                      "     mov     %w[v], %w[o]        \n"
                      "2:                               \n"
            : [mem] "+m" (*ptr), [v] "=&r" (val)
            : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val)
            : );
    }
    else
    {
        asm volatile ("cas %w[o], %w[n], [%[b]]"
                      : [mem] "+m" (*ptr), [o] "+r" (val)
                      : [b] "r" (ptr), [n] "r" (new_val)
                      : );
    }
    return old_val == val;
}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does memory synchronization that is required to use this as a locking primitive.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline));
static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
{
    uint32_t val = old_val;

    /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
       Don't use compare and swap on these chips */
    if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
    {
        asm volatile ("1:   ldaxr   %w[v], [%[b]]       \n"
                      "     cmp     %w[v], %w[o]        \n"
                      "     b.ne    2f                  \n"
                      "     stlxr   %w[v], %w[n], [%[b]]\n" /* Returns zero on success */
                      "     cbnz    %w[v], 1b           \n"
                      "     mov     %w[v], %w[o]        \n"
                      "2:                               \n"
            : [mem] "+m" (*ptr), [v] "=&r" (val)
            : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val)
            : );
    }
    else
    {
        asm volatile ("casal %w[o], %w[n], [%[b]]"
                      : [mem] "+m" (*ptr), [o] "+r" (val)
                      : [b] "r" (ptr), [n] "r" (new_val)
                      : );
    }
    return old_val == val;
}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does no memory synchronization.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline));
static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
{
    uint32_t val = old_val;

    /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
       Don't use compare and swap on these chips */
    if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
    {
        asm volatile ("1:   ldxr    %x[v], [%[b]]       \n"
                      "     cmp     %x[v], %x[o]        \n"
                      "     b.ne    2f                  \n"
                      "     stxr    %x[v], %x[n], [%[b]]\n" /* Returns zero on success */
                      "     cbnz    %x[v], 1b           \n"
                      "     mov     %x[v], %x[o]        \n"
                      "2:                               \n"
            : [mem] "+m" (*ptr), [v] "=&r" (val)
            : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val)
            : );
    }
    else
    {
        asm volatile ("cas %x[o], %x[n], [%[b]]"
                      : [mem] "+m" (*ptr), [o] "+r" (val)
                      : [b] "r" (ptr), [n] "r" (new_val)
                      : );
    }
    return old_val == val;
}

/**
 * Atomically compares the old value with the value at ptr, and if they match,
 * stores new_val to ptr.
 * If *ptr and old don't match, function returns failure immediately.
 * If *ptr and old match, function spins until *ptr updated to new atomically, or
 *  until *ptr and old no longer match
 *
 * Does memory synchronization that is required to use this as a locking primitive.
 *
 * @return 1 on success (match and store)
 *         0 on no match
 */
static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline));
static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
{
    uint32_t val = old_val;

    /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
       Don't use compare and swap on these chips */
    if (CAVIUM_IS_MODEL(CAVIUM_CN88XX_PASS1_X))
    {
        asm volatile ("1:   ldaxr   %x[v], [%[b]]       \n"
                      "     cmp     %x[v], %x[o]        \n"
                      "     b.ne    2f                  \n"
                      "     stlxr   %x[v], %x[n], [%[b]]\n" /* Returns zero on success */
                      "     cbnz    %x[v], 1b           \n"
                      "     mov     %x[v], %x[o]        \n"
                      "2:                               \n"
            : [mem] "+m" (*ptr), [v] "=&r" (val)
            : [b] "r" (ptr), [n] "r" (new_val), [o] "r" (old_val)
            : );
    }
    else
    {
        asm volatile ("casal %x[o], %x[n], [%[b]]"
                      : [mem] "+m" (*ptr), [o] "+r" (val)
                      : [b] "r" (ptr), [n] "r" (new_val)
                      : );
    }
    return old_val == val;
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int64_t bdk_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
{
    int64_t result;
    /* Atomic add with no ordering */
    asm volatile ("ldadd %x[i], %x[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically adds a signed value to a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int64_t bdk_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
{
    int64_t result;
    /* Atomic add with acquire/release */
    asm volatile ("ldaddal %x[i], %x[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.  (This should NOT be used for reference counting -
 * use the standard version instead.)
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int32_t bdk_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
{
    int32_t result;
    /* Atomic add with no ordering */
    asm volatile ("ldadd %w[i], %w[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically adds a signed value to a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * Memory access ordering is enforced before/after the atomic operation,
 * so no additional 'sync' instructions are required.
 *
 * @param ptr    address in memory to add incr to
 * @param incr   amount to increment memory location by (signed)
 *
 * @return Value of memory location before increment
 */
static inline int32_t bdk_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
{
    int32_t result;
    /* Atomic add with acquire/release */
    asm volatile ("ldaddal %w[i], %w[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (incr), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically set bits in a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to set
 *
 * @return Value of memory location before setting bits
 */
static inline uint64_t bdk_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
{
    uint64_t result;
    /* Atomic or with no ordering */
    asm volatile ("ldset %x[i], %x[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (mask), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically set bits in a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to set
 *
 * @return Value of memory location before setting bits
 */
static inline uint32_t bdk_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
{
    uint32_t result;
    /* Atomic or with no ordering */
    asm volatile ("ldset %w[i], %w[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (mask), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically clear bits in a 64 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to clear
 *
 * @return Value of memory location before clearing bits
 */
static inline uint64_t bdk_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
{
    uint64_t result;
    /* Atomic and with no ordering */
    asm volatile ("ldclr %x[i], %x[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (mask), [b] "r" (ptr)
                  : "memory");
    return result;
}

/**
 * Atomically clear bits in a 32 bit (aligned) memory location,
 * and returns previous value.
 *
 * This version does not perform 'sync' operations to enforce memory
 * operations.  This should only be used when there are no memory operation
 * ordering constraints.
 *
 * @param ptr    address in memory
 * @param mask   mask of bits to clear
 *
 * @return Value of memory location before clearing bits
 */
static inline uint32_t bdk_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
{
    uint32_t result;
    /* Atomic and with no ordering */
    asm volatile ("ldclr %w[i], %w[r], [%[b]]"
                  : [r] "=r" (result), "+m" (*ptr)
                  : [i] "r" (mask), [b] "r" (ptr)
                  : "memory");
    return result;
}

/** @} */