diff options
| author | Linus Torvalds | 2021-06-28 11:45:29 -0700 |
|---|---|---|
| committer | Linus Torvalds | 2021-06-28 11:45:29 -0700 |
| commit | a15286c63d113d4296c58867994cd266a28f5d6d (patch) | |
| tree | aaebbc86918c0c1943c26115d5bb1dd6c2fc2d9b | |
| parent | b89c07dea16137696d0f2d479ef665ef7c1022ab (diff) | |
| parent | 0e8a89d49d45197770f2e57fb15f1bc9ded96eb0 (diff) | |
Merge tag 'locking-core-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar:
- Core locking & atomics:
- Convert all architectures to ARCH_ATOMIC: move every architecture
to ARCH_ATOMIC, then get rid of ARCH_ATOMIC and all the
transitory facilities and #ifdefs.
Much reduction in complexity from that series:
63 files changed, 756 insertions(+), 4094 deletions(-)
- Self-test enhancements
- Futexes:
- Add the new FUTEX_LOCK_PI2 ABI, which is a variant that doesn't
set FLAGS_CLOCKRT (.e. uses CLOCK_MONOTONIC).
[ The temptation to repurpose FUTEX_LOCK_PI's implicit setting of
FLAGS_CLOCKRT & invert the flag's meaning to avoid having to
introduce a new variant was resisted successfully. ]
- Enhance futex self-tests
- Lockdep:
- Fix dependency path printouts
- Optimize trace saving
- Broaden & fix wait-context checks
- Misc cleanups and fixes.
* tag 'locking-core-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (52 commits)
locking/lockdep: Correct the description error for check_redundant()
futex: Provide FUTEX_LOCK_PI2 to support clock selection
futex: Prepare futex_lock_pi() for runtime clock selection
lockdep/selftest: Remove wait-type RCU_CALLBACK tests
lockdep/selftests: Fix selftests vs PROVE_RAW_LOCK_NESTING
lockdep: Fix wait-type for empty stack
locking/selftests: Add a selftest for check_irq_usage()
lockding/lockdep: Avoid to find wrong lock dep path in check_irq_usage()
locking/lockdep: Remove the unnecessary trace saving
locking/lockdep: Fix the dep path printing for backwards BFS
selftests: futex: Add futex compare requeue test
selftests: futex: Add futex wait test
seqlock: Remove trailing semicolon in macros
locking/lockdep: Reduce LOCKDEP dependency list
locking/lockdep,doc: Improve readability of the block matrix
locking/atomics: atomic-instrumented: simplify ifdeffery
locking/atomic: delete !ARCH_ATOMIC remnants
locking/atomic: xtensa: move to ARCH_ATOMIC
locking/atomic: sparc: move to ARCH_ATOMIC
locking/atomic: sh: move to ARCH_ATOMIC
...
77 files changed, 1395 insertions, 4157 deletions
diff --git a/Documentation/locking/lockdep-design.rst b/Documentation/locking/lockdep-design.rst index 9f3cfca9f8a4..82f36cab61bd 100644 --- a/Documentation/locking/lockdep-design.rst +++ b/Documentation/locking/lockdep-design.rst @@ -453,9 +453,9 @@ There are simply four block conditions: Block condition matrix, Y means the row blocks the column, and N means otherwise. +---+---+---+---+ - | | E | r | R | + | | W | r | R | +---+---+---+---+ - | E | Y | Y | Y | + | W | Y | Y | Y | +---+---+---+---+ | r | Y | Y | N | +---+---+---+---+ diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index e41c113c6688..f2861a43a61e 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -26,11 +26,11 @@ #define ATOMIC64_INIT(i) { (i) } -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic64_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) -#define atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) -#define atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) +#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) +#define arch_atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) /* * To get proper branch prediction for the main line, we must branch @@ -39,7 +39,7 @@ */ #define ATOMIC_OP(op, asm_op) \ -static __inline__ void atomic_##op(int i, atomic_t * v) \ +static __inline__ void arch_atomic_##op(int i, atomic_t * v) \ { \ unsigned long temp; \ __asm__ __volatile__( \ @@ -55,7 +55,7 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ } \ #define ATOMIC_OP_RETURN(op, asm_op) \ -static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ long temp, result; \ __asm__ __volatile__( \ @@ -74,7 +74,7 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, asm_op) \ -static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ { \ long temp, result; \ __asm__ __volatile__( \ @@ -92,7 +92,7 @@ static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ } #define ATOMIC64_OP(op, asm_op) \ -static __inline__ void atomic64_##op(s64 i, atomic64_t * v) \ +static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \ { \ s64 temp; \ __asm__ __volatile__( \ @@ -108,7 +108,8 @@ static __inline__ void atomic64_##op(s64 i, atomic64_t * v) \ } \ #define ATOMIC64_OP_RETURN(op, asm_op) \ -static __inline__ s64 atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ +static __inline__ s64 \ +arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ { \ s64 temp, result; \ __asm__ __volatile__( \ @@ -127,7 +128,8 @@ static __inline__ s64 atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ } #define ATOMIC64_FETCH_OP(op, asm_op) \ -static __inline__ s64 atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ +static __inline__ s64 \ +arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ { \ s64 temp, result; \ __asm__ __volatile__( \ @@ -155,18 +157,18 @@ static __inline__ s64 atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed -#define atomic_andnot atomic_andnot -#define atomic64_andnot atomic64_andnot +#define arch_atomic_andnot arch_atomic_andnot +#define arch_atomic64_andnot arch_atomic64_andnot #undef ATOMIC_OPS #define ATOMIC_OPS(op, asm) \ @@ -180,15 +182,15 @@ ATOMIC_OPS(andnot, bic) ATOMIC_OPS(or, bis) ATOMIC_OPS(xor, xor) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed #undef ATOMIC_OPS #undef ATOMIC64_FETCH_OP @@ -198,14 +200,18 @@ ATOMIC_OPS(xor, xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic64_cmpxchg(v, old, new) \ + (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic64_xchg(v, new) \ + (arch_xchg(&((v)->counter), new)) -#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, old, new) \ + (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic_xchg(v, new) \ + (arch_xchg(&((v)->counter), new)) /** - * atomic_fetch_add_unless - add unless the number is a given value + * arch_atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -213,7 +219,7 @@ ATOMIC_OPS(xor, xor) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int c, new, old; smp_mb(); @@ -234,10 +240,10 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) smp_mb(); return old; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless /** - * atomic64_fetch_add_unless - add unless the number is a given value + * arch_atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -245,7 +251,7 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 c, new, old; smp_mb(); @@ -266,16 +272,16 @@ static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) smp_mb(); return old; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless /* - * atomic64_dec_if_positive - decrement by 1 if old value positive + * arch_atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t * * The function returns the old value of *v minus 1, even if * the atomic variable, v, was not decremented. */ -static inline s64 atomic64_dec_if_positive(atomic64_t *v) +static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) { s64 old, tmp; smp_mb(); @@ -295,6 +301,6 @@ static inline s64 atomic64_dec_if_positive(atomic64_t *v) smp_mb(); return old - 1; } -#define atomic64_dec_if_positive atomic64_dec_if_positive +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive #endif /* _ALPHA_ATOMIC_H */ diff --git a/arch/alpha/include/asm/cmpxchg.h b/arch/alpha/include/asm/cmpxchg.h index 6c7c39452471..6e0a850aa9d3 100644 --- a/arch/alpha/include/asm/cmpxchg.h +++ b/arch/alpha/include/asm/cmpxchg.h @@ -17,7 +17,7 @@ sizeof(*(ptr))); \ }) -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -26,7 +26,7 @@ sizeof(*(ptr))); \ }) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg_local((ptr), (o), (n)); \ @@ -42,7 +42,7 @@ * The leading and the trailing memory barriers guarantee that these * operations are fully ordered. */ -#define xchg(ptr, x) \ +#define arch_xchg(ptr, x) \ ({ \ __typeof__(*(ptr)) __ret; \ __typeof__(*(ptr)) _x_ = (x); \ @@ -53,7 +53,7 @@ __ret; \ }) -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) __ret; \ __typeof__(*(ptr)) _o_ = (o); \ @@ -65,10 +65,10 @@ __ret; \ }) -#define cmpxchg64(ptr, o, n) \ +#define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) #undef ____cmpxchg diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 5afc79c9b2f5..7a36d79b5b2f 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -14,14 +14,14 @@ #include <asm/barrier.h> #include <asm/smp.h> -#define atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) #ifdef CONFIG_ARC_HAS_LLSC -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) #define ATOMIC_OP(op, c_op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned int val; \ \ @@ -37,7 +37,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned int val; \ \ @@ -63,7 +63,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned int val, orig; \ \ @@ -94,11 +94,11 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #ifndef CONFIG_SMP /* violating atomic_xxx API locking protocol in UP for optimization sake */ -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) #else -static inline void atomic_set(atomic_t *v, int i) +static inline void arch_atomic_set(atomic_t *v, int i) { /* * Independent of hardware support, all of the atomic_xxx() APIs need @@ -116,7 +116,7 @@ static inline void atomic_set(atomic_t *v, int i) atomic_ops_unlock(flags); } -#define atomic_set_release(v, i) atomic_set((v), (i)) +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) #endif @@ -126,7 +126,7 @@ static inline void atomic_set(atomic_t *v, int i) */ #define ATOMIC_OP(op, c_op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -136,7 +136,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long flags; \ unsigned long temp; \ @@ -154,7 +154,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ unsigned long orig; \ @@ -180,9 +180,6 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define atomic_andnot atomic_andnot -#define atomic_fetch_andnot atomic_fetch_andnot - #undef ATOMIC_OPS #define ATOMIC_OPS(op, c_op, asm_op) \ ATOMIC_OP(op, c_op, asm_op) \ @@ -193,6 +190,9 @@ ATOMIC_OPS(andnot, &= ~, bic) ATOMIC_OPS(or, |=, or) ATOMIC_OPS(xor, ^=, xor) +#define arch_atomic_andnot arch_atomic_andnot +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot + #undef ATOMIC_OPS #undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN @@ -220,7 +220,7 @@ typedef struct { #define ATOMIC64_INIT(a) { (a) } -static inline s64 atomic64_read(const atomic64_t *v) +static inline s64 arch_atomic64_read(const atomic64_t *v) { s64 val; @@ -232,7 +232,7 @@ static inline s64 atomic64_read(const atomic64_t *v) return val; } -static inline void atomic64_set(atomic64_t *v, s64 a) +static inline void arch_atomic64_set(atomic64_t *v, s64 a) { /* * This could have been a simple assignment in "C" but would need @@ -253,7 +253,7 @@ static inline void atomic64_set(atomic64_t *v, s64 a) } #define ATOMIC64_OP(op, op1, op2) \ -static inline void atomic64_##op(s64 a, atomic64_t *v) \ +static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ { \ s64 val; \ \ @@ -270,7 +270,7 @@ static inline void atomic64_##op(s64 a, atomic64_t *v) \ } \ #define ATOMIC64_OP_RETURN(op, op1, op2) \ -static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ s64 val; \ \ @@ -293,7 +293,7 @@ static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ } #define ATOMIC64_FETCH_OP(op, op1, op2) \ -static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ s64 val, orig; \ \ @@ -320,9 +320,6 @@ static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ ATOMIC64_OP_RETURN(op, op1, op2) \ ATOMIC64_FETCH_OP(op, op1, op2) -#define atomic64_andnot atomic64_andnot -#define atomic64_fetch_andnot atomic64_fetch_andnot - ATOMIC64_OPS(add, add.f, adc) ATOMIC64_OPS(sub, sub.f, sbc) ATOMIC64_OPS(and, and, and) @@ -330,13 +327,16 @@ ATOMIC64_OPS(andnot, bic, bic) ATOMIC64_OPS(or, or, or) ATOMIC64_OPS(xor, xor, xor) +#define arch_atomic64_andnot arch_atomic64_andnot +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot + #undef ATOMIC64_OPS #undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP static inline s64 -atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) +arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) { s64 prev; @@ -358,7 +358,7 @@ atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) return prev; } -static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new) +static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) { s64 prev; @@ -379,14 +379,14 @@ static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new) } /** - * atomic64_dec_if_positive - decrement by 1 if old value positive + * arch_atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic64_t * * The function returns the old value of *v minus 1, even if * the atomic variable, v, was not decremented. */ -static inline s64 atomic64_dec_if_positive(atomic64_t *v) +static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) { s64 val; @@ -408,10 +408,10 @@ static inline s64 atomic64_dec_if_positive(atomic64_t *v) return val; } -#define atomic64_dec_if_positive atomic64_dec_if_positive +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive /** - * atomic64_fetch_add_unless - add unless the number is a given value + * arch_atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -419,7 +419,7 @@ static inline s64 atomic64_dec_if_positive(atomic64_t *v) * Atomically adds @a to @v, if it was not @u. * Returns the old value of @v */ -static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 old, temp; @@ -443,7 +443,7 @@ static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return old; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless #endif /* !CONFIG_GENERIC_ATOMIC64 */ diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h index dfeffa25499b..d42917e803e1 100644 --- a/arch/arc/include/asm/cmpxchg.h +++ b/arch/arc/include/asm/cmpxchg.h @@ -63,7 +63,7 @@ __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new) #endif -#define cmpxchg(ptr, o, n) ({ \ +#define arch_cmpxchg(ptr, o, n) ({ \ (typeof(*(ptr)))__cmpxchg((ptr), \ (unsigned long)(o), \ (unsigned long)(n)); \ @@ -75,7 +75,7 @@ __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new) * !LLSC: cmpxchg() has to use an external lock atomic_ops_lock to guarantee * semantics, and this lock also happens to be used by atomic_*() */ -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n))) /* @@ -123,7 +123,7 @@ static inline unsigned long __xchg(unsigned long val, volatile void *ptr, #if !defined(CONFIG_ARC_HAS_LLSC) && defined(CONFIG_SMP) -#define xchg(ptr, with) \ +#define arch_xchg(ptr, with) \ ({ \ unsigned long flags; \ typeof(*(ptr)) old_val; \ @@ -136,7 +136,7 @@ static inline unsigned long __xchg(unsigned long val, volatile void *ptr, #else -#define xchg(ptr, with) _xchg(ptr, with) +#define arch_xchg(ptr, with) _xchg(ptr, with) #endif @@ -153,6 +153,6 @@ static inline unsigned long __xchg(unsigned long val, volatile void *ptr, * can't be clobbered by others. Thus no serialization required when * atomic_xchg is involved. */ -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) #endif diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 455eb19a5ac1..db8512d9a918 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -22,8 +22,8 @@ * strex/ldrex monitor on some implementations. The reason we can use it for * atomic_set() is the clrex or dummy strex done on every exception return. */ -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic_set(v,i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v,i) WRITE_ONCE(((v)->counter), (i)) #if __LINUX_ARM_ARCH__ >= 6 @@ -34,7 +34,7 @@ */ #define ATOMIC_OP(op, c_op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ @@ -52,7 +52,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ @@ -73,7 +73,7 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result, val; \ @@ -93,17 +93,17 @@ static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ return result; \ } -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed -static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) +static inline int arch_atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) { int oldval; unsigned long res; @@ -123,9 +123,9 @@ static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) return oldval; } -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed +#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed -static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int oldval, newval; unsigned long tmp; @@ -151,7 +151,7 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) return oldval; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless #else /* ARM_ARCH_6 */ @@ -160,7 +160,7 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) #endif #define ATOMIC_OP(op, c_op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -170,7 +170,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long flags; \ int val; \ @@ -184,7 +184,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ int val; \ @@ -197,7 +197,7 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ return val; \ } -static inline int atomic_cmpxchg(atomic_t *v, int old, int new) +static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; unsigned long flags; @@ -211,7 +211,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -#define atomic_fetch_andnot atomic_fetch_andnot +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot #endif /* __LINUX_ARM_ARCH__ */ @@ -223,7 +223,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define atomic_andnot atomic_andnot +#define arch_atomic_andnot arch_atomic_andnot #undef ATOMIC_OPS #define ATOMIC_OPS(op, c_op, asm_op) \ @@ -240,7 +240,7 @@ ATOMIC_OPS(xor, ^=, eor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct { @@ -250,7 +250,7 @@ typedef struct { #define ATOMIC64_INIT(i) { (i) } #ifdef CONFIG_ARM_LPAE -static inline s64 atomic64_read(const atomic64_t *v) +static inline s64 arch_atomic64_read(const atomic64_t *v) { s64 result; @@ -263,7 +263,7 @@ static inline s64 atomic64_read(const atomic64_t *v) return result; } -static inline void atomic64_set(atomic64_t *v, s64 i) +static inline void arch_atomic64_set(atomic64_t *v, s64 i) { __asm__ __volatile__("@ atomic64_set\n" " strd %2, %H2, [%1]" @@ -272,7 +272,7 @@ static inline void atomic64_set(atomic64_t *v, s64 i) ); } #else -static inline s64 atomic64_read(const atomic64_t *v) +static inline s64 arch_atomic64_read(const atomic64_t *v) { s64 result; @@ -285,7 +285,7 @@ static inline s64 atomic64_read(const atomic64_t *v) return result; } -static inline void atomic64_set(atomic64_t *v, s64 i) +static inline void arch_atomic64_set(atomic64_t *v, s64 i) { s64 tmp; @@ -302,7 +302,7 @@ static inline void atomic64_set(atomic64_t *v, s64 i) #endif #define ATOMIC64_OP(op, op1, op2) \ -static inline void atomic64_##op(s64 i, atomic64_t *v) \ +static inline void arch_atomic64_##op(s64 i, atomic64_t *v) \ { \ s64 result; \ unsigned long tmp; \ @@ -322,7 +322,7 @@ static inline void atomic64_##op(s64 i, atomic64_t *v) \ #define ATOMIC64_OP_RETURN(op, op1, op2) \ static inline s64 \ -atomic64_##op##_return_relaxed(s64 i, atomic64_t *v) \ +arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t *v) \ { \ s64 result; \ unsigned long tmp; \ @@ -345,7 +345,7 @@ atomic64_##op##_return_relaxed(s64 i, atomic64_t *v) \ #define ATOMIC64_FETCH_OP(op, op1, op2) \ static inline s64 \ -atomic64_fetch_##op##_relaxed(s64 i, atomic64_t *v) \ +arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t *v) \ { \ s64 result, val; \ unsigned long tmp; \ @@ -374,34 +374,34 @@ atomic64_fetch_##op##_relaxed(s64 i, atomic64_t *v) \ ATOMIC64_OPS(add, adds, adc) ATOMIC64_OPS(sub, subs, sbc) -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed #undef ATOMIC64_OPS #define ATOMIC64_OPS(op, op1, op2) \ ATOMIC64_OP(op, op1, op2) \ ATOMIC64_FETCH_OP(op, op1, op2) -#define atomic64_andnot atomic64_andnot +#define arch_atomic64_andnot arch_atomic64_andnot ATOMIC64_OPS(and, and, and) ATOMIC64_OPS(andnot, bic, bic) ATOMIC64_OPS(or, orr, orr) ATOMIC64_OPS(xor, eor, eor) -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed #undef ATOMIC64_OPS #undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -static inline s64 atomic64_cmpxchg_relaxed(atomic64_t *ptr, s64 old, s64 new) +static inline s64 arch_atomic64_cmpxchg_relaxed(atomic64_t *ptr, s64 old, s64 new) { s64 oldval; unsigned long res; @@ -422,9 +422,9 @@ static inline s64 atomic64_cmpxchg_relaxed(atomic64_t *ptr, s64 old, s64 new) return oldval; } -#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed +#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed -static inline s64 atomic64_xchg_relaxed(atomic64_t *ptr, s64 new) +static inline s64 arch_atomic64_xchg_relaxed(atomic64_t *ptr, s64 new) { s64 result; unsigned long tmp; @@ -442,9 +442,9 @@ static inline s64 atomic64_xchg_relaxed(atomic64_t *ptr, s64 new) return result; } -#define atomic64_xchg_relaxed atomic64_xchg_relaxed +#define arch_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed -static inline s64 atomic64_dec_if_positive(atomic64_t *v) +static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) { s64 result; unsigned long tmp; @@ -470,9 +470,9 @@ static inline s64 atomic64_dec_if_positive(atomic64_t *v) return result; } -#define atomic64_dec_if_positive atomic64_dec_if_positive +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive -static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 oldval, newval; unsigned long tmp; @@ -500,7 +500,7 @@ static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return oldval; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless #endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index 8b701f8e175c..4dfe538dfc68 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -114,7 +114,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size return ret; } -#define xchg_relaxed(ptr, x) ({ \ +#define arch_xchg_relaxed(ptr, x) ({ \ (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ sizeof(*(ptr))); \ }) @@ -128,20 +128,20 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #error "SMP is not supported on this platform" #endif -#define xchg xchg_relaxed +#define arch_xchg arch_xchg_relaxed /* * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ -#define cmpxchg_local(ptr, o, n) ({ \ - (__typeof(*ptr))__cmpxchg_local_generic((ptr), \ +#define arch_cmpxchg_local(ptr, o, n) ({ \ + (__typeof(*ptr))__generic_cmpxchg_local((ptr), \ (unsigned long)(o), \ (unsigned long)(n), \ sizeof(*(ptr))); \ }) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #include <asm-generic/cmpxchg.h> @@ -207,7 +207,7 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, return oldval; } -#define cmpxchg_relaxed(ptr,o,n) ({ \ +#define arch_cmpxchg_relaxed(ptr,o,n) ({ \ (__typeof__(*(ptr)))__cmpxchg((ptr), \ (unsigned long)(o), \ (unsigned long)(n), \ @@ -224,7 +224,7 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ case 1: case 2: - ret = __cmpxchg_local_generic(ptr, old, new, size); + ret = __generic_cmpxchg_local(ptr, old, new, size); break; #endif default: @@ -234,7 +234,7 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, return ret; } -#define cmpxchg_local(ptr, o, n) ({ \ +#define arch_cmpxchg_local(ptr, o, n) ({ \ (__typeof(*ptr))__cmpxchg_local((ptr), \ (unsigned long)(o), \ (unsigned long)(n), \ @@ -266,13 +266,13 @@ static inline unsigned long long __cmpxchg64(unsigned long long *ptr, return oldval; } -#define cmpxchg64_relaxed(ptr, o, n) ({ \ +#define arch_cmpxchg64_relaxed(ptr, o, n) ({ \ (__typeof__(*(ptr)))__cmpxchg64((ptr), \ (unsigned long long)(o), \ (unsigned long long)(n)); \ }) -#define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n)) #endif /* __LINUX_ARM_ARCH__ >= 6 */ diff --git a/arch/arm/include/asm/sync_bitops.h b/arch/arm/include/asm/sync_bitops.h index 39ff217136d1..6f5d627c44a3 100644 --- a/arch/arm/include/asm/sync_bitops.h +++ b/arch/arm/include/asm/sync_bitops.h @@ -21,7 +21,7 @@ #define sync_test_and_clear_bit(nr, p) _test_and_clear_bit(nr, p) #define sync_test_and_change_bit(nr, p) _test_and_change_bit(nr, p) #define sync_test_bit(nr, addr) test_bit(nr, addr) -#define sync_cmpxchg cmpxchg +#define arch_sync_cmpxchg arch_cmpxchg #endif diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index b56a4b2bc248..c9979273d389 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -223,6 +223,4 @@ static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v) #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive -#define ARCH_ATOMIC - #endif /* __ASM_ATOMIC_H */ diff --git a/arch/csky/include/asm/cmpxchg.h b/arch/csky/include/asm/cmpxchg.h index dabc8e46ce7b..d1bef11f8dc9 100644 --- a/arch/csky/include/asm/cmpxchg.h +++ b/arch/csky/include/asm/cmpxchg.h @@ -31,7 +31,7 @@ extern void __bad_xchg(void); __ret; \ }) -#define xchg_relaxed(ptr, x) \ +#define arch_xchg_relaxed(ptr, x) \ (__xchg_relaxed((x), (ptr), sizeof(*(ptr)))) #define __cmpxchg_relaxed(ptr, old, new, size) \ @@ -61,14 +61,14 @@ extern void __bad_xchg(void); __ret; \ }) -#define cmpxchg_relaxed(ptr, o, n) \ +#define arch_cmpxchg_relaxed(ptr, o, n) \ (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) __ret; \ __smp_release_fence(); \ - __ret = cmpxchg_relaxed(ptr, o, n); \ + __ret = arch_cmpxchg_relaxed(ptr, o, n); \ __smp_acquire_fence(); \ __ret; \ }) diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild index 60ee7f0d60a8..e23139c8fc0d 100644 --- a/arch/h8300/include/asm/Kbuild +++ b/arch/h8300/include/asm/Kbuild @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += asm-offsets.h +generic-y += cmpxchg.h generic-y += extable.h generic-y += kvm_para.h generic-y += mcs_spinlock.h diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h deleted file mode 100644 index a990d151f163..000000000000 --- a/arch/h8300/include/asm/atomic.h +++ /dev/null @@ -1,97 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ARCH_H8300_ATOMIC__ -#define __ARCH_H8300_ATOMIC__ - -#include <linux/compiler.h> -#include <linux/types.h> -#include <asm/cmpxchg.h> -#include <asm/irqflags.h> - -/* - * Atomic operations that C can't guarantee us. Useful for - * resource counting etc.. - */ - -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) - -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - h8300flags flags; \ - int ret; \ - \ - flags = arch_local_irq_save(); \ - ret = v->counter c_op i; \ - arch_local_irq_restore(flags); \ - return ret; \ -} - -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - h8300flags flags; \ - int ret; \ - \ - flags = arch_local_irq_save(); \ - ret = v->counter; \ - v->counter c_op i; \ - arch_local_irq_restore(flags); \ - return ret; \ -} - -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - h8300flags flags; \ - \ - flags = arch_local_irq_save(); \ - v->counter c_op i; \ - arch_local_irq_restore(flags); \ -} - -ATOMIC_OP_RETURN(add, +=) -ATOMIC_OP_RETURN(sub, -=) - -#define ATOMIC_OPS(op, c_op) \ - ATOMIC_OP(op, c_op) \ - ATOMIC_FETCH_OP(op, c_op) - -ATOMIC_OPS(and, &=) -ATOMIC_OPS(or, |=) -ATOMIC_OPS(xor, ^=) -ATOMIC_OPS(add, +=) -ATOMIC_OPS(sub, -=) - -#undef ATOMIC_OPS -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - -static inline int atomic_cmpxchg(atomic_t *v, int old, int new) -{ - int ret; - h8300flags flags; - - flags = arch_local_irq_save(); - ret = v->counter; - if (likely(ret == old)) - v->counter = new; - arch_local_irq_restore(flags); - return ret; -} - -static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ - int ret; - h8300flags flags; - - flags = arch_local_irq_save(); - ret = v->counter; - if (ret != u) - v->counter += a; - arch_local_irq_restore(flags); - return ret; -} -#define atomic_fetch_add_unless atomic_fetch_add_unless - -#endif /* __ARCH_H8300_ATOMIC __ */ diff --git a/arch/h8300/include/asm/cmpxchg.h b/arch/h8300/include/asm/cmpxchg.h deleted file mode 100644 index c64bb38ce242..000000000000 --- a/arch/h8300/include/asm/cmpxchg.h +++ /dev/null @@ -1,66 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ARCH_H8300_CMPXCHG__ -#define __ARCH_H8300_CMPXCHG__ - -#include <linux/irqflags.h> - -#define xchg(ptr, x) \ - ((__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ - sizeof(*(ptr)))) - -struct __xchg_dummy { unsigned long a[100]; }; -#define __xg(x) ((volatile struct __xchg_dummy *)(x)) - -static inline unsigned long __xchg(unsigned long x, - volatile void *ptr, int size) -{ - unsigned long tmp, flags; - - local_irq_save(flags); - - switch (size) { - case 1: - __asm__ __volatile__ - ("mov.b %2,%0\n\t" - "mov.b %1,%2" - : "=&r" (tmp) : "r" (x), "m" (*__xg(ptr))); - break; - case 2: - __asm__ __volatile__ - ("mov.w %2,%0\n\t" - "mov.w %1,%2" - : "=&r" (tmp) : "r" (x), "m" (*__xg(ptr))); - break; - case 4: - __asm__ __volatile__ - ("mov.l %2,%0\n\t" - "mov.l %1,%2" - : "=&r" (tmp) : "r" (x), "m" (*__xg(ptr))); - break; - default: - tmp = 0; - } - local_irq_restore(flags); - return tmp; -} - -#include <asm-generic/cmpxchg-local.h> - -/* - * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make - * them available. - */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), \ - (unsigned long)(o), \ - (unsigned long)(n), \ - sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) - -#ifndef CONFIG_SMP -#include <asm-generic/cmpxchg.h> -#endif - -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) - -#endif /* __ARCH_H8300_CMPXCHG__ */ diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 4ab895d7111f..6e94f8d04146 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -14,7 +14,7 @@ /* Normal writes in our arch don't clear lock reservations */ -static inline void atomic_set(atomic_t *v, int new) +static inline void arch_atomic_set(atomic_t *v, int new) { asm volatile( "1: r6 = memw_locked(%0);\n" @@ -26,26 +26,26 @@ static inline void atomic_set(atomic_t *v, int new) ); } -#define atomic_set_release(v, i) atomic_set((v), (i)) +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) /** - * atomic_read - reads a word, atomically + * arch_atomic_read - reads a word, atomically * @v: pointer to atomic value * * Assumes all word reads on our architecture are atomic. */ -#define atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) /** - * atomic_xchg - atomic + * arch_atomic_xchg - atomic * @v: pointer to memory to change * @new: new value (technically passed in a register -- see xchg) */ -#define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new))) /** - * atomic_cmpxchg - atomic compare-and-exchange values + * arch_atomic_cmpxchg - atomic compare-and-exchange values * @v: pointer to value to change * @old: desired old value to match * @new: new value to put in @@ -61,7 +61,7 @@ static inline void atomic_set(atomic_t *v, int new) * * "old" is "expected" old val, __oldval is actual old value */ -static inline int atomic_cmpxchg(atomic_t *v, int old, int new) +static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { int __oldval; @@ -81,7 +81,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) } #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ int output; \ \ @@ -97,7 +97,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ int output; \ \ @@ -114,7 +114,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ int output, val; \ \ @@ -148,7 +148,7 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP /** - * atomic_fetch_add_unless - add unless the number is a given value + * arch_atomic_fetch_add_unless - add unless the number is a given value * @v: pointer to value * @a: amount to add * @u: unless value is equal to u @@ -157,7 +157,7 @@ ATOMIC_OPS(xor) * */ -static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int __oldval; register int tmp; @@ -180,6 +180,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) ); return __oldval; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless #endif diff --git a/arch/hexagon/include/asm/cmpxchg.h b/arch/hexagon/include/asm/cmpxchg.h index 92b8a02e588a..cdb705e1496a 100644 --- a/arch/hexagon/include/asm/cmpxchg.h +++ b/arch/hexagon/include/asm/cmpxchg.h @@ -42,7 +42,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, * Atomically swap the contents of a register with memory. Should be atomic * between multiple CPU's and within interrupts on the same CPU. */ -#define xchg(ptr, v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), \ +#define arch_xchg(ptr, v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), \ sizeof(*(ptr)))) /* @@ -51,7 +51,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, * variable casting. */ -#define cmpxchg(ptr, old, new) \ +#define arch_cmpxchg(ptr, old, new) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(*(ptr)) __old = (old); \ diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index f267d956458f..266c429b9137 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -21,11 +21,11 @@ #define ATOMIC64_INIT(i) { (i) } -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic64_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) -#define atomic_set(v,i) WRITE_ONCE(((v)->counter), (i)) -#define atomic64_set(v,i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_set(v,i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic64_set(v,i) WRITE_ONCE(((v)->counter), (i)) #define ATOMIC_OP(op, c_op) \ static __inline__ int \ @@ -36,7 +36,7 @@ ia64_atomic_##op (int i, atomic_t *v) \ \ do { \ CMPXCHG_BUGCHECK(v); \ - old = atomic_read(v); \ + old = arch_atomic_read(v); \ new = old c_op i; \ } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \ return new; \ @@ -51,7 +51,7 @@ ia64_atomic_fetch_##op (int i, atomic_t *v) \ \ do { \ CMPXCHG_BUGCHECK(v); \ - old = atomic_read(v); \ + old = arch_atomic_read(v); \ new = old c_op i; \ } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \ return old; \ @@ -74,7 +74,7 @@ ATOMIC_OPS(sub, -) #define __ia64_atomic_const(i) 0 #endif -#define atomic_add_return(i,v) \ +#define arch_atomic_add_return(i,v) \ ({ \ int __ia64_aar_i = (i); \ __ia64_atomic_const(i) \ @@ -82,7 +82,7 @@ ATOMIC_OPS(sub, -) : ia64_atomic_add(__ia64_aar_i, v); \ }) -#define atomic_sub_return(i,v) \ +#define arch_atomic_sub_return(i,v) \ ({ \ int __ia64_asr_i = (i); \ __ia64_atomic_const(i) \ @@ -90,7 +90,7 @@ ATOMIC_OPS(sub, -) : ia64_atomic_sub(__ia64_asr_i, v); \ }) -#define atomic_fetch_add(i,v) \ +#define arch_atomic_fetch_add(i,v) \ ({ \ int __ia64_aar_i = (i); \ __ia64_atomic_const(i) \ @@ -98,7 +98,7 @@ ATOMIC_OPS(sub, -) : ia64_atomic_fetch_add(__ia64_aar_i, v); \ }) -#define atomic_fetch_sub(i,v) \ +#define arch_atomic_fetch_sub(i,v) \ ({ \ int __ia64_asr_i = (i); \ __ia64_atomic_const(i) \ @@ -110,13 +110,13 @@ ATOMIC_FETCH_OP(and, &) ATOMIC_FETCH_OP(or, |) ATOMIC_FETCH_OP(xor, ^) -#define atomic_and(i,v) (void)ia64_atomic_fetch_and(i,v) -#define atomic_or(i,v) (void)ia64_atomic_fetch_or(i,v) -#define atomic_xor(i,v) (void)ia64_atomic_fetch_xor(i,v) +#define arch_atomic_and(i,v) (void)ia64_atomic_fetch_and(i,v) +#define arch_atomic_or(i,v) (void)ia64_atomic_fetch_or(i,v) +#define arch_atomic_xor(i,v) (void)ia64_atomic_fetch_xor(i,v) -#define atomic_fetch_and(i,v) ia64_atomic_fetch_and(i,v) -#define atomic_fetch_or(i,v) ia64_atomic_fetch_or(i,v) -#define atomic_fetch_xor(i,v) ia64_atomic_fetch_xor(i,v) +#define arch_atomic_fetch_and(i,v) ia64_atomic_fetch_and(i,v) +#define arch_atomic_fetch_or(i,v) ia64_atomic_fetch_or(i,v) +#define arch_atomic_fetch_xor(i,v) ia64_atomic_fetch_xor(i,v) #undef ATOMIC_OPS #undef ATOMIC_FETCH_OP @@ -131,7 +131,7 @@ ia64_atomic64_##op (s64 i, atomic64_t *v) \ \ do { \ CMPXCHG_BUGCHECK(v); \ - old = atomic64_read(v); \ + old = arch_atomic64_read(v); \ new = old c_op i; \ } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic64_t)) != old); \ return new; \ @@ -146,7 +146,7 @@ ia64_atomic64_fetch_##op (s64 i, atomic64_t *v) \ \ do { \ CMPXCHG_BUGCHECK(v); \ - old = atomic64_read(v); \ + old = arch_atomic64_read(v); \ new = old c_op i; \ } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic64_t)) != old); \ return old; \ @@ -159,7 +159,7 @@ ia64_atomic64_fetch_##op (s64 i, atomic64_t *v) \ ATOMIC64_OPS(add, +) ATOMIC64_OPS(sub, -) -#define atomic64_add_return(i,v) \ +#define arch_atomic64_add_return(i,v) \ ({ \ s64 __ia64_aar_i = (i); \ __ia64_atomic_const(i) \ @@ -167,7 +167,7 @@ ATOMIC64_OPS(sub, -) : ia64_atomic64_add(__ia64_aar_i, v); \ }) -#define atomic64_sub_return(i,v) \ +#define arch_atomic64_sub_return(i,v) \ ({ \ s64 __ia64_asr_i = (i); \ __ia64_atomic_const(i) \ @@ -175,7 +175,7 @@ ATOMIC64_OPS(sub, -) : ia64_atomic64_sub(__ia64_asr_i, v); \ }) -#define atomic64_fetch_add(i,v) \ +#define arch_atomic64_fetch_add(i,v) \ ({ \ s64 __ia64_aar_i = (i); \ __ia64_atomic_const(i) \ @@ -183,7 +183,7 @@ ATOMIC64_OPS(sub, -) : ia64_atomic64_fetch_add(__ia64_aar_i, v); \ }) -#define atomic64_fetch_sub(i,v) \ +#define arch_atomic64_fetch_sub(i,v) \ ({ \ s64 __ia64_asr_i = (i); \ __ia64_atomic_const(i) \ @@ -195,29 +195,29 @@ ATOMIC64_FETCH_OP(and, &) ATOMIC64_FETCH_OP(or, |) ATOMIC64_FETCH_OP(xor, ^) -#define atomic64_and(i,v) (void)ia64_atomic64_fetch_and(i,v) -#define atomic64_or(i,v) (void)ia64_atomic64_fetch_or(i,v) -#define atomic64_xor(i,v) (void)ia64_atomic64_fetch_xor(i,v) +#define arch_atomic64_and(i,v) (void)ia64_atomic64_fetch_and(i,v) +#define arch_atomic64_or(i,v) (void)ia64_atomic64_fetch_or(i,v) +#define arch_atomic64_xor(i,v) (void)ia64_atomic64_fetch_xor(i,v) -#define atomic64_fetch_and(i,v) ia64_atomic64_fetch_and(i,v) -#define atomic64_fetch_or(i,v) ia64_atomic64_fetch_or(i,v) -#define atomic64_fetch_xor(i,v) ia64_atomic64_fetch_xor(i,v) +#define arch_atomic64_fetch_and(i,v) ia64_atomic64_fetch_and(i,v) +#define arch_atomic64_fetch_or(i,v) ia64_atomic64_fetch_or(i,v) +#define arch_atomic64_fetch_xor(i,v) ia64_atomic64_fetch_xor(i,v) #undef ATOMIC64_OPS #undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP -#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) -#define atomic64_cmpxchg(v, old, new) \ - (cmpxchg(&((v)->counter), old, new)) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic64_cmpxchg(v, old, new) \ + (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) -#define atomic_add(i,v) (void)atomic_add_return((i), (v)) -#define atomic_sub(i,v) (void)atomic_sub_return((i), (v)) +#define arch_atomic_add(i,v) (void)arch_atomic_add_return((i), (v)) +#define arch_atomic_sub(i,v) (void)arch_atomic_sub_return((i), (v)) -#define atomic64_add(i,v) (void)atomic64_add_return((i), (v)) -#define atomic64_sub(i,v) (void)atomic64_sub_return((i), (v)) +#define arch_atomic64_add(i,v) (void)arch_atomic64_add_return((i), (v)) +#define arch_atomic64_sub(i,v) (void)arch_atomic64_sub_return((i), (v)) #endif /* _ASM_IA64_ATOMIC_H */ diff --git a/arch/ia64/include/asm/cmpxchg.h b/arch/ia64/include/asm/cmpxchg.h new file mode 100644 index 000000000000..94ef84429843 --- /dev/null +++ b/arch/ia64/include/asm/cmpxchg.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_IA64_CMPXCHG_H +#define _ASM_IA64_CMPXCHG_H + +#include <uapi/asm/cmpxchg.h> + +#define arch_xchg(ptr, x) \ +({(__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));}) + +#define arch_cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) +#define arch_cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) + +#define arch_cmpxchg_local arch_cmpxchg +#define arch_cmpxchg64_local arch_cmpxchg64 + +#endif /* _ASM_IA64_CMPXCHG_H */ diff --git a/arch/ia64/include/uapi/asm/cmpxchg.h b/arch/ia64/include/uapi/asm/cmpxchg.h index 5d90307fd6e0..926c6cb1e029 100644 --- a/arch/ia64/include/uapi/asm/cmpxchg.h +++ b/arch/ia64/include/uapi/asm/cmpxchg.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _ASM_IA64_CMPXCHG_H -#define _ASM_IA64_CMPXCHG_H +#ifndef _UAPI_ASM_IA64_CMPXCHG_H +#define _UAPI_ASM_IA64_CMPXCHG_H /* * Compare/Exchange, forked from asm/intrinsics.h @@ -53,8 +53,10 @@ extern void ia64_xchg_called_with_bad_pointer(void); __xchg_result; \ }) +#ifndef __KERNEL__ #define xchg(ptr, x) \ ({(__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)));}) +#endif /* * Atomic compare and exchange. Compare OLD with MEM, if identical, @@ -126,12 +128,14 @@ extern long ia64_cmpxchg_called_with_bad_pointer(void); * we had to back-pedal and keep the "legacy" behavior of a full fence :-( */ +#ifndef __KERNEL__ /* for compatibility with other platforms: */ #define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) #define cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) #define cmpxchg_local cmpxchg #define cmpxchg64_local cmpxchg64 +#endif #ifdef CONFIG_IA64_DEBUG_CMPXCHG # define CMPXCHG_BUGCHECK_DECL int _cmpxchg_bugcheck_count = 128; @@ -152,4 +156,4 @@ do { \ #endif /* !__ASSEMBLY__ */ -#endif /* _ASM_IA64_CMPXCHG_H */ +#endif /* _UAPI_ASM_IA64_CMPXCHG_H */ diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index 756c5cc58f94..8637bf8a2f65 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -16,8 +16,8 @@ * We do not have SMP m68k systems, so we don't have to deal with that. */ -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) /* * The ColdFire parts cannot do some immediate to memory operations, @@ -30,7 +30,7 @@ #endif #define ATOMIC_OP(op, c_op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ __asm__ __volatile__(#asm_op "l %1,%0" : "+m" (*v) : ASM_DI (i));\ } \ @@ -38,7 +38,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ #ifdef CONFIG_RMW_INSNS #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ int t, tmp; \ \ @@ -48,12 +48,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ " casl %2,%1,%0\n" \ " jne 1b" \ : "+m" (*v), "=&d" (t), "=&d" (tmp) \ - : "g" (i), "2" (atomic_read(v))); \ + : "g" (i), "2" (arch_atomic_read(v))); \ return t; \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ int t, tmp; \ \ @@ -63,14 +63,14 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ " casl %2,%1,%0\n" \ " jne 1b" \ : "+m" (*v), "=&d" (t), "=&d" (tmp) \ - : "g" (i), "2" (atomic_read(v))); \ + : "g" (i), "2" (arch_atomic_read(v))); \ return tmp; \ } #else #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t * v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t * v) \ { \ unsigned long flags; \ int t; \ @@ -83,7 +83,7 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ } #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ -static inline int atomic_fetch_##op(int i, atomic_t * v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ { \ unsigned long flags; \ int t; \ @@ -120,27 +120,27 @@ ATOMIC_OPS(xor, ^=, eor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_inc(atomic_t *v) +static inline void arch_atomic_inc(atomic_t *v) { __asm__ __volatile__("addql #1,%0" : "+m" (*v)); } -#define atomic_inc atomic_inc +#define arch_atomic_inc arch_atomic_inc -static inline void atomic_dec(atomic_t *v) +static inline void arch_atomic_dec(atomic_t *v) { __asm__ __volatile__("subql #1,%0" : "+m" (*v)); } -#define atomic_dec atomic_dec +#define arch_atomic_dec arch_atomic_dec -static inline int atomic_dec_and_test(atomic_t *v) +static inline int arch_atomic_dec_and_test(atomic_t *v) { char c; __asm__ __volatile__("subql #1,%1; seq %0" : "=d" (c), "+m" (*v)); return c != 0; } -#define atomic_dec_and_test atomic_dec_and_test +#define arch_atomic_dec_and_test arch_atomic_dec_and_test -static inline int atomic_dec_and_test_lt(atomic_t *v) +static inline int arch_atomic_dec_and_test_lt(atomic_t *v) { char c; __asm__ __volatile__( @@ -150,49 +150,49 @@ static inline int atomic_dec_and_test_lt(atomic_t *v) return c != 0; } -static inline int atomic_inc_and_test(atomic_t *v) +static inline int arch_atomic_inc_and_test(atomic_t *v) { char c; __asm__ __volatile__("addql #1,%1; seq %0" : "=d" (c), "+m" (*v)); return c != 0; } -#define atomic_inc_and_test atomic_inc_and_test +#define arch_atomic_inc_and_test arch_atomic_inc_and_test #ifdef CONFIG_RMW_INSNS -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) #else /* !CONFIG_RMW_INSNS */ -static inline int atomic_cmpxchg(atomic_t *v, int old, int new) +static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { unsigned long flags; int prev; local_irq_save(flags); - prev = atomic_read(v); + prev = arch_atomic_read(v); if (prev == old) - atomic_set(v, new); + arch_atomic_set(v, new); local_irq_restore(flags); return prev; } -static inline int atomic_xchg(atomic_t *v, int new) +static inline int arch_atomic_xchg(atomic_t *v, int new) { unsigned long flags; int prev; local_irq_save(flags); - prev = atomic_read(v); - atomic_set(v, new); + prev = arch_atomic_read(v); + arch_atomic_set(v, new); local_irq_restore(flags); return prev; } #endif /* !CONFIG_RMW_INSNS */ -static inline int atomic_sub_and_test(int i, atomic_t *v) +static inline int arch_atomic_sub_and_test(int i, atomic_t *v) { char c; __asm__ __volatile__("subl %2,%1; seq %0" @@ -200,9 +200,9 @@ static inline int atomic_sub_and_test(int i, atomic_t *v) : ASM_DI (i)); return c != 0; } -#define atomic_sub_and_test atomic_sub_and_test +#define arch_atomic_sub_and_test arch_atomic_sub_and_test -static inline int atomic_add_negative(int i, atomic_t *v) +static inline int arch_atomic_add_negative(int i, atomic_t *v) { char c; __asm__ __volatile__("addl %2,%1; smi %0" @@ -210,6 +210,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) : ASM_DI (i)); return c != 0; } -#define atomic_add_negative atomic_add_negative +#define arch_atomic_add_negative arch_atomic_add_negative #endif /* __ARCH_M68K_ATOMIC __ */ diff --git a/arch/m68k/include/asm/cmpxchg.h b/arch/m68k/include/asm/cmpxchg.h index a4aa82021d3b..e8ca4b0ccefa 100644 --- a/arch/m68k/include/asm/cmpxchg.h +++ b/arch/m68k/include/asm/cmpxchg.h @@ -76,11 +76,11 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz } #endif -#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) +#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) #include <asm-generic/cmpxchg-local.h> -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) extern unsigned long __invalid_cmpxchg_size(volatile void *, unsigned long, unsigned long, int); @@ -118,14 +118,14 @@ static inline unsigned long __cmpxchg(volatile void *p, unsigned long old, return old; } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr)));}) -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr)));}) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) #else diff --git a/arch/m68k/include/asm/mmu_context.h b/arch/m68k/include/asm/mmu_context.h index a5d358855878..8ed6ac14d99f 100644 --- a/arch/m68k/include/asm/mmu_context.h +++ b/arch/m68k/include/asm/mmu_context.h @@ -31,7 +31,7 @@ static inline void get_mmu_context(struct mm_struct *mm) if (mm->context != NO_CONTEXT) return; - while (atomic_dec_and_test_lt(&nr_free_contexts)) { + while (arch_atomic_dec_and_test_lt(&nr_free_contexts)) { atomic_inc(&nr_free_contexts); steal_context(); } diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index 29b0e557aa7c..a055f5dbe00a 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += syscall_table.h +generic-y += cmpxchg.h generic-y += extable.h generic-y += kvm_para.h generic-y += mcs_spinlock.h diff --git a/arch/microblaze/include/asm/atomic.h b/arch/microblaze/include/asm/atomic.h deleted file mode 100644 index 41e9aff23a62..000000000000 --- a/arch/microblaze/include/asm/atomic.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_MICROBLAZE_ATOMIC_H -#define _ASM_MICROBLAZE_ATOMIC_H - -#include <asm/cmpxchg.h> -#include <asm-generic/atomic.h> -#include <asm-generic/atomic64.h> - -/* - * Atomically test *v and decrement if it is greater than 0. - * The function returns the old value of *v minus 1. - */ -static inline int atomic_dec_if_positive(atomic_t *v) -{ - unsigned long flags; - int res; - - local_irq_save(flags); - res = v->counter - 1; - if (res >= 0) - v->counter = res; - local_irq_restore(flags); - - return res; -} -#define atomic_dec_if_positive atomic_dec_if_positive - -#endif /* _ASM_MICROBLAZE_ATOMIC_H */ diff --git a/arch/microblaze/include/asm/cmpxchg.h b/arch/microblaze/include/asm/cmpxchg.h deleted file mode 100644 index 3523b51aab36..000000000000 --- a/arch/microblaze/include/asm/cmpxchg.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_MICROBLAZE_CMPXCHG_H -#define _ASM_MICROBLAZE_CMPXCHG_H - -#ifndef CONFIG_SMP -# include <asm-generic/cmpxchg.h> -#endif - -#endif /* _ASM_MICROBLAZE_CMPXCHG_H */ diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 27ad76791539..95e1f7f3597f 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -25,24 +25,25 @@ #include <asm/war.h> #define ATOMIC_OPS(pfx, type) \ -static __always_inline type pfx##_read(const pfx##_t *v) \ +static __always_inline type arch_##pfx##_read(const pfx##_t *v) \ { \ return READ_ONCE(v->counter); \ } \ \ -static __always_inline void pfx##_set(pfx##_t *v, type i) \ +static __always_inline void arch_##pfx##_set(pfx##_t *v, type i) \ { \ WRITE_ONCE(v->counter, i); \ } \ \ -static __always_inline type pfx##_cmpxchg(pfx##_t *v, type o, type n) \ +static __always_inline type \ +arch_##pfx##_cmpxchg(pfx##_t *v, type o, type n) \ { \ - return cmpxchg(&v->counter, o, n); \ + return arch_cmpxchg(&v->counter, o, n); \ } \ \ -static __always_inline type pfx##_xchg(pfx##_t *v, type n) \ +static __always_inline type arch_##pfx##_xchg(pfx##_t *v, type n) \ { \ - return xchg(&v->counter, n); \ + return arch_xchg(&v->counter, n); \ } ATOMIC_OPS(atomic, int) @@ -53,7 +54,7 @@ ATOMIC_OPS(atomic64, s64) #endif #define ATOMIC_OP(pfx, op, type, c_op, asm_op, ll, sc) \ -static __inline__ void pfx##_##op(type i, pfx##_t * v) \ +static __inline__ void arch_##pfx##_##op(type i, pfx##_t * v) \ { \ type temp; \ \ @@ -80,7 +81,8 @@ static __inline__ void pfx##_##op(type i, pfx##_t * v) \ } #define ATOMIC_OP_RETURN(pfx, op, type, c_op, asm_op, ll, sc) \ -static __inline__ type pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ +static __inline__ type \ +arch_##pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ { \ type temp, result; \ \ @@ -113,7 +115,8 @@ static __inline__ type pfx##_##op##_return_relaxed(type i, pfx##_t * v) \ } #define ATOMIC_FETCH_OP(pfx, op, type, c_op, asm_op, ll, sc) \ -static __inline__ type pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ +static __inline__ type \ +arch_##pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ { \ int temp, result; \ \ @@ -153,18 +156,18 @@ static __inline__ type pfx##_fetch_##op##_relaxed(type i, pfx##_t * v) \ ATOMIC_OPS(atomic, add, int, +=, addu, ll, sc) ATOMIC_OPS(atomic, sub, int, -=, subu, ll, sc) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed #ifdef CONFIG_64BIT ATOMIC_OPS(atomic64, add, s64, +=, daddu, lld, scd) ATOMIC_OPS(atomic64, sub, s64, -=, dsubu, lld, scd) -# define atomic64_add_return_relaxed atomic64_add_return_relaxed -# define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -# define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -# define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +# define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +# define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +# define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +# define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed #endif /* CONFIG_64BIT */ #undef ATOMIC_OPS @@ -176,17 +179,17 @@ ATOMIC_OPS(atomic, and, int, &=, and, ll, sc) ATOMIC_OPS(atomic, or, int, |=, or, ll, sc) ATOMIC_OPS(atomic, xor, int, ^=, xor, ll, sc) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed #ifdef CONFIG_64BIT ATOMIC_OPS(atomic64, and, s64, &=, and, lld, scd) ATOMIC_OPS(atomic64, or, s64, |=, or, lld, scd) ATOMIC_OPS(atomic64, xor, s64, ^=, xor, lld, scd) -# define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -# define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -# define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +# define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +# define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +# define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed #endif #undef ATOMIC_OPS @@ -203,7 +206,7 @@ ATOMIC_OPS(atomic64, xor, s64, ^=, xor, lld, scd) * The function returns the old value of @v minus @i. */ #define ATOMIC_SIP_OP(pfx, type, op, ll, sc) \ -static __inline__ int pfx##_sub_if_positive(type i, pfx##_t * v) \ +static __inline__ int arch_##pfx##_sub_if_positive(type i, pfx##_t * v) \ { \ type temp, result; \ \ @@ -255,11 +258,11 @@ static __inline__ int pfx##_sub_if_positive(type i, pfx##_t * v) \ } ATOMIC_SIP_OP(atomic, int, subu, ll, sc) -#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) +#define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v) #ifdef CONFIG_64BIT ATOMIC_SIP_OP(atomic64, s64, dsubu, lld, scd) -#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v) +#define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v) #endif #undef ATOMIC_SIP_OP diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h index ed8f3f3c4304..0b983800f48b 100644 --- a/arch/mips/include/asm/cmpxchg.h +++ b/arch/mips/include/asm/cmpxchg.h @@ -90,7 +90,7 @@ unsigned long __xchg(volatile void *ptr, unsigned long x, int size) } } -#define xchg(ptr, x) \ +#define arch_xchg(ptr, x) \ ({ \ __typeof__(*(ptr)) __res; \ \ @@ -175,14 +175,14 @@ unsigned long __cmpxchg(volatile void *ptr, unsigned long old, } } -#define cmpxchg_local(ptr, old, new) \ +#define arch_cmpxchg_local(ptr, old, new) \ ((__typeof__(*(ptr))) \ __cmpxchg((ptr), \ (unsigned long)(__typeof__(*(ptr)))(old), \ (unsigned long)(__typeof__(*(ptr)))(new), \ sizeof(*(ptr)))) -#define cmpxchg(ptr, old, new) \ +#define arch_cmpxchg(ptr, old, new) \ ({ \ __typeof__(*(ptr)) __res; \ \ @@ -194,7 +194,7 @@ unsigned long __cmpxchg(volatile void *ptr, unsigned long old, if (__SYNC_loongson3_war == 0) \ smp_mb__before_llsc(); \ \ - __res = cmpxchg_local((ptr), (old), (new)); \ + __res = arch_cmpxchg_local((ptr), (old), (new)); \ \ /* \ * In the Loongson3 workaround case __cmpxchg_asm() already \ @@ -208,21 +208,21 @@ unsigned long __cmpxchg(volatile void *ptr, unsigned long old, }) #ifdef CONFIG_64BIT -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) -#define cmpxchg64(ptr, o, n) \ +#define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) #else # include <asm-generic/cmpxchg-local.h> -# define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +# define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) # ifdef CONFIG_SMP @@ -294,7 +294,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, return ret; } -# define cmpxchg64(ptr, o, n) ({ \ +# define arch_cmpxchg64(ptr, o, n) ({ \ unsigned long long __old = (__typeof__(*(ptr)))(o); \ unsigned long long __new = (__typeof__(*(ptr)))(n); \ __typeof__(*(ptr)) __res; \ @@ -317,7 +317,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, }) # else /* !CONFIG_SMP */ -# define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +# define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) # endif /* !CONFIG_SMP */ #endif /* !CONFIG_64BIT */ diff --git a/arch/mips/kernel/cmpxchg.c b/arch/mips/kernel/cmpxchg.c index 89107deb03fc..ac9c8cfb2ba9 100644 --- a/arch/mips/kernel/cmpxchg.c +++ b/arch/mips/kernel/cmpxchg.c @@ -41,7 +41,7 @@ unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int s do { old32 = load32; new32 = (load32 & ~mask) | (val << shift); - load32 = cmpxchg(ptr32, old32, new32); + load32 = arch_cmpxchg(ptr32, old32, new32); } while (load32 != old32); return (load32 & mask) >> shift; @@ -97,7 +97,7 @@ unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, */ old32 = (load32 & ~mask) | (old << shift); new32 = (load32 & ~mask) | (new << shift); - load32 = cmpxchg(ptr32, old32, new32); + load32 = arch_cmpxchg(ptr32, old32, new32); if (load32 == old32) return old; } diff --git a/arch/openrisc/include/asm/atomic.h b/arch/openrisc/include/asm/atomic.h index b589fac39b92..326167e4783a 100644 --- a/arch/openrisc/include/asm/atomic.h +++ b/arch/openrisc/include/asm/atomic.h @@ -13,7 +13,7 @@ /* Atomically perform op with v->counter and i */ #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ int tmp; \ \ @@ -30,7 +30,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ /* Atomically perform op with v->counter and i, return the result */ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ int tmp; \ \ @@ -49,7 +49,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ /* Atomically perform op with v->counter and i, return orig v->counter */ #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ int tmp, old; \ \ @@ -75,6 +75,8 @@ ATOMIC_FETCH_OP(and) ATOMIC_FETCH_OP(or) ATOMIC_FETCH_OP(xor) +ATOMIC_OP(add) +ATOMIC_OP(sub) ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) @@ -83,16 +85,18 @@ ATOMIC_OP(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_add_return atomic_add_return -#define atomic_sub_return atomic_sub_return -#define atomic_fetch_add atomic_fetch_add -#define atomic_fetch_sub atomic_fetch_sub -#define atomic_fetch_and atomic_fetch_and -#define atomic_fetch_or atomic_fetch_or -#define atomic_fetch_xor atomic_fetch_xor -#define atomic_and atomic_and -#define atomic_or atomic_or -#define atomic_xor atomic_xor +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor +#define arch_atomic_add arch_atomic_add +#define arch_atomic_sub arch_atomic_sub +#define arch_atomic_and arch_atomic_and +#define arch_atomic_or arch_atomic_or +#define arch_atomic_xor arch_atomic_xor /* * Atomically add a to v->counter as long as v is not already u. @@ -100,7 +104,7 @@ ATOMIC_OP(xor) * * This is often used through atomic_inc_not_zero() */ -static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int old, tmp; @@ -119,8 +123,14 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) return old; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless -#include <asm-generic/atomic.h> +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) + +#include <asm/cmpxchg.h> + +#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (v))) +#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (old), (new))) #endif /* __ASM_OPENRISC_ATOMIC_H */ diff --git a/arch/openrisc/include/asm/cmpxchg.h b/arch/openrisc/include/asm/cmpxchg.h index f9cd43a39d72..79fd16162ccb 100644 --- a/arch/openrisc/include/asm/cmpxchg.h +++ b/arch/openrisc/include/asm/cmpxchg.h @@ -132,7 +132,7 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, } } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ (__typeof__(*(ptr))) __cmpxchg((ptr), \ (unsigned long)(o), \ @@ -161,7 +161,7 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long with, } } -#define xchg(ptr, with) \ +#define arch_xchg(ptr, with) \ ({ \ (__typeof__(*(ptr))) __xchg((ptr), \ (unsigned long)(with), \ diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 21b375c67e53..dd5a299ada69 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -56,7 +56,7 @@ extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned; * are atomic, so a reader never sees inconsistent values. */ -static __inline__ void atomic_set(atomic_t *v, int i) +static __inline__ void arch_atomic_set(atomic_t *v, int i) { unsigned long flags; _atomic_spin_lock_irqsave(v, flags); @@ -66,19 +66,19 @@ static __inline__ void atomic_set(atomic_t *v, int i) _atomic_spin_unlock_irqrestore(v, flags); } -#define atomic_set_release(v, i) atomic_set((v), (i)) +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) -static __inline__ int atomic_read(const atomic_t *v) +static __inline__ int arch_atomic_read(const atomic_t *v) { return READ_ONCE((v)->counter); } /* exported interface */ -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) #define ATOMIC_OP(op, c_op) \ -static __inline__ void atomic_##op(int i, atomic_t *v) \ +static __inline__ void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -88,7 +88,7 @@ static __inline__ void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ +static __inline__ int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -101,7 +101,7 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static __inline__ int atomic_fetch_##op(int i, atomic_t *v) \ +static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -141,7 +141,7 @@ ATOMIC_OPS(xor, ^=) #define ATOMIC64_INIT(i) { (i) } #define ATOMIC64_OP(op, c_op) \ -static __inline__ void atomic64_##op(s64 i, atomic64_t *v) \ +static __inline__ void arch_atomic64_##op(s64 i, atomic64_t *v) \ { \ unsigned long flags; \ \ @@ -151,7 +151,7 @@ static __inline__ void atomic64_##op(s64 i, atomic64_t *v) \ } #define ATOMIC64_OP_RETURN(op, c_op) \ -static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ +static __inline__ s64 arch_atomic64_##op##_return(s64 i, atomic64_t *v) \ { \ unsigned long flags; \ s64 ret; \ @@ -164,7 +164,7 @@ static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ } #define ATOMIC64_FETCH_OP(op, c_op) \ -static __inline__ s64 atomic64_fetch_##op(s64 i, atomic64_t *v) \ +static __inline__ s64 arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \ { \ unsigned long flags; \ s64 ret; \ @@ -200,7 +200,7 @@ ATOMIC64_OPS(xor, ^=) #undef ATOMIC64_OP static __inline__ void -atomic64_set(atomic64_t *v, s64 i) +arch_atomic64_set(atomic64_t *v, s64 i) { unsigned long flags; _atomic_spin_lock_irqsave(v, flags); @@ -210,18 +210,18 @@ atomic64_set(atomic64_t *v, s64 i) _atomic_spin_unlock_irqrestore(v, flags); } -#define atomic64_set_release(v, i) atomic64_set((v), (i)) +#define arch_atomic64_set_release(v, i) arch_atomic64_set((v), (i)) static __inline__ s64 -atomic64_read(const atomic64_t *v) +arch_atomic64_read(const atomic64_t *v) { return READ_ONCE((v)->counter); } /* exported interface */ -#define atomic64_cmpxchg(v, o, n) \ - ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic64_cmpxchg(v, o, n) \ + ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) #endif /* !CONFIG_64BIT */ diff --git a/arch/parisc/include/asm/cmpxchg.h b/arch/parisc/include/asm/cmpxchg.h index 84ee232278a6..5f274be10567 100644 --- a/arch/parisc/include/asm/cmpxchg.h +++ b/arch/parisc/include/asm/cmpxchg.h @@ -44,7 +44,7 @@ __xchg(unsigned long x, volatile void *ptr, int size) ** if (((unsigned long)p & 0xf) == 0) ** return __ldcw(p); */ -#define xchg(ptr, x) \ +#define arch_xchg(ptr, x) \ ({ \ __typeof__(*(ptr)) __ret; \ __typeof__(*(ptr)) _x_ = (x); \ @@ -78,7 +78,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) return old; } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -98,7 +98,7 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, #endif case 4: return __cmpxchg_u32(ptr, old, new_); default: - return __cmpxchg_local_generic(ptr, old, new_, size); + return __generic_cmpxchg_local(ptr, old, new_, size); } } @@ -106,19 +106,19 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr)))) #ifdef CONFIG_64BIT -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg_local((ptr), (o), (n)); \ }) #else -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif -#define cmpxchg64(ptr, o, n) __cmpxchg_u64(ptr, o, n) +#define arch_cmpxchg64(ptr, o, n) __cmpxchg_u64(ptr, o, n) #endif /* _ASM_PARISC_CMPXCHG_H_ */ diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 61c6e8b200e8..a1732a79e92a 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -23,7 +23,7 @@ #define __atomic_release_fence() \ __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") -static __inline__ int atomic_read(const atomic_t *v) +static __inline__ int arch_atomic_read(const atomic_t *v) { int t; @@ -32,13 +32,13 @@ static __inline__ int atomic_read(const atomic_t *v) return t; } -static __inline__ void atomic_set(atomic_t *v, int i) +static __inline__ void arch_atomic_set(atomic_t *v, int i) { __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"UPD_CONSTR(v->counter) : "r"(i)); } #define ATOMIC_OP(op, asm_op) \ -static __inline__ void atomic_##op(int a, atomic_t *v) \ +static __inline__ void arch_atomic_##op(int a, atomic_t *v) \ { \ int t; \ \ @@ -53,7 +53,7 @@ static __inline__ void atomic_##op(int a, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ -static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ +static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \ { \ int t; \ \ @@ -70,7 +70,7 @@ static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ } #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ -static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ +static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ { \ int res, t; \ \ @@ -94,11 +94,11 @@ static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, subf) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed #undef ATOMIC_OPS #define ATOMIC_OPS(op, asm_op) \ @@ -109,16 +109,16 @@ ATOMIC_OPS(and, and) ATOMIC_OPS(or, or) ATOMIC_OPS(xor, xor) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed #undef ATOMIC_OPS #undef ATOMIC_FETCH_OP_RELAXED #undef ATOMIC_OP_RETURN_RELAXED #undef ATOMIC_OP -static __inline__ void atomic_inc(atomic_t *v) +static __inline__ void arch_atomic_inc(atomic_t *v) { int t; @@ -131,9 +131,9 @@ static __inline__ void atomic_inc(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } -#define atomic_inc atomic_inc +#define arch_atomic_inc arch_atomic_inc -static __inline__ int atomic_inc_return_relaxed(atomic_t *v) +static __inline__ int arch_atomic_inc_return_relaxed(atomic_t *v) { int t; @@ -149,7 +149,7 @@ static __inline__ int atomic_inc_return_relaxed(atomic_t *v) return t; } -static __inline__ void atomic_dec(atomic_t *v) +static __inline__ void arch_atomic_dec(atomic_t *v) { int t; @@ -162,9 +162,9 @@ static __inline__ void atomic_dec(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } -#define atomic_dec atomic_dec +#define arch_atomic_dec arch_atomic_dec -static __inline__ int atomic_dec_return_relaxed(atomic_t *v) +static __inline__ int arch_atomic_dec_return_relaxed(atomic_t *v) { int t; @@ -180,17 +180,20 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) return t; } -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#define atomic_dec_return_relaxed atomic_dec_return_relaxed +#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed +#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_cmpxchg_relaxed(v, o, n) \ - cmpxchg_relaxed(&((v)->counter), (o), (n)) -#define atomic_cmpxchg_acquire(v, o, n) \ - cmpxchg_acquire(&((v)->counter), (o), (n)) +#define arch_atomic_cmpxchg(v, o, n) \ + (arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_cmpxchg_relaxed(v, o, n) \ + arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) +#define arch_atomic_cmpxchg_acquire(v, o, n) \ + arch_cmpxchg_acquire(&((v)->counter), (o), (n)) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) +#define arch_atomic_xchg(v, new) \ + (arch_xchg(&((v)->counter), new)) +#define arch_atomic_xchg_relaxed(v, new) \ + arch_xchg_relaxed(&((v)->counter), (new)) /* * Don't want to override the generic atomic_try_cmpxchg_acquire, because @@ -199,7 +202,7 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) * would be a surprise). */ static __always_inline bool -atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) +arch_atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) { int r, o = *old; @@ -229,7 +232,7 @@ atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int t; @@ -250,7 +253,7 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) return t; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless /** * atomic_inc_not_zero - increment unless the number is zero @@ -259,7 +262,7 @@ static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) * Atomically increments @v by 1, so long as @v is non-zero. * Returns non-zero if @v was non-zero, and zero otherwise. */ -static __inline__ int atomic_inc_not_zero(atomic_t *v) +static __inline__ int arch_atomic_inc_not_zero(atomic_t *v) { int t1, t2; @@ -280,14 +283,14 @@ static __inline__ int atomic_inc_not_zero(atomic_t *v) return t1; } -#define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) +#define arch_atomic_inc_not_zero(v) arch_atomic_inc_not_zero((v)) /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1, even if * the atomic variable, v, was not decremented. */ -static __inline__ int atomic_dec_if_positive(atomic_t *v) +static __inline__ int arch_atomic_dec_if_positive(atomic_t *v) { int t; @@ -307,13 +310,13 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) return t; } -#define atomic_dec_if_positive atomic_dec_if_positive +#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive #ifdef __powerpc64__ #define ATOMIC64_INIT(i) { (i) } -static __inline__ s64 atomic64_read(const atomic64_t *v) +static __inline__ s64 arch_atomic64_read(const atomic64_t *v) { s64 t; @@ -322,13 +325,13 @@ static __inline__ s64 atomic64_read(const atomic64_t *v) return t; } -static __inline__ void atomic64_set(atomic64_t *v, s64 i) +static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i) { __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"UPD_CONSTR(v->counter) : "r"(i)); } #define ATOMIC64_OP(op, asm_op) \ -static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ +static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v) \ { \ s64 t; \ \ @@ -344,7 +347,7 @@ static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ static inline s64 \ -atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ +arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ { \ s64 t; \ \ @@ -362,7 +365,7 @@ atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ static inline s64 \ -atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ +arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ { \ s64 res, t; \ \ @@ -386,11 +389,11 @@ atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ ATOMIC64_OPS(add, add) ATOMIC64_OPS(sub, subf) -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed #undef ATOMIC64_OPS #define ATOMIC64_OPS(op, asm_op) \ @@ -401,16 +404,16 @@ ATOMIC64_OPS(and, and) ATOMIC64_OPS(or, or) ATOMIC64_OPS(xor, xor) -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed #undef ATOPIC64_OPS #undef ATOMIC64_FETCH_OP_RELAXED #undef ATOMIC64_OP_RETURN_RELAXED #undef ATOMIC64_OP -static __inline__ void atomic64_inc(atomic64_t *v) +static __inline__ void arch_atomic64_inc(atomic64_t *v) { s64 t; @@ -423,9 +426,9 @@ static __inline__ void atomic64_inc(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } -#define atomic64_inc atomic64_inc +#define arch_atomic64_inc arch_atomic64_inc -static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) +static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v) { s64 t; @@ -441,7 +444,7 @@ static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) return t; } -static __inline__ void atomic64_dec(atomic64_t *v) +static __inline__ void arch_atomic64_dec(atomic64_t *v) { s64 t; @@ -454,9 +457,9 @@ static __inline__ void atomic64_dec(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } -#define atomic64_dec atomic64_dec +#define arch_atomic64_dec arch_atomic64_dec -static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) +static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v) { s64 t; @@ -472,14 +475,14 @@ static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) return t; } -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed +#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed +#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1. */ -static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) +static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) { s64 t; @@ -498,16 +501,19 @@ static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) return t; } -#define atomic64_dec_if_positive atomic64_dec_if_positive +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive -#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_cmpxchg_relaxed(v, o, n) \ - cmpxchg_relaxed(&((v)->counter), (o), (n)) -#define atomic64_cmpxchg_acquire(v, o, n) \ - cmpxchg_acquire(&((v)->counter), (o), (n)) +#define arch_atomic64_cmpxchg(v, o, n) \ + (arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic64_cmpxchg_relaxed(v, o, n) \ + arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) +#define arch_atomic64_cmpxchg_acquire(v, o, n) \ + arch_cmpxchg_acquire(&((v)->counter), (o), (n)) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) +#define arch_atomic64_xchg(v, new) \ + (arch_xchg(&((v)->counter), new)) +#define arch_atomic64_xchg_relaxed(v, new) \ + arch_xchg_relaxed(&((v)->counter), (new)) /** * atomic64_fetch_add_unless - add unless the number is a given value @@ -518,7 +524,7 @@ static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 t; @@ -539,7 +545,7 @@ static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return t; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless /** * atomic_inc64_not_zero - increment unless the number is zero @@ -548,7 +554,7 @@ static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) * Atomically increments @v by 1, so long as @v is non-zero. * Returns non-zero if @v was non-zero, and zero otherwise. */ -static __inline__ int atomic64_inc_not_zero(atomic64_t *v) +static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v) { s64 t1, t2; @@ -569,7 +575,7 @@ static __inline__ int atomic64_inc_not_zero(atomic64_t *v) return t1 != 0; } -#define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) +#define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v)) #endif /* __powerpc64__ */ diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h index cf091c4c22e5..05f246c0e36e 100644 --- a/arch/powerpc/include/asm/cmpxchg.h +++ b/arch/powerpc/include/asm/cmpxchg.h @@ -185,14 +185,14 @@ __xchg_relaxed(void *ptr, unsigned long x, unsigned int size) BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local"); return x; } -#define xchg_local(ptr,x) \ +#define arch_xchg_local(ptr,x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_local((ptr), \ (unsigned long)_x_, sizeof(*(ptr))); \ }) -#define xchg_relaxed(ptr, x) \ +#define arch_xchg_relaxed(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ @@ -467,7 +467,7 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); return old; } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -476,7 +476,7 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, }) -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -484,7 +484,7 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, (unsigned long)_n_, sizeof(*(ptr))); \ }) -#define cmpxchg_relaxed(ptr, o, n) \ +#define arch_cmpxchg_relaxed(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -493,7 +493,7 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, sizeof(*(ptr))); \ }) -#define cmpxchg_acquire(ptr, o, n) \ +#define arch_cmpxchg_acquire(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -502,29 +502,29 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, sizeof(*(ptr))); \ }) #ifdef CONFIG_PPC64 -#define cmpxchg64(ptr, o, n) \ +#define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) -#define cmpxchg64_relaxed(ptr, o, n) \ +#define arch_cmpxchg64_relaxed(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_relaxed((ptr), (o), (n)); \ + arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) -#define cmpxchg64_acquire(ptr, o, n) \ +#define arch_cmpxchg64_acquire(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_acquire((ptr), (o), (n)); \ + arch_cmpxchg_acquire((ptr), (o), (n)); \ }) #else #include <asm-generic/cmpxchg-local.h> -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h index 07318bc63e3d..b676c4fb90fd 100644 --- a/arch/powerpc/include/asm/qspinlock.h +++ b/arch/powerpc/include/asm/qspinlock.h @@ -37,7 +37,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) { u32 val = 0; - if (likely(atomic_try_cmpxchg_lock(&lock->val, &val, _Q_LOCKED_VAL))) + if (likely(arch_atomic_try_cmpxchg_lock(&lock->val, &val, _Q_LOCKED_VAL))) return; queued_spin_lock_slowpath(lock, val); diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 400a8c8b6de7..ac9bdf4fc404 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -25,22 +25,22 @@ #define __atomic_release_fence() \ __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); -static __always_inline int atomic_read(const atomic_t *v) +static __always_inline int arch_atomic_read(const atomic_t *v) { return READ_ONCE(v->counter); } -static __always_inline void atomic_set(atomic_t *v, int i) +static __always_inline void arch_atomic_set(atomic_t *v, int i) { WRITE_ONCE(v->counter, i); } #ifndef CONFIG_GENERIC_ATOMIC64 #define ATOMIC64_INIT(i) { (i) } -static __always_inline s64 atomic64_read(const atomic64_t *v) +static __always_inline s64 arch_atomic64_read(const atomic64_t *v) { return READ_ONCE(v->counter); } -static __always_inline void atomic64_set(atomic64_t *v, s64 i) +static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) { WRITE_ONCE(v->counter, i); } @@ -53,7 +53,7 @@ static __always_inline void atomic64_set(atomic64_t *v, s64 i) */ #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ static __always_inline \ -void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ +void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ { \ __asm__ __volatile__ ( \ " amo" #asm_op "." #asm_type " zero, %1, %0" \ @@ -87,7 +87,7 @@ ATOMIC_OPS(xor, xor, i) */ #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ register c_type ret; \ @@ -99,7 +99,7 @@ c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \ return ret; \ } \ static __always_inline \ -c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ { \ register c_type ret; \ __asm__ __volatile__ ( \ @@ -112,15 +112,15 @@ c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \ static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(c_type i, \ +c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ } \ static __always_inline \ -c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ +c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ { \ - return atomic##prefix##_fetch_##op(i, v) c_op I; \ + return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ } #ifdef CONFIG_GENERIC_ATOMIC64 @@ -138,26 +138,26 @@ c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ ATOMIC_OPS(add, add, +, i) ATOMIC_OPS(sub, add, +, -i) -#define atomic_add_return_relaxed atomic_add_return_relaxed -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#define atomic_add_return atomic_add_return -#define atomic_sub_return atomic_sub_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed -#define atomic_fetch_add atomic_fetch_add -#define atomic_fetch_sub atomic_fetch_sub +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub #ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#define atomic64_add_return atomic64_add_return -#define atomic64_sub_return atomic64_sub_return - -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed -#define atomic64_fetch_add atomic64_fetch_add -#define atomic64_fetch_sub atomic64_fetch_sub +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return + +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub #endif #undef ATOMIC_OPS @@ -175,20 +175,20 @@ ATOMIC_OPS(and, and, i) ATOMIC_OPS( or, or, i) ATOMIC_OPS(xor, xor, i) -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed -#define atomic_fetch_and atomic_fetch_and -#define atomic_fetch_or atomic_fetch_or -#define atomic_fetch_xor atomic_fetch_xor +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor #ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed -#define atomic64_fetch_and atomic64_fetch_and -#define atomic64_fetch_or atomic64_fetch_or -#define atomic64_fetch_xor atomic64_fetch_xor +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor #endif #undef ATOMIC_OPS @@ -197,7 +197,7 @@ ATOMIC_OPS(xor, xor, i) #undef ATOMIC_OP_RETURN /* This is required to provide a full barrier on success. */ -static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int prev, rc; @@ -214,10 +214,10 @@ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "memory"); return prev; } -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless #ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 prev; long rc; @@ -235,7 +235,7 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u : "memory"); return prev; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless #endif /* @@ -244,45 +244,45 @@ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u */ #define ATOMIC_OP(c_t, prefix, size) \ static __always_inline \ -c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ +c_t arch_atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ { \ return __xchg_relaxed(&(v->counter), n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ +c_t arch_atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ { \ return __xchg_acquire(&(v->counter), n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ +c_t arch_atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ { \ return __xchg_release(&(v->counter), n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ +c_t arch_atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ { \ return __xchg(&(v->counter), n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ +c_t arch_atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ c_t o, c_t n) \ { \ return __cmpxchg_relaxed(&(v->counter), o, n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ +c_t arch_atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ c_t o, c_t n) \ { \ return __cmpxchg_acquire(&(v->counter), o, n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ +c_t arch_atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ c_t o, c_t n) \ { \ return __cmpxchg_release(&(v->counter), o, n, size); \ } \ static __always_inline \ -c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ +c_t arch_atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ { \ return __cmpxchg(&(v->counter), o, n, size); \ } @@ -298,19 +298,19 @@ c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ ATOMIC_OPS() -#define atomic_xchg_relaxed atomic_xchg_relaxed -#define atomic_xchg_acquire atomic_xchg_acquire -#define atomic_xchg_release atomic_xchg_release -#define atomic_xchg atomic_xchg -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#define atomic_cmpxchg_release atomic_cmpxchg_release -#define atomic_cmpxchg atomic_cmpxchg +#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed +#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire +#define arch_atomic_xchg_release arch_atomic_xchg_release +#define arch_atomic_xchg arch_atomic_xchg +#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire +#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release +#define arch_atomic_cmpxchg arch_atomic_cmpxchg #undef ATOMIC_OPS #undef ATOMIC_OP -static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) +static __always_inline int arch_atomic_sub_if_positive(atomic_t *v, int offset) { int prev, rc; @@ -328,10 +328,10 @@ static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset) return prev - offset; } -#define atomic_dec_if_positive(v) atomic_sub_if_positive(v, 1) +#define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(v, 1) #ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) +static __always_inline s64 arch_atomic64_sub_if_positive(atomic64_t *v, s64 offset) { s64 prev; long rc; @@ -350,7 +350,7 @@ static __always_inline s64 atomic64_sub_if_positive(atomic64_t *v, s64 offset) return prev - offset; } -#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(v, 1) +#define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(v, 1) #endif #endif /* _ASM_RISCV_ATOMIC_H */ diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 262e5bbb2776..36dc962f6343 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -37,7 +37,7 @@ __ret; \ }) -#define xchg_relaxed(ptr, x) \ +#define arch_xchg_relaxed(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ @@ -72,7 +72,7 @@ __ret; \ }) -#define xchg_acquire(ptr, x) \ +#define arch_xchg_acquire(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_acquire((ptr), \ @@ -107,7 +107,7 @@ __ret; \ }) -#define xchg_release(ptr, x) \ +#define arch_xchg_release(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_release((ptr), \ @@ -140,7 +140,7 @@ __ret; \ }) -#define xchg(ptr, x) \ +#define arch_xchg(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ @@ -149,13 +149,13 @@ #define xchg32(ptr, x) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - xchg((ptr), (x)); \ + arch_xchg((ptr), (x)); \ }) #define xchg64(ptr, x) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - xchg((ptr), (x)); \ + arch_xchg((ptr), (x)); \ }) /* @@ -199,7 +199,7 @@ __ret; \ }) -#define cmpxchg_relaxed(ptr, o, n) \ +#define arch_cmpxchg_relaxed(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -245,7 +245,7 @@ __ret; \ }) -#define cmpxchg_acquire(ptr, o, n) \ +#define arch_cmpxchg_acquire(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -291,7 +291,7 @@ __ret; \ }) -#define cmpxchg_release(ptr, o, n) \ +#define arch_cmpxchg_release(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -337,7 +337,7 @@ __ret; \ }) -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -345,31 +345,31 @@ _o_, _n_, sizeof(*(ptr))); \ }) -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) #define cmpxchg32(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) #define cmpxchg32_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ - cmpxchg_relaxed((ptr), (o), (n)) \ + arch_cmpxchg_relaxed((ptr), (o), (n)) \ }) -#define cmpxchg64(ptr, o, n) \ +#define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_relaxed((ptr), (o), (n)); \ + arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) #endif /* _ASM_RISCV_CMPXCHG_H */ diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 7c93c6573524..7138d189cc42 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -147,6 +147,4 @@ ATOMIC64_OPS(xor) #define arch_atomic64_fetch_sub(_i, _v) arch_atomic64_fetch_add(-(s64)(_i), _v) #define arch_atomic64_sub(_i, _v) arch_atomic64_add(-(s64)(_i), _v) -#define ARCH_ATOMIC - #endif /* __ARCH_S390_ATOMIC__ */ diff --git a/arch/sh/include/asm/atomic-grb.h b/arch/sh/include/asm/atomic-grb.h index aace62d42288..059791fd394f 100644 --- a/arch/sh/include/asm/atomic-grb.h +++ b/arch/sh/include/asm/atomic-grb.h @@ -3,7 +3,7 @@ #define __ASM_SH_ATOMIC_GRB_H #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ int tmp; \ \ @@ -23,7 +23,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ int tmp; \ \ @@ -45,7 +45,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ int res, tmp; \ \ diff --git a/arch/sh/include/asm/atomic-irq.h b/arch/sh/include/asm/atomic-irq.h index ee523bd2120f..7665de9d00d0 100644 --- a/arch/sh/include/asm/atomic-irq.h +++ b/arch/sh/include/asm/atomic-irq.h @@ -11,7 +11,7 @@ */ #define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -21,7 +21,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long temp, flags; \ \ @@ -35,7 +35,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long temp, flags; \ \ diff --git a/arch/sh/include/asm/atomic-llsc.h b/arch/sh/include/asm/atomic-llsc.h index 1d06e4d288dc..b63dcfbfa14e 100644 --- a/arch/sh/include/asm/atomic-llsc.h +++ b/arch/sh/include/asm/atomic-llsc.h @@ -17,7 +17,7 @@ */ #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ \ @@ -32,7 +32,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long temp; \ \ @@ -50,7 +50,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long res, temp; \ \ diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index 7c2a8a703b9a..528bfeda78f5 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -19,8 +19,8 @@ #include <asm/cmpxchg.h> #include <asm/barrier.h> -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) #if defined(CONFIG_GUSA_RB) #include <asm/atomic-grb.h> @@ -30,8 +30,8 @@ #include <asm/atomic-irq.h> #endif -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n))) #endif /* CONFIG_CPU_J2 */ diff --git a/arch/sh/include/asm/cmpxchg.h b/arch/sh/include/asm/cmpxchg.h index e9501d85c278..0ed9b3f4a577 100644 --- a/arch/sh/include/asm/cmpxchg.h +++ b/arch/sh/include/asm/cmpxchg.h @@ -45,7 +45,7 @@ extern void __xchg_called_with_bad_pointer(void); __xchg__res; \ }) -#define xchg(ptr,x) \ +#define arch_xchg(ptr,x) \ ((__typeof__(*(ptr)))__xchg((ptr),(unsigned long)(x), sizeof(*(ptr)))) /* This function doesn't exist, so you'll get a linker error @@ -63,7 +63,7 @@ static inline unsigned long __cmpxchg(volatile void * ptr, unsigned long old, return old; } -#define cmpxchg(ptr,o,n) \ +#define arch_cmpxchg(ptr,o,n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index efad5532f169..d775daa83d12 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -18,30 +18,30 @@ #include <asm/barrier.h> #include <asm-generic/atomic64.h> -int atomic_add_return(int, atomic_t *); -int atomic_fetch_add(int, atomic_t *); -int atomic_fetch_and(int, atomic_t *); -int atomic_fetch_or(int, atomic_t *); -int atomic_fetch_xor(int, atomic_t *); -int atomic_cmpxchg(atomic_t *, int, int); -int atomic_xchg(atomic_t *, int); -int atomic_fetch_add_unless(atomic_t *, int, int); -void atomic_set(atomic_t *, int); +int arch_atomic_add_return(int, atomic_t *); +int arch_atomic_fetch_add(int, atomic_t *); +int arch_atomic_fetch_and(int, atomic_t *); +int arch_atomic_fetch_or(int, atomic_t *); +int arch_atomic_fetch_xor(int, atomic_t *); +int arch_atomic_cmpxchg(atomic_t *, int, int); +int arch_atomic_xchg(atomic_t *, int); +int arch_atomic_fetch_add_unless(atomic_t *, int, int); +void arch_atomic_set(atomic_t *, int); -#define atomic_fetch_add_unless atomic_fetch_add_unless +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless -#define atomic_set_release(v, i) atomic_set((v), (i)) +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) -#define atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) -#define atomic_add(i, v) ((void)atomic_add_return( (int)(i), (v))) -#define atomic_sub(i, v) ((void)atomic_add_return(-(int)(i), (v))) +#define arch_atomic_add(i, v) ((void)arch_atomic_add_return( (int)(i), (v))) +#define arch_atomic_sub(i, v) ((void)arch_atomic_add_return(-(int)(i), (v))) -#define atomic_and(i, v) ((void)atomic_fetch_and((i), (v))) -#define atomic_or(i, v) ((void)atomic_fetch_or((i), (v))) -#define atomic_xor(i, v) ((void)atomic_fetch_xor((i), (v))) +#define arch_atomic_and(i, v) ((void)arch_atomic_fetch_and((i), (v))) +#define arch_atomic_or(i, v) ((void)arch_atomic_fetch_or((i), (v))) +#define arch_atomic_xor(i, v) ((void)arch_atomic_fetch_xor((i), (v))) -#define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v))) -#define atomic_fetch_sub(i, v) (atomic_fetch_add (-(int)(i), (v))) +#define arch_atomic_sub_return(i, v) (arch_atomic_add_return(-(int)(i), (v))) +#define arch_atomic_fetch_sub(i, v) (arch_atomic_fetch_add (-(int)(i), (v))) #endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index 6b235d3d1d9d..077891686715 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -14,23 +14,23 @@ #define ATOMIC64_INIT(i) { (i) } -#define atomic_read(v) READ_ONCE((v)->counter) -#define atomic64_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) -#define atomic64_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic64_set(v, i) WRITE_ONCE(((v)->counter), (i)) #define ATOMIC_OP(op) \ -void atomic_##op(int, atomic_t *); \ -void atomic64_##op(s64, atomic64_t *); +void arch_atomic_##op(int, atomic_t *); \ +void arch_atomic64_##op(s64, atomic64_t *); #define ATOMIC_OP_RETURN(op) \ -int atomic_##op##_return(int, atomic_t *); \ -s64 atomic64_##op##_return(s64, atomic64_t *); +int arch_atomic_##op##_return(int, atomic_t *); \ +s64 arch_atomic64_##op##_return(s64, atomic64_t *); #define ATOMIC_FETCH_OP(op) \ -int atomic_fetch_##op(int, atomic_t *); \ -s64 atomic64_fetch_##op(s64, atomic64_t *); +int arch_atomic_fetch_##op(int, atomic_t *); \ +s64 arch_atomic64_fetch_##op(s64, atomic64_t *); #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) @@ -49,18 +49,18 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n))) -static inline int atomic_xchg(atomic_t *v, int new) +static inline int arch_atomic_xchg(atomic_t *v, int new) { - return xchg(&v->counter, new); + return arch_xchg(&v->counter, new); } -#define atomic64_cmpxchg(v, o, n) \ - ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic64_cmpxchg(v, o, n) \ + ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) -s64 atomic64_dec_if_positive(atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive +s64 arch_atomic64_dec_if_positive(atomic64_t *v); +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive #endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h index a53d744d4212..27a57a3a7597 100644 --- a/arch/sparc/include/asm/cmpxchg_32.h +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -25,7 +25,7 @@ static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int return x; } -#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) +#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) /* Emulate cmpxchg() the same way we emulate atomics, * by hashing the object address and indexing into an array @@ -55,7 +55,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) return old; } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -64,7 +64,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) }) u64 __cmpxchg_u64(u64 *ptr, u64 old, u64 new); -#define cmpxchg64(ptr, old, new) __cmpxchg_u64(ptr, old, new) +#define arch_cmpxchg64(ptr, old, new) __cmpxchg_u64(ptr, old, new) #include <asm-generic/cmpxchg-local.h> @@ -72,9 +72,9 @@ u64 __cmpxchg_u64(u64 *ptr, u64 old, u64 new); * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o),\ (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif /* __ARCH_SPARC_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/cmpxchg_64.h b/arch/sparc/include/asm/cmpxchg_64.h index 316faa0130ba..8c39a9981187 100644 --- a/arch/sparc/include/asm/cmpxchg_64.h +++ b/arch/sparc/include/asm/cmpxchg_64.h @@ -52,7 +52,7 @@ static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long return val; } -#define xchg(ptr,x) \ +#define arch_xchg(ptr,x) \ ({ __typeof__(*(ptr)) __ret; \ __ret = (__typeof__(*(ptr))) \ __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ @@ -168,7 +168,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) return old; } -#define cmpxchg(ptr,o,n) \ +#define arch_cmpxchg(ptr,o,n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -189,20 +189,20 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, case 4: case 8: return __cmpxchg(ptr, old, new, size); default: - return __cmpxchg_local_generic(ptr, old, new, size); + return __generic_cmpxchg_local(ptr, old, new, size); } return old; } -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ cmpxchg_local((ptr), (o), (n)); \ }) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) #endif /* __ARCH_SPARC64_CMPXCHG__ */ diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 281fa634bb1a..8b81d0f00c97 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -29,7 +29,7 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ #define ATOMIC_FETCH_OP(op, c_op) \ -int atomic_fetch_##op(int i, atomic_t *v) \ +int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ @@ -41,10 +41,10 @@ int atomic_fetch_##op(int i, atomic_t *v) \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ -EXPORT_SYMBOL(atomic_fetch_##op); +EXPORT_SYMBOL(arch_atomic_fetch_##op); #define ATOMIC_OP_RETURN(op, c_op) \ -int atomic_##op##_return(int i, atomic_t *v) \ +int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ @@ -55,7 +55,7 @@ int atomic_##op##_return(int i, atomic_t *v) \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ -EXPORT_SYMBOL(atomic_##op##_return); +EXPORT_SYMBOL(arch_atomic_##op##_return); ATOMIC_OP_RETURN(add, +=) @@ -67,7 +67,7 @@ ATOMIC_FETCH_OP(xor, ^=) #undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN -int atomic_xchg(atomic_t *v, int new) +int arch_atomic_xchg(atomic_t *v, int new) { int ret; unsigned long flags; @@ -78,9 +78,9 @@ int atomic_xchg(atomic_t *v, int new) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(atomic_xchg); +EXPORT_SYMBOL(arch_atomic_xchg); -int atomic_cmpxchg(atomic_t *v, int old, int new) +int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; unsigned long flags; @@ -93,9 +93,9 @@ int atomic_cmpxchg(atomic_t *v, int old, int new) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(atomic_cmpxchg); +EXPORT_SYMBOL(arch_atomic_cmpxchg); -int atomic_fetch_add_unless(atomic_t *v, int a, int u) +int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int ret; unsigned long flags; @@ -107,10 +107,10 @@ int atomic_fetch_add_unless(atomic_t *v, int a, int u) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(atomic_fetch_add_unless); +EXPORT_SYMBOL(arch_atomic_fetch_add_unless); /* Atomic operations are already serializing */ -void atomic_set(atomic_t *v, int i) +void arch_atomic_set(atomic_t *v, int i) { unsigned long flags; @@ -118,7 +118,7 @@ void atomic_set(atomic_t *v, int i) v->counter = i; spin_unlock_irqrestore(ATOMIC_HASH(v), flags); } -EXPORT_SYMBOL(atomic_set); +EXPORT_SYMBOL(arch_atomic_set); unsigned long ___set_bit(unsigned long *addr, unsigned long mask) { diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index 456b65a30ecf..8245d4a97301 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -19,7 +19,7 @@ */ #define ATOMIC_OP(op) \ -ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ BACKOFF_SETUP(%o2); \ 1: lduw [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -30,11 +30,11 @@ ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ nop; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic_##op); \ -EXPORT_SYMBOL(atomic_##op); +ENDPROC(arch_atomic_##op); \ +EXPORT_SYMBOL(arch_atomic_##op); #define ATOMIC_OP_RETURN(op) \ -ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */\ BACKOFF_SETUP(%o2); \ 1: lduw [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -45,11 +45,11 @@ ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ sra %g1, 0, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic_##op##_return); \ -EXPORT_SYMBOL(atomic_##op##_return); +ENDPROC(arch_atomic_##op##_return); \ +EXPORT_SYMBOL(arch_atomic_##op##_return); #define ATOMIC_FETCH_OP(op) \ -ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ BACKOFF_SETUP(%o2); \ 1: lduw [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -60,8 +60,8 @@ ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ sra %g1, 0, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic_fetch_##op); \ -EXPORT_SYMBOL(atomic_fetch_##op); +ENDPROC(arch_atomic_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic_fetch_##op); ATOMIC_OP(add) ATOMIC_OP_RETURN(add) @@ -85,7 +85,7 @@ ATOMIC_FETCH_OP(xor) #undef ATOMIC_OP #define ATOMIC64_OP(op) \ -ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ BACKOFF_SETUP(%o2); \ 1: ldx [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -96,11 +96,11 @@ ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ nop; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic64_##op); \ -EXPORT_SYMBOL(atomic64_##op); +ENDPROC(arch_atomic64_##op); \ +EXPORT_SYMBOL(arch_atomic64_##op); #define ATOMIC64_OP_RETURN(op) \ -ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ BACKOFF_SETUP(%o2); \ 1: ldx [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -111,11 +111,11 @@ ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ op %g1, %o0, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic64_##op##_return); \ -EXPORT_SYMBOL(atomic64_##op##_return); +ENDPROC(arch_atomic64_##op##_return); \ +EXPORT_SYMBOL(arch_atomic64_##op##_return); #define ATOMIC64_FETCH_OP(op) \ -ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ +ENTRY(arch_atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ BACKOFF_SETUP(%o2); \ 1: ldx [%o1], %g1; \ op %g1, %o0, %g7; \ @@ -126,8 +126,8 @@ ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ mov %g1, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic64_fetch_##op); \ -EXPORT_SYMBOL(atomic64_fetch_##op); +ENDPROC(arch_atomic64_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic64_fetch_##op); ATOMIC64_OP(add) ATOMIC64_OP_RETURN(add) @@ -150,7 +150,7 @@ ATOMIC64_FETCH_OP(xor) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ +ENTRY(arch_atomic64_dec_if_positive) /* %o0 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o0], %g1 brlez,pn %g1, 3f @@ -162,5 +162,5 @@ ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ 3: retl sub %g1, 1, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_dec_if_positive) -EXPORT_SYMBOL(atomic64_dec_if_positive) +ENDPROC(arch_atomic64_dec_if_positive) +EXPORT_SYMBOL(arch_atomic64_dec_if_positive) diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index f732741ad7c7..5e754e895767 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -269,6 +269,4 @@ static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v) # include <asm/atomic64_64.h> #endif -#define ARCH_ATOMIC - #endif /* _ASM_X86_ATOMIC_H */ diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index 744c2f463845..4361fe4247e3 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -43,7 +43,7 @@ * * Atomically reads the value of @v. */ -#define atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) /** * atomic_set - set atomic variable @@ -52,11 +52,11 @@ * * Atomically sets the value of @v to @i. */ -#define atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) +#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) #if XCHAL_HAVE_EXCLUSIVE #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ @@ -74,7 +74,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ @@ -95,7 +95,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ @@ -116,7 +116,7 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #elif XCHAL_HAVE_S32C1I #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t * v) \ +static inline void arch_atomic_##op(int i, atomic_t * v) \ { \ unsigned long tmp; \ int result; \ @@ -135,7 +135,7 @@ static inline void atomic_##op(int i, atomic_t * v) \ } \ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t * v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t * v) \ { \ unsigned long tmp; \ int result; \ @@ -157,7 +157,7 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t * v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ { \ unsigned long tmp; \ int result; \ @@ -180,7 +180,7 @@ static inline int atomic_fetch_##op(int i, atomic_t * v) \ #else /* XCHAL_HAVE_S32C1I */ #define ATOMIC_OP(op) \ -static inline void atomic_##op(int i, atomic_t * v) \ +static inline void arch_atomic_##op(int i, atomic_t * v) \ { \ unsigned int vval; \ \ @@ -198,7 +198,7 @@ static inline void atomic_##op(int i, atomic_t * v) \ } \ #define ATOMIC_OP_RETURN(op) \ -static inline int atomic_##op##_return(int i, atomic_t * v) \ +static inline int arch_atomic_##op##_return(int i, atomic_t * v) \ { \ unsigned int vval; \ \ @@ -218,7 +218,7 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ } #define ATOMIC_FETCH_OP(op) \ -static inline int atomic_fetch_##op(int i, atomic_t * v) \ +static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ { \ unsigned int tmp, vval; \ \ @@ -257,7 +257,7 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n))) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) #endif /* _XTENSA_ATOMIC_H */ diff --git a/arch/xtensa/include/asm/cmpxchg.h b/arch/xtensa/include/asm/cmpxchg.h index a175f8aec3fb..3699e2818efb 100644 --- a/arch/xtensa/include/asm/cmpxchg.h +++ b/arch/xtensa/include/asm/cmpxchg.h @@ -80,7 +80,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) } } -#define cmpxchg(ptr,o,n) \ +#define arch_cmpxchg(ptr,o,n) \ ({ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ @@ -97,7 +97,7 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, case 4: return __cmpxchg_u32(ptr, old, new); default: - return __cmpxchg_local_generic(ptr, old, new, size); + return __generic_cmpxchg_local(ptr, old, new, size); } return old; @@ -107,11 +107,11 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o),\ (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) /* * xchg_u32 @@ -169,7 +169,7 @@ static inline unsigned long xchg_u32(volatile int * m, unsigned long val) #endif } -#define xchg(ptr,x) \ +#define arch_xchg(ptr,x) \ ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) static inline u32 xchg_small(volatile void *ptr, u32 x, int size) diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h index 888b6cfeed91..bc45af52c93b 100644 --- a/include/asm-generic/atomic-instrumented.h +++ b/include/asm-generic/atomic-instrumented.h @@ -27,17 +27,13 @@ atomic_read(const atomic_t *v) instrument_atomic_read(v, sizeof(*v)); return arch_atomic_read(v); } -#define atomic_read atomic_read -#if defined(arch_atomic_read_acquire) static __always_inline int atomic_read_acquire(const atomic_t *v) { instrument_atomic_read(v, sizeof(*v)); return arch_atomic_read_acquire(v); } -#define atomic_read_acquire atomic_read_acquire -#endif static __always_inline void atomic_set(atomic_t *v, int i) @@ -45,17 +41,13 @@ atomic_set(atomic_t *v, int i) instrument_atomic_write(v, sizeof(*v)); arch_atomic_set(v, i); } -#define atomic_set atomic_set -#if defined(arch_atomic_set_release) static __always_inline void atomic_set_release(atomic_t *v, int i) { instrument_atomic_write(v, sizeof(*v)); arch_atomic_set_release(v, i); } -#define atomic_set_release atomic_set_release -#endif static __always_inline void atomic_add(int i, atomic_t *v) @@ -63,87 +55,62 @@ atomic_add(int i, atomic_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_add(i, v); } -#define atomic_add atomic_add -#if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return) static __always_inline int atomic_add_return(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return(i, v); } -#define atomic_add_return atomic_add_return -#endif -#if defined(arch_atomic_add_return_acquire) static __always_inline int atomic_add_return_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return_acquire(i, v); } -#define atomic_add_return_acquire atomic_add_return_acquire -#endif -#if defined(arch_atomic_add_return_release) static __always_inline int atomic_add_return_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return_release(i, v); } -#define atomic_add_return_release atomic_add_return_release -#endif -#if defined(arch_atomic_add_return_relaxed) static __always_inline int atomic_add_return_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_return_relaxed(i, v); } -#define atomic_add_return_relaxed atomic_add_return_relaxed -#endif -#if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add) static __always_inline int atomic_fetch_add(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add(i, v); } -#define atomic_fetch_add atomic_fetch_add -#endif -#if defined(arch_atomic_fetch_add_acquire) static __always_inline int atomic_fetch_add_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_acquire(i, v); } -#define atomic_fetch_add_acquire atomic_fetch_add_acquire -#endif -#if defined(arch_atomic_fetch_add_release) static __always_inline int atomic_fetch_add_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_release(i, v); } -#define atomic_fetch_add_release atomic_fetch_add_release -#endif -#if defined(arch_atomic_fetch_add_relaxed) static __always_inline int atomic_fetch_add_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_relaxed(i, v); } -#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed -#endif static __always_inline void atomic_sub(int i, atomic_t *v) @@ -151,267 +118,188 @@ atomic_sub(int i, atomic_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_sub(i, v); } -#define atomic_sub atomic_sub -#if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return) static __always_inline int atomic_sub_return(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return(i, v); } -#define atomic_sub_return atomic_sub_return -#endif -#if defined(arch_atomic_sub_return_acquire) static __always_inline int atomic_sub_return_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return_acquire(i, v); } -#define atomic_sub_return_acquire atomic_sub_return_acquire -#endif -#if defined(arch_atomic_sub_return_release) static __always_inline int atomic_sub_return_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return_release(i, v); } -#define atomic_sub_return_release atomic_sub_return_release -#endif -#if defined(arch_atomic_sub_return_relaxed) static __always_inline int atomic_sub_return_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_return_relaxed(i, v); } -#define atomic_sub_return_relaxed atomic_sub_return_relaxed -#endif -#if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub) static __always_inline int atomic_fetch_sub(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub(i, v); } -#define atomic_fetch_sub atomic_fetch_sub -#endif -#if defined(arch_atomic_fetch_sub_acquire) static __always_inline int atomic_fetch_sub_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub_acquire(i, v); } -#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire -#endif -#if defined(arch_atomic_fetch_sub_release) static __always_inline int atomic_fetch_sub_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub_release(i, v); } -#define atomic_fetch_sub_release atomic_fetch_sub_release -#endif -#if defined(arch_atomic_fetch_sub_relaxed) static __always_inline int atomic_fetch_sub_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_sub_relaxed(i, v); } -#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed -#endif -#if defined(arch_atomic_inc) static __always_inline void atomic_inc(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_inc(v); } -#define atomic_inc atomic_inc -#endif -#if defined(arch_atomic_inc_return) static __always_inline int atomic_inc_return(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return(v); } -#define atomic_inc_return atomic_inc_return -#endif -#if defined(arch_atomic_inc_return_acquire) static __always_inline int atomic_inc_return_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return_acquire(v); } -#define atomic_inc_return_acquire atomic_inc_return_acquire -#endif -#if defined(arch_atomic_inc_return_release) static __always_inline int atomic_inc_return_release(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return_release(v); } -#define atomic_inc_return_release atomic_inc_return_release -#endif -#if defined(arch_atomic_inc_return_relaxed) static __always_inline int atomic_inc_return_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_return_relaxed(v); } -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#endif -#if defined(arch_atomic_fetch_inc) static __always_inline int atomic_fetch_inc(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc(v); } -#define atomic_fetch_inc atomic_fetch_inc -#endif -#if defined(arch_atomic_fetch_inc_acquire) static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc_acquire(v); } -#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire -#endif -#if defined(arch_atomic_fetch_inc_release) static __always_inline int atomic_fetch_inc_release(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc_release(v); } -#define atomic_fetch_inc_release atomic_fetch_inc_release -#endif -#if defined(arch_atomic_fetch_inc_relaxed) static __always_inline int atomic_fetch_inc_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_inc_relaxed(v); } -#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed -#endif -#if defined(arch_atomic_dec) static __always_inline void atomic_dec(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_dec(v); } -#define atomic_dec atomic_dec -#endif -#if defined(arch_atomic_dec_return) static __always_inline int atomic_dec_return(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return(v); } -#define atomic_dec_return atomic_dec_return -#endif -#if defined(arch_atomic_dec_return_acquire) static __always_inline int atomic_dec_return_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return_acquire(v); } -#define atomic_dec_return_acquire atomic_dec_return_acquire -#endif -#if defined(arch_atomic_dec_return_release) static __always_inline int atomic_dec_return_release(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return_release(v); } -#define atomic_dec_return_release atomic_dec_return_release -#endif -#if defined(arch_atomic_dec_return_relaxed) static __always_inline int atomic_dec_return_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_return_relaxed(v); } -#define atomic_dec_return_relaxed atomic_dec_return_relaxed -#endif -#if defined(arch_atomic_fetch_dec) static __always_inline int atomic_fetch_dec(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec(v); } -#define atomic_fetch_dec atomic_fetch_dec -#endif -#if defined(arch_atomic_fetch_dec_acquire) static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec_acquire(v); } -#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire -#endif -#if defined(arch_atomic_fetch_dec_release) static __always_inline int atomic_fetch_dec_release(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec_release(v); } -#define atomic_fetch_dec_release atomic_fetch_dec_release -#endif -#if defined(arch_atomic_fetch_dec_relaxed) static __always_inline int atomic_fetch_dec_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_dec_relaxed(v); } -#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed -#endif static __always_inline void atomic_and(int i, atomic_t *v) @@ -419,97 +307,69 @@ atomic_and(int i, atomic_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_and(i, v); } -#define atomic_and atomic_and -#if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and) static __always_inline int atomic_fetch_and(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and(i, v); } -#define atomic_fetch_and atomic_fetch_and -#endif -#if defined(arch_atomic_fetch_and_acquire) static __always_inline int atomic_fetch_and_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and_acquire(i, v); } -#define atomic_fetch_and_acquire atomic_fetch_and_acquire -#endif -#if defined(arch_atomic_fetch_and_release) static __always_inline int atomic_fetch_and_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and_release(i, v); } -#define atomic_fetch_and_release atomic_fetch_and_release -#endif -#if defined(arch_atomic_fetch_and_relaxed) static __always_inline int atomic_fetch_and_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_and_relaxed(i, v); } -#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed -#endif -#if defined(arch_atomic_andnot) static __always_inline void atomic_andnot(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_andnot(i, v); } -#define atomic_andnot atomic_andnot -#endif -#if defined(arch_atomic_fetch_andnot) static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot(i, v); } -#define atomic_fetch_andnot atomic_fetch_andnot -#endif -#if defined(arch_atomic_fetch_andnot_acquire) static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_acquire(i, v); } -#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire -#endif -#if defined(arch_atomic_fetch_andnot_release) static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_release(i, v); } -#define atomic_fetch_andnot_release atomic_fetch_andnot_release -#endif -#if defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_relaxed(i, v); } -#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed -#endif static __always_inline void atomic_or(int i, atomic_t *v) @@ -517,47 +377,34 @@ atomic_or(int i, atomic_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_or(i, v); } -#define atomic_or atomic_or -#if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or) static __always_inline int atomic_fetch_or(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or(i, v); } -#define atomic_fetch_or atomic_fetch_or -#endif -#if defined(arch_atomic_fetch_or_acquire) static __always_inline int atomic_fetch_or_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or_acquire(i, v); } -#define atomic_fetch_or_acquire atomic_fetch_or_acquire -#endif -#if defined(arch_atomic_fetch_or_release) static __always_inline int atomic_fetch_or_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or_release(i, v); } -#define atomic_fetch_or_release atomic_fetch_or_release -#endif -#if defined(arch_atomic_fetch_or_relaxed) static __always_inline int atomic_fetch_or_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_or_relaxed(i, v); } -#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed -#endif static __always_inline void atomic_xor(int i, atomic_t *v) @@ -565,129 +412,91 @@ atomic_xor(int i, atomic_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic_xor(i, v); } -#define atomic_xor atomic_xor -#if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor) static __always_inline int atomic_fetch_xor(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor(i, v); } -#define atomic_fetch_xor atomic_fetch_xor -#endif -#if defined(arch_atomic_fetch_xor_acquire) static __always_inline int atomic_fetch_xor_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor_acquire(i, v); } -#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire -#endif -#if defined(arch_atomic_fetch_xor_release) static __always_inline int atomic_fetch_xor_release(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor_release(i, v); } -#define atomic_fetch_xor_release atomic_fetch_xor_release -#endif -#if defined(arch_atomic_fetch_xor_relaxed) static __always_inline int atomic_fetch_xor_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_xor_relaxed(i, v); } -#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed -#endif -#if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg) static __always_inline int atomic_xchg(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg(v, i); } -#define atomic_xchg atomic_xchg -#endif -#if defined(arch_atomic_xchg_acquire) static __always_inline int atomic_xchg_acquire(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg_acquire(v, i); } -#define atomic_xchg_acquire atomic_xchg_acquire -#endif -#if defined(arch_atomic_xchg_release) static __always_inline int atomic_xchg_release(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg_release(v, i); } -#define atomic_xchg_release atomic_xchg_release -#endif -#if defined(arch_atomic_xchg_relaxed) static __always_inline int atomic_xchg_relaxed(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_xchg_relaxed(v, i); } -#define atomic_xchg_relaxed atomic_xchg_relaxed -#endif -#if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg(v, old, new); } -#define atomic_cmpxchg atomic_cmpxchg -#endif -#if defined(arch_atomic_cmpxchg_acquire) static __always_inline int atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg_acquire(v, old, new); } -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#endif -#if defined(arch_atomic_cmpxchg_release) static __always_inline int atomic_cmpxchg_release(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg_release(v, old, new); } -#define atomic_cmpxchg_release atomic_cmpxchg_release -#endif -#if defined(arch_atomic_cmpxchg_relaxed) static __always_inline int atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_cmpxchg_relaxed(v, old, new); } -#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -#endif -#if defined(arch_atomic_try_cmpxchg) static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { @@ -695,10 +504,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg(v, old, new); } -#define atomic_try_cmpxchg atomic_try_cmpxchg -#endif -#if defined(arch_atomic_try_cmpxchg_acquire) static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { @@ -706,10 +512,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_acquire(v, old, new); } -#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire -#endif -#if defined(arch_atomic_try_cmpxchg_release) static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { @@ -717,10 +520,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_release(v, old, new); } -#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release -#endif -#if defined(arch_atomic_try_cmpxchg_relaxed) static __always_inline bool atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { @@ -728,108 +528,76 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_relaxed(v, old, new); } -#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed -#endif -#if defined(arch_atomic_sub_and_test) static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_sub_and_test(i, v); } -#define atomic_sub_and_test atomic_sub_and_test -#endif -#if defined(arch_atomic_dec_and_test) static __always_inline bool atomic_dec_and_test(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_and_test(v); } -#define atomic_dec_and_test atomic_dec_and_test -#endif -#if defined(arch_atomic_inc_and_test) static __always_inline bool atomic_inc_and_test(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_and_test(v); } -#define atomic_inc_and_test atomic_inc_and_test -#endif -#if defined(arch_atomic_add_negative) static __always_inline bool atomic_add_negative(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_negative(i, v); } -#define atomic_add_negative atomic_add_negative -#endif -#if defined(arch_atomic_fetch_add_unless) static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_fetch_add_unless(v, a, u); } -#define atomic_fetch_add_unless atomic_fetch_add_unless -#endif -#if defined(arch_atomic_add_unless) static __always_inline bool atomic_add_unless(atomic_t *v, int a, int u) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_add_unless(v, a, u); } -#define atomic_add_unless atomic_add_unless -#endif -#if defined(arch_atomic_inc_not_zero) static __always_inline bool atomic_inc_not_zero(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_not_zero(v); } -#define atomic_inc_not_zero atomic_inc_not_zero -#endif -#if defined(arch_atomic_inc_unless_negative) static __always_inline bool atomic_inc_unless_negative(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_inc_unless_negative(v); } -#define atomic_inc_unless_negative atomic_inc_unless_negative -#endif -#if defined(arch_atomic_dec_unless_positive) static __always_inline bool atomic_dec_unless_positive(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_unless_positive(v); } -#define atomic_dec_unless_positive atomic_dec_unless_positive -#endif -#if defined(arch_atomic_dec_if_positive) static __always_inline int atomic_dec_if_positive(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic_dec_if_positive(v); } -#define atomic_dec_if_positive atomic_dec_if_positive -#endif static __always_inline s64 atomic64_read(const atomic64_t *v) @@ -837,17 +605,13 @@ atomic64_read(const atomic64_t *v) instrument_atomic_read(v, sizeof(*v)); return arch_atomic64_read(v); } -#define atomic64_read atomic64_read -#if defined(arch_atomic64_read_acquire) static __always_inline s64 atomic64_read_acquire(const atomic64_t *v) { instrument_atomic_read(v, sizeof(*v)); return arch_atomic64_read_acquire(v); } -#define atomic64_read_acquire atomic64_read_acquire -#endif static __always_inline void atomic64_set(atomic64_t *v, s64 i) @@ -855,17 +619,13 @@ atomic64_set(atomic64_t *v, s64 i) instrument_atomic_write(v, sizeof(*v)); arch_atomic64_set(v, i); } -#define atomic64_set atomic64_set -#if defined(arch_atomic64_set_release) static __always_inline void atomic64_set_release(atomic64_t *v, s64 i) { instrument_atomic_write(v, sizeof(*v)); arch_atomic64_set_release(v, i); } -#define atomic64_set_release atomic64_set_release -#endif static __always_inline void atomic64_add(s64 i, atomic64_t *v) @@ -873,87 +633,62 @@ atomic64_add(s64 i, atomic64_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_add(i, v); } -#define atomic64_add atomic64_add -#if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return) static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return(i, v); } -#define atomic64_add_return atomic64_add_return -#endif -#if defined(arch_atomic64_add_return_acquire) static __always_inline s64 atomic64_add_return_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return_acquire(i, v); } -#define atomic64_add_return_acquire atomic64_add_return_acquire -#endif -#if defined(arch_atomic64_add_return_release) static __always_inline s64 atomic64_add_return_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return_release(i, v); } -#define atomic64_add_return_release atomic64_add_return_release -#endif -#if defined(arch_atomic64_add_return_relaxed) static __always_inline s64 atomic64_add_return_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_return_relaxed(i, v); } -#define atomic64_add_return_relaxed atomic64_add_return_relaxed -#endif -#if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add) static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add(i, v); } -#define atomic64_fetch_add atomic64_fetch_add -#endif -#if defined(arch_atomic64_fetch_add_acquire) static __always_inline s64 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_acquire(i, v); } -#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire -#endif -#if defined(arch_atomic64_fetch_add_release) static __always_inline s64 atomic64_fetch_add_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_release(i, v); } -#define atomic64_fetch_add_release atomic64_fetch_add_release -#endif -#if defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_relaxed(i, v); } -#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed -#endif static __always_inline void atomic64_sub(s64 i, atomic64_t *v) @@ -961,267 +696,188 @@ atomic64_sub(s64 i, atomic64_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_sub(i, v); } -#define atomic64_sub atomic64_sub -#if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return) static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return(i, v); } -#define atomic64_sub_return atomic64_sub_return -#endif -#if defined(arch_atomic64_sub_return_acquire) static __always_inline s64 atomic64_sub_return_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return_acquire(i, v); } -#define atomic64_sub_return_acquire atomic64_sub_return_acquire -#endif -#if defined(arch_atomic64_sub_return_release) static __always_inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return_release(i, v); } -#define atomic64_sub_return_release atomic64_sub_return_release -#endif -#if defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_return_relaxed(i, v); } -#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed -#endif -#if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub) static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub(i, v); } -#define atomic64_fetch_sub atomic64_fetch_sub -#endif -#if defined(arch_atomic64_fetch_sub_acquire) static __always_inline s64 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_acquire(i, v); } -#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire -#endif -#if defined(arch_atomic64_fetch_sub_release) static __always_inline s64 atomic64_fetch_sub_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_release(i, v); } -#define atomic64_fetch_sub_release atomic64_fetch_sub_release -#endif -#if defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_relaxed(i, v); } -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed -#endif -#if defined(arch_atomic64_inc) static __always_inline void atomic64_inc(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_inc(v); } -#define atomic64_inc atomic64_inc -#endif -#if defined(arch_atomic64_inc_return) static __always_inline s64 atomic64_inc_return(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return(v); } -#define atomic64_inc_return atomic64_inc_return -#endif -#if defined(arch_atomic64_inc_return_acquire) static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return_acquire(v); } -#define atomic64_inc_return_acquire atomic64_inc_return_acquire -#endif -#if defined(arch_atomic64_inc_return_release) static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return_release(v); } -#define atomic64_inc_return_release atomic64_inc_return_release -#endif -#if defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 atomic64_inc_return_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_return_relaxed(v); } -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#endif -#if defined(arch_atomic64_fetch_inc) static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc(v); } -#define atomic64_fetch_inc atomic64_fetch_inc -#endif -#if defined(arch_atomic64_fetch_inc_acquire) static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_acquire(v); } -#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire -#endif -#if defined(arch_atomic64_fetch_inc_release) static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_release(v); } -#define atomic64_fetch_inc_release atomic64_fetch_inc_release -#endif -#if defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 atomic64_fetch_inc_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_relaxed(v); } -#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed -#endif -#if defined(arch_atomic64_dec) static __always_inline void atomic64_dec(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_dec(v); } -#define atomic64_dec atomic64_dec -#endif -#if defined(arch_atomic64_dec_return) static __always_inline s64 atomic64_dec_return(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return(v); } -#define atomic64_dec_return atomic64_dec_return -#endif -#if defined(arch_atomic64_dec_return_acquire) static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return_acquire(v); } -#define atomic64_dec_return_acquire atomic64_dec_return_acquire -#endif -#if defined(arch_atomic64_dec_return_release) static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return_release(v); } -#define atomic64_dec_return_release atomic64_dec_return_release -#endif -#if defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 atomic64_dec_return_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_return_relaxed(v); } -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#endif -#if defined(arch_atomic64_fetch_dec) static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec(v); } -#define atomic64_fetch_dec atomic64_fetch_dec -#endif -#if defined(arch_atomic64_fetch_dec_acquire) static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_acquire(v); } -#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire -#endif -#if defined(arch_atomic64_fetch_dec_release) static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_release(v); } -#define atomic64_fetch_dec_release atomic64_fetch_dec_release -#endif -#if defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 atomic64_fetch_dec_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_relaxed(v); } -#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed -#endif static __always_inline void atomic64_and(s64 i, atomic64_t *v) @@ -1229,97 +885,69 @@ atomic64_and(s64 i, atomic64_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_and(i, v); } -#define atomic64_and atomic64_and -#if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and) static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and(i, v); } -#define atomic64_fetch_and atomic64_fetch_and -#endif -#if defined(arch_atomic64_fetch_and_acquire) static __always_inline s64 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and_acquire(i, v); } -#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire -#endif -#if defined(arch_atomic64_fetch_and_release) static __always_inline s64 atomic64_fetch_and_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and_release(i, v); } -#define atomic64_fetch_and_release atomic64_fetch_and_release -#endif -#if defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_and_relaxed(i, v); } -#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed -#endif -#if defined(arch_atomic64_andnot) static __always_inline void atomic64_andnot(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_andnot(i, v); } -#define atomic64_andnot atomic64_andnot -#endif -#if defined(arch_atomic64_fetch_andnot) static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot(i, v); } -#define atomic64_fetch_andnot atomic64_fetch_andnot -#endif -#if defined(arch_atomic64_fetch_andnot_acquire) static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_acquire(i, v); } -#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire -#endif -#if defined(arch_atomic64_fetch_andnot_release) static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_release(i, v); } -#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release -#endif -#if defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_relaxed(i, v); } -#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed -#endif static __always_inline void atomic64_or(s64 i, atomic64_t *v) @@ -1327,47 +955,34 @@ atomic64_or(s64 i, atomic64_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_or(i, v); } -#define atomic64_or atomic64_or -#if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or) static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or(i, v); } -#define atomic64_fetch_or atomic64_fetch_or -#endif -#if defined(arch_atomic64_fetch_or_acquire) static __always_inline s64 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or_acquire(i, v); } -#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire -#endif -#if defined(arch_atomic64_fetch_or_release) static __always_inline s64 atomic64_fetch_or_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or_release(i, v); } -#define atomic64_fetch_or_release atomic64_fetch_or_release -#endif -#if defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_or_relaxed(i, v); } -#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed -#endif static __always_inline void atomic64_xor(s64 i, atomic64_t *v) @@ -1375,129 +990,91 @@ atomic64_xor(s64 i, atomic64_t *v) instrument_atomic_read_write(v, sizeof(*v)); arch_atomic64_xor(i, v); } -#define atomic64_xor atomic64_xor -#if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor) static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor(i, v); } -#define atomic64_fetch_xor atomic64_fetch_xor -#endif -#if defined(arch_atomic64_fetch_xor_acquire) static __always_inline s64 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_acquire(i, v); } -#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire -#endif -#if defined(arch_atomic64_fetch_xor_release) static __always_inline s64 atomic64_fetch_xor_release(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_release(i, v); } -#define atomic64_fetch_xor_release atomic64_fetch_xor_release -#endif -#if defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_relaxed(i, v); } -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed -#endif -#if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg) static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg(v, i); } -#define atomic64_xchg atomic64_xchg -#endif -#if defined(arch_atomic64_xchg_acquire) static __always_inline s64 atomic64_xchg_acquire(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg_acquire(v, i); } -#define atomic64_xchg_acquire atomic64_xchg_acquire -#endif -#if defined(arch_atomic64_xchg_release) static __always_inline s64 atomic64_xchg_release(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg_release(v, i); } -#define atomic64_xchg_release atomic64_xchg_release -#endif -#if defined(arch_atomic64_xchg_relaxed) static __always_inline s64 atomic64_xchg_relaxed(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_xchg_relaxed(v, i); } -#define atomic64_xchg_relaxed atomic64_xchg_relaxed -#endif -#if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg) static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg(v, old, new); } -#define atomic64_cmpxchg atomic64_cmpxchg -#endif -#if defined(arch_atomic64_cmpxchg_acquire) static __always_inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_acquire(v, old, new); } -#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire -#endif -#if defined(arch_atomic64_cmpxchg_release) static __always_inline s64 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_release(v, old, new); } -#define atomic64_cmpxchg_release atomic64_cmpxchg_release -#endif -#if defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_relaxed(v, old, new); } -#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed -#endif -#if defined(arch_atomic64_try_cmpxchg) static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { @@ -1505,10 +1082,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg(v, old, new); } -#define atomic64_try_cmpxchg atomic64_try_cmpxchg -#endif -#if defined(arch_atomic64_try_cmpxchg_acquire) static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { @@ -1516,10 +1090,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_acquire(v, old, new); } -#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire -#endif -#if defined(arch_atomic64_try_cmpxchg_release) static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { @@ -1527,10 +1098,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_release(v, old, new); } -#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release -#endif -#if defined(arch_atomic64_try_cmpxchg_relaxed) static __always_inline bool atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { @@ -1538,218 +1106,161 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) instrument_atomic_read_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_relaxed(v, old, new); } -#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed -#endif -#if defined(arch_atomic64_sub_and_test) static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_sub_and_test(i, v); } -#define atomic64_sub_and_test atomic64_sub_and_test -#endif -#if defined(arch_atomic64_dec_and_test) static __always_inline bool atomic64_dec_and_test(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_and_test(v); } -#define atomic64_dec_and_test atomic64_dec_and_test -#endif -#if defined(arch_atomic64_inc_and_test) static __always_inline bool atomic64_inc_and_test(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_and_test(v); } -#define atomic64_inc_and_test atomic64_inc_and_test -#endif -#if defined(arch_atomic64_add_negative) static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_negative(i, v); } -#define atomic64_add_negative atomic64_add_negative -#endif -#if defined(arch_atomic64_fetch_add_unless) static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_fetch_add_unless(v, a, u); } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#endif -#if defined(arch_atomic64_add_unless) static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_add_unless(v, a, u); } -#define atomic64_add_unless atomic64_add_unless -#endif -#if defined(arch_atomic64_inc_not_zero) static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_not_zero(v); } -#define atomic64_inc_not_zero atomic64_inc_not_zero -#endif -#if defined(arch_atomic64_inc_unless_negative) static __always_inline bool atomic64_inc_unless_negative(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_inc_unless_negative(v); } -#define atomic64_inc_unless_negative atomic64_inc_unless_negative -#endif -#if defined(arch_atomic64_dec_unless_positive) static __always_inline bool atomic64_dec_unless_positive(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_unless_positive(v); } -#define atomic64_dec_unless_positive atomic64_dec_unless_positive -#endif -#if defined(arch_atomic64_dec_if_positive) static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); return arch_atomic64_dec_if_positive(v); } -#define atomic64_dec_if_positive atomic64_dec_if_positive -#endif -#if !defined(arch_xchg_relaxed) || defined(arch_xchg) #define xchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_xchg_acquire) #define xchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_xchg_release) #define xchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_release(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_xchg_relaxed) #define xchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) #define cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg_acquire) #define cmpxchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg_release) #define cmpxchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg_relaxed) #define cmpxchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64) #define cmpxchg64(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg64_acquire) #define cmpxchg64_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg64_release) #define cmpxchg64_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if defined(arch_cmpxchg64_relaxed) #define cmpxchg64_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) -#endif -#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) #define try_cmpxchg(ptr, oldp, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -1758,9 +1269,7 @@ atomic64_dec_if_positive(atomic64_t *v) instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#endif -#if defined(arch_try_cmpxchg_acquire) #define try_cmpxchg_acquire(ptr, oldp, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -1769,9 +1278,7 @@ atomic64_dec_if_positive(atomic64_t *v) instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#endif -#if defined(arch_try_cmpxchg_release) #define try_cmpxchg_release(ptr, oldp, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -1780,9 +1287,7 @@ atomic64_dec_if_positive(atomic64_t *v) instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#endif -#if defined(arch_try_cmpxchg_relaxed) #define try_cmpxchg_relaxed(ptr, oldp, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -1791,7 +1296,6 @@ atomic64_dec_if_positive(atomic64_t *v) instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#endif #define cmpxchg_local(ptr, ...) \ ({ \ @@ -1830,4 +1334,4 @@ atomic64_dec_if_positive(atomic64_t *v) }) #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// 4bec382e44520f4d8267e42620054db26a659ea3 +// 1d7c3a25aca5c7fb031c307be4c3d24c7b48fcd5 diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 11f96f40f4a7..04b8be9f1a77 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Generic C implementation of atomic counter operations. Usable on - * UP systems only. Do not include in machine independent code. + * Generic C implementation of atomic counter operations. Do not include in + * machine independent code. * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) @@ -12,56 +12,39 @@ #include <asm/cmpxchg.h> #include <asm/barrier.h> -/* - * atomic_$op() - $op integer to atomic variable - * @i: integer value to $op - * @v: pointer to the atomic variable - * - * Atomically $ops @i to @v. Does not strictly guarantee a memory-barrier, use - * smp_mb__{before,after}_atomic(). - */ - -/* - * atomic_$op_return() - $op interer to atomic variable and returns the result - * @i: integer value to $op - * @v: pointer to the atomic variable - * - * Atomically $ops @i to @v. Does imply a full memory barrier. - */ - #ifdef CONFIG_SMP /* we can build all atomic primitives from cmpxchg */ #define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void generic_atomic_##op(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int generic_atomic_##op##_return(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ \ return c c_op i; \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \ { \ int c, old; \ \ c = v->counter; \ - while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \ + while ((old = arch_cmpxchg(&v->counter, c, c c_op i)) != c) \ c = old; \ \ return c; \ @@ -72,7 +55,7 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #include <linux/irqflags.h> #define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ +static inline void generic_atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ @@ -82,7 +65,7 @@ static inline void atomic_##op(int i, atomic_t *v) \ } #define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int generic_atomic_##op##_return(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -95,7 +78,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int generic_atomic_fetch_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ int ret; \ @@ -110,87 +93,44 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ #endif /* CONFIG_SMP */ -#ifndef atomic_add_return ATOMIC_OP_RETURN(add, +) -#endif - -#ifndef atomic_sub_return ATOMIC_OP_RETURN(sub, -) -#endif -#ifndef atomic_fetch_add ATOMIC_FETCH_OP(add, +) -#endif - -#ifndef atomic_fetch_sub ATOMIC_FETCH_OP(sub, -) -#endif - -#ifndef atomic_fetch_and ATOMIC_FETCH_OP(and, &) -#endif - -#ifndef atomic_fetch_or ATOMIC_FETCH_OP(or, |) -#endif - -#ifndef atomic_fetch_xor ATOMIC_FETCH_OP(xor, ^) -#endif -#ifndef atomic_and +ATOMIC_OP(add, +) +ATOMIC_OP(sub, -) ATOMIC_OP(and, &) -#endif - -#ifndef atomic_or ATOMIC_OP(or, |) -#endif - -#ifndef atomic_xor ATOMIC_OP(xor, ^) -#endif #undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -/* - * Atomic operations that C can't guarantee us. Useful for - * resource counting etc.. - */ - -/** - * atomic_read - read atomic variable - * @v: pointer of type atomic_t - * - * Atomically reads the value of @v. - */ -#ifndef atomic_read -#define atomic_read(v) READ_ONCE((v)->counter) -#endif - -/** - * atomic_set - set atomic variable - * @v: pointer of type atomic_t - * @i: required value - * - * Atomically sets the value of @v to @i. - */ -#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_add_return generic_atomic_add_return +#define arch_atomic_sub_return generic_atomic_sub_return -#include <linux/irqflags.h> +#define arch_atomic_fetch_add generic_atomic_fetch_add +#define arch_atomic_fetch_sub generic_atomic_fetch_sub +#define arch_atomic_fetch_and generic_atomic_fetch_and +#define arch_atomic_fetch_or generic_atomic_fetch_or +#define arch_atomic_fetch_xor generic_atomic_fetch_xor -static inline void atomic_add(int i, atomic_t *v) -{ - atomic_add_return(i, v); -} +#define arch_atomic_add generic_atomic_add +#define arch_atomic_sub generic_atomic_sub +#define arch_atomic_and generic_atomic_and +#define arch_atomic_or generic_atomic_or +#define arch_atomic_xor generic_atomic_xor -static inline void atomic_sub(int i, atomic_t *v) -{ - atomic_sub_return(i, v); -} +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) -#define atomic_xchg(ptr, v) (xchg(&(ptr)->counter, (v))) -#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) +#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (v))) +#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (old), (new))) #endif /* __ASM_GENERIC_ATOMIC_H */ diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h index 370f01d4450f..100d24b02e52 100644 --- a/include/asm-generic/atomic64.h +++ b/include/asm-generic/atomic64.h @@ -15,19 +15,17 @@ typedef struct { #define ATOMIC64_INIT(i) { (i) } -extern s64 atomic64_read(const atomic64_t *v); -extern void atomic64_set(atomic64_t *v, s64 i); - -#define atomic64_set_release(v, i) atomic64_set((v), (i)) +extern s64 generic_atomic64_read(const atomic64_t *v); +extern void generic_atomic64_set(atomic64_t *v, s64 i); #define ATOMIC64_OP(op) \ -extern void atomic64_##op(s64 a, atomic64_t *v); +extern void generic_atomic64_##op(s64 a, atomic64_t *v); #define ATOMIC64_OP_RETURN(op) \ -extern s64 atomic64_##op##_return(s64 a, atomic64_t *v); +extern s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v); #define ATOMIC64_FETCH_OP(op) \ -extern s64 atomic64_fetch_##op(s64 a, atomic64_t *v); +extern s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v); #define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) @@ -46,11 +44,32 @@ ATOMIC64_OPS(xor) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -extern s64 atomic64_dec_if_positive(atomic64_t *v); -#define atomic64_dec_if_positive atomic64_dec_if_positive -extern s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n); -extern s64 atomic64_xchg(atomic64_t *v, s64 new); -extern s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u); -#define atomic64_fetch_add_unless atomic64_fetch_add_unless +extern s64 generic_atomic64_dec_if_positive(atomic64_t *v); +extern s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n); +extern s64 generic_atomic64_xchg(atomic64_t *v, s64 new); +extern s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u); + +#define arch_atomic64_read generic_atomic64_read +#define arch_atomic64_set generic_atomic64_set +#define arch_atomic64_set_release generic_atomic64_set + +#define arch_atomic64_add generic_atomic64_add +#define arch_atomic64_add_return generic_atomic64_add_return +#define arch_atomic64_fetch_add generic_atomic64_fetch_add +#define arch_atomic64_sub generic_atomic64_sub +#define arch_atomic64_sub_return generic_atomic64_sub_return +#define arch_atomic64_fetch_sub generic_atomic64_fetch_sub + +#define arch_atomic64_and generic_atomic64_and +#define arch_atomic64_fetch_and generic_atomic64_fetch_and +#define arch_atomic64_or generic_atomic64_or +#define arch_atomic64_fetch_or generic_atomic64_fetch_or +#define arch_atomic64_xor generic_atomic64_xor +#define arch_atomic64_fetch_xor generic_atomic64_fetch_xor + +#define arch_atomic64_dec_if_positive generic_atomic64_dec_if_positive +#define arch_atomic64_cmpxchg generic_atomic64_cmpxchg +#define arch_atomic64_xchg generic_atomic64_xchg +#define arch_atomic64_fetch_add_unless generic_atomic64_fetch_add_unless #endif /* _ASM_GENERIC_ATOMIC64_H */ diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h index f17f14f84d09..380cdc824e4b 100644 --- a/include/asm-generic/cmpxchg-local.h +++ b/include/asm-generic/cmpxchg-local.h @@ -12,7 +12,7 @@ extern unsigned long wrong_size_cmpxchg(volatile void *ptr) * Generic version of __cmpxchg_local (disables interrupts). Takes an unsigned * long parameter, supporting various types of architectures. */ -static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, +static inline unsigned long __generic_cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, int size) { unsigned long flags, prev; @@ -51,7 +51,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, /* * Generic version of __cmpxchg64_local. Takes an u64 parameter. */ -static inline u64 __cmpxchg64_local_generic(volatile void *ptr, +static inline u64 __generic_cmpxchg64_local(volatile void *ptr, u64 old, u64 new) { u64 prev; diff --git a/include/asm-generic/cmpxchg.h b/include/asm-generic/cmpxchg.h index 9a24510cd8c1..dca4419922a9 100644 --- a/include/asm-generic/cmpxchg.h +++ b/include/asm-generic/cmpxchg.h @@ -14,16 +14,14 @@ #include <linux/types.h> #include <linux/irqflags.h> -#ifndef xchg - /* * This function doesn't exist, so you'll get a linker error if * something tries to do an invalidly-sized xchg(). */ -extern void __xchg_called_with_bad_pointer(void); +extern void __generic_xchg_called_with_bad_pointer(void); static inline -unsigned long __xchg(unsigned long x, volatile void *ptr, int size) +unsigned long __generic_xchg(unsigned long x, volatile void *ptr, int size) { unsigned long ret, flags; @@ -75,35 +73,43 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size) #endif /* CONFIG_64BIT */ default: - __xchg_called_with_bad_pointer(); + __generic_xchg_called_with_bad_pointer(); return x; } } -#define xchg(ptr, x) ({ \ - ((__typeof__(*(ptr))) \ - __xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ +#define generic_xchg(ptr, x) ({ \ + ((__typeof__(*(ptr))) \ + __generic_xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ }) -#endif /* xchg */ - /* * Atomic compare and exchange. */ #include <asm-generic/cmpxchg-local.h> -#ifndef cmpxchg_local -#define cmpxchg_local(ptr, o, n) ({ \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ - (unsigned long)(n), sizeof(*(ptr)))); \ +#define generic_cmpxchg_local(ptr, o, n) ({ \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)))); \ }) + +#define generic_cmpxchg64_local(ptr, o, n) \ + __generic_cmpxchg64_local((ptr), (o), (n)) + + +#ifndef arch_xchg +#define arch_xchg generic_xchg +#endif + +#ifndef arch_cmpxchg_local +#define arch_cmpxchg_local generic_cmpxchg_local #endif -#ifndef cmpxchg64_local -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#ifndef arch_cmpxchg64_local +#define arch_cmpxchg64_local generic_cmpxchg64_local #endif -#define cmpxchg(ptr, o, n) cmpxchg_local((ptr), (o), (n)) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg arch_cmpxchg_local +#define arch_cmpxchg64 arch_cmpxchg64_local #endif /* __ASM_GENERIC_CMPXCHG_H */ diff --git a/include/linux/atomic-fallback.h b/include/linux/atomic-fallback.h deleted file mode 100644 index 2a3f55d98be9..000000000000 --- a/include/linux/atomic-fallback.h +++ /dev/null @@ -1,2595 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// Generated by scripts/atomic/gen-atomic-fallback.sh -// DO NOT MODIFY THIS FILE DIRECTLY - -#ifndef _LINUX_ATOMIC_FALLBACK_H -#define _LINUX_ATOMIC_FALLBACK_H - -#include <linux/compiler.h> - -#ifndef xchg_relaxed -#define xchg_acquire xchg -#define xchg_release xchg -#define xchg_relaxed xchg -#else /* xchg_relaxed */ - -#ifndef xchg_acquire -#define xchg_acquire(...) \ - __atomic_op_acquire(xchg, __VA_ARGS__) -#endif - -#ifndef xchg_release -#define xchg_release(...) \ - __atomic_op_release(xchg, __VA_ARGS__) -#endif - -#ifndef xchg -#define xchg(...) \ - __atomic_op_fence(xchg, __VA_ARGS__) -#endif - -#endif /* xchg_relaxed */ - -#ifndef cmpxchg_relaxed -#define cmpxchg_acquire cmpxchg -#define cmpxchg_release cmpxchg -#define cmpxchg_relaxed cmpxchg -#else /* cmpxchg_relaxed */ - -#ifndef cmpxchg_acquire -#define cmpxchg_acquire(...) \ - __atomic_op_acquire(cmpxchg, __VA_ARGS__) -#endif - -#ifndef cmpxchg_release -#define cmpxchg_release(...) \ - __atomic_op_release(cmpxchg, __VA_ARGS__) -#endif - -#ifndef cmpxchg -#define cmpxchg(...) \ - __atomic_op_fence(cmpxchg, __VA_ARGS__) -#endif - -#endif /* cmpxchg_relaxed */ - -#ifndef cmpxchg64_relaxed -#define cmpxchg64_acquire cmpxchg64 -#define cmpxchg64_release cmpxchg64 -#define cmpxchg64_relaxed cmpxchg64 -#else /* cmpxchg64_relaxed */ - -#ifndef cmpxchg64_acquire -#define cmpxchg64_acquire(...) \ - __atomic_op_acquire(cmpxchg64, __VA_ARGS__) -#endif - -#ifndef cmpxchg64_release -#define cmpxchg64_release(...) \ - __atomic_op_release(cmpxchg64, __VA_ARGS__) -#endif - -#ifndef cmpxchg64 -#define cmpxchg64(...) \ - __atomic_op_fence(cmpxchg64, __VA_ARGS__) -#endif - -#endif /* cmpxchg64_relaxed */ - -#ifndef try_cmpxchg_relaxed -#ifdef try_cmpxchg -#define try_cmpxchg_acquire try_cmpxchg -#define try_cmpxchg_release try_cmpxchg -#define try_cmpxchg_relaxed try_cmpxchg -#endif /* try_cmpxchg */ - -#ifndef try_cmpxchg -#define try_cmpxchg(_ptr, _oldp, _new) \ -({ \ - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = cmpxchg((_ptr), ___o, (_new)); \ - if (unlikely(___r != ___o)) \ - *___op = ___r; \ - likely(___r == ___o); \ -}) -#endif /* try_cmpxchg */ - -#ifndef try_cmpxchg_acquire -#define try_cmpxchg_acquire(_ptr, _oldp, _new) \ -({ \ - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = cmpxchg_acquire((_ptr), ___o, (_new)); \ - if (unlikely(___r != ___o)) \ - *___op = ___r; \ - likely(___r == ___o); \ -}) -#endif /* try_cmpxchg_acquire */ - -#ifndef try_cmpxchg_release -#define try_cmpxchg_release(_ptr, _oldp, _new) \ -({ \ - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = cmpxchg_release((_ptr), ___o, (_new)); \ - if (unlikely(___r != ___o)) \ - *___op = ___r; \ - likely(___r == ___o); \ -}) -#endif /* try_cmpxchg_release */ - -#ifndef try_cmpxchg_relaxed -#define try_cmpxchg_relaxed(_ptr, _oldp, _new) \ -({ \ - typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = cmpxchg_relaxed((_ptr), ___o, (_new)); \ - if (unlikely(___r != ___o)) \ - *___op = ___r; \ - likely(___r == ___o); \ -}) -#endif /* try_cmpxchg_relaxed */ - -#else /* try_cmpxchg_relaxed */ - -#ifndef try_cmpxchg_acquire -#define try_cmpxchg_acquire(...) \ - __atomic_op_acquire(try_cmpxchg, __VA_ARGS__) -#endif - -#ifndef try_cmpxchg_release -#define try_cmpxchg_release(...) \ - __atomic_op_release(try_cmpxchg, __VA_ARGS__) -#endif - -#ifndef try_cmpxchg -#define try_cmpxchg(...) \ - __atomic_op_fence(try_cmpxchg, __VA_ARGS__) -#endif - -#endif /* try_cmpxchg_relaxed */ - -#define arch_atomic_read atomic_read -#define arch_atomic_read_acquire atomic_read_acquire - -#ifndef atomic_read_acquire -static __always_inline int -atomic_read_acquire(const atomic_t *v) -{ - return smp_load_acquire(&(v)->counter); -} -#define atomic_read_acquire atomic_read_acquire -#endif - -#define arch_atomic_set atomic_set -#define arch_atomic_set_release atomic_set_release - -#ifndef atomic_set_release -static __always_inline void -atomic_set_release(atomic_t *v, int i) -{ - smp_store_release(&(v)->counter, i); -} -#define atomic_set_release atomic_set_release -#endif - -#define arch_atomic_add atomic_add - -#define arch_atomic_add_return atomic_add_return -#define arch_atomic_add_return_acquire atomic_add_return_acquire -#define arch_atomic_add_return_release atomic_add_return_release -#define arch_atomic_add_return_relaxed atomic_add_return_relaxed - -#ifndef atomic_add_return_relaxed -#define atomic_add_return_acquire atomic_add_return -#define atomic_add_return_release atomic_add_return -#define atomic_add_return_relaxed atomic_add_return -#else /* atomic_add_return_relaxed */ - -#ifndef atomic_add_return_acquire -static __always_inline int -atomic_add_return_acquire(int i, atomic_t *v) -{ - int ret = atomic_add_return_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_add_return_acquire atomic_add_return_acquire -#endif - -#ifndef atomic_add_return_release -static __always_inline int -atomic_add_return_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_add_return_relaxed(i, v); -} -#define atomic_add_return_release atomic_add_return_release -#endif - -#ifndef atomic_add_return -static __always_inline int -atomic_add_return(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_add_return_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_add_return atomic_add_return -#endif - -#endif /* atomic_add_return_relaxed */ - -#define arch_atomic_fetch_add atomic_fetch_add -#define arch_atomic_fetch_add_acquire atomic_fetch_add_acquire -#define arch_atomic_fetch_add_release atomic_fetch_add_release -#define arch_atomic_fetch_add_relaxed atomic_fetch_add_relaxed - -#ifndef atomic_fetch_add_relaxed -#define atomic_fetch_add_acquire atomic_fetch_add -#define atomic_fetch_add_release atomic_fetch_add -#define atomic_fetch_add_relaxed atomic_fetch_add -#else /* atomic_fetch_add_relaxed */ - -#ifndef atomic_fetch_add_acquire -static __always_inline int -atomic_fetch_add_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_add_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_add_acquire atomic_fetch_add_acquire -#endif - -#ifndef atomic_fetch_add_release -static __always_inline int -atomic_fetch_add_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_add_relaxed(i, v); -} -#define atomic_fetch_add_release atomic_fetch_add_release -#endif - -#ifndef atomic_fetch_add -static __always_inline int -atomic_fetch_add(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_add_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_add atomic_fetch_add -#endif - -#endif /* atomic_fetch_add_relaxed */ - -#define arch_atomic_sub atomic_sub - -#define arch_atomic_sub_return atomic_sub_return -#define arch_atomic_sub_return_acquire atomic_sub_return_acquire -#define arch_atomic_sub_return_release atomic_sub_return_release -#define arch_atomic_sub_return_relaxed atomic_sub_return_relaxed - -#ifndef atomic_sub_return_relaxed -#define atomic_sub_return_acquire atomic_sub_return -#define atomic_sub_return_release atomic_sub_return -#define atomic_sub_return_relaxed atomic_sub_return -#else /* atomic_sub_return_relaxed */ - -#ifndef atomic_sub_return_acquire -static __always_inline int -atomic_sub_return_acquire(int i, atomic_t *v) -{ - int ret = atomic_sub_return_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_sub_return_acquire atomic_sub_return_acquire -#endif - -#ifndef atomic_sub_return_release -static __always_inline int -atomic_sub_return_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_sub_return_relaxed(i, v); -} -#define atomic_sub_return_release atomic_sub_return_release -#endif - -#ifndef atomic_sub_return -static __always_inline int -atomic_sub_return(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_sub_return_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_sub_return atomic_sub_return -#endif - -#endif /* atomic_sub_return_relaxed */ - -#define arch_atomic_fetch_sub atomic_fetch_sub -#define arch_atomic_fetch_sub_acquire atomic_fetch_sub_acquire -#define arch_atomic_fetch_sub_release atomic_fetch_sub_release -#define arch_atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed - -#ifndef atomic_fetch_sub_relaxed -#define atomic_fetch_sub_acquire atomic_fetch_sub -#define atomic_fetch_sub_release atomic_fetch_sub -#define atomic_fetch_sub_relaxed atomic_fetch_sub -#else /* atomic_fetch_sub_relaxed */ - -#ifndef atomic_fetch_sub_acquire -static __always_inline int -atomic_fetch_sub_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_sub_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire -#endif - -#ifndef atomic_fetch_sub_release -static __always_inline int -atomic_fetch_sub_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_sub_relaxed(i, v); -} -#define atomic_fetch_sub_release atomic_fetch_sub_release -#endif - -#ifndef atomic_fetch_sub -static __always_inline int -atomic_fetch_sub(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_sub_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_sub atomic_fetch_sub -#endif - -#endif /* atomic_fetch_sub_relaxed */ - -#define arch_atomic_inc atomic_inc - -#ifndef atomic_inc -static __always_inline void -atomic_inc(atomic_t *v) -{ - atomic_add(1, v); -} -#define atomic_inc atomic_inc -#endif - -#define arch_atomic_inc_return atomic_inc_return -#define arch_atomic_inc_return_acquire atomic_inc_return_acquire -#define arch_atomic_inc_return_release atomic_inc_return_release -#define arch_atomic_inc_return_relaxed atomic_inc_return_relaxed - -#ifndef atomic_inc_return_relaxed -#ifdef atomic_inc_return -#define atomic_inc_return_acquire atomic_inc_return -#define atomic_inc_return_release atomic_inc_return -#define atomic_inc_return_relaxed atomic_inc_return -#endif /* atomic_inc_return */ - -#ifndef atomic_inc_return -static __always_inline int -atomic_inc_return(atomic_t *v) -{ - return atomic_add_return(1, v); -} -#define atomic_inc_return atomic_inc_return -#endif - -#ifndef atomic_inc_return_acquire -static __always_inline int -atomic_inc_return_acquire(atomic_t *v) -{ - return atomic_add_return_acquire(1, v); -} -#define atomic_inc_return_acquire atomic_inc_return_acquire -#endif - -#ifndef atomic_inc_return_release -static __always_inline int -atomic_inc_return_release(atomic_t *v) -{ - return atomic_add_return_release(1, v); -} -#define atomic_inc_return_release atomic_inc_return_release -#endif - -#ifndef atomic_inc_return_relaxed -static __always_inline int -atomic_inc_return_relaxed(atomic_t *v) -{ - return atomic_add_return_relaxed(1, v); -} -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#endif - -#else /* atomic_inc_return_relaxed */ - -#ifndef atomic_inc_return_acquire -static __always_inline int -atomic_inc_return_acquire(atomic_t *v) -{ - int ret = atomic_inc_return_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_inc_return_acquire atomic_inc_return_acquire -#endif - -#ifndef atomic_inc_return_release -static __always_inline int -atomic_inc_return_release(atomic_t *v) -{ - __atomic_release_fence(); - return atomic_inc_return_relaxed(v); -} -#define atomic_inc_return_release atomic_inc_return_release -#endif - -#ifndef atomic_inc_return -static __always_inline int -atomic_inc_return(atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_inc_return_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_inc_return atomic_inc_return -#endif - -#endif /* atomic_inc_return_relaxed */ - -#define arch_atomic_fetch_inc atomic_fetch_inc -#define arch_atomic_fetch_inc_acquire atomic_fetch_inc_acquire -#define arch_atomic_fetch_inc_release atomic_fetch_inc_release -#define arch_atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed - -#ifndef atomic_fetch_inc_relaxed -#ifdef atomic_fetch_inc -#define atomic_fetch_inc_acquire atomic_fetch_inc -#define atomic_fetch_inc_release atomic_fetch_inc -#define atomic_fetch_inc_relaxed atomic_fetch_inc -#endif /* atomic_fetch_inc */ - -#ifndef atomic_fetch_inc -static __always_inline int -atomic_fetch_inc(atomic_t *v) -{ - return atomic_fetch_add(1, v); -} -#define atomic_fetch_inc atomic_fetch_inc -#endif - -#ifndef atomic_fetch_inc_acquire -static __always_inline int -atomic_fetch_inc_acquire(atomic_t *v) -{ - return atomic_fetch_add_acquire(1, v); -} -#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire -#endif - -#ifndef atomic_fetch_inc_release -static __always_inline int -atomic_fetch_inc_release(atomic_t *v) -{ - return atomic_fetch_add_release(1, v); -} -#define atomic_fetch_inc_release atomic_fetch_inc_release -#endif - -#ifndef atomic_fetch_inc_relaxed -static __always_inline int -atomic_fetch_inc_relaxed(atomic_t *v) -{ - return atomic_fetch_add_relaxed(1, v); -} -#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed -#endif - -#else /* atomic_fetch_inc_relaxed */ - -#ifndef atomic_fetch_inc_acquire -static __always_inline int -atomic_fetch_inc_acquire(atomic_t *v) -{ - int ret = atomic_fetch_inc_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire -#endif - -#ifndef atomic_fetch_inc_release -static __always_inline int -atomic_fetch_inc_release(atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_inc_relaxed(v); -} -#define atomic_fetch_inc_release atomic_fetch_inc_release -#endif - -#ifndef atomic_fetch_inc -static __always_inline int -atomic_fetch_inc(atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_inc_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_inc atomic_fetch_inc -#endif - -#endif /* atomic_fetch_inc_relaxed */ - -#define arch_atomic_dec atomic_dec - -#ifndef atomic_dec -static __always_inline void -atomic_dec(atomic_t *v) -{ - atomic_sub(1, v); -} -#define atomic_dec atomic_dec -#endif - -#define arch_atomic_dec_return atomic_dec_return -#define arch_atomic_dec_return_acquire atomic_dec_return_acquire -#define arch_atomic_dec_return_release atomic_dec_return_release -#define arch_atomic_dec_return_relaxed atomic_dec_return_relaxed - -#ifndef atomic_dec_return_relaxed -#ifdef atomic_dec_return -#define atomic_dec_return_acquire atomic_dec_return -#define atomic_dec_return_release atomic_dec_return -#define atomic_dec_return_relaxed atomic_dec_return -#endif /* atomic_dec_return */ - -#ifndef atomic_dec_return -static __always_inline int -atomic_dec_return(atomic_t *v) -{ - return atomic_sub_return(1, v); -} -#define atomic_dec_return atomic_dec_return -#endif - -#ifndef atomic_dec_return_acquire -static __always_inline int -atomic_dec_return_acquire(atomic_t *v) -{ - return atomic_sub_return_acquire(1, v); -} -#define atomic_dec_return_acquire atomic_dec_return_acquire -#endif - -#ifndef atomic_dec_return_release -static __always_inline int -atomic_dec_return_release(atomic_t *v) -{ - return atomic_sub_return_release(1, v); -} -#define atomic_dec_return_release atomic_dec_return_release -#endif - -#ifndef atomic_dec_return_relaxed -static __always_inline int -atomic_dec_return_relaxed(atomic_t *v) -{ - return atomic_sub_return_relaxed(1, v); -} -#define atomic_dec_return_relaxed atomic_dec_return_relaxed -#endif - -#else /* atomic_dec_return_relaxed */ - -#ifndef atomic_dec_return_acquire -static __always_inline int -atomic_dec_return_acquire(atomic_t *v) -{ - int ret = atomic_dec_return_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_dec_return_acquire atomic_dec_return_acquire -#endif - -#ifndef atomic_dec_return_release -static __always_inline int -atomic_dec_return_release(atomic_t *v) -{ - __atomic_release_fence(); - return atomic_dec_return_relaxed(v); -} -#define atomic_dec_return_release atomic_dec_return_release -#endif - -#ifndef atomic_dec_return -static __always_inline int -atomic_dec_return(atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_dec_return_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_dec_return atomic_dec_return -#endif - -#endif /* atomic_dec_return_relaxed */ - -#define arch_atomic_fetch_dec atomic_fetch_dec -#define arch_atomic_fetch_dec_acquire atomic_fetch_dec_acquire -#define arch_atomic_fetch_dec_release atomic_fetch_dec_release -#define arch_atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed - -#ifndef atomic_fetch_dec_relaxed -#ifdef atomic_fetch_dec -#define atomic_fetch_dec_acquire atomic_fetch_dec -#define atomic_fetch_dec_release atomic_fetch_dec -#define atomic_fetch_dec_relaxed atomic_fetch_dec -#endif /* atomic_fetch_dec */ - -#ifndef atomic_fetch_dec -static __always_inline int -atomic_fetch_dec(atomic_t *v) -{ - return atomic_fetch_sub(1, v); -} -#define atomic_fetch_dec atomic_fetch_dec -#endif - -#ifndef atomic_fetch_dec_acquire -static __always_inline int -atomic_fetch_dec_acquire(atomic_t *v) -{ - return atomic_fetch_sub_acquire(1, v); -} -#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire -#endif - -#ifndef atomic_fetch_dec_release -static __always_inline int -atomic_fetch_dec_release(atomic_t *v) -{ - return atomic_fetch_sub_release(1, v); -} -#define atomic_fetch_dec_release atomic_fetch_dec_release -#endif - -#ifndef atomic_fetch_dec_relaxed -static __always_inline int -atomic_fetch_dec_relaxed(atomic_t *v) -{ - return atomic_fetch_sub_relaxed(1, v); -} -#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed -#endif - -#else /* atomic_fetch_dec_relaxed */ - -#ifndef atomic_fetch_dec_acquire -static __always_inline int -atomic_fetch_dec_acquire(atomic_t *v) -{ - int ret = atomic_fetch_dec_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire -#endif - -#ifndef atomic_fetch_dec_release -static __always_inline int -atomic_fetch_dec_release(atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_dec_relaxed(v); -} -#define atomic_fetch_dec_release atomic_fetch_dec_release -#endif - -#ifndef atomic_fetch_dec -static __always_inline int -atomic_fetch_dec(atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_dec_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_dec atomic_fetch_dec -#endif - -#endif /* atomic_fetch_dec_relaxed */ - -#define arch_atomic_and atomic_and - -#define arch_atomic_fetch_and atomic_fetch_and -#define arch_atomic_fetch_and_acquire atomic_fetch_and_acquire -#define arch_atomic_fetch_and_release atomic_fetch_and_release -#define arch_atomic_fetch_and_relaxed atomic_fetch_and_relaxed - -#ifndef atomic_fetch_and_relaxed -#define atomic_fetch_and_acquire atomic_fetch_and -#define atomic_fetch_and_release atomic_fetch_and -#define atomic_fetch_and_relaxed atomic_fetch_and -#else /* atomic_fetch_and_relaxed */ - -#ifndef atomic_fetch_and_acquire -static __always_inline int -atomic_fetch_and_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_and_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_and_acquire atomic_fetch_and_acquire -#endif - -#ifndef atomic_fetch_and_release -static __always_inline int -atomic_fetch_and_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_and_relaxed(i, v); -} -#define atomic_fetch_and_release atomic_fetch_and_release -#endif - -#ifndef atomic_fetch_and -static __always_inline int -atomic_fetch_and(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_and_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_and atomic_fetch_and -#endif - -#endif /* atomic_fetch_and_relaxed */ - -#define arch_atomic_andnot atomic_andnot - -#ifndef atomic_andnot -static __always_inline void -atomic_andnot(int i, atomic_t *v) -{ - atomic_and(~i, v); -} -#define atomic_andnot atomic_andnot -#endif - -#define arch_atomic_fetch_andnot atomic_fetch_andnot -#define arch_atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire -#define arch_atomic_fetch_andnot_release atomic_fetch_andnot_release -#define arch_atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed - -#ifndef atomic_fetch_andnot_relaxed -#ifdef atomic_fetch_andnot -#define atomic_fetch_andnot_acquire atomic_fetch_andnot -#define atomic_fetch_andnot_release atomic_fetch_andnot -#define atomic_fetch_andnot_relaxed atomic_fetch_andnot -#endif /* atomic_fetch_andnot */ - -#ifndef atomic_fetch_andnot -static __always_inline int -atomic_fetch_andnot(int i, atomic_t *v) -{ - return atomic_fetch_and(~i, v); -} -#define atomic_fetch_andnot atomic_fetch_andnot -#endif - -#ifndef atomic_fetch_andnot_acquire -static __always_inline int -atomic_fetch_andnot_acquire(int i, atomic_t *v) -{ - return atomic_fetch_and_acquire(~i, v); -} -#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire -#endif - -#ifndef atomic_fetch_andnot_release -static __always_inline int -atomic_fetch_andnot_release(int i, atomic_t *v) -{ - return atomic_fetch_and_release(~i, v); -} -#define atomic_fetch_andnot_release atomic_fetch_andnot_release -#endif - -#ifndef atomic_fetch_andnot_relaxed -static __always_inline int -atomic_fetch_andnot_relaxed(int i, atomic_t *v) -{ - return atomic_fetch_and_relaxed(~i, v); -} -#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed -#endif - -#else /* atomic_fetch_andnot_relaxed */ - -#ifndef atomic_fetch_andnot_acquire -static __always_inline int -atomic_fetch_andnot_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_andnot_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire -#endif - -#ifndef atomic_fetch_andnot_release -static __always_inline int -atomic_fetch_andnot_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_andnot_relaxed(i, v); -} -#define atomic_fetch_andnot_release atomic_fetch_andnot_release -#endif - -#ifndef atomic_fetch_andnot -static __always_inline int -atomic_fetch_andnot(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_andnot_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_andnot atomic_fetch_andnot -#endif - -#endif /* atomic_fetch_andnot_relaxed */ - -#define arch_atomic_or atomic_or - -#define arch_atomic_fetch_or atomic_fetch_or -#define arch_atomic_fetch_or_acquire atomic_fetch_or_acquire -#define arch_atomic_fetch_or_release atomic_fetch_or_release -#define arch_atomic_fetch_or_relaxed atomic_fetch_or_relaxed - -#ifndef atomic_fetch_or_relaxed -#define atomic_fetch_or_acquire atomic_fetch_or -#define atomic_fetch_or_release atomic_fetch_or -#define atomic_fetch_or_relaxed atomic_fetch_or -#else /* atomic_fetch_or_relaxed */ - -#ifndef atomic_fetch_or_acquire -static __always_inline int -atomic_fetch_or_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_or_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_or_acquire atomic_fetch_or_acquire -#endif - -#ifndef atomic_fetch_or_release -static __always_inline int -atomic_fetch_or_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_or_relaxed(i, v); -} -#define atomic_fetch_or_release atomic_fetch_or_release -#endif - -#ifndef atomic_fetch_or -static __always_inline int -atomic_fetch_or(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_or_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_or atomic_fetch_or -#endif - -#endif /* atomic_fetch_or_relaxed */ - -#define arch_atomic_xor atomic_xor - -#define arch_atomic_fetch_xor atomic_fetch_xor -#define arch_atomic_fetch_xor_acquire atomic_fetch_xor_acquire -#define arch_atomic_fetch_xor_release atomic_fetch_xor_release -#define arch_atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed - -#ifndef atomic_fetch_xor_relaxed -#define atomic_fetch_xor_acquire atomic_fetch_xor -#define atomic_fetch_xor_release atomic_fetch_xor -#define atomic_fetch_xor_relaxed atomic_fetch_xor -#else /* atomic_fetch_xor_relaxed */ - -#ifndef atomic_fetch_xor_acquire -static __always_inline int -atomic_fetch_xor_acquire(int i, atomic_t *v) -{ - int ret = atomic_fetch_xor_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire -#endif - -#ifndef atomic_fetch_xor_release -static __always_inline int -atomic_fetch_xor_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return atomic_fetch_xor_relaxed(i, v); -} -#define atomic_fetch_xor_release atomic_fetch_xor_release -#endif - -#ifndef atomic_fetch_xor -static __always_inline int -atomic_fetch_xor(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_fetch_xor_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic_fetch_xor atomic_fetch_xor -#endif - -#endif /* atomic_fetch_xor_relaxed */ - -#define arch_atomic_xchg atomic_xchg -#define arch_atomic_xchg_acquire atomic_xchg_acquire -#define arch_atomic_xchg_release atomic_xchg_release -#define arch_atomic_xchg_relaxed atomic_xchg_relaxed - -#ifndef atomic_xchg_relaxed -#define atomic_xchg_acquire atomic_xchg -#define atomic_xchg_release atomic_xchg -#define atomic_xchg_relaxed atomic_xchg -#else /* atomic_xchg_relaxed */ - -#ifndef atomic_xchg_acquire -static __always_inline int -atomic_xchg_acquire(atomic_t *v, int i) -{ - int ret = atomic_xchg_relaxed(v, i); - __atomic_acquire_fence(); - return ret; -} -#define atomic_xchg_acquire atomic_xchg_acquire -#endif - -#ifndef atomic_xchg_release -static __always_inline int -atomic_xchg_release(atomic_t *v, int i) -{ - __atomic_release_fence(); - return atomic_xchg_relaxed(v, i); -} -#define atomic_xchg_release atomic_xchg_release -#endif - -#ifndef atomic_xchg -static __always_inline int -atomic_xchg(atomic_t *v, int i) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_xchg_relaxed(v, i); - __atomic_post_full_fence(); - return ret; -} -#define atomic_xchg atomic_xchg -#endif - -#endif /* atomic_xchg_relaxed */ - -#define arch_atomic_cmpxchg atomic_cmpxchg -#define arch_atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#define arch_atomic_cmpxchg_release atomic_cmpxchg_release -#define arch_atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed - -#ifndef atomic_cmpxchg_relaxed -#define atomic_cmpxchg_acquire atomic_cmpxchg -#define atomic_cmpxchg_release atomic_cmpxchg -#define atomic_cmpxchg_relaxed atomic_cmpxchg -#else /* atomic_cmpxchg_relaxed */ - -#ifndef atomic_cmpxchg_acquire -static __always_inline int -atomic_cmpxchg_acquire(atomic_t *v, int old, int new) -{ - int ret = atomic_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire -#endif - -#ifndef atomic_cmpxchg_release -static __always_inline int -atomic_cmpxchg_release(atomic_t *v, int old, int new) -{ - __atomic_release_fence(); - return atomic_cmpxchg_relaxed(v, old, new); -} -#define atomic_cmpxchg_release atomic_cmpxchg_release -#endif - -#ifndef atomic_cmpxchg -static __always_inline int -atomic_cmpxchg(atomic_t *v, int old, int new) -{ - int ret; - __atomic_pre_full_fence(); - ret = atomic_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define atomic_cmpxchg atomic_cmpxchg -#endif - -#endif /* atomic_cmpxchg_relaxed */ - -#define arch_atomic_try_cmpxchg atomic_try_cmpxchg -#define arch_atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire -#define arch_atomic_try_cmpxchg_release atomic_try_cmpxchg_release -#define arch_atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed - -#ifndef atomic_try_cmpxchg_relaxed -#ifdef atomic_try_cmpxchg -#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg -#define atomic_try_cmpxchg_release atomic_try_cmpxchg -#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg -#endif /* atomic_try_cmpxchg */ - -#ifndef atomic_try_cmpxchg -static __always_inline bool -atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ - int r, o = *old; - r = atomic_cmpxchg(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic_try_cmpxchg atomic_try_cmpxchg -#endif - -#ifndef atomic_try_cmpxchg_acquire -static __always_inline bool -atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) -{ - int r, o = *old; - r = atomic_cmpxchg_acquire(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire -#endif - -#ifndef atomic_try_cmpxchg_release -static __always_inline bool -atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) -{ - int r, o = *old; - r = atomic_cmpxchg_release(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release -#endif - -#ifndef atomic_try_cmpxchg_relaxed -static __always_inline bool -atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) -{ - int r, o = *old; - r = atomic_cmpxchg_relaxed(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed -#endif - -#else /* atomic_try_cmpxchg_relaxed */ - -#ifndef atomic_try_cmpxchg_acquire -static __always_inline bool -atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) -{ - bool ret = atomic_try_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire -#endif - -#ifndef atomic_try_cmpxchg_release -static __always_inline bool -atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) -{ - __atomic_release_fence(); - return atomic_try_cmpxchg_relaxed(v, old, new); -} -#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release -#endif - -#ifndef atomic_try_cmpxchg -static __always_inline bool -atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ - bool ret; - __atomic_pre_full_fence(); - ret = atomic_try_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define atomic_try_cmpxchg atomic_try_cmpxchg -#endif - -#endif /* atomic_try_cmpxchg_relaxed */ - -#define arch_atomic_sub_and_test atomic_sub_and_test - -#ifndef atomic_sub_and_test -/** - * atomic_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -static __always_inline bool -atomic_sub_and_test(int i, atomic_t *v) -{ - return atomic_sub_return(i, v) == 0; -} -#define atomic_sub_and_test atomic_sub_and_test -#endif - -#define arch_atomic_dec_and_test atomic_dec_and_test - -#ifndef atomic_dec_and_test -/** - * atomic_dec_and_test - decrement and test - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -static __always_inline bool -atomic_dec_and_test(atomic_t *v) -{ - return atomic_dec_return(v) == 0; -} -#define atomic_dec_and_test atomic_dec_and_test -#endif - -#define arch_atomic_inc_and_test atomic_inc_and_test - -#ifndef atomic_inc_and_test -/** - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -static __always_inline bool -atomic_inc_and_test(atomic_t *v) -{ - return atomic_inc_return(v) == 0; -} -#define atomic_inc_and_test atomic_inc_and_test -#endif - -#define arch_atomic_add_negative atomic_add_negative - -#ifndef atomic_add_negative -/** - * atomic_add_negative - add and test if negative - * @i: integer value to add - * @v: pointer of type atomic_t - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -static __always_inline bool -atomic_add_negative(int i, atomic_t *v) -{ - return atomic_add_return(i, v) < 0; -} -#define atomic_add_negative atomic_add_negative -#endif - -#define arch_atomic_fetch_add_unless atomic_fetch_add_unless - -#ifndef atomic_fetch_add_unless -/** - * atomic_fetch_add_unless - add unless the number is already a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as @v was not already @u. - * Returns original value of @v - */ -static __always_inline int -atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ - int c = atomic_read(v); - - do { - if (unlikely(c == u)) - break; - } while (!atomic_try_cmpxchg(v, &c, c + a)); - - return c; -} -#define atomic_fetch_add_unless atomic_fetch_add_unless -#endif - -#define arch_atomic_add_unless atomic_add_unless - -#ifndef atomic_add_unless -/** - * atomic_add_unless - add unless the number is already a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, if @v was not already @u. - * Returns true if the addition was done. - */ -static __always_inline bool -atomic_add_unless(atomic_t *v, int a, int u) -{ - return atomic_fetch_add_unless(v, a, u) != u; -} -#define atomic_add_unless atomic_add_unless -#endif - -#define arch_atomic_inc_not_zero atomic_inc_not_zero - -#ifndef atomic_inc_not_zero -/** - * atomic_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1, if @v is non-zero. - * Returns true if the increment was done. - */ -static __always_inline bool -atomic_inc_not_zero(atomic_t *v) -{ - return atomic_add_unless(v, 1, 0); -} -#define atomic_inc_not_zero atomic_inc_not_zero -#endif - -#define arch_atomic_inc_unless_negative atomic_inc_unless_negative - -#ifndef atomic_inc_unless_negative -static __always_inline bool -atomic_inc_unless_negative(atomic_t *v) -{ - int c = atomic_read(v); - - do { - if (unlikely(c < 0)) - return false; - } while (!atomic_try_cmpxchg(v, &c, c + 1)); - - return true; -} -#define atomic_inc_unless_negative atomic_inc_unless_negative -#endif - -#define arch_atomic_dec_unless_positive atomic_dec_unless_positive - -#ifndef atomic_dec_unless_positive -static __always_inline bool -atomic_dec_unless_positive(atomic_t *v) -{ - int c = atomic_read(v); - - do { - if (unlikely(c > 0)) - return false; - } while (!atomic_try_cmpxchg(v, &c, c - 1)); - - return true; -} -#define atomic_dec_unless_positive atomic_dec_unless_positive -#endif - -#define arch_atomic_dec_if_positive atomic_dec_if_positive - -#ifndef atomic_dec_if_positive -static __always_inline int -atomic_dec_if_positive(atomic_t *v) -{ - int dec, c = atomic_read(v); - - do { - dec = c - 1; - if (unlikely(dec < 0)) - break; - } while (!atomic_try_cmpxchg(v, &c, dec)); - - return dec; -} -#define atomic_dec_if_positive atomic_dec_if_positive -#endif - -#ifdef CONFIG_GENERIC_ATOMIC64 -#include <asm-generic/atomic64.h> -#endif - -#define arch_atomic64_read atomic64_read -#define arch_atomic64_read_acquire atomic64_read_acquire - -#ifndef atomic64_read_acquire -static __always_inline s64 -atomic64_read_acquire(const atomic64_t *v) -{ - return smp_load_acquire(&(v)->counter); -} -#define atomic64_read_acquire atomic64_read_acquire -#endif - -#define arch_atomic64_set atomic64_set -#define arch_atomic64_set_release atomic64_set_release - -#ifndef atomic64_set_release -static __always_inline void -atomic64_set_release(atomic64_t *v, s64 i) -{ - smp_store_release(&(v)->counter, i); -} -#define atomic64_set_release atomic64_set_release -#endif - -#define arch_atomic64_add atomic64_add - -#define arch_atomic64_add_return atomic64_add_return -#define arch_atomic64_add_return_acquire atomic64_add_return_acquire -#define arch_atomic64_add_return_release atomic64_add_return_release -#define arch_atomic64_add_return_relaxed atomic64_add_return_relaxed - -#ifndef atomic64_add_return_relaxed -#define atomic64_add_return_acquire atomic64_add_return -#define atomic64_add_return_release atomic64_add_return -#define atomic64_add_return_relaxed atomic64_add_return -#else /* atomic64_add_return_relaxed */ - -#ifndef atomic64_add_return_acquire -static __always_inline s64 -atomic64_add_return_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_add_return_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_add_return_acquire atomic64_add_return_acquire -#endif - -#ifndef atomic64_add_return_release -static __always_inline s64 -atomic64_add_return_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_add_return_relaxed(i, v); -} -#define atomic64_add_return_release atomic64_add_return_release -#endif - -#ifndef atomic64_add_return -static __always_inline s64 -atomic64_add_return(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_add_return_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_add_return atomic64_add_return -#endif - -#endif /* atomic64_add_return_relaxed */ - -#define arch_atomic64_fetch_add atomic64_fetch_add -#define arch_atomic64_fetch_add_acquire atomic64_fetch_add_acquire -#define arch_atomic64_fetch_add_release atomic64_fetch_add_release -#define arch_atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed - -#ifndef atomic64_fetch_add_relaxed -#define atomic64_fetch_add_acquire atomic64_fetch_add -#define atomic64_fetch_add_release atomic64_fetch_add -#define atomic64_fetch_add_relaxed atomic64_fetch_add -#else /* atomic64_fetch_add_relaxed */ - -#ifndef atomic64_fetch_add_acquire -static __always_inline s64 -atomic64_fetch_add_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_add_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire -#endif - -#ifndef atomic64_fetch_add_release -static __always_inline s64 -atomic64_fetch_add_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_add_relaxed(i, v); -} -#define atomic64_fetch_add_release atomic64_fetch_add_release -#endif - -#ifndef atomic64_fetch_add -static __always_inline s64 -atomic64_fetch_add(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_add_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_add atomic64_fetch_add -#endif - -#endif /* atomic64_fetch_add_relaxed */ - -#define arch_atomic64_sub atomic64_sub - -#define arch_atomic64_sub_return atomic64_sub_return -#define arch_atomic64_sub_return_acquire atomic64_sub_return_acquire -#define arch_atomic64_sub_return_release atomic64_sub_return_release -#define arch_atomic64_sub_return_relaxed atomic64_sub_return_relaxed - -#ifndef atomic64_sub_return_relaxed -#define atomic64_sub_return_acquire atomic64_sub_return -#define atomic64_sub_return_release atomic64_sub_return -#define atomic64_sub_return_relaxed atomic64_sub_return -#else /* atomic64_sub_return_relaxed */ - -#ifndef atomic64_sub_return_acquire -static __always_inline s64 -atomic64_sub_return_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_sub_return_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_sub_return_acquire atomic64_sub_return_acquire -#endif - -#ifndef atomic64_sub_return_release -static __always_inline s64 -atomic64_sub_return_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_sub_return_relaxed(i, v); -} -#define atomic64_sub_return_release atomic64_sub_return_release -#endif - -#ifndef atomic64_sub_return -static __always_inline s64 -atomic64_sub_return(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_sub_return_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_sub_return atomic64_sub_return -#endif - -#endif /* atomic64_sub_return_relaxed */ - -#define arch_atomic64_fetch_sub atomic64_fetch_sub -#define arch_atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire -#define arch_atomic64_fetch_sub_release atomic64_fetch_sub_release -#define arch_atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed - -#ifndef atomic64_fetch_sub_relaxed -#define atomic64_fetch_sub_acquire atomic64_fetch_sub -#define atomic64_fetch_sub_release atomic64_fetch_sub -#define atomic64_fetch_sub_relaxed atomic64_fetch_sub -#else /* atomic64_fetch_sub_relaxed */ - -#ifndef atomic64_fetch_sub_acquire -static __always_inline s64 -atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_sub_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire -#endif - -#ifndef atomic64_fetch_sub_release -static __always_inline s64 -atomic64_fetch_sub_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_sub_relaxed(i, v); -} -#define atomic64_fetch_sub_release atomic64_fetch_sub_release -#endif - -#ifndef atomic64_fetch_sub -static __always_inline s64 -atomic64_fetch_sub(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_sub_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_sub atomic64_fetch_sub -#endif - -#endif /* atomic64_fetch_sub_relaxed */ - -#define arch_atomic64_inc atomic64_inc - -#ifndef atomic64_inc -static __always_inline void -atomic64_inc(atomic64_t *v) -{ - atomic64_add(1, v); -} -#define atomic64_inc atomic64_inc -#endif - -#define arch_atomic64_inc_return atomic64_inc_return -#define arch_atomic64_inc_return_acquire atomic64_inc_return_acquire -#define arch_atomic64_inc_return_release atomic64_inc_return_release -#define arch_atomic64_inc_return_relaxed atomic64_inc_return_relaxed - -#ifndef atomic64_inc_return_relaxed -#ifdef atomic64_inc_return -#define atomic64_inc_return_acquire atomic64_inc_return -#define atomic64_inc_return_release atomic64_inc_return -#define atomic64_inc_return_relaxed atomic64_inc_return -#endif /* atomic64_inc_return */ - -#ifndef atomic64_inc_return -static __always_inline s64 -atomic64_inc_return(atomic64_t *v) -{ - return atomic64_add_return(1, v); -} -#define atomic64_inc_return atomic64_inc_return -#endif - -#ifndef atomic64_inc_return_acquire -static __always_inline s64 -atomic64_inc_return_acquire(atomic64_t *v) -{ - return atomic64_add_return_acquire(1, v); -} -#define atomic64_inc_return_acquire atomic64_inc_return_acquire -#endif - -#ifndef atomic64_inc_return_release -static __always_inline s64 -atomic64_inc_return_release(atomic64_t *v) -{ - return atomic64_add_return_release(1, v); -} -#define atomic64_inc_return_release atomic64_inc_return_release -#endif - -#ifndef atomic64_inc_return_relaxed -static __always_inline s64 -atomic64_inc_return_relaxed(atomic64_t *v) -{ - return atomic64_add_return_relaxed(1, v); -} -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#endif - -#else /* atomic64_inc_return_relaxed */ - -#ifndef atomic64_inc_return_acquire -static __always_inline s64 -atomic64_inc_return_acquire(atomic64_t *v) -{ - s64 ret = atomic64_inc_return_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_inc_return_acquire atomic64_inc_return_acquire -#endif - -#ifndef atomic64_inc_return_release -static __always_inline s64 -atomic64_inc_return_release(atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_inc_return_relaxed(v); -} -#define atomic64_inc_return_release atomic64_inc_return_release -#endif - -#ifndef atomic64_inc_return -static __always_inline s64 -atomic64_inc_return(atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_inc_return_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_inc_return atomic64_inc_return -#endif - -#endif /* atomic64_inc_return_relaxed */ - -#define arch_atomic64_fetch_inc atomic64_fetch_inc -#define arch_atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire -#define arch_atomic64_fetch_inc_release atomic64_fetch_inc_release -#define arch_atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed - -#ifndef atomic64_fetch_inc_relaxed -#ifdef atomic64_fetch_inc -#define atomic64_fetch_inc_acquire atomic64_fetch_inc -#define atomic64_fetch_inc_release atomic64_fetch_inc -#define atomic64_fetch_inc_relaxed atomic64_fetch_inc -#endif /* atomic64_fetch_inc */ - -#ifndef atomic64_fetch_inc -static __always_inline s64 -atomic64_fetch_inc(atomic64_t *v) -{ - return atomic64_fetch_add(1, v); -} -#define atomic64_fetch_inc atomic64_fetch_inc -#endif - -#ifndef atomic64_fetch_inc_acquire -static __always_inline s64 -atomic64_fetch_inc_acquire(atomic64_t *v) -{ - return atomic64_fetch_add_acquire(1, v); -} -#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire -#endif - -#ifndef atomic64_fetch_inc_release -static __always_inline s64 -atomic64_fetch_inc_release(atomic64_t *v) -{ - return atomic64_fetch_add_release(1, v); -} -#define atomic64_fetch_inc_release atomic64_fetch_inc_release -#endif - -#ifndef atomic64_fetch_inc_relaxed -static __always_inline s64 -atomic64_fetch_inc_relaxed(atomic64_t *v) -{ - return atomic64_fetch_add_relaxed(1, v); -} -#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed -#endif - -#else /* atomic64_fetch_inc_relaxed */ - -#ifndef atomic64_fetch_inc_acquire -static __always_inline s64 -atomic64_fetch_inc_acquire(atomic64_t *v) -{ - s64 ret = atomic64_fetch_inc_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire -#endif - -#ifndef atomic64_fetch_inc_release -static __always_inline s64 -atomic64_fetch_inc_release(atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_inc_relaxed(v); -} -#define atomic64_fetch_inc_release atomic64_fetch_inc_release -#endif - -#ifndef atomic64_fetch_inc -static __always_inline s64 -atomic64_fetch_inc(atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_inc_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_inc atomic64_fetch_inc -#endif - -#endif /* atomic64_fetch_inc_relaxed */ - -#define arch_atomic64_dec atomic64_dec - -#ifndef atomic64_dec -static __always_inline void -atomic64_dec(atomic64_t *v) -{ - atomic64_sub(1, v); -} -#define atomic64_dec atomic64_dec -#endif - -#define arch_atomic64_dec_return atomic64_dec_return -#define arch_atomic64_dec_return_acquire atomic64_dec_return_acquire -#define arch_atomic64_dec_return_release atomic64_dec_return_release -#define arch_atomic64_dec_return_relaxed atomic64_dec_return_relaxed - -#ifndef atomic64_dec_return_relaxed -#ifdef atomic64_dec_return -#define atomic64_dec_return_acquire atomic64_dec_return -#define atomic64_dec_return_release atomic64_dec_return -#define atomic64_dec_return_relaxed atomic64_dec_return -#endif /* atomic64_dec_return */ - -#ifndef atomic64_dec_return -static __always_inline s64 -atomic64_dec_return(atomic64_t *v) -{ - return atomic64_sub_return(1, v); -} -#define atomic64_dec_return atomic64_dec_return -#endif - -#ifndef atomic64_dec_return_acquire -static __always_inline s64 -atomic64_dec_return_acquire(atomic64_t *v) -{ - return atomic64_sub_return_acquire(1, v); -} -#define atomic64_dec_return_acquire atomic64_dec_return_acquire -#endif - -#ifndef atomic64_dec_return_release -static __always_inline s64 -atomic64_dec_return_release(atomic64_t *v) -{ - return atomic64_sub_return_release(1, v); -} -#define atomic64_dec_return_release atomic64_dec_return_release -#endif - -#ifndef atomic64_dec_return_relaxed -static __always_inline s64 -atomic64_dec_return_relaxed(atomic64_t *v) -{ - return atomic64_sub_return_relaxed(1, v); -} -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#endif - -#else /* atomic64_dec_return_relaxed */ - -#ifndef atomic64_dec_return_acquire -static __always_inline s64 -atomic64_dec_return_acquire(atomic64_t *v) -{ - s64 ret = atomic64_dec_return_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_dec_return_acquire atomic64_dec_return_acquire -#endif - -#ifndef atomic64_dec_return_release -static __always_inline s64 -atomic64_dec_return_release(atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_dec_return_relaxed(v); -} -#define atomic64_dec_return_release atomic64_dec_return_release -#endif - -#ifndef atomic64_dec_return -static __always_inline s64 -atomic64_dec_return(atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_dec_return_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_dec_return atomic64_dec_return -#endif - -#endif /* atomic64_dec_return_relaxed */ - -#define arch_atomic64_fetch_dec atomic64_fetch_dec -#define arch_atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire -#define arch_atomic64_fetch_dec_release atomic64_fetch_dec_release -#define arch_atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed - -#ifndef atomic64_fetch_dec_relaxed -#ifdef atomic64_fetch_dec -#define atomic64_fetch_dec_acquire atomic64_fetch_dec -#define atomic64_fetch_dec_release atomic64_fetch_dec -#define atomic64_fetch_dec_relaxed atomic64_fetch_dec -#endif /* atomic64_fetch_dec */ - -#ifndef atomic64_fetch_dec -static __always_inline s64 -atomic64_fetch_dec(atomic64_t *v) -{ - return atomic64_fetch_sub(1, v); -} -#define atomic64_fetch_dec atomic64_fetch_dec -#endif - -#ifndef atomic64_fetch_dec_acquire -static __always_inline s64 -atomic64_fetch_dec_acquire(atomic64_t *v) -{ - return atomic64_fetch_sub_acquire(1, v); -} -#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire -#endif - -#ifndef atomic64_fetch_dec_release -static __always_inline s64 -atomic64_fetch_dec_release(atomic64_t *v) -{ - return atomic64_fetch_sub_release(1, v); -} -#define atomic64_fetch_dec_release atomic64_fetch_dec_release -#endif - -#ifndef atomic64_fetch_dec_relaxed -static __always_inline s64 -atomic64_fetch_dec_relaxed(atomic64_t *v) -{ - return atomic64_fetch_sub_relaxed(1, v); -} -#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed -#endif - -#else /* atomic64_fetch_dec_relaxed */ - -#ifndef atomic64_fetch_dec_acquire -static __always_inline s64 -atomic64_fetch_dec_acquire(atomic64_t *v) -{ - s64 ret = atomic64_fetch_dec_relaxed(v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire -#endif - -#ifndef atomic64_fetch_dec_release -static __always_inline s64 -atomic64_fetch_dec_release(atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_dec_relaxed(v); -} -#define atomic64_fetch_dec_release atomic64_fetch_dec_release -#endif - -#ifndef atomic64_fetch_dec -static __always_inline s64 -atomic64_fetch_dec(atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_dec_relaxed(v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_dec atomic64_fetch_dec -#endif - -#endif /* atomic64_fetch_dec_relaxed */ - -#define arch_atomic64_and atomic64_and - -#define arch_atomic64_fetch_and atomic64_fetch_and -#define arch_atomic64_fetch_and_acquire atomic64_fetch_and_acquire -#define arch_atomic64_fetch_and_release atomic64_fetch_and_release -#define arch_atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed - -#ifndef atomic64_fetch_and_relaxed -#define atomic64_fetch_and_acquire atomic64_fetch_and -#define atomic64_fetch_and_release atomic64_fetch_and -#define atomic64_fetch_and_relaxed atomic64_fetch_and -#else /* atomic64_fetch_and_relaxed */ - -#ifndef atomic64_fetch_and_acquire -static __always_inline s64 -atomic64_fetch_and_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_and_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire -#endif - -#ifndef atomic64_fetch_and_release -static __always_inline s64 -atomic64_fetch_and_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_and_relaxed(i, v); -} -#define atomic64_fetch_and_release atomic64_fetch_and_release -#endif - -#ifndef atomic64_fetch_and -static __always_inline s64 -atomic64_fetch_and(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_and_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_and atomic64_fetch_and -#endif - -#endif /* atomic64_fetch_and_relaxed */ - -#define arch_atomic64_andnot atomic64_andnot - -#ifndef atomic64_andnot -static __always_inline void -atomic64_andnot(s64 i, atomic64_t *v) -{ - atomic64_and(~i, v); -} -#define atomic64_andnot atomic64_andnot -#endif - -#define arch_atomic64_fetch_andnot atomic64_fetch_andnot -#define arch_atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire -#define arch_atomic64_fetch_andnot_release atomic64_fetch_andnot_release -#define arch_atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed - -#ifndef atomic64_fetch_andnot_relaxed -#ifdef atomic64_fetch_andnot -#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot -#define atomic64_fetch_andnot_release atomic64_fetch_andnot -#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot -#endif /* atomic64_fetch_andnot */ - -#ifndef atomic64_fetch_andnot -static __always_inline s64 -atomic64_fetch_andnot(s64 i, atomic64_t *v) -{ - return atomic64_fetch_and(~i, v); -} -#define atomic64_fetch_andnot atomic64_fetch_andnot -#endif - -#ifndef atomic64_fetch_andnot_acquire -static __always_inline s64 -atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) -{ - return atomic64_fetch_and_acquire(~i, v); -} -#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire -#endif - -#ifndef atomic64_fetch_andnot_release -static __always_inline s64 -atomic64_fetch_andnot_release(s64 i, atomic64_t *v) -{ - return atomic64_fetch_and_release(~i, v); -} -#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release -#endif - -#ifndef atomic64_fetch_andnot_relaxed -static __always_inline s64 -atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) -{ - return atomic64_fetch_and_relaxed(~i, v); -} -#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed -#endif - -#else /* atomic64_fetch_andnot_relaxed */ - -#ifndef atomic64_fetch_andnot_acquire -static __always_inline s64 -atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_andnot_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire -#endif - -#ifndef atomic64_fetch_andnot_release -static __always_inline s64 -atomic64_fetch_andnot_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_andnot_relaxed(i, v); -} -#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release -#endif - -#ifndef atomic64_fetch_andnot -static __always_inline s64 -atomic64_fetch_andnot(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_andnot_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_andnot atomic64_fetch_andnot -#endif - -#endif /* atomic64_fetch_andnot_relaxed */ - -#define arch_atomic64_or atomic64_or - -#define arch_atomic64_fetch_or atomic64_fetch_or -#define arch_atomic64_fetch_or_acquire atomic64_fetch_or_acquire -#define arch_atomic64_fetch_or_release atomic64_fetch_or_release -#define arch_atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed - -#ifndef atomic64_fetch_or_relaxed -#define atomic64_fetch_or_acquire atomic64_fetch_or -#define atomic64_fetch_or_release atomic64_fetch_or -#define atomic64_fetch_or_relaxed atomic64_fetch_or -#else /* atomic64_fetch_or_relaxed */ - -#ifndef atomic64_fetch_or_acquire -static __always_inline s64 -atomic64_fetch_or_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_or_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire -#endif - -#ifndef atomic64_fetch_or_release -static __always_inline s64 -atomic64_fetch_or_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_or_relaxed(i, v); -} -#define atomic64_fetch_or_release atomic64_fetch_or_release -#endif - -#ifndef atomic64_fetch_or -static __always_inline s64 -atomic64_fetch_or(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_or_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_or atomic64_fetch_or -#endif - -#endif /* atomic64_fetch_or_relaxed */ - -#define arch_atomic64_xor atomic64_xor - -#define arch_atomic64_fetch_xor atomic64_fetch_xor -#define arch_atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire -#define arch_atomic64_fetch_xor_release atomic64_fetch_xor_release -#define arch_atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed - -#ifndef atomic64_fetch_xor_relaxed -#define atomic64_fetch_xor_acquire atomic64_fetch_xor -#define atomic64_fetch_xor_release atomic64_fetch_xor -#define atomic64_fetch_xor_relaxed atomic64_fetch_xor -#else /* atomic64_fetch_xor_relaxed */ - -#ifndef atomic64_fetch_xor_acquire -static __always_inline s64 -atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) -{ - s64 ret = atomic64_fetch_xor_relaxed(i, v); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire -#endif - -#ifndef atomic64_fetch_xor_release -static __always_inline s64 -atomic64_fetch_xor_release(s64 i, atomic64_t *v) -{ - __atomic_release_fence(); - return atomic64_fetch_xor_relaxed(i, v); -} -#define atomic64_fetch_xor_release atomic64_fetch_xor_release -#endif - -#ifndef atomic64_fetch_xor -static __always_inline s64 -atomic64_fetch_xor(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_fetch_xor_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_fetch_xor atomic64_fetch_xor -#endif - -#endif /* atomic64_fetch_xor_relaxed */ - -#define arch_atomic64_xchg atomic64_xchg -#define arch_atomic64_xchg_acquire atomic64_xchg_acquire -#define arch_atomic64_xchg_release atomic64_xchg_release -#define arch_atomic64_xchg_relaxed atomic64_xchg_relaxed - -#ifndef atomic64_xchg_relaxed -#define atomic64_xchg_acquire atomic64_xchg -#define atomic64_xchg_release atomic64_xchg -#define atomic64_xchg_relaxed atomic64_xchg -#else /* atomic64_xchg_relaxed */ - -#ifndef atomic64_xchg_acquire -static __always_inline s64 -atomic64_xchg_acquire(atomic64_t *v, s64 i) -{ - s64 ret = atomic64_xchg_relaxed(v, i); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_xchg_acquire atomic64_xchg_acquire -#endif - -#ifndef atomic64_xchg_release -static __always_inline s64 -atomic64_xchg_release(atomic64_t *v, s64 i) -{ - __atomic_release_fence(); - return atomic64_xchg_relaxed(v, i); -} -#define atomic64_xchg_release atomic64_xchg_release -#endif - -#ifndef atomic64_xchg -static __always_inline s64 -atomic64_xchg(atomic64_t *v, s64 i) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_xchg_relaxed(v, i); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_xchg atomic64_xchg -#endif - -#endif /* atomic64_xchg_relaxed */ - -#define arch_atomic64_cmpxchg atomic64_cmpxchg -#define arch_atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire -#define arch_atomic64_cmpxchg_release atomic64_cmpxchg_release -#define arch_atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed - -#ifndef atomic64_cmpxchg_relaxed -#define atomic64_cmpxchg_acquire atomic64_cmpxchg -#define atomic64_cmpxchg_release atomic64_cmpxchg -#define atomic64_cmpxchg_relaxed atomic64_cmpxchg -#else /* atomic64_cmpxchg_relaxed */ - -#ifndef atomic64_cmpxchg_acquire -static __always_inline s64 -atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) -{ - s64 ret = atomic64_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire -#endif - -#ifndef atomic64_cmpxchg_release -static __always_inline s64 -atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) -{ - __atomic_release_fence(); - return atomic64_cmpxchg_relaxed(v, old, new); -} -#define atomic64_cmpxchg_release atomic64_cmpxchg_release -#endif - -#ifndef atomic64_cmpxchg -static __always_inline s64 -atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = atomic64_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_cmpxchg atomic64_cmpxchg -#endif - -#endif /* atomic64_cmpxchg_relaxed */ - -#define arch_atomic64_try_cmpxchg atomic64_try_cmpxchg -#define arch_atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire -#define arch_atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release -#define arch_atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed - -#ifndef atomic64_try_cmpxchg_relaxed -#ifdef atomic64_try_cmpxchg -#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg -#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg -#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg -#endif /* atomic64_try_cmpxchg */ - -#ifndef atomic64_try_cmpxchg -static __always_inline bool -atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ - s64 r, o = *old; - r = atomic64_cmpxchg(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic64_try_cmpxchg atomic64_try_cmpxchg -#endif - -#ifndef atomic64_try_cmpxchg_acquire -static __always_inline bool -atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) -{ - s64 r, o = *old; - r = atomic64_cmpxchg_acquire(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire -#endif - -#ifndef atomic64_try_cmpxchg_release -static __always_inline bool -atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) -{ - s64 r, o = *old; - r = atomic64_cmpxchg_release(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release -#endif - -#ifndef atomic64_try_cmpxchg_relaxed -static __always_inline bool -atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) -{ - s64 r, o = *old; - r = atomic64_cmpxchg_relaxed(v, o, new); - if (unlikely(r != o)) - *old = r; - return likely(r == o); -} -#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed -#endif - -#else /* atomic64_try_cmpxchg_relaxed */ - -#ifndef atomic64_try_cmpxchg_acquire -static __always_inline bool -atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) -{ - bool ret = atomic64_try_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire -#endif - -#ifndef atomic64_try_cmpxchg_release -static __always_inline bool -atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) -{ - __atomic_release_fence(); - return atomic64_try_cmpxchg_relaxed(v, old, new); -} -#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release -#endif - -#ifndef atomic64_try_cmpxchg -static __always_inline bool -atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ - bool ret; - __atomic_pre_full_fence(); - ret = atomic64_try_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define atomic64_try_cmpxchg atomic64_try_cmpxchg -#endif - -#endif /* atomic64_try_cmpxchg_relaxed */ - -#define arch_atomic64_sub_and_test atomic64_sub_and_test - -#ifndef atomic64_sub_and_test -/** - * atomic64_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic64_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -static __always_inline bool -atomic64_sub_and_test(s64 i, atomic64_t *v) -{ - return atomic64_sub_return(i, v) == 0; -} -#define atomic64_sub_and_test atomic64_sub_and_test -#endif - -#define arch_atomic64_dec_and_test atomic64_dec_and_test - -#ifndef atomic64_dec_and_test -/** - * atomic64_dec_and_test - decrement and test - * @v: pointer of type atomic64_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -static __always_inline bool -atomic64_dec_and_test(atomic64_t *v) -{ - return atomic64_dec_return(v) == 0; -} -#define atomic64_dec_and_test atomic64_dec_and_test -#endif - -#define arch_atomic64_inc_and_test atomic64_inc_and_test - -#ifndef atomic64_inc_and_test -/** - * atomic64_inc_and_test - increment and test - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -static __always_inline bool -atomic64_inc_and_test(atomic64_t *v) -{ - return atomic64_inc_return(v) == 0; -} -#define atomic64_inc_and_test atomic64_inc_and_test -#endif - -#define arch_atomic64_add_negative atomic64_add_negative - -#ifndef atomic64_add_negative -/** - * atomic64_add_negative - add and test if negative - * @i: integer value to add - * @v: pointer of type atomic64_t - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -static __always_inline bool -atomic64_add_negative(s64 i, atomic64_t *v) -{ - return atomic64_add_return(i, v) < 0; -} -#define atomic64_add_negative atomic64_add_negative -#endif - -#define arch_atomic64_fetch_add_unless atomic64_fetch_add_unless - -#ifndef atomic64_fetch_add_unless -/** - * atomic64_fetch_add_unless - add unless the number is already a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as @v was not already @u. - * Returns original value of @v - */ -static __always_inline s64 -atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) -{ - s64 c = atomic64_read(v); - - do { - if (unlikely(c == u)) - break; - } while (!atomic64_try_cmpxchg(v, &c, c + a)); - - return c; -} -#define atomic64_fetch_add_unless atomic64_fetch_add_unless -#endif - -#define arch_atomic64_add_unless atomic64_add_unless - -#ifndef atomic64_add_unless -/** - * atomic64_add_unless - add unless the number is already a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, if @v was not already @u. - * Returns true if the addition was done. - */ -static __always_inline bool -atomic64_add_unless(atomic64_t *v, s64 a, s64 u) -{ - return atomic64_fetch_add_unless(v, a, u) != u; -} -#define atomic64_add_unless atomic64_add_unless -#endif - -#define arch_atomic64_inc_not_zero atomic64_inc_not_zero - -#ifndef atomic64_inc_not_zero -/** - * atomic64_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1, if @v is non-zero. - * Returns true if the increment was done. - */ -static __always_inline bool -atomic64_inc_not_zero(atomic64_t *v) -{ - return atomic64_add_unless(v, 1, 0); -} -#define atomic64_inc_not_zero atomic64_inc_not_zero -#endif - -#define arch_atomic64_inc_unless_negative atomic64_inc_unless_negative - -#ifndef atomic64_inc_unless_negative -static __always_inline bool -atomic64_inc_unless_negative(atomic64_t *v) -{ - s64 c = atomic64_read(v); - - do { - if (unlikely(c < 0)) - return false; - } while (!atomic64_try_cmpxchg(v, &c, c + 1)); - - return true; -} -#define atomic64_inc_unless_negative atomic64_inc_unless_negative -#endif - -#define arch_atomic64_dec_unless_positive atomic64_dec_unless_positive - -#ifndef atomic64_dec_unless_positive -static __always_inline bool -atomic64_dec_unless_positive(atomic64_t *v) -{ - s64 c = atomic64_read(v); - - do { - if (unlikely(c > 0)) - return false; - } while (!atomic64_try_cmpxchg(v, &c, c - 1)); - - return true; -} -#define atomic64_dec_unless_positive atomic64_dec_unless_positive -#endif - -#define arch_atomic64_dec_if_positive atomic64_dec_if_positive - -#ifndef atomic64_dec_if_positive -static __always_inline s64 -atomic64_dec_if_positive(atomic64_t *v) -{ - s64 dec, c = atomic64_read(v); - - do { - dec = c - 1; - if (unlikely(dec < 0)) - break; - } while (!atomic64_try_cmpxchg(v, &c, dec)); - - return dec; -} -#define atomic64_dec_if_positive atomic64_dec_if_positive -#endif - -#endif /* _LINUX_ATOMIC_FALLBACK_H */ -// d78e6c293c661c15188f0ec05bce45188c8d5892 diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 571a11008ab5..ed1d3ffd5b9d 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -77,12 +77,8 @@ __ret; \ }) -#ifdef ARCH_ATOMIC #include <linux/atomic-arch-fallback.h> #include <asm-generic/atomic-instrumented.h> -#else -#include <linux/atomic-fallback.h> -#endif #include <asm-generic/atomic-long.h> diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index 2ec9ff5a7fff..3e726ace5c62 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -52,7 +52,7 @@ enum lockdep_lock_type { * NR_LOCKDEP_CACHING_CLASSES ... Number of classes * cached in the instance of lockdep_map * - * Currently main class (subclass == 0) and signle depth subclass + * Currently main class (subclass == 0) and single depth subclass * are cached in lockdep_map. This optimization is mainly targeting * on rq->lock. double_rq_lock() acquires this highly competitive with * single depth. diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index f61e34fbaaea..37ded6b8fee6 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -182,9 +182,9 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s) #define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock) #define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock) -#define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock); -#define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex); -#define seqcount_ww_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, ww_mutex); +#define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock) +#define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex) +#define seqcount_ww_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, ww_mutex) /* * SEQCOUNT_LOCKNAME() - Instantiate seqcount_LOCKNAME_t and helpers diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index a89eb0accd5e..235e5b2facaa 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -21,6 +21,7 @@ #define FUTEX_WAKE_BITSET 10 #define FUTEX_WAIT_REQUEUE_PI 11 #define FUTEX_CMP_REQUEUE_PI 12 +#define FUTEX_LOCK_PI2 13 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 @@ -32,6 +33,7 @@ #define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG) #define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG) +#define FUTEX_LOCK_PI2_PRIVATE (FUTEX_LOCK_PI2 | FUTEX_PRIVATE_FLAG) #define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG) #define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG) diff --git a/kernel/futex.c b/kernel/futex.c index 408cad5e8968..2ecb07575055 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1727,12 +1727,9 @@ retry_private: return ret; } - if (!(flags & FLAGS_SHARED)) { - cond_resched(); - goto retry_private; - } - cond_resched(); + if (!(flags & FLAGS_SHARED)) + goto retry_private; goto retry; } @@ -1873,7 +1870,7 @@ futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1, * If the caller intends to requeue more than 1 waiter to pifutex, * force futex_lock_pi_atomic() to set the FUTEX_WAITERS bit now, * as we have means to handle the possible fault. If not, don't set - * the bit unecessarily as it will force the subsequent unlock to enter + * the bit unnecessarily as it will force the subsequent unlock to enter * the kernel. */ top_waiter = futex_top_waiter(hb1, key1); @@ -2102,7 +2099,7 @@ retry_private: continue; /* - * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always + * FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI should always * be paired with each other and no other futex ops. * * We should never be requeueing a futex_q with a pi_state, @@ -2317,7 +2314,7 @@ retry: } /* - * PI futexes can not be requeued and must remove themself from the + * PI futexes can not be requeued and must remove themselves from the * hash bucket. The hash bucket lock (i.e. lock_ptr) is held. */ static void unqueue_me_pi(struct futex_q *q) @@ -2785,7 +2782,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, if (refill_pi_state_cache()) return -ENOMEM; - to = futex_setup_timer(time, &timeout, FLAGS_CLOCKRT, 0); + to = futex_setup_timer(time, &timeout, flags, 0); retry: ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE); @@ -2902,7 +2899,7 @@ no_block: */ res = fixup_owner(uaddr, &q, !ret); /* - * If fixup_owner() returned an error, proprogate that. If it acquired + * If fixup_owner() returned an error, propagate that. If it acquired * the lock, clear our -ETIMEDOUT or -EINTR. */ if (res) @@ -3279,7 +3276,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, */ res = fixup_owner(uaddr2, &q, !ret); /* - * If fixup_owner() returned an error, proprogate that. If it + * If fixup_owner() returned an error, propagate that. If it * acquired the lock, clear -ETIMEDOUT or -EINTR. */ if (res) @@ -3677,7 +3674,7 @@ void futex_exec_release(struct task_struct *tsk) { /* * The state handling is done for consistency, but in the case of - * exec() there is no way to prevent futher damage as the PID stays + * exec() there is no way to prevent further damage as the PID stays * the same. But for the unlikely and arguably buggy case that a * futex is held on exec(), this provides at least as much state * consistency protection which is possible. @@ -3709,12 +3706,14 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, if (op & FUTEX_CLOCK_REALTIME) { flags |= FLAGS_CLOCKRT; - if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI) + if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI && + cmd != FUTEX_LOCK_PI2) return -ENOSYS; } switch (cmd) { case FUTEX_LOCK_PI: + case FUTEX_LOCK_PI2: case FUTEX_UNLOCK_PI: case FUTEX_TRYLOCK_PI: case FUTEX_WAIT_REQUEUE_PI: @@ -3741,6 +3740,9 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, case FUTEX_WAKE_OP: return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3); case FUTEX_LOCK_PI: + flags |= FLAGS_CLOCKRT; + fallthrough; + case FUTEX_LOCK_PI2: return futex_lock_pi(uaddr, flags, timeout, 0); case FUTEX_UNLOCK_PI: return futex_unlock_pi(uaddr, flags); @@ -3761,6 +3763,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd) switch (cmd) { case FUTEX_WAIT: case FUTEX_LOCK_PI: + case FUTEX_LOCK_PI2: case FUTEX_WAIT_BITSET: case FUTEX_WAIT_REQUEUE_PI: return true; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index e32313072506..0c0524bfff99 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -2306,7 +2306,56 @@ static void print_lock_class_header(struct lock_class *class, int depth) } /* - * printk the shortest lock dependencies from @start to @end in reverse order: + * Dependency path printing: + * + * After BFS we get a lock dependency path (linked via ->parent of lock_list), + * printing out each lock in the dependency path will help on understanding how + * the deadlock could happen. Here are some details about dependency path + * printing: + * + * 1) A lock_list can be either forwards or backwards for a lock dependency, + * for a lock dependency A -> B, there are two lock_lists: + * + * a) lock_list in the ->locks_after list of A, whose ->class is B and + * ->links_to is A. In this case, we can say the lock_list is + * "A -> B" (forwards case). + * + * b) lock_list in the ->locks_before list of B, whose ->class is A + * and ->links_to is B. In this case, we can say the lock_list is + * "B <- A" (bacwards case). + * + * The ->trace of both a) and b) point to the call trace where B was + * acquired with A held. + * + * 2) A "helper" lock_list is introduced during BFS, this lock_list doesn't + * represent a certain lock dependency, it only provides an initial entry + * for BFS. For example, BFS may introduce a "helper" lock_list whose + * ->class is A, as a result BFS will search all dependencies starting with + * A, e.g. A -> B or A -> C. + * + * The notation of a forwards helper lock_list is like "-> A", which means + * we should search the forwards dependencies starting with "A", e.g A -> B + * or A -> C. + * + * The notation of a bacwards helper lock_list is like "<- B", which means + * we should search the backwards dependencies ending with "B", e.g. + * B <- A or B <- C. + */ + +/* + * printk the shortest lock dependencies from @root to @leaf in reverse order. + * + * We have a lock dependency path as follow: + * + * @root @leaf + * | | + * V V + * ->parent ->parent + * | lock_list | <--------- | lock_list | ... | lock_list | <--------- | lock_list | + * | -> L1 | | L1 -> L2 | ... |Ln-2 -> Ln-1| | Ln-1 -> Ln| + * + * , so it's natural that we start from @leaf and print every ->class and + * ->trace until we reach the @root. */ static void __used print_shortest_lock_dependencies(struct lock_list *leaf, @@ -2334,6 +2383,61 @@ print_shortest_lock_dependencies(struct lock_list *leaf, } while (entry && (depth >= 0)); } +/* + * printk the shortest lock dependencies from @leaf to @root. + * + * We have a lock dependency path (from a backwards search) as follow: + * + * @leaf @root + * | | + * V V + * ->parent ->parent + * | lock_list | ---------> | lock_list | ... | lock_list | ---------> | lock_list | + * | L2 <- L1 | | L3 <- L2 | ... | Ln <- Ln-1 | | <- Ln | + * + * , so when we iterate from @leaf to @root, we actually print the lock + * dependency path L1 -> L2 -> .. -> Ln in the non-reverse order. + * + * Another thing to notice here is that ->class of L2 <- L1 is L1, while the + * ->trace of L2 <- L1 is the call trace of L2, in fact we don't have the call + * trace of L1 in the dependency path, which is alright, because most of the + * time we can figure out where L1 is held from the call trace of L2. + */ +static void __used +print_shortest_lock_dependencies_backwards(struct lock_list *leaf, + struct lock_list *root) +{ + struct lock_list *entry = leaf; + const struct lock_trace *trace = NULL; + int depth; + + /*compute depth from generated tree by BFS*/ + depth = get_lock_depth(leaf); + + do { + print_lock_class_header(entry->class, depth); + if (trace) { + printk("%*s ... acquired at:\n", depth, ""); + print_lock_trace(trace, 2); + printk("\n"); + } + + /* + * Record the pointer to the trace for the next lock_list + * entry, see the comments for the function. + */ + trace = entry->trace; + + if (depth == 0 && (entry != root)) { + printk("lockdep:%s bad path found in chain graph\n", __func__); + break; + } + + entry = get_lock_parent(entry); + depth--; + } while (entry && (depth >= 0)); +} + static void print_irq_lock_scenario(struct lock_list *safe_entry, struct lock_list *unsafe_entry, @@ -2448,10 +2552,7 @@ print_bad_irq_dependency(struct task_struct *curr, lockdep_print_held_locks(curr); pr_warn("\nthe dependencies between %s-irq-safe lock and the holding lock:\n", irqclass); - prev_root->trace = save_trace(); - if (!prev_root->trace) - return; - print_shortest_lock_dependencies(backwards_entry, prev_root); + print_shortest_lock_dependencies_backwards(backwards_entry, prev_root); pr_warn("\nthe dependencies between the lock to be acquired"); pr_warn(" and %s-irq-unsafe lock:\n", irqclass); @@ -2669,8 +2770,18 @@ static int check_irq_usage(struct task_struct *curr, struct held_lock *prev, * Step 3: we found a bad match! Now retrieve a lock from the backward * list whose usage mask matches the exclusive usage mask from the * lock found on the forward list. + * + * Note, we should only keep the LOCKF_ENABLED_IRQ_ALL bits, considering + * the follow case: + * + * When trying to add A -> B to the graph, we find that there is a + * hardirq-safe L, that L -> ... -> A, and another hardirq-unsafe M, + * that B -> ... -> M. However M is **softirq-safe**, if we use exact + * invert bits of M's usage_mask, we will find another lock N that is + * **softirq-unsafe** and N -> ... -> A, however N -> .. -> M will not + * cause a inversion deadlock. */ - backward_mask = original_mask(target_entry1->class->usage_mask); + backward_mask = original_mask(target_entry1->class->usage_mask & LOCKF_ENABLED_IRQ_ALL); ret = find_usage_backwards(&this, backward_mask, &target_entry); if (bfs_error(ret)) { @@ -2720,7 +2831,7 @@ static inline bool usage_skip(struct lock_list *entry, void *mask) * <target> or not. If it can, <src> -> <target> dependency is already * in the graph. * - * Return BFS_RMATCH if it does, or BFS_RMATCH if it does not, return BFS_E* if + * Return BFS_RMATCH if it does, or BFS_RNOMATCH if it does not, return BFS_E* if * any error appears in the bfs search. */ static noinline enum bfs_result @@ -4579,7 +4690,7 @@ static int check_wait_context(struct task_struct *curr, struct held_lock *next) u8 curr_inner; int depth; - if (!curr->lockdep_depth || !next_inner || next->trylock) + if (!next_inner || next->trylock) return 0; if (!next_outer) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 678c13967580..1e1bd6f4a13d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1372,7 +1372,6 @@ config LOCKDEP bool depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select STACKTRACE - depends on FRAME_POINTER || MIPS || PPC || S390 || MICROBLAZE || ARM || ARC || X86 select KALLSYMS select KALLSYMS_ALL diff --git a/lib/atomic64.c b/lib/atomic64.c index e98c85a99787..3df653994177 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -42,7 +42,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v) return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; } -s64 atomic64_read(const atomic64_t *v) +s64 generic_atomic64_read(const atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -53,9 +53,9 @@ s64 atomic64_read(const atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_read); +EXPORT_SYMBOL(generic_atomic64_read); -void atomic64_set(atomic64_t *v, s64 i) +void generic_atomic64_set(atomic64_t *v, s64 i) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -64,10 +64,10 @@ void atomic64_set(atomic64_t *v, s64 i) v->counter = i; raw_spin_unlock_irqrestore(lock, flags); } -EXPORT_SYMBOL(atomic64_set); +EXPORT_SYMBOL(generic_atomic64_set); #define ATOMIC64_OP(op, c_op) \ -void atomic64_##op(s64 a, atomic64_t *v) \ +void generic_atomic64_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -76,10 +76,10 @@ void atomic64_##op(s64 a, atomic64_t *v) \ v->counter c_op a; \ raw_spin_unlock_irqrestore(lock, flags); \ } \ -EXPORT_SYMBOL(atomic64_##op); +EXPORT_SYMBOL(generic_atomic64_##op); #define ATOMIC64_OP_RETURN(op, c_op) \ -s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ +s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -90,10 +90,10 @@ s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_##op##_return); +EXPORT_SYMBOL(generic_atomic64_##op##_return); #define ATOMIC64_FETCH_OP(op, c_op) \ -s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ +s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ { \ unsigned long flags; \ raw_spinlock_t *lock = lock_addr(v); \ @@ -105,7 +105,7 @@ s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ raw_spin_unlock_irqrestore(lock, flags); \ return val; \ } \ -EXPORT_SYMBOL(atomic64_fetch_##op); +EXPORT_SYMBOL(generic_atomic64_fetch_##op); #define ATOMIC64_OPS(op, c_op) \ ATOMIC64_OP(op, c_op) \ @@ -130,7 +130,7 @@ ATOMIC64_OPS(xor, ^=) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -s64 atomic64_dec_if_positive(atomic64_t *v) +s64 generic_atomic64_dec_if_positive(atomic64_t *v) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -143,9 +143,9 @@ s64 atomic64_dec_if_positive(atomic64_t *v) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_dec_if_positive); +EXPORT_SYMBOL(generic_atomic64_dec_if_positive); -s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) +s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -158,9 +158,9 @@ s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_cmpxchg); +EXPORT_SYMBOL(generic_atomic64_cmpxchg); -s64 atomic64_xchg(atomic64_t *v, s64 new) +s64 generic_atomic64_xchg(atomic64_t *v, s64 new) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -172,9 +172,9 @@ s64 atomic64_xchg(atomic64_t *v, s64 new) raw_spin_unlock_irqrestore(lock, flags); return val; } -EXPORT_SYMBOL(atomic64_xchg); +EXPORT_SYMBOL(generic_atomic64_xchg); -s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { unsigned long flags; raw_spinlock_t *lock = lock_addr(v); @@ -188,4 +188,4 @@ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return val; } -EXPORT_SYMBOL(atomic64_fetch_add_unless); +EXPORT_SYMBOL(generic_atomic64_fetch_add_unless); diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 2d85abac1744..161108e5d2fe 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -53,6 +53,7 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose); #define LOCKTYPE_WW 0x10 #define LOCKTYPE_RTMUTEX 0x20 #define LOCKTYPE_LL 0x40 +#define LOCKTYPE_SPECIAL 0x80 static struct ww_acquire_ctx t, t2; static struct ww_mutex o, o2, o3; @@ -194,6 +195,7 @@ static void init_shared_classes(void) #define HARDIRQ_ENTER() \ local_irq_disable(); \ __irq_enter(); \ + lockdep_hardirq_threaded(); \ WARN_ON(!in_irq()); #define HARDIRQ_EXIT() \ @@ -2492,16 +2494,6 @@ static void rcu_sched_exit(int *_) int rcu_sched_guard_##name __guard(rcu_sched_exit); \ rcu_read_lock_sched(); -static void rcu_callback_exit(int *_) -{ - rcu_lock_release(&rcu_callback_map); -} - -#define RCU_CALLBACK_CONTEXT(name, ...) \ - int rcu_callback_guard_##name __guard(rcu_callback_exit); \ - rcu_lock_acquire(&rcu_callback_map); - - static void raw_spinlock_exit(raw_spinlock_t **lock) { raw_spin_unlock(*lock); @@ -2558,8 +2550,6 @@ static void __maybe_unused inner##_in_##outer(void) \ * ---------------+-------+----------+------+------- * RCU_BH | o | o | o | x * ---------------+-------+----------+------+------- - * RCU_CALLBACK | o | o | o | x - * ---------------+-------+----------+------+------- * RCU_SCHED | o | o | x | x * ---------------+-------+----------+------+------- * RAW_SPIN | o | o | x | x @@ -2576,7 +2566,6 @@ GENERATE_2_CONTEXT_TESTCASE(NOTTHREADED_HARDIRQ, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(SOFTIRQ, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU_BH, , inner, inner_lock) \ -GENERATE_2_CONTEXT_TESTCASE(RCU_CALLBACK, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RCU_SCHED, , inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(RAW_SPINLOCK, raw_lock_A, inner, inner_lock) \ GENERATE_2_CONTEXT_TESTCASE(SPINLOCK, lock_A, inner, inner_lock) \ @@ -2638,10 +2627,6 @@ static void wait_context_tests(void) DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_BH); pr_cont("\n"); - print_testname("in RCU callback context"); - DO_CONTEXT_TESTCASE_OUTER_LIMITED_PREEMPTIBLE(RCU_CALLBACK); - pr_cont("\n"); - print_testname("in RCU-sched context"); DO_CONTEXT_TESTCASE_OUTER_NOT_PREEMPTIBLE(RCU_SCHED); pr_cont("\n"); @@ -2744,6 +2729,66 @@ static void local_lock_tests(void) pr_cont("\n"); } +static void hardirq_deadlock_softirq_not_deadlock(void) +{ + /* mutex_A is hardirq-unsafe and softirq-unsafe */ + /* mutex_A -> lock_C */ + mutex_lock(&mutex_A); + HARDIRQ_DISABLE(); + spin_lock(&lock_C); + spin_unlock(&lock_C); + HARDIRQ_ENABLE(); + mutex_unlock(&mutex_A); + + /* lock_A is hardirq-safe */ + HARDIRQ_ENTER(); + spin_lock(&lock_A); + spin_unlock(&lock_A); + HARDIRQ_EXIT(); + + /* lock_A -> lock_B */ + HARDIRQ_DISABLE(); + spin_lock(&lock_A); + spin_lock(&lock_B); + spin_unlock(&lock_B); + spin_unlock(&lock_A); + HARDIRQ_ENABLE(); + + /* lock_B -> lock_C */ + HARDIRQ_DISABLE(); + spin_lock(&lock_B); + spin_lock(&lock_C); + spin_unlock(&lock_C); + spin_unlock(&lock_B); + HARDIRQ_ENABLE(); + + /* lock_D is softirq-safe */ + SOFTIRQ_ENTER(); + spin_lock(&lock_D); + spin_unlock(&lock_D); + SOFTIRQ_EXIT(); + + /* And lock_D is hardirq-unsafe */ + SOFTIRQ_DISABLE(); + spin_lock(&lock_D); + spin_unlock(&lock_D); + SOFTIRQ_ENABLE(); + + /* + * mutex_A -> lock_C -> lock_D is softirq-unsafe -> softirq-safe, not + * deadlock. + * + * lock_A -> lock_B -> lock_C -> lock_D is hardirq-safe -> + * hardirq-unsafe, deadlock. + */ + HARDIRQ_DISABLE(); + spin_lock(&lock_C); + spin_lock(&lock_D); + spin_unlock(&lock_D); + spin_unlock(&lock_C); + HARDIRQ_ENABLE(); +} + void locking_selftest(void) { /* @@ -2872,6 +2917,10 @@ void locking_selftest(void) local_lock_tests(); + print_testname("hardirq_unsafe_softirq_safe"); + dotest(hardirq_deadlock_softirq_not_deadlock, FAILURE, LOCKTYPE_SPECIAL); + pr_cont("\n"); + if (unexpected_testcase_failures) { printk("-----------------------------------------------------------------\n"); debug_locks = 0; diff --git a/net/core/filter.c b/net/core/filter.c index 65ab4e21c087..d81352ca1b5c 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -17,6 +17,7 @@ * Kris Katterjohn - Added many additional checks in bpf_check_classic() */ +#include <linux/atomic.h> #include <linux/module.h> #include <linux/types.h> #include <linux/mm.h> @@ -41,7 +42,6 @@ #include <linux/timer.h> #include <linux/uaccess.h> #include <asm/unaligned.h> -#include <asm/cmpxchg.h> #include <linux/filter.h> #include <linux/ratelimit.h> #include <linux/seccomp.h> diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 78c075a68c04..1b4073131c6f 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -7,13 +7,13 @@ * Trond Myklebust <trond.myklebust@primarydata.com> * */ +#include <linux/atomic.h> #include <linux/types.h> #include <linux/kref.h> #include <linux/list.h> #include <linux/rcupdate.h> #include <linux/rculist.h> #include <linux/slab.h> -#include <asm/cmpxchg.h> #include <linux/spinlock.h> #include <linux/sunrpc/xprt.h> #include <linux/sunrpc/addr.h> diff --git a/scripts/atomic/check-atomics.sh b/scripts/atomic/check-atomics.sh index 82748d42ecc5..9c7fbd4bcbce 100755 --- a/scripts/atomic/check-atomics.sh +++ b/scripts/atomic/check-atomics.sh @@ -17,7 +17,6 @@ cat <<EOF | asm-generic/atomic-instrumented.h asm-generic/atomic-long.h linux/atomic-arch-fallback.h -linux/atomic-fallback.h EOF while read header; do OLDSUM="$(tail -n 1 ${LINUXDIR}/include/${header})" diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index 5766ffcec7c5..b0c45aee19d7 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -41,34 +41,6 @@ gen_params_checks() done } -# gen_guard(meta, atomic, pfx, name, sfx, order) -gen_guard() -{ - local meta="$1"; shift - local atomic="$1"; shift - local pfx="$1"; shift - local name="$1"; shift - local sfx="$1"; shift - local order="$1"; shift - - local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}" - - local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" - - # We definitely need a preprocessor symbol for this atomic if it is an - # ordering variant, or if there's a generic fallback. - if [ ! -z "${order}" ] || [ ! -z "${template}" ]; then - printf "defined(${atomicname})" - return - fi - - # If this is a base variant, but a relaxed variant *may* exist, then we - # only have a preprocessor symbol if the relaxed variant isn't defined - if meta_has_relaxed "${meta}"; then - printf "!defined(${atomicname}_relaxed) || defined(${atomicname})" - fi -} - #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) gen_proto_order_variant() { @@ -82,16 +54,12 @@ gen_proto_order_variant() local atomicname="${atomic}_${pfx}${name}${sfx}${order}" - local guard="$(gen_guard "${meta}" "${atomic}" "${pfx}" "${name}" "${sfx}" "${order}")" - local ret="$(gen_ret_type "${meta}" "${int}")" local params="$(gen_params "${int}" "${atomic}" "$@")" local checks="$(gen_params_checks "${meta}" "$@")" local args="$(gen_args "$@")" local retstmt="$(gen_ret_stmt "${meta}")" - [ ! -z "${guard}" ] && printf "#if ${guard}\n" - cat <<EOF static __always_inline ${ret} ${atomicname}(${params}) @@ -99,11 +67,8 @@ ${atomicname}(${params}) ${checks} ${retstmt}arch_${atomicname}(${args}); } -#define ${atomicname} ${atomicname} EOF - [ ! -z "${guard}" ] && printf "#endif\n" - printf "\n" } @@ -139,19 +104,6 @@ EOF fi } -gen_optional_xchg() -{ - local name="$1"; shift - local sfx="$1"; shift - local guard="defined(arch_${name}${sfx})" - - [ -z "${sfx}" ] && guard="!defined(arch_${name}_relaxed) || defined(arch_${name})" - - printf "#if ${guard}\n" - gen_xchg "${name}${sfx}" "" - printf "#endif\n\n" -} - cat << EOF // SPDX-License-Identifier: GPL-2.0 @@ -188,7 +140,8 @@ done for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do for order in "" "_acquire" "_release" "_relaxed"; do - gen_optional_xchg "${xchg}" "${order}" + gen_xchg "${xchg}${order}" "" + printf "\n" done done diff --git a/scripts/atomic/gen-atomics.sh b/scripts/atomic/gen-atomics.sh index d29e159ef489..f776a574224d 100755 --- a/scripts/atomic/gen-atomics.sh +++ b/scripts/atomic/gen-atomics.sh @@ -11,7 +11,6 @@ cat <<EOF | gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h gen-atomic-long.sh asm-generic/atomic-long.h gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_ -gen-atomic-fallback.sh linux/atomic-fallback.h EOF while read script header args; do /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header} diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore index 0efcd494daab..0e78b49d0f2f 100644 --- a/tools/testing/selftests/futex/functional/.gitignore +++ b/tools/testing/selftests/futex/functional/.gitignore @@ -6,3 +6,5 @@ futex_wait_private_mapped_file futex_wait_timeout futex_wait_uninitialized_heap futex_wait_wouldblock +futex_wait +futex_requeue diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index 23207829ec75..bd1fec59e010 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -INCLUDES := -I../include -I../../ +INCLUDES := -I../include -I../../ -I../../../../../usr/include/ \ + -I$(KBUILD_OUTPUT)/kselftest/usr/include CFLAGS := $(CFLAGS) -g -O2 -Wall -D_GNU_SOURCE -pthread $(INCLUDES) LDLIBS := -lpthread -lrt @@ -14,7 +15,9 @@ TEST_GEN_FILES := \ futex_requeue_pi_signal_restart \ futex_requeue_pi_mismatched_ops \ futex_wait_uninitialized_heap \ - futex_wait_private_mapped_file + futex_wait_private_mapped_file \ + futex_wait \ + futex_requeue TEST_PROGS := run.sh diff --git a/tools/testing/selftests/futex/functional/futex_requeue.c b/tools/testing/selftests/futex/functional/futex_requeue.c new file mode 100644 index 000000000000..51485be6eb2f --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex_requeue.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright Collabora Ltd., 2021 + * + * futex cmp requeue test by André Almeida <andrealmeid@collabora.com> + */ + +#include <pthread.h> +#include <limits.h> +#include "logging.h" +#include "futextest.h" + +#define TEST_NAME "futex-requeue" +#define timeout_ns 30000000 +#define WAKE_WAIT_US 10000 + +volatile futex_t *f1; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +void *waiterfn(void *arg) +{ + struct timespec to; + + to.tv_sec = 0; + to.tv_nsec = timeout_ns; + + if (futex_wait(f1, *f1, &to, 0)) + printf("waiter failed errno %d\n", errno); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + pthread_t waiter[10]; + int res, ret = RET_PASS; + int c, i; + volatile futex_t _f1 = 0; + volatile futex_t f2 = 0; + + f1 = &_f1; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(2); + ksft_print_msg("%s: Test futex_requeue\n", + basename(argv[0])); + + /* + * Requeue a waiter from f1 to f2, and wake f2. + */ + if (pthread_create(&waiter[0], NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Requeuing 1 futex from f1 to f2\n"); + res = futex_cmp_requeue(f1, 0, &f2, 0, 1, 0); + if (res != 1) { + ksft_test_result_fail("futex_requeue simple returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } + + + info("Waking 1 futex at f2\n"); + res = futex_wake(&f2, 1, 0); + if (res != 1) { + ksft_test_result_fail("futex_requeue simple returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_requeue simple succeeds\n"); + } + + + /* + * Create 10 waiters at f1. At futex_requeue, wake 3 and requeue 7. + * At futex_wake, wake INT_MAX (should be exactly 7). + */ + for (i = 0; i < 10; i++) { + if (pthread_create(&waiter[i], NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + } + + usleep(WAKE_WAIT_US); + + info("Waking 3 futexes at f1 and requeuing 7 futexes from f1 to f2\n"); + res = futex_cmp_requeue(f1, 0, &f2, 3, 7, 0); + if (res != 10) { + ksft_test_result_fail("futex_requeue many returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } + + info("Waking INT_MAX futexes at f2\n"); + res = futex_wake(&f2, INT_MAX, 0); + if (res != 7) { + ksft_test_result_fail("futex_requeue many returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_requeue many succeeds\n"); + } + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/functional/futex_wait.c b/tools/testing/selftests/futex/functional/futex_wait.c new file mode 100644 index 000000000000..685140d9b93d --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex_wait.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright Collabora Ltd., 2021 + * + * futex cmp requeue test by André Almeida <andrealmeid@collabora.com> + */ + +#include <pthread.h> +#include <sys/shm.h> +#include <sys/mman.h> +#include <fcntl.h> +#include "logging.h" +#include "futextest.h" + +#define TEST_NAME "futex-wait" +#define timeout_ns 30000000 +#define WAKE_WAIT_US 10000 +#define SHM_PATH "futex_shm_file" + +void *futex; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +static void *waiterfn(void *arg) +{ + struct timespec to; + unsigned int flags = 0; + + if (arg) + flags = *((unsigned int *) arg); + + to.tv_sec = 0; + to.tv_nsec = timeout_ns; + + if (futex_wait(futex, 0, &to, flags)) + printf("waiter failed errno %d\n", errno); + + return NULL; +} + +int main(int argc, char *argv[]) +{ + int res, ret = RET_PASS, fd, c, shm_id; + u_int32_t f_private = 0, *shared_data; + unsigned int flags = FUTEX_PRIVATE_FLAG; + pthread_t waiter; + void *shm; + + futex = &f_private; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(3); + ksft_print_msg("%s: Test futex_wait\n", basename(argv[0])); + + /* Testing a private futex */ + info("Calling private futex_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, (void *) &flags)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling private futex_wake on futex: %p\n", futex); + res = futex_wake(futex, 1, FUTEX_PRIVATE_FLAG); + if (res != 1) { + ksft_test_result_fail("futex_wake private returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_wake private succeeds\n"); + } + + /* Testing an anon page shared memory */ + shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666); + if (shm_id < 0) { + perror("shmget"); + exit(1); + } + + shared_data = shmat(shm_id, NULL, 0); + + *shared_data = 0; + futex = shared_data; + + info("Calling shared (page anon) futex_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling shared (page anon) futex_wake on futex: %p\n", futex); + res = futex_wake(futex, 1, 0); + if (res != 1) { + ksft_test_result_fail("futex_wake shared (page anon) returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_wake shared (page anon) succeeds\n"); + } + + + /* Testing a file backed shared memory */ + fd = open(SHM_PATH, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + if (fd < 0) { + perror("open"); + exit(1); + } + + if (ftruncate(fd, sizeof(f_private))) { + perror("ftruncate"); + exit(1); + } + + shm = mmap(NULL, sizeof(f_private), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (shm == MAP_FAILED) { + perror("mmap"); + exit(1); + } + + memcpy(shm, &f_private, sizeof(f_private)); + + futex = shm; + + info("Calling shared (file backed) futex_wait on futex: %p\n", futex); + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + info("Calling shared (file backed) futex_wake on futex: %p\n", futex); + res = futex_wake(shm, 1, 0); + if (res != 1) { + ksft_test_result_fail("futex_wake shared (file backed) returned: %d %s\n", + errno, strerror(errno)); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex_wake shared (file backed) succeeds\n"); + } + + /* Freeing resources */ + shmdt(shared_data); + munmap(shm, sizeof(f_private)); + remove(SHM_PATH); + close(fd); + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c index ee55e6d389a3..1f8f6daaf1e7 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c +++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c @@ -11,21 +11,18 @@ * * HISTORY * 2009-Nov-6: Initial version by Darren Hart <dvhart@linux.intel.com> + * 2021-Apr-26: More test cases by André Almeida <andrealmeid@collabora.com> * *****************************************************************************/ -#include <errno.h> -#include <getopt.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> +#include <pthread.h> #include "futextest.h" #include "logging.h" #define TEST_NAME "futex-wait-timeout" static long timeout_ns = 100000; /* 100us default timeout */ +static futex_t futex_pi; void usage(char *prog) { @@ -37,11 +34,67 @@ void usage(char *prog) VQUIET, VCRITICAL, VINFO); } +/* + * Get a PI lock and hold it forever, so the main thread lock_pi will block + * and we can test the timeout + */ +void *get_pi_lock(void *arg) +{ + int ret; + volatile futex_t lock = 0; + + ret = futex_lock_pi(&futex_pi, NULL, 0, 0); + if (ret != 0) + error("futex_lock_pi failed\n", ret); + + /* Blocks forever */ + ret = futex_wait(&lock, 0, NULL, 0); + error("futex_wait failed\n", ret); + + return NULL; +} + +/* + * Check if the function returned the expected error + */ +static void test_timeout(int res, int *ret, char *test_name, int err) +{ + if (!res || errno != err) { + ksft_test_result_fail("%s returned %d\n", test_name, + res < 0 ? errno : res); + *ret = RET_FAIL; + } else { + ksft_test_result_pass("%s succeeds\n", test_name); + } +} + +/* + * Calculate absolute timeout and correct overflow + */ +static int futex_get_abs_timeout(clockid_t clockid, struct timespec *to, + long timeout_ns) +{ + if (clock_gettime(clockid, to)) { + error("clock_gettime failed\n", errno); + return errno; + } + + to->tv_nsec += timeout_ns; + + if (to->tv_nsec >= 1000000000) { + to->tv_sec++; + to->tv_nsec -= 1000000000; + } + + return 0; +} + int main(int argc, char *argv[]) { futex_t f1 = FUTEX_INITIALIZER; - struct timespec to; int res, ret = RET_PASS; + struct timespec to; + pthread_t thread; int c; while ((c = getopt(argc, argv, "cht:v:")) != -1) { @@ -65,22 +118,63 @@ int main(int argc, char *argv[]) } ksft_print_header(); - ksft_set_plan(1); + ksft_set_plan(7); ksft_print_msg("%s: Block on a futex and wait for timeout\n", basename(argv[0])); ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns); - /* initialize timeout */ + pthread_create(&thread, NULL, get_pi_lock, NULL); + + /* initialize relative timeout */ to.tv_sec = 0; to.tv_nsec = timeout_ns; - info("Calling futex_wait on f1: %u @ %p\n", f1, &f1); - res = futex_wait(&f1, f1, &to, FUTEX_PRIVATE_FLAG); - if (!res || errno != ETIMEDOUT) { - fail("futex_wait returned %d\n", ret < 0 ? errno : ret); - ret = RET_FAIL; - } + res = futex_wait(&f1, f1, &to, 0); + test_timeout(res, &ret, "futex_wait relative", ETIMEDOUT); + + /* FUTEX_WAIT_BITSET with CLOCK_REALTIME */ + if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns)) + return RET_FAIL; + res = futex_wait_bitset(&f1, f1, &to, 1, FUTEX_CLOCK_REALTIME); + test_timeout(res, &ret, "futex_wait_bitset realtime", ETIMEDOUT); + + /* FUTEX_WAIT_BITSET with CLOCK_MONOTONIC */ + if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns)) + return RET_FAIL; + res = futex_wait_bitset(&f1, f1, &to, 1, 0); + test_timeout(res, &ret, "futex_wait_bitset monotonic", ETIMEDOUT); + + /* FUTEX_WAIT_REQUEUE_PI with CLOCK_REALTIME */ + if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns)) + return RET_FAIL; + res = futex_wait_requeue_pi(&f1, f1, &futex_pi, &to, FUTEX_CLOCK_REALTIME); + test_timeout(res, &ret, "futex_wait_requeue_pi realtime", ETIMEDOUT); + + /* FUTEX_WAIT_REQUEUE_PI with CLOCK_MONOTONIC */ + if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns)) + return RET_FAIL; + res = futex_wait_requeue_pi(&f1, f1, &futex_pi, &to, 0); + test_timeout(res, &ret, "futex_wait_requeue_pi monotonic", ETIMEDOUT); + + /* + * FUTEX_LOCK_PI with CLOCK_REALTIME + * Due to historical reasons, FUTEX_LOCK_PI supports only realtime + * clock, but requires the caller to not set CLOCK_REALTIME flag. + * + * If you call FUTEX_LOCK_PI with a monotonic clock, it'll be + * interpreted as a realtime clock, and (unless you mess your machine's + * time or your time machine) the monotonic clock value is always + * smaller than realtime and the syscall will timeout immediately. + */ + if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns)) + return RET_FAIL; + res = futex_lock_pi(&futex_pi, &to, 0, 0); + test_timeout(res, &ret, "futex_lock_pi realtime", ETIMEDOUT); + + /* Test operations that don't support FUTEX_CLOCK_REALTIME */ + res = futex_lock_pi(&futex_pi, NULL, 0, FUTEX_CLOCK_REALTIME); + test_timeout(res, &ret, "futex_lock_pi invalid timeout flag", ENOSYS); - print_result(TEST_NAME, ret); + ksft_print_cnts(); return ret; } diff --git a/tools/testing/selftests/futex/functional/run.sh b/tools/testing/selftests/futex/functional/run.sh index 1acb6ace1680..11a9d62290f5 100755 --- a/tools/testing/selftests/futex/functional/run.sh +++ b/tools/testing/selftests/futex/functional/run.sh @@ -73,3 +73,9 @@ echo echo ./futex_wait_uninitialized_heap $COLOR ./futex_wait_private_mapped_file $COLOR + +echo +./futex_wait $COLOR + +echo +./futex_requeue $COLOR |