From 02c503ff237cdcd8e012a122a638295550db10a5 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Mon, 28 Nov 2016 15:50:48 +0100 Subject: s390/spinlock: use atomic primitives for spinlocks Add a couple more __atomic_xxx function to atomic_ops.h and use them to replace the compare-and-swap inlines in the spinlock code. This changes the type of the lock value from unsigned int to int. Signed-off-by: Martin Schwidefsky --- arch/s390/lib/spinlock.c | 73 +++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 38 deletions(-) (limited to 'arch/s390/lib') diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index ba427eb6f14c..3f4d0c69bbfe 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -32,23 +32,22 @@ static int __init spin_retry_setup(char *str) } __setup("spin_retry=", spin_retry_setup); -static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) +static inline void compare_and_delay(int *lock, int old) { asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); } void arch_spin_lock_wait(arch_spinlock_t *lp) { - unsigned int cpu = SPINLOCK_LOCKVAL; - unsigned int owner; - int count, first_diag; + int cpu = SPINLOCK_LOCKVAL; + int owner, count, first_diag; first_diag = 1; while (1) { owner = ACCESS_ONCE(lp->lock); /* Try to get the lock if it is free. */ if (!owner) { - if (_raw_compare_and_swap(&lp->lock, 0, cpu)) + if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) return; continue; } @@ -62,7 +61,7 @@ void arch_spin_lock_wait(arch_spinlock_t *lp) count = spin_retry; do { if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&lp->lock, owner); + compare_and_delay(&lp->lock, owner); owner = ACCESS_ONCE(lp->lock); } while (owner && count-- > 0); if (!owner) @@ -82,9 +81,8 @@ EXPORT_SYMBOL(arch_spin_lock_wait); void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) { - unsigned int cpu = SPINLOCK_LOCKVAL; - unsigned int owner; - int count, first_diag; + int cpu = SPINLOCK_LOCKVAL; + int owner, count, first_diag; local_irq_restore(flags); first_diag = 1; @@ -93,7 +91,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) /* Try to get the lock if it is free. */ if (!owner) { local_irq_disable(); - if (_raw_compare_and_swap(&lp->lock, 0, cpu)) + if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) return; local_irq_restore(flags); continue; @@ -108,7 +106,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) count = spin_retry; do { if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&lp->lock, owner); + compare_and_delay(&lp->lock, owner); owner = ACCESS_ONCE(lp->lock); } while (owner && count-- > 0); if (!owner) @@ -128,18 +126,17 @@ EXPORT_SYMBOL(arch_spin_lock_wait_flags); int arch_spin_trylock_retry(arch_spinlock_t *lp) { - unsigned int cpu = SPINLOCK_LOCKVAL; - unsigned int owner; - int count; + int cpu = SPINLOCK_LOCKVAL; + int owner, count; for (count = spin_retry; count > 0; count--) { owner = READ_ONCE(lp->lock); /* Try to get the lock if it is free. */ if (!owner) { - if (_raw_compare_and_swap(&lp->lock, 0, cpu)) + if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) return 1; } else if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&lp->lock, owner); + compare_and_delay(&lp->lock, owner); } return 0; } @@ -147,8 +144,8 @@ EXPORT_SYMBOL(arch_spin_trylock_retry); void _raw_read_lock_wait(arch_rwlock_t *rw) { - unsigned int owner, old; int count = spin_retry; + int owner, old; #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); @@ -162,12 +159,12 @@ void _raw_read_lock_wait(arch_rwlock_t *rw) } old = ACCESS_ONCE(rw->lock); owner = ACCESS_ONCE(rw->owner); - if ((int) old < 0) { + if (old < 0) { if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&rw->lock, old); + compare_and_delay(&rw->lock, old); continue; } - if (_raw_compare_and_swap(&rw->lock, old, old + 1)) + if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1)) return; } } @@ -175,17 +172,17 @@ EXPORT_SYMBOL(_raw_read_lock_wait); int _raw_read_trylock_retry(arch_rwlock_t *rw) { - unsigned int old; int count = spin_retry; + int old; while (count-- > 0) { old = ACCESS_ONCE(rw->lock); - if ((int) old < 0) { + if (old < 0) { if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&rw->lock, old); + compare_and_delay(&rw->lock, old); continue; } - if (_raw_compare_and_swap(&rw->lock, old, old + 1)) + if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1)) return 1; } return 0; @@ -194,10 +191,10 @@ EXPORT_SYMBOL(_raw_read_trylock_retry); #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES -void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) +void _raw_write_lock_wait(arch_rwlock_t *rw, int prev) { - unsigned int owner, old; int count = spin_retry; + int owner, old; owner = 0; while (1) { @@ -209,14 +206,14 @@ void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) old = ACCESS_ONCE(rw->lock); owner = ACCESS_ONCE(rw->owner); smp_mb(); - if ((int) old >= 0) { + if (old >= 0) { prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); old = prev; } - if ((old & 0x7fffffff) == 0 && (int) prev >= 0) + if ((old & 0x7fffffff) == 0 && prev >= 0) break; if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&rw->lock, old); + compare_and_delay(&rw->lock, old); } } EXPORT_SYMBOL(_raw_write_lock_wait); @@ -225,8 +222,8 @@ EXPORT_SYMBOL(_raw_write_lock_wait); void _raw_write_lock_wait(arch_rwlock_t *rw) { - unsigned int owner, old, prev; int count = spin_retry; + int owner, old, prev; prev = 0x80000000; owner = 0; @@ -238,15 +235,15 @@ void _raw_write_lock_wait(arch_rwlock_t *rw) } old = ACCESS_ONCE(rw->lock); owner = ACCESS_ONCE(rw->owner); - if ((int) old >= 0 && - _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) + if (old >= 0 && + __atomic_cmpxchg_bool(&rw->lock, old, old | 0x80000000)) prev = old; else smp_mb(); - if ((old & 0x7fffffff) == 0 && (int) prev >= 0) + if ((old & 0x7fffffff) == 0 && prev >= 0) break; if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&rw->lock, old); + compare_and_delay(&rw->lock, old); } } EXPORT_SYMBOL(_raw_write_lock_wait); @@ -255,24 +252,24 @@ EXPORT_SYMBOL(_raw_write_lock_wait); int _raw_write_trylock_retry(arch_rwlock_t *rw) { - unsigned int old; int count = spin_retry; + int old; while (count-- > 0) { old = ACCESS_ONCE(rw->lock); if (old) { if (MACHINE_HAS_CAD) - _raw_compare_and_delay(&rw->lock, old); + compare_and_delay(&rw->lock, old); continue; } - if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) + if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000)) return 1; } return 0; } EXPORT_SYMBOL(_raw_write_trylock_retry); -void arch_lock_relax(unsigned int cpu) +void arch_lock_relax(int cpu) { if (!cpu) return; -- cgit v1.2.3 From b13de4b7adeb7a5e37a5aa78d5a4926c3cd4e131 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Fri, 24 Mar 2017 17:00:45 +0100 Subject: s390/spinlock: remove compare and delay instruction The CAD instruction never worked quite as expected for the spinlock code. It has been disabled by default with git commit 61b0b01686d48220, if the "cad" kernel parameter is specified it is enabled for both user space and the spinlock code. Leave the option to enable the instruction for user space but remove it from the spinlock code. Signed-off-by: Martin Schwidefsky --- arch/s390/lib/spinlock.c | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) (limited to 'arch/s390/lib') diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 3f4d0c69bbfe..ffb15bd4c593 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -17,7 +17,7 @@ int spin_retry = -1; static int __init spin_retry_init(void) { if (spin_retry < 0) - spin_retry = MACHINE_HAS_CAD ? 10 : 1000; + spin_retry = 1000; return 0; } early_initcall(spin_retry_init); @@ -32,11 +32,6 @@ static int __init spin_retry_setup(char *str) } __setup("spin_retry=", spin_retry_setup); -static inline void compare_and_delay(int *lock, int old) -{ - asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); -} - void arch_spin_lock_wait(arch_spinlock_t *lp) { int cpu = SPINLOCK_LOCKVAL; @@ -60,8 +55,6 @@ void arch_spin_lock_wait(arch_spinlock_t *lp) /* Loop for a while on the lock value. */ count = spin_retry; do { - if (MACHINE_HAS_CAD) - compare_and_delay(&lp->lock, owner); owner = ACCESS_ONCE(lp->lock); } while (owner && count-- > 0); if (!owner) @@ -105,8 +98,6 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) /* Loop for a while on the lock value. */ count = spin_retry; do { - if (MACHINE_HAS_CAD) - compare_and_delay(&lp->lock, owner); owner = ACCESS_ONCE(lp->lock); } while (owner && count-- > 0); if (!owner) @@ -135,8 +126,7 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp) if (!owner) { if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu)) return 1; - } else if (MACHINE_HAS_CAD) - compare_and_delay(&lp->lock, owner); + } } return 0; } @@ -159,11 +149,8 @@ void _raw_read_lock_wait(arch_rwlock_t *rw) } old = ACCESS_ONCE(rw->lock); owner = ACCESS_ONCE(rw->owner); - if (old < 0) { - if (MACHINE_HAS_CAD) - compare_and_delay(&rw->lock, old); + if (old < 0) continue; - } if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1)) return; } @@ -177,11 +164,8 @@ int _raw_read_trylock_retry(arch_rwlock_t *rw) while (count-- > 0) { old = ACCESS_ONCE(rw->lock); - if (old < 0) { - if (MACHINE_HAS_CAD) - compare_and_delay(&rw->lock, old); + if (old < 0) continue; - } if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1)) return 1; } @@ -212,8 +196,6 @@ void _raw_write_lock_wait(arch_rwlock_t *rw, int prev) } if ((old & 0x7fffffff) == 0 && prev >= 0) break; - if (MACHINE_HAS_CAD) - compare_and_delay(&rw->lock, old); } } EXPORT_SYMBOL(_raw_write_lock_wait); @@ -242,8 +224,6 @@ void _raw_write_lock_wait(arch_rwlock_t *rw) smp_mb(); if ((old & 0x7fffffff) == 0 && prev >= 0) break; - if (MACHINE_HAS_CAD) - compare_and_delay(&rw->lock, old); } } EXPORT_SYMBOL(_raw_write_lock_wait); @@ -257,11 +237,8 @@ int _raw_write_trylock_retry(arch_rwlock_t *rw) while (count-- > 0) { old = ACCESS_ONCE(rw->lock); - if (old) { - if (MACHINE_HAS_CAD) - compare_and_delay(&rw->lock, old); + if (old) continue; - } if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000)) return 1; } -- cgit v1.2.3