diff options
Diffstat (limited to 'kernel/locking')
| -rw-r--r-- | kernel/locking/mutex.c | 25 | ||||
| -rw-r--r-- | kernel/locking/rwsem-xadd.c | 26 |
2 files changed, 32 insertions, 19 deletions
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 57e28af96c5b..6dba40af542c 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -427,21 +427,31 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, { bool ret = true; - rcu_read_lock(); - while (__mutex_owner(lock) == owner) { + for (;;) { + unsigned int cpu; + bool same_owner; + /* - * Ensure we emit the owner->on_cpu, dereference _after_ - * checking lock->owner still matches owner. If that fails, + * Ensure lock->owner still matches owner. If that fails, * owner might point to freed memory. If it still matches, * the rcu_read_lock() ensures the memory stays valid. */ - barrier(); + rcu_read_lock(); + same_owner = __mutex_owner(lock) == owner; + if (same_owner) { + ret = owner->on_cpu; + if (ret) + cpu = task_cpu(owner); + } + rcu_read_unlock(); + + if (!ret || !same_owner) + break; /* * Use vcpu_is_preempted to detect lock holder preemption issue. */ - if (!owner->on_cpu || need_resched() || - vcpu_is_preempted(task_cpu(owner))) { + if (need_resched() || vcpu_is_preempted(cpu)) { ret = false; break; } @@ -453,7 +463,6 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, cpu_relax(); } - rcu_read_unlock(); return ret; } diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c index 350861f0da16..8917e6ceb3d0 100644 --- a/kernel/locking/rwsem-xadd.c +++ b/kernel/locking/rwsem-xadd.c @@ -400,32 +400,36 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem) { struct task_struct *owner = READ_ONCE(sem->owner); - if (!is_rwsem_owner_spinnable(owner)) + if (!owner || !is_rwsem_owner_spinnable(owner)) return false; - rcu_read_lock(); - while (owner && (READ_ONCE(sem->owner) == owner)) { + while (true) { + bool on_cpu, same_owner; + /* - * Ensure we emit the owner->on_cpu, dereference _after_ - * checking sem->owner still matches owner, if that fails, + * Ensure sem->owner still matches owner. If that fails, * owner might point to free()d memory, if it still matches, * the rcu_read_lock() ensures the memory stays valid. */ - barrier(); + rcu_read_lock(); + same_owner = sem->owner == owner; + if (same_owner) + on_cpu = owner->on_cpu && + !vcpu_is_preempted(task_cpu(owner)); + rcu_read_unlock(); + + if (!same_owner) + break; /* * abort spinning when need_resched or owner is not running or * owner's cpu is preempted. */ - if (!owner->on_cpu || need_resched() || - vcpu_is_preempted(task_cpu(owner))) { - rcu_read_unlock(); + if (!on_cpu || need_resched()) return false; - } cpu_relax(); } - rcu_read_unlock(); /* * If there is a new owner or the owner is not set, we continue |