mirror of
https://github.com/torvalds/linux
synced 2024-10-10 13:24:04 +00:00
f233f7f158
We use the regular paravirt call patching to switch between: native_queued_spin_lock_slowpath() __pv_queued_spin_lock_slowpath() native_queued_spin_unlock() __pv_queued_spin_unlock() We use a callee saved call for the unlock function which reduces the i-cache footprint and allows 'inlining' of SPIN_UNLOCK functions again. We further optimize the unlock path by patching the direct call with a "movb $0,%arg1" if we are indeed using the native unlock code. This makes the unlock code almost as fast as the !PARAVIRT case. This significantly lowers the overhead of having CONFIG_PARAVIRT_SPINLOCKS enabled, even for native code. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Waiman Long <Waiman.Long@hp.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Daniel J Blueman <daniel@numascale.com> Cc: David Vrabel <david.vrabel@citrix.com> Cc: Douglas Hatch <doug.hatch@hp.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Paolo Bonzini <paolo.bonzini@gmail.com> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com> Cc: Rik van Riel <riel@redhat.com> Cc: Scott J Norton <scott.norton@hp.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: virtualization@lists.linux-foundation.org Cc: xen-devel@lists.xenproject.org Link: http://lkml.kernel.org/r/1429901803-29771-10-git-send-email-Waiman.Long@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
43 lines
1.1 KiB
C
43 lines
1.1 KiB
C
/*
|
|
* Split spinlock implementation out into its own file, so it can be
|
|
* compiled in a FTRACE-compatible way.
|
|
*/
|
|
#include <linux/spinlock.h>
|
|
#include <linux/module.h>
|
|
#include <linux/jump_label.h>
|
|
|
|
#include <asm/paravirt.h>
|
|
|
|
#ifdef CONFIG_QUEUED_SPINLOCK
|
|
__visible void __native_queued_spin_unlock(struct qspinlock *lock)
|
|
{
|
|
native_queued_spin_unlock(lock);
|
|
}
|
|
|
|
PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock);
|
|
|
|
bool pv_is_native_spin_unlock(void)
|
|
{
|
|
return pv_lock_ops.queued_spin_unlock.func ==
|
|
__raw_callee_save___native_queued_spin_unlock;
|
|
}
|
|
#endif
|
|
|
|
struct pv_lock_ops pv_lock_ops = {
|
|
#ifdef CONFIG_SMP
|
|
#ifdef CONFIG_QUEUED_SPINLOCK
|
|
.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
|
|
.queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock),
|
|
.wait = paravirt_nop,
|
|
.kick = paravirt_nop,
|
|
#else /* !CONFIG_QUEUED_SPINLOCK */
|
|
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
|
|
.unlock_kick = paravirt_nop,
|
|
#endif /* !CONFIG_QUEUED_SPINLOCK */
|
|
#endif /* SMP */
|
|
};
|
|
EXPORT_SYMBOL(pv_lock_ops);
|
|
|
|
struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
|
|
EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
|