linux/arch/ia64/include/asm/paravirt.h
Ingo Molnar c5905afb0e static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.

Typical usage scenarios:

        #include <linux/static_key.h>

        struct static_key key = STATIC_KEY_INIT_TRUE;

        if (static_key_false(&key))
                do unlikely code
        else
                do likely code

Or:

        if (static_key_true(&key))
                do likely code
        else
                do unlikely code

The static key is modified via:

        static_key_slow_inc(&key);
        ...
        static_key_slow_dec(&key);

The 'slow' prefix makes it abundantly clear that this is an
expensive operation.

I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.

On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Jason Baron <jbaron@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: a.p.zijlstra@chello.nl
Cc: mathieu.desnoyers@efficios.com
Cc: davem@davemloft.net
Cc: ddaney.cavm@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-24 10:05:59 +01:00

323 lines
7.6 KiB
C

/******************************************************************************
* Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
* VA Linux Systems Japan K.K.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*
*/
#ifndef __ASM_PARAVIRT_H
#define __ASM_PARAVIRT_H
#ifndef __ASSEMBLY__
/******************************************************************************
* fsys related addresses
*/
struct pv_fsys_data {
unsigned long *fsyscall_table;
void *fsys_bubble_down;
};
extern struct pv_fsys_data pv_fsys_data;
unsigned long *paravirt_get_fsyscall_table(void);
char *paravirt_get_fsys_bubble_down(void);
/******************************************************************************
* patchlist addresses for gate page
*/
enum pv_gate_patchlist {
PV_GATE_START_FSYSCALL,
PV_GATE_END_FSYSCALL,
PV_GATE_START_BRL_FSYS_BUBBLE_DOWN,
PV_GATE_END_BRL_FSYS_BUBBLE_DOWN,
PV_GATE_START_VTOP,
PV_GATE_END_VTOP,
PV_GATE_START_MCKINLEY_E9,
PV_GATE_END_MCKINLEY_E9,
};
struct pv_patchdata {
unsigned long start_fsyscall_patchlist;
unsigned long end_fsyscall_patchlist;
unsigned long start_brl_fsys_bubble_down_patchlist;
unsigned long end_brl_fsys_bubble_down_patchlist;
unsigned long start_vtop_patchlist;
unsigned long end_vtop_patchlist;
unsigned long start_mckinley_e9_patchlist;
unsigned long end_mckinley_e9_patchlist;
void *gate_section;
};
extern struct pv_patchdata pv_patchdata;
unsigned long paravirt_get_gate_patchlist(enum pv_gate_patchlist type);
void *paravirt_get_gate_section(void);
#endif
#ifdef CONFIG_PARAVIRT_GUEST
#define PARAVIRT_HYPERVISOR_TYPE_DEFAULT 0
#define PARAVIRT_HYPERVISOR_TYPE_XEN 1
#ifndef __ASSEMBLY__
#include <asm/hw_irq.h>
#include <asm/meminit.h>
/******************************************************************************
* general info
*/
struct pv_info {
unsigned int kernel_rpl;
int paravirt_enabled;
const char *name;
};
extern struct pv_info pv_info;
static inline int paravirt_enabled(void)
{
return pv_info.paravirt_enabled;
}
static inline unsigned int get_kernel_rpl(void)
{
return pv_info.kernel_rpl;
}
/******************************************************************************
* initialization hooks.
*/
struct rsvd_region;
struct pv_init_ops {
void (*banner)(void);
int (*reserve_memory)(struct rsvd_region *region);
void (*arch_setup_early)(void);
void (*arch_setup_console)(char **cmdline_p);
int (*arch_setup_nomca)(void);
void (*post_smp_prepare_boot_cpu)(void);
#ifdef ASM_SUPPORTED
unsigned long (*patch_bundle)(void *sbundle, void *ebundle,
unsigned long type);
unsigned long (*patch_inst)(unsigned long stag, unsigned long etag,
unsigned long type);
#endif
void (*patch_branch)(unsigned long tag, unsigned long type);
};
extern struct pv_init_ops pv_init_ops;
static inline void paravirt_banner(void)
{
if (pv_init_ops.banner)
pv_init_ops.banner();
}
static inline int paravirt_reserve_memory(struct rsvd_region *region)
{
if (pv_init_ops.reserve_memory)
return pv_init_ops.reserve_memory(region);
return 0;
}
static inline void paravirt_arch_setup_early(void)
{
if (pv_init_ops.arch_setup_early)
pv_init_ops.arch_setup_early();
}
static inline void paravirt_arch_setup_console(char **cmdline_p)
{
if (pv_init_ops.arch_setup_console)
pv_init_ops.arch_setup_console(cmdline_p);
}
static inline int paravirt_arch_setup_nomca(void)
{
if (pv_init_ops.arch_setup_nomca)
return pv_init_ops.arch_setup_nomca();
return 0;
}
static inline void paravirt_post_smp_prepare_boot_cpu(void)
{
if (pv_init_ops.post_smp_prepare_boot_cpu)
pv_init_ops.post_smp_prepare_boot_cpu();
}
/******************************************************************************
* replacement of iosapic operations.
*/
struct pv_iosapic_ops {
void (*pcat_compat_init)(void);
struct irq_chip *(*__get_irq_chip)(unsigned long trigger);
unsigned int (*__read)(char __iomem *iosapic, unsigned int reg);
void (*__write)(char __iomem *iosapic, unsigned int reg, u32 val);
};
extern struct pv_iosapic_ops pv_iosapic_ops;
static inline void
iosapic_pcat_compat_init(void)
{
if (pv_iosapic_ops.pcat_compat_init)
pv_iosapic_ops.pcat_compat_init();
}
static inline struct irq_chip*
iosapic_get_irq_chip(unsigned long trigger)
{
return pv_iosapic_ops.__get_irq_chip(trigger);
}
static inline unsigned int
__iosapic_read(char __iomem *iosapic, unsigned int reg)
{
return pv_iosapic_ops.__read(iosapic, reg);
}
static inline void
__iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
{
return pv_iosapic_ops.__write(iosapic, reg, val);
}
/******************************************************************************
* replacement of irq operations.
*/
struct pv_irq_ops {
void (*register_ipi)(void);
int (*assign_irq_vector)(int irq);
void (*free_irq_vector)(int vector);
void (*register_percpu_irq)(ia64_vector vec,
struct irqaction *action);
void (*resend_irq)(unsigned int vector);
};
extern struct pv_irq_ops pv_irq_ops;
static inline void
ia64_register_ipi(void)
{
pv_irq_ops.register_ipi();
}
static inline int
assign_irq_vector(int irq)
{
return pv_irq_ops.assign_irq_vector(irq);
}
static inline void
free_irq_vector(int vector)
{
return pv_irq_ops.free_irq_vector(vector);
}
static inline void
register_percpu_irq(ia64_vector vec, struct irqaction *action)
{
pv_irq_ops.register_percpu_irq(vec, action);
}
static inline void
ia64_resend_irq(unsigned int vector)
{
pv_irq_ops.resend_irq(vector);
}
/******************************************************************************
* replacement of time operations.
*/
extern struct itc_jitter_data_t itc_jitter_data;
extern volatile int time_keeper_id;
struct pv_time_ops {
void (*init_missing_ticks_accounting)(int cpu);
int (*do_steal_accounting)(unsigned long *new_itm);
void (*clocksource_resume)(void);
unsigned long long (*sched_clock)(void);
};
extern struct pv_time_ops pv_time_ops;
static inline void
paravirt_init_missing_ticks_accounting(int cpu)
{
if (pv_time_ops.init_missing_ticks_accounting)
pv_time_ops.init_missing_ticks_accounting(cpu);
}
struct static_key;
extern struct static_key paravirt_steal_enabled;
extern struct static_key paravirt_steal_rq_enabled;
static inline int
paravirt_do_steal_accounting(unsigned long *new_itm)
{
return pv_time_ops.do_steal_accounting(new_itm);
}
static inline unsigned long long paravirt_sched_clock(void)
{
return pv_time_ops.sched_clock();
}
#endif /* !__ASSEMBLY__ */
#else
/* fallback for native case */
#ifndef __ASSEMBLY__
#define paravirt_banner() do { } while (0)
#define paravirt_reserve_memory(region) 0
#define paravirt_arch_setup_early() do { } while (0)
#define paravirt_arch_setup_console(cmdline_p) do { } while (0)
#define paravirt_arch_setup_nomca() 0
#define paravirt_post_smp_prepare_boot_cpu() do { } while (0)
#define paravirt_init_missing_ticks_accounting(cpu) do { } while (0)
#define paravirt_do_steal_accounting(new_itm) 0
#endif /* __ASSEMBLY__ */
#endif /* CONFIG_PARAVIRT_GUEST */
#endif /* __ASM_PARAVIRT_H */