mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
static_call: Allow early init
In order to use static_call() to wire up x86_pmu, we need to initialize earlier, specifically before memory allocation works; copy some of the tricks from jump_label to enable this. Primarily we overload key->next to store a sites pointer when there are no modules, this avoids having to use kmalloc() to initialize the sites and allows us to run much earlier. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Link: https://lore.kernel.org/r/20200818135805.220737930@infradead.org
This commit is contained in:
parent
6c3fce794e
commit
a945c8345e
|
@ -19,6 +19,7 @@
|
|||
#include <linux/hugetlb.h>
|
||||
#include <linux/tboot.h>
|
||||
#include <linux/usb/xhci-dbgp.h>
|
||||
#include <linux/static_call.h>
|
||||
|
||||
#include <uapi/linux/mount.h>
|
||||
|
||||
|
@ -849,6 +850,7 @@ void __init setup_arch(char **cmdline_p)
|
|||
early_cpu_init();
|
||||
arch_init_ideal_nops();
|
||||
jump_label_init();
|
||||
static_call_init();
|
||||
early_ioremap_init();
|
||||
|
||||
setup_olpc_ofw_pgd();
|
||||
|
|
|
@ -11,7 +11,7 @@ enum insn_type {
|
|||
RET = 3, /* tramp / site cond-tail-call */
|
||||
};
|
||||
|
||||
static void __static_call_transform(void *insn, enum insn_type type, void *func)
|
||||
static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
|
||||
{
|
||||
int size = CALL_INSN_SIZE;
|
||||
const void *code;
|
||||
|
@ -38,6 +38,9 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func)
|
|||
if (memcmp(insn, code, size) == 0)
|
||||
return;
|
||||
|
||||
if (unlikely(system_state == SYSTEM_BOOTING))
|
||||
return text_poke_early(insn, code, size);
|
||||
|
||||
text_poke_bp(insn, code, size, NULL);
|
||||
}
|
||||
|
||||
|
|
|
@ -136,6 +136,8 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool
|
|||
|
||||
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
|
||||
|
||||
extern void __init static_call_init(void);
|
||||
|
||||
struct static_call_mod {
|
||||
struct static_call_mod *next;
|
||||
struct module *mod; /* for vmlinux, mod == NULL */
|
||||
|
@ -144,7 +146,12 @@ struct static_call_mod {
|
|||
|
||||
struct static_call_key {
|
||||
void *func;
|
||||
struct static_call_mod *mods;
|
||||
union {
|
||||
/* bit 0: 0 = mods, 1 = sites */
|
||||
unsigned long type;
|
||||
struct static_call_mod *mods;
|
||||
struct static_call_site *sites;
|
||||
};
|
||||
};
|
||||
|
||||
extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
|
||||
|
@ -155,7 +162,7 @@ extern int static_call_text_reserved(void *start, void *end);
|
|||
DECLARE_STATIC_CALL(name, _func); \
|
||||
struct static_call_key STATIC_CALL_KEY(name) = { \
|
||||
.func = _func, \
|
||||
.mods = NULL, \
|
||||
.type = 1, \
|
||||
}; \
|
||||
ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
|
||||
|
||||
|
@ -180,6 +187,8 @@ extern int static_call_text_reserved(void *start, void *end);
|
|||
|
||||
#elif defined(CONFIG_HAVE_STATIC_CALL)
|
||||
|
||||
static inline void static_call_init(void) { }
|
||||
|
||||
struct static_call_key {
|
||||
void *func;
|
||||
};
|
||||
|
@ -225,6 +234,8 @@ static inline int static_call_text_reserved(void *start, void *end)
|
|||
|
||||
#else /* Generic implementation */
|
||||
|
||||
static inline void static_call_init(void) { }
|
||||
|
||||
struct static_call_key {
|
||||
void *func;
|
||||
};
|
||||
|
|
|
@ -94,10 +94,31 @@ static inline void static_call_sort_entries(struct static_call_site *start,
|
|||
static_call_site_cmp, static_call_site_swap);
|
||||
}
|
||||
|
||||
static inline bool static_call_key_has_mods(struct static_call_key *key)
|
||||
{
|
||||
return !(key->type & 1);
|
||||
}
|
||||
|
||||
static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
|
||||
{
|
||||
if (!static_call_key_has_mods(key))
|
||||
return NULL;
|
||||
|
||||
return key->mods;
|
||||
}
|
||||
|
||||
static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
|
||||
{
|
||||
if (static_call_key_has_mods(key))
|
||||
return NULL;
|
||||
|
||||
return (struct static_call_site *)(key->type & ~1);
|
||||
}
|
||||
|
||||
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
|
||||
{
|
||||
struct static_call_site *site, *stop;
|
||||
struct static_call_mod *site_mod;
|
||||
struct static_call_mod *site_mod, first;
|
||||
|
||||
cpus_read_lock();
|
||||
static_call_lock();
|
||||
|
@ -116,13 +137,22 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
|
|||
if (WARN_ON_ONCE(!static_call_initialized))
|
||||
goto done;
|
||||
|
||||
for (site_mod = key->mods; site_mod; site_mod = site_mod->next) {
|
||||
first = (struct static_call_mod){
|
||||
.next = static_call_key_next(key),
|
||||
.mod = NULL,
|
||||
.sites = static_call_key_sites(key),
|
||||
};
|
||||
|
||||
for (site_mod = &first; site_mod; site_mod = site_mod->next) {
|
||||
struct module *mod = site_mod->mod;
|
||||
|
||||
if (!site_mod->sites) {
|
||||
/*
|
||||
* This can happen if the static call key is defined in
|
||||
* a module which doesn't use it.
|
||||
*
|
||||
* It also happens in the has_mods case, where the
|
||||
* 'first' entry has no sites associated with it.
|
||||
*/
|
||||
continue;
|
||||
}
|
||||
|
@ -192,16 +222,48 @@ static int __static_call_init(struct module *mod,
|
|||
if (key != prev_key) {
|
||||
prev_key = key;
|
||||
|
||||
/*
|
||||
* For vmlinux (!mod) avoid the allocation by storing
|
||||
* the sites pointer in the key itself. Also see
|
||||
* __static_call_update()'s @first.
|
||||
*
|
||||
* This allows architectures (eg. x86) to call
|
||||
* static_call_init() before memory allocation works.
|
||||
*/
|
||||
if (!mod) {
|
||||
key->sites = site;
|
||||
key->type |= 1;
|
||||
goto do_transform;
|
||||
}
|
||||
|
||||
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
||||
if (!site_mod)
|
||||
return -ENOMEM;
|
||||
|
||||
/*
|
||||
* When the key has a direct sites pointer, extract
|
||||
* that into an explicit struct static_call_mod, so we
|
||||
* can have a list of modules.
|
||||
*/
|
||||
if (static_call_key_sites(key)) {
|
||||
site_mod->mod = NULL;
|
||||
site_mod->next = NULL;
|
||||
site_mod->sites = static_call_key_sites(key);
|
||||
|
||||
key->mods = site_mod;
|
||||
|
||||
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
||||
if (!site_mod)
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
site_mod->mod = mod;
|
||||
site_mod->sites = site;
|
||||
site_mod->next = key->mods;
|
||||
site_mod->next = static_call_key_next(key);
|
||||
key->mods = site_mod;
|
||||
}
|
||||
|
||||
do_transform:
|
||||
arch_static_call_transform(site_addr, NULL, key->func,
|
||||
static_call_is_tail(site));
|
||||
}
|
||||
|
@ -348,7 +410,7 @@ int static_call_text_reserved(void *start, void *end)
|
|||
return __static_call_mod_text_reserved(start, end);
|
||||
}
|
||||
|
||||
static void __init static_call_init(void)
|
||||
void __init static_call_init(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
|
|
Loading…
Reference in a new issue