mirror of
https://github.com/torvalds/linux
synced 2024-09-06 09:51:23 +00:00
mm/slub: perform free consistency checks before call_rcu
For SLAB_TYPESAFE_BY_RCU caches we use call_rcu to perform empty slab freeing. The rcu callback rcu_free_slab() calls __free_slab() that currently includes checking the slab consistency for caches with SLAB_CONSISTENCY_CHECKS flags. This check needs the slab->objects field to be intact. Because in the next patch we want to allow rcu_head in struct slab to become larger in debug configurations and thus potentially overwrite more fields through a union than slab_list, we want to limit the fields used in rcu_free_slab(). Thus move the consistency checks to free_slab() before call_rcu(). This can be done safely even for SLAB_TYPESAFE_BY_RCU caches where accesses to the objects can still occur after freeing them. As a result, only the slab->slab_cache field has to be physically separate from rcu_head for the freeing callback to work. We also save some cycles in the rcu callback for caches with consistency checks enabled. Signed-off-by: Vlastimil Babka <vbabka@suse.cz> Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
This commit is contained in:
parent
247f34f7b8
commit
bc29d5bd2b
20
mm/slub.c
20
mm/slub.c
|
@ -1999,14 +1999,6 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)
|
|||
int order = folio_order(folio);
|
||||
int pages = 1 << order;
|
||||
|
||||
if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
|
||||
void *p;
|
||||
|
||||
slab_pad_check(s, slab);
|
||||
for_each_object(p, s, slab_address(slab), slab->objects)
|
||||
check_object(s, slab, p, SLUB_RED_INACTIVE);
|
||||
}
|
||||
|
||||
__slab_clear_pfmemalloc(slab);
|
||||
__folio_clear_slab(folio);
|
||||
folio->mapping = NULL;
|
||||
|
@ -2025,9 +2017,17 @@ static void rcu_free_slab(struct rcu_head *h)
|
|||
|
||||
static void free_slab(struct kmem_cache *s, struct slab *slab)
|
||||
{
|
||||
if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
|
||||
if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
|
||||
void *p;
|
||||
|
||||
slab_pad_check(s, slab);
|
||||
for_each_object(p, s, slab_address(slab), slab->objects)
|
||||
check_object(s, slab, p, SLUB_RED_INACTIVE);
|
||||
}
|
||||
|
||||
if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
|
||||
call_rcu(&slab->rcu_head, rcu_free_slab);
|
||||
} else
|
||||
else
|
||||
__free_slab(s, slab);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue