mirror of
https://github.com/torvalds/linux
synced 2024-09-06 09:51:23 +00:00
mm/slub: refactor deactivate_slab()
Simplify deactivate_slab() by unlocking n->list_lock and retrying cmpxchg_double() when cmpxchg_double() fails, and perform add_{partial,full} only when it succeed. Releasing and taking n->list_lock again here is not harmful as SLUB avoids deactivating slabs as much as possible. [ vbabka@suse.cz: perform add_{partial,full} when cmpxchg_double() succeed. count deactivating full slabs even if debugging flag is not set. ] Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Reviewed-by: Vlastimil Babka <vbabka@suse.cz> Reviewed-by: Roman Gushchin <roman.gushchin@linux.dev> Signed-off-by: Vlastimil Babka <vbabka@suse.cz> Link: https://lore.kernel.org/r/20220307074057.902222-3-42.hyeyoo@gmail.com
This commit is contained in:
parent
5182f3c918
commit
6d3a16d09b
91
mm/slub.c
91
mm/slub.c
|
@ -2348,10 +2348,10 @@ static void init_kmem_cache_cpus(struct kmem_cache *s)
|
|||
static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
|
||||
void *freelist)
|
||||
{
|
||||
enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
|
||||
enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE, M_FULL_NOLIST };
|
||||
struct kmem_cache_node *n = get_node(s, slab_nid(slab));
|
||||
int lock = 0, free_delta = 0;
|
||||
enum slab_modes l = M_NONE, m = M_NONE;
|
||||
int free_delta = 0;
|
||||
enum slab_modes mode = M_NONE;
|
||||
void *nextfree, *freelist_iter, *freelist_tail;
|
||||
int tail = DEACTIVATE_TO_HEAD;
|
||||
unsigned long flags = 0;
|
||||
|
@ -2393,14 +2393,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
|
|||
* Ensure that the slab is unfrozen while the list presence
|
||||
* reflects the actual number of objects during unfreeze.
|
||||
*
|
||||
* We setup the list membership and then perform a cmpxchg
|
||||
* with the count. If there is a mismatch then the slab
|
||||
* is not unfrozen but the slab is on the wrong list.
|
||||
*
|
||||
* Then we restart the process which may have to remove
|
||||
* the slab from the list that we just put it on again
|
||||
* because the number of objects in the slab may have
|
||||
* changed.
|
||||
* We first perform cmpxchg holding lock and insert to list
|
||||
* when it succeed. If there is mismatch then the slab is not
|
||||
* unfrozen and number of objects in the slab may have changed.
|
||||
* Then release lock and retry cmpxchg again.
|
||||
*/
|
||||
redo:
|
||||
|
||||
|
@ -2419,61 +2415,52 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
|
|||
|
||||
new.frozen = 0;
|
||||
|
||||
if (!new.inuse && n->nr_partial >= s->min_partial)
|
||||
m = M_FREE;
|
||||
else if (new.freelist) {
|
||||
m = M_PARTIAL;
|
||||
if (!lock) {
|
||||
lock = 1;
|
||||
/*
|
||||
* Taking the spinlock removes the possibility that
|
||||
* acquire_slab() will see a slab that is frozen
|
||||
*/
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
}
|
||||
if (!new.inuse && n->nr_partial >= s->min_partial) {
|
||||
mode = M_FREE;
|
||||
} else if (new.freelist) {
|
||||
mode = M_PARTIAL;
|
||||
/*
|
||||
* Taking the spinlock removes the possibility that
|
||||
* acquire_slab() will see a slab that is frozen
|
||||
*/
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
} else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
|
||||
mode = M_FULL;
|
||||
/*
|
||||
* This also ensures that the scanning of full
|
||||
* slabs from diagnostic functions will not see
|
||||
* any frozen slabs.
|
||||
*/
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
} else {
|
||||
m = M_FULL;
|
||||
if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {
|
||||
lock = 1;
|
||||
/*
|
||||
* This also ensures that the scanning of full
|
||||
* slabs from diagnostic functions will not see
|
||||
* any frozen slabs.
|
||||
*/
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
}
|
||||
mode = M_FULL_NOLIST;
|
||||
}
|
||||
|
||||
if (l != m) {
|
||||
if (l == M_PARTIAL)
|
||||
remove_partial(n, slab);
|
||||
else if (l == M_FULL)
|
||||
remove_full(s, n, slab);
|
||||
|
||||
if (m == M_PARTIAL)
|
||||
add_partial(n, slab, tail);
|
||||
else if (m == M_FULL)
|
||||
add_full(s, n, slab);
|
||||
}
|
||||
|
||||
l = m;
|
||||
if (!cmpxchg_double_slab(s, slab,
|
||||
old.freelist, old.counters,
|
||||
new.freelist, new.counters,
|
||||
"unfreezing slab"))
|
||||
"unfreezing slab")) {
|
||||
if (mode == M_PARTIAL || mode == M_FULL)
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
goto redo;
|
||||
}
|
||||
|
||||
if (lock)
|
||||
|
||||
if (mode == M_PARTIAL) {
|
||||
add_partial(n, slab, tail);
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
|
||||
if (m == M_PARTIAL)
|
||||
stat(s, tail);
|
||||
else if (m == M_FULL)
|
||||
stat(s, DEACTIVATE_FULL);
|
||||
else if (m == M_FREE) {
|
||||
} else if (mode == M_FREE) {
|
||||
stat(s, DEACTIVATE_EMPTY);
|
||||
discard_slab(s, slab);
|
||||
stat(s, FREE_SLAB);
|
||||
} else if (mode == M_FULL) {
|
||||
add_full(s, n, slab);
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
stat(s, DEACTIVATE_FULL);
|
||||
} else if (mode == M_FULL_NOLIST) {
|
||||
stat(s, DEACTIVATE_FULL);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue