mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
percpu: set PCPU_BITMAP_BLOCK_SIZE to PAGE_SIZE
Previously, block size was flexible based on the constraint that the GCD(PCPU_BITMAP_BLOCK_SIZE, PAGE_SIZE) > 1. However, this carried the overhead that keeping a floating number of populated free pages required scanning over the free regions of a chunk. Setting the block size to be fixed at PAGE_SIZE lets us know when an empty page becomes used as we will break a full contig_hint of a block. This means we no longer have to scan the whole chunk upon breaking a contig_hint which empty page management piggybacked off. A later patch takes advantage of this to optimize the allocation path by only scanning forward using the scan_hint introduced later too. Signed-off-by: Dennis Zhou <dennis@kernel.org> Reviewed-by: Peng Fan <peng.fan@nxp.com>
This commit is contained in:
parent
8744d85942
commit
b239f7daf5
|
@ -26,16 +26,10 @@
|
|||
#define PCPU_MIN_ALLOC_SHIFT 2
|
||||
#define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT)
|
||||
|
||||
/* number of bits per page, used to trigger a scan if blocks are > PAGE_SIZE */
|
||||
#define PCPU_BITS_PER_PAGE (PAGE_SIZE >> PCPU_MIN_ALLOC_SHIFT)
|
||||
|
||||
/*
|
||||
* This determines the size of each metadata block. There are several subtle
|
||||
* constraints around this constant. The reserved region must be a multiple of
|
||||
* PCPU_BITMAP_BLOCK_SIZE. Additionally, PCPU_BITMAP_BLOCK_SIZE must be a
|
||||
* multiple of PAGE_SIZE or PAGE_SIZE must be a multiple of
|
||||
* PCPU_BITMAP_BLOCK_SIZE to align with the populated page map. The unit_size
|
||||
* also has to be a multiple of PCPU_BITMAP_BLOCK_SIZE to ensure full blocks.
|
||||
* The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the
|
||||
* updating of hints is used to manage the nr_empty_pop_pages in both
|
||||
* the chunk and globally.
|
||||
*/
|
||||
#define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE
|
||||
#define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \
|
||||
|
|
|
@ -70,7 +70,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
|
|||
chunk->base_addr = page_address(pages);
|
||||
|
||||
spin_lock_irqsave(&pcpu_lock, flags);
|
||||
pcpu_chunk_populated(chunk, 0, nr_pages, false);
|
||||
pcpu_chunk_populated(chunk, 0, nr_pages);
|
||||
spin_unlock_irqrestore(&pcpu_lock, flags);
|
||||
|
||||
pcpu_stats_chunk_alloc();
|
||||
|
|
110
mm/percpu.c
110
mm/percpu.c
|
@ -527,37 +527,20 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
|
|||
__pcpu_chunk_move(chunk, nslot, oslot < nslot);
|
||||
}
|
||||
|
||||
/**
|
||||
* pcpu_cnt_pop_pages- counts populated backing pages in range
|
||||
/*
|
||||
* pcpu_update_empty_pages - update empty page counters
|
||||
* @chunk: chunk of interest
|
||||
* @bit_off: start offset
|
||||
* @bits: size of area to check
|
||||
* @nr: nr of empty pages
|
||||
*
|
||||
* Calculates the number of populated pages in the region
|
||||
* [page_start, page_end). This keeps track of how many empty populated
|
||||
* pages are available and decide if async work should be scheduled.
|
||||
*
|
||||
* RETURNS:
|
||||
* The nr of populated pages.
|
||||
* This is used to keep track of the empty pages now based on the premise
|
||||
* a md_block covers a page. The hint update functions recognize if a block
|
||||
* is made full or broken to calculate deltas for keeping track of free pages.
|
||||
*/
|
||||
static inline int pcpu_cnt_pop_pages(struct pcpu_chunk *chunk, int bit_off,
|
||||
int bits)
|
||||
static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
|
||||
{
|
||||
int page_start = PFN_UP(bit_off * PCPU_MIN_ALLOC_SIZE);
|
||||
int page_end = PFN_DOWN((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);
|
||||
|
||||
if (page_start >= page_end)
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* bitmap_weight counts the number of bits set in a bitmap up to
|
||||
* the specified number of bits. This is counting the populated
|
||||
* pages up to page_end and then subtracting the populated pages
|
||||
* up to page_start to count the populated pages in
|
||||
* [page_start, page_end).
|
||||
*/
|
||||
return bitmap_weight(chunk->populated, page_end) -
|
||||
bitmap_weight(chunk->populated, page_start);
|
||||
chunk->nr_empty_pop_pages += nr;
|
||||
if (chunk != pcpu_reserved_chunk)
|
||||
pcpu_nr_empty_pop_pages += nr;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -608,36 +591,19 @@ static void pcpu_chunk_update(struct pcpu_chunk *chunk, int bit_off, int bits)
|
|||
* Updates:
|
||||
* chunk->contig_bits
|
||||
* chunk->contig_bits_start
|
||||
* nr_empty_pop_pages (chunk and global)
|
||||
*/
|
||||
static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)
|
||||
{
|
||||
int bit_off, bits, nr_empty_pop_pages;
|
||||
int bit_off, bits;
|
||||
|
||||
/* clear metadata */
|
||||
chunk->contig_bits = 0;
|
||||
|
||||
bit_off = chunk->first_bit;
|
||||
bits = nr_empty_pop_pages = 0;
|
||||
bits = 0;
|
||||
pcpu_for_each_md_free_region(chunk, bit_off, bits) {
|
||||
pcpu_chunk_update(chunk, bit_off, bits);
|
||||
|
||||
nr_empty_pop_pages += pcpu_cnt_pop_pages(chunk, bit_off, bits);
|
||||
}
|
||||
|
||||
/*
|
||||
* Keep track of nr_empty_pop_pages.
|
||||
*
|
||||
* The chunk maintains the previous number of free pages it held,
|
||||
* so the delta is used to update the global counter. The reserved
|
||||
* chunk is not part of the free page count as they are populated
|
||||
* at init and are special to serving reserved allocations.
|
||||
*/
|
||||
if (chunk != pcpu_reserved_chunk)
|
||||
pcpu_nr_empty_pop_pages +=
|
||||
(nr_empty_pop_pages - chunk->nr_empty_pop_pages);
|
||||
|
||||
chunk->nr_empty_pop_pages = nr_empty_pop_pages;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -709,6 +675,7 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index)
|
|||
static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
||||
int bits)
|
||||
{
|
||||
int nr_empty_pages = 0;
|
||||
struct pcpu_block_md *s_block, *e_block, *block;
|
||||
int s_index, e_index; /* block indexes of the freed allocation */
|
||||
int s_off, e_off; /* block offsets of the freed allocation */
|
||||
|
@ -733,6 +700,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
|||
* If the allocation breaks the contig_hint, a scan is required to
|
||||
* restore this hint.
|
||||
*/
|
||||
if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
|
||||
nr_empty_pages++;
|
||||
|
||||
if (s_off == s_block->first_free)
|
||||
s_block->first_free = find_next_zero_bit(
|
||||
pcpu_index_alloc_map(chunk, s_index),
|
||||
|
@ -760,6 +730,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
|||
* Update e_block.
|
||||
*/
|
||||
if (s_index != e_index) {
|
||||
if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
|
||||
nr_empty_pages++;
|
||||
|
||||
/*
|
||||
* When the allocation is across blocks, the end is along
|
||||
* the left part of the e_block.
|
||||
|
@ -784,6 +757,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
|||
}
|
||||
|
||||
/* update in-between md_blocks */
|
||||
nr_empty_pages += (e_index - s_index - 1);
|
||||
for (block = s_block + 1; block < e_block; block++) {
|
||||
block->contig_hint = 0;
|
||||
block->left_free = 0;
|
||||
|
@ -791,6 +765,9 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
|||
}
|
||||
}
|
||||
|
||||
if (nr_empty_pages)
|
||||
pcpu_update_empty_pages(chunk, -nr_empty_pages);
|
||||
|
||||
/*
|
||||
* The only time a full chunk scan is required is if the chunk
|
||||
* contig hint is broken. Otherwise, it means a smaller space
|
||||
|
@ -823,6 +800,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
|
|||
static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
|
||||
int bits)
|
||||
{
|
||||
int nr_empty_pages = 0;
|
||||
struct pcpu_block_md *s_block, *e_block, *block;
|
||||
int s_index, e_index; /* block indexes of the freed allocation */
|
||||
int s_off, e_off; /* block offsets of the freed allocation */
|
||||
|
@ -876,14 +854,19 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
|
|||
|
||||
/* update s_block */
|
||||
e_off = (s_index == e_index) ? end : PCPU_BITMAP_BLOCK_BITS;
|
||||
if (!start && e_off == PCPU_BITMAP_BLOCK_BITS)
|
||||
nr_empty_pages++;
|
||||
pcpu_block_update(s_block, start, e_off);
|
||||
|
||||
/* freeing in the same block */
|
||||
if (s_index != e_index) {
|
||||
/* update e_block */
|
||||
if (end == PCPU_BITMAP_BLOCK_BITS)
|
||||
nr_empty_pages++;
|
||||
pcpu_block_update(e_block, 0, end);
|
||||
|
||||
/* reset md_blocks in the middle */
|
||||
nr_empty_pages += (e_index - s_index - 1);
|
||||
for (block = s_block + 1; block < e_block; block++) {
|
||||
block->first_free = 0;
|
||||
block->contig_hint_start = 0;
|
||||
|
@ -893,15 +876,16 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
|
|||
}
|
||||
}
|
||||
|
||||
if (nr_empty_pages)
|
||||
pcpu_update_empty_pages(chunk, nr_empty_pages);
|
||||
|
||||
/*
|
||||
* Refresh chunk metadata when the free makes a page free, a block
|
||||
* free, or spans across blocks. The contig hint may be off by up to
|
||||
* a page, but if the hint is contained in a block, it will be accurate
|
||||
* with the else condition below.
|
||||
* Refresh chunk metadata when the free makes a block free or spans
|
||||
* across blocks. The contig_hint may be off by up to a page, but if
|
||||
* the contig_hint is contained in a block, it will be accurate with
|
||||
* the else condition below.
|
||||
*/
|
||||
if ((ALIGN_DOWN(end, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS)) >
|
||||
ALIGN(start, min(PCPU_BITS_PER_PAGE, PCPU_BITMAP_BLOCK_BITS))) ||
|
||||
s_index != e_index)
|
||||
if (((end - start) >= PCPU_BITMAP_BLOCK_BITS) || s_index != e_index)
|
||||
pcpu_chunk_refresh_hint(chunk);
|
||||
else
|
||||
pcpu_chunk_update(chunk, pcpu_block_off_to_off(s_index, start),
|
||||
|
@ -1178,9 +1162,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
|
|||
chunk->immutable = true;
|
||||
bitmap_fill(chunk->populated, chunk->nr_pages);
|
||||
chunk->nr_populated = chunk->nr_pages;
|
||||
chunk->nr_empty_pop_pages =
|
||||
pcpu_cnt_pop_pages(chunk, start_offset / PCPU_MIN_ALLOC_SIZE,
|
||||
map_size / PCPU_MIN_ALLOC_SIZE);
|
||||
chunk->nr_empty_pop_pages = chunk->nr_pages;
|
||||
|
||||
chunk->contig_bits = map_size / PCPU_MIN_ALLOC_SIZE;
|
||||
chunk->free_bytes = map_size;
|
||||
|
@ -1275,7 +1257,6 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk)
|
|||
* @chunk: pcpu_chunk which got populated
|
||||
* @page_start: the start page
|
||||
* @page_end: the end page
|
||||
* @for_alloc: if this is to populate for allocation
|
||||
*
|
||||
* Pages in [@page_start,@page_end) have been populated to @chunk. Update
|
||||
* the bookkeeping information accordingly. Must be called after each
|
||||
|
@ -1285,7 +1266,7 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk)
|
|||
* is to serve an allocation in that area.
|
||||
*/
|
||||
static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
|
||||
int page_end, bool for_alloc)
|
||||
int page_end)
|
||||
{
|
||||
int nr = page_end - page_start;
|
||||
|
||||
|
@ -1295,10 +1276,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk, int page_start,
|
|||
chunk->nr_populated += nr;
|
||||
pcpu_nr_populated += nr;
|
||||
|
||||
if (!for_alloc) {
|
||||
chunk->nr_empty_pop_pages += nr;
|
||||
pcpu_nr_empty_pop_pages += nr;
|
||||
}
|
||||
pcpu_update_empty_pages(chunk, nr);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1320,9 +1298,9 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
|
|||
|
||||
bitmap_clear(chunk->populated, page_start, nr);
|
||||
chunk->nr_populated -= nr;
|
||||
chunk->nr_empty_pop_pages -= nr;
|
||||
pcpu_nr_empty_pop_pages -= nr;
|
||||
pcpu_nr_populated -= nr;
|
||||
|
||||
pcpu_update_empty_pages(chunk, -nr);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1537,7 +1515,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
|
|||
err = "failed to populate";
|
||||
goto fail_unlock;
|
||||
}
|
||||
pcpu_chunk_populated(chunk, rs, re, true);
|
||||
pcpu_chunk_populated(chunk, rs, re);
|
||||
spin_unlock_irqrestore(&pcpu_lock, flags);
|
||||
}
|
||||
|
||||
|
@ -1736,7 +1714,7 @@ static void pcpu_balance_workfn(struct work_struct *work)
|
|||
if (!ret) {
|
||||
nr_to_pop -= nr;
|
||||
spin_lock_irq(&pcpu_lock);
|
||||
pcpu_chunk_populated(chunk, rs, rs + nr, false);
|
||||
pcpu_chunk_populated(chunk, rs, rs + nr);
|
||||
spin_unlock_irq(&pcpu_lock);
|
||||
} else {
|
||||
nr_to_pop = 0;
|
||||
|
|
Loading…
Reference in a new issue