mirror of
https://github.com/torvalds/linux
synced 2024-09-06 09:51:23 +00:00
![Ming Lei](/assets/img/avatar_default.png)
sg_alloc_table_chained() currently allows the caller to provide one preallocated SGL and returns if the requested number isn't bigger than size of that SGL. This is used to inline an SGL for an IO request. However, scattergather code only allows that size of the 1st preallocated SGL to be SG_CHUNK_SIZE(128). This means a substantial amount of memory (4KB) is claimed for the SGL for each IO request. If the I/O is small, it would be prudent to allocate a smaller SGL. Introduce an extra parameter to sg_alloc_table_chained() and sg_free_table_chained() for specifying size of the preallocated SGL. Both __sg_free_table() and __sg_alloc_table() assume that each SGL has the same size except for the last one. Change the code to allow both functions to accept a variable size for the 1st preallocated SGL. [mkp: attempted to clarify commit desc] Cc: Christoph Hellwig <hch@lst.de> Cc: Bart Van Assche <bvanassche@acm.org> Cc: Ewan D. Milne <emilne@redhat.com> Cc: Hannes Reinecke <hare@suse.com> Cc: Sagi Grimberg <sagi@grimberg.me> Cc: Chuck Lever <chuck.lever@oracle.com> Cc: netdev@vger.kernel.org Cc: linux-nvme@lists.infradead.org Suggested-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Ming Lei <ming.lei@redhat.com> Reviewed-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
189 lines
4.1 KiB
C
189 lines
4.1 KiB
C
#include <linux/module.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/mempool.h>
|
|
#include <linux/slab.h>
|
|
|
|
#define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools)
|
|
#define SG_MEMPOOL_SIZE 2
|
|
|
|
struct sg_pool {
|
|
size_t size;
|
|
char *name;
|
|
struct kmem_cache *slab;
|
|
mempool_t *pool;
|
|
};
|
|
|
|
#define SP(x) { .size = x, "sgpool-" __stringify(x) }
|
|
#if (SG_CHUNK_SIZE < 32)
|
|
#error SG_CHUNK_SIZE is too small (must be 32 or greater)
|
|
#endif
|
|
static struct sg_pool sg_pools[] = {
|
|
SP(8),
|
|
SP(16),
|
|
#if (SG_CHUNK_SIZE > 32)
|
|
SP(32),
|
|
#if (SG_CHUNK_SIZE > 64)
|
|
SP(64),
|
|
#if (SG_CHUNK_SIZE > 128)
|
|
SP(128),
|
|
#if (SG_CHUNK_SIZE > 256)
|
|
#error SG_CHUNK_SIZE is too large (256 MAX)
|
|
#endif
|
|
#endif
|
|
#endif
|
|
#endif
|
|
SP(SG_CHUNK_SIZE)
|
|
};
|
|
#undef SP
|
|
|
|
static inline unsigned int sg_pool_index(unsigned short nents)
|
|
{
|
|
unsigned int index;
|
|
|
|
BUG_ON(nents > SG_CHUNK_SIZE);
|
|
|
|
if (nents <= 8)
|
|
index = 0;
|
|
else
|
|
index = get_count_order(nents) - 3;
|
|
|
|
return index;
|
|
}
|
|
|
|
static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
|
|
{
|
|
struct sg_pool *sgp;
|
|
|
|
sgp = sg_pools + sg_pool_index(nents);
|
|
mempool_free(sgl, sgp->pool);
|
|
}
|
|
|
|
static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
|
|
{
|
|
struct sg_pool *sgp;
|
|
|
|
sgp = sg_pools + sg_pool_index(nents);
|
|
return mempool_alloc(sgp->pool, gfp_mask);
|
|
}
|
|
|
|
/**
|
|
* sg_free_table_chained - Free a previously mapped sg table
|
|
* @table: The sg table header to use
|
|
* @nents_first_chunk: size of the first_chunk SGL passed to
|
|
* sg_alloc_table_chained
|
|
*
|
|
* Description:
|
|
* Free an sg table previously allocated and setup with
|
|
* sg_alloc_table_chained().
|
|
*
|
|
* @nents_first_chunk has to be same with that same parameter passed
|
|
* to sg_alloc_table_chained().
|
|
*
|
|
**/
|
|
void sg_free_table_chained(struct sg_table *table,
|
|
unsigned nents_first_chunk)
|
|
{
|
|
if (table->orig_nents <= nents_first_chunk)
|
|
return;
|
|
|
|
if (nents_first_chunk == 1)
|
|
nents_first_chunk = 0;
|
|
|
|
__sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free);
|
|
}
|
|
EXPORT_SYMBOL_GPL(sg_free_table_chained);
|
|
|
|
/**
|
|
* sg_alloc_table_chained - Allocate and chain SGLs in an sg table
|
|
* @table: The sg table header to use
|
|
* @nents: Number of entries in sg list
|
|
* @first_chunk: first SGL
|
|
* @nents_first_chunk: number of the SGL of @first_chunk
|
|
*
|
|
* Description:
|
|
* Allocate and chain SGLs in an sg table. If @nents@ is larger than
|
|
* @nents_first_chunk a chained sg table will be setup.
|
|
*
|
|
**/
|
|
int sg_alloc_table_chained(struct sg_table *table, int nents,
|
|
struct scatterlist *first_chunk, unsigned nents_first_chunk)
|
|
{
|
|
int ret;
|
|
|
|
BUG_ON(!nents);
|
|
|
|
if (first_chunk && nents_first_chunk) {
|
|
if (nents <= nents_first_chunk) {
|
|
table->nents = table->orig_nents = nents;
|
|
sg_init_table(table->sgl, nents);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/* User supposes that the 1st SGL includes real entry */
|
|
if (nents_first_chunk == 1) {
|
|
first_chunk = NULL;
|
|
nents_first_chunk = 0;
|
|
}
|
|
|
|
ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
|
|
first_chunk, nents_first_chunk,
|
|
GFP_ATOMIC, sg_pool_alloc);
|
|
if (unlikely(ret))
|
|
sg_free_table_chained(table, nents_first_chunk);
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
|
|
|
|
static __init int sg_pool_init(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < SG_MEMPOOL_NR; i++) {
|
|
struct sg_pool *sgp = sg_pools + i;
|
|
int size = sgp->size * sizeof(struct scatterlist);
|
|
|
|
sgp->slab = kmem_cache_create(sgp->name, size, 0,
|
|
SLAB_HWCACHE_ALIGN, NULL);
|
|
if (!sgp->slab) {
|
|
printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
|
|
sgp->name);
|
|
goto cleanup_sdb;
|
|
}
|
|
|
|
sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
|
|
sgp->slab);
|
|
if (!sgp->pool) {
|
|
printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
|
|
sgp->name);
|
|
goto cleanup_sdb;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
|
|
cleanup_sdb:
|
|
for (i = 0; i < SG_MEMPOOL_NR; i++) {
|
|
struct sg_pool *sgp = sg_pools + i;
|
|
|
|
mempool_destroy(sgp->pool);
|
|
kmem_cache_destroy(sgp->slab);
|
|
}
|
|
|
|
return -ENOMEM;
|
|
}
|
|
|
|
static __exit void sg_pool_exit(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < SG_MEMPOOL_NR; i++) {
|
|
struct sg_pool *sgp = sg_pools + i;
|
|
mempool_destroy(sgp->pool);
|
|
kmem_cache_destroy(sgp->slab);
|
|
}
|
|
}
|
|
|
|
module_init(sg_pool_init);
|
|
module_exit(sg_pool_exit);
|