From 453b329be5eacfc48dd43035af82bc7f28ecfedf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 24 May 2022 21:43:10 -0600 Subject: [PATCH] io_uring: separate out file table handling code Signed-off-by: Jens Axboe --- io_uring/Makefile | 2 +- io_uring/filetable.c | 57 ++++++++++++++++++++++++++ io_uring/filetable.h | 58 ++++++++++++++++++++++++++ io_uring/io_uring.c | 86 --------------------------------------- io_uring/io_uring_types.h | 7 +--- 5 files changed, 117 insertions(+), 93 deletions(-) create mode 100644 io_uring/filetable.c create mode 100644 io_uring/filetable.h diff --git a/io_uring/Makefile b/io_uring/Makefile index 4492aa24397e..5efc4fe565a1 100644 --- a/io_uring/Makefile +++ b/io_uring/Makefile @@ -3,5 +3,5 @@ # Makefile for io_uring obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ - sync.o advise.o + sync.o advise.o filetable.o obj-$(CONFIG_IO_WQ) += io-wq.o diff --git a/io_uring/filetable.c b/io_uring/filetable.c new file mode 100644 index 000000000000..560629a93c04 --- /dev/null +++ b/io_uring/filetable.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include + +#include "io_uring_types.h" +#include "io_uring.h" + +int io_file_bitmap_get(struct io_ring_ctx *ctx) +{ + struct io_file_table *table = &ctx->file_table; + unsigned long nr = ctx->nr_user_files; + int ret; + + do { + ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); + if (ret != nr) + return ret; + + if (!table->alloc_hint) + break; + + nr = table->alloc_hint; + table->alloc_hint = 0; + } while (1); + + return -ENFILE; +} + +bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) +{ + table->files = kvcalloc(nr_files, sizeof(table->files[0]), + GFP_KERNEL_ACCOUNT); + if (unlikely(!table->files)) + return false; + + table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); + if (unlikely(!table->bitmap)) { + kvfree(table->files); + return false; + } + + return true; +} + +void io_free_file_tables(struct io_file_table *table) +{ + kvfree(table->files); + bitmap_free(table->bitmap); + table->files = NULL; + table->bitmap = NULL; +} diff --git a/io_uring/filetable.h b/io_uring/filetable.h new file mode 100644 index 000000000000..fe1ec581958d --- /dev/null +++ b/io_uring/filetable.h @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef IOU_FILE_TABLE_H +#define IOU_FILE_TABLE_H + +struct io_ring_ctx; + +/* + * FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0 + * and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we + * can't safely always dereference the file when the task has exited and ring + * cleanup is done. If a file is tracked and part of SCM, then unix gc on + * process exit may reap it before __io_sqe_files_unregister() is run. + */ +#define FFS_NOWAIT 0x1UL +#define FFS_ISREG 0x2UL +#if defined(CONFIG_64BIT) +#define FFS_SCM 0x4UL +#else +#define IO_URING_SCM_ALL +#define FFS_SCM 0x0UL +#endif +#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM) + +struct io_fixed_file { + /* file * with additional FFS_* flags */ + unsigned long file_ptr; +}; + +struct io_file_table { + struct io_fixed_file *files; + unsigned long *bitmap; + unsigned int alloc_hint; +}; + +bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files); +void io_free_file_tables(struct io_file_table *table); +int io_file_bitmap_get(struct io_ring_ctx *ctx); + +static inline void io_file_bitmap_clear(struct io_file_table *table, int bit) +{ + __clear_bit(bit, table->bitmap); + table->alloc_hint = bit; +} + +static inline void io_file_bitmap_set(struct io_file_table *table, int bit) +{ + WARN_ON_ONCE(test_bit(bit, table->bitmap)); + __set_bit(bit, table->bitmap); + table->alloc_hint = bit + 1; +} + +static inline struct io_fixed_file * +io_fixed_file_slot(struct io_file_table *table, unsigned i) +{ + return &table->files[i]; +} + +#endif diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c2041fb10aa2..4b4d6fd509d1 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -146,28 +146,6 @@ struct io_overflow_cqe { struct io_uring_cqe cqe; }; -/* - * FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0 - * and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we - * can't safely always dereference the file when the task has exited and ring - * cleanup is done. If a file is tracked and part of SCM, then unix gc on - * process exit may reap it before __io_sqe_files_unregister() is run. - */ -#define FFS_NOWAIT 0x1UL -#define FFS_ISREG 0x2UL -#if defined(CONFIG_64BIT) -#define FFS_SCM 0x4UL -#else -#define IO_URING_SCM_ALL -#define FFS_SCM 0x0UL -#endif -#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM) - -struct io_fixed_file { - /* file * with additional FFS_* flags */ - unsigned long file_ptr; -}; - struct io_rsrc_put { struct list_head list; u64 tag; @@ -3983,27 +3961,6 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return __io_openat_prep(req, sqe); } -static int io_file_bitmap_get(struct io_ring_ctx *ctx) -{ - struct io_file_table *table = &ctx->file_table; - unsigned long nr = ctx->nr_user_files; - int ret; - - do { - ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); - if (ret != nr) - return ret; - - if (!table->alloc_hint) - break; - - nr = table->alloc_hint; - table->alloc_hint = 0; - } while (1); - - return -ENFILE; -} - /* * Note when io_fixed_fd_install() returns error value, it will ensure * fput() is called correspondingly. @@ -6832,12 +6789,6 @@ static void io_wq_submit_work(struct io_wq_work *work) io_req_task_queue_fail(req, ret); } -static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table, - unsigned i) -{ - return &table->files[i]; -} - static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, int index) { @@ -7934,43 +7885,6 @@ static __cold int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_pu return ret; } -static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) -{ - table->files = kvcalloc(nr_files, sizeof(table->files[0]), - GFP_KERNEL_ACCOUNT); - if (unlikely(!table->files)) - return false; - - table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT); - if (unlikely(!table->bitmap)) { - kvfree(table->files); - return false; - } - - return true; -} - -static void io_free_file_tables(struct io_file_table *table) -{ - kvfree(table->files); - bitmap_free(table->bitmap); - table->files = NULL; - table->bitmap = NULL; -} - -static inline void io_file_bitmap_set(struct io_file_table *table, int bit) -{ - WARN_ON_ONCE(test_bit(bit, table->bitmap)); - __set_bit(bit, table->bitmap); - table->alloc_hint = bit + 1; -} - -static inline void io_file_bitmap_clear(struct io_file_table *table, int bit) -{ - __clear_bit(bit, table->bitmap); - table->alloc_hint = bit; -} - static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) { #if !defined(IO_URING_SCM_ALL) diff --git a/io_uring/io_uring_types.h b/io_uring/io_uring_types.h index 1a0f592ff6fc..dba72113c59d 100644 --- a/io_uring/io_uring_types.h +++ b/io_uring/io_uring_types.h @@ -5,6 +5,7 @@ #include #include "io-wq.h" +#include "filetable.h" struct io_uring { u32 head ____cacheline_aligned_in_smp; @@ -122,12 +123,6 @@ struct io_ev_fd { struct rcu_head rcu; }; -struct io_file_table { - struct io_fixed_file *files; - unsigned long *bitmap; - unsigned int alloc_hint; -}; - struct io_ring_ctx { /* const or read-mostly hot data */ struct {