From c8576f3e612d3c5b01d434ae296b9b76d7907708 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:16 +0000 Subject: io_uring: refactor req allocation Follow the io_get_sqe pattern returning the result via a pointer and hide request cache refill inside io_alloc_req(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8c37c2e8a3cb5e4cd6a8ae3b91371227a92708a6.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 7 +++---- io_uring/io_uring.h | 19 +++++++++++-------- io_uring/notif.c | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) (limited to 'io_uring') diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c1f27626c205..2b046a80f75b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2415,9 +2415,8 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) const struct io_uring_sqe *sqe; struct io_kiocb *req; - if (unlikely(!io_alloc_req_refill(ctx))) + if (unlikely(!io_alloc_req(ctx, &req))) break; - req = io_alloc_req(ctx); if (unlikely(!io_get_sqe(ctx, &sqe))) { io_req_add_to_cache(req, ctx); break; @@ -2736,14 +2735,14 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) static void io_req_caches_free(struct io_ring_ctx *ctx) { + struct io_kiocb *req; int nr = 0; mutex_lock(&ctx->uring_lock); io_flush_cached_locked_reqs(ctx, &ctx->submit_state); while (!io_req_cache_empty(ctx)) { - struct io_kiocb *req = io_alloc_req(ctx); - + req = io_extract_req(ctx); kmem_cache_free(req_cachep, req); nr++; } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 9270156288aa..f90816aac896 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -341,16 +341,9 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) return !ctx->submit_state.free_list.next; } -static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx) -{ - if (unlikely(io_req_cache_empty(ctx))) - return __io_alloc_req_refill(ctx); - return true; -} - extern struct kmem_cache *req_cachep; -static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) { struct io_kiocb *req; @@ -360,6 +353,16 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return req; } +static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) +{ + if (unlikely(io_req_cache_empty(ctx))) { + if (!__io_alloc_req_refill(ctx)) + return false; + } + *req = io_extract_req(ctx); + return true; +} + static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) { return likely(ctx->submitter_task == current); diff --git a/io_uring/notif.c b/io_uring/notif.c index c4bb793ebf0e..09dfd0832d19 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -68,9 +68,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) struct io_kiocb *notif; struct io_notif_data *nd; - if (unlikely(!io_alloc_req_refill(ctx))) + if (unlikely(!io_alloc_req(ctx, ¬if))) return NULL; - notif = io_alloc_req(ctx); notif->opcode = IORING_OP_NOP; notif->flags = 0; notif->file = NULL; -- cgit v1.2.3