io_uring: convert timeout path to use io_cmd_type

Signed-off-by: Jens Axboe <axboe@kernel.dk>

1 files changed, 68 insertions, 49 deletions

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index ef7b5430bb14..d0daa65f7138 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -979,8 +979,6 @@ struct io_kiocb {
 		 */
 		struct file		*file;
 		struct io_cmd_data	cmd;
-		struct io_timeout	timeout;
-		struct io_timeout_rem	timeout_rem;
 		struct io_open		open;
 		struct io_close		close;
 		struct io_rsrc_update	rsrc_update;
@@ -1652,7 +1650,9 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
 
 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
 {
-	return !req->timeout.off;
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
+
+	return !timeout->off;
 }
 
 static __cold void io_fallback_req_func(struct work_struct *work)
@@ -1898,11 +1898,13 @@ static void io_kill_timeout(struct io_kiocb *req, int status)
 	struct io_timeout_data *io = req->async_data;
 
 	if (hrtimer_try_to_cancel(&io->timer) != -1) {
+		struct io_timeout *timeout = io_kiocb_to_cmd(req);
+
 		if (status)
 			req_set_fail(req);
 		atomic_set(&req->ctx->cq_timeouts,
 			atomic_read(&req->ctx->cq_timeouts) + 1);
-		list_del_init(&req->timeout.list);
+		list_del_init(&timeout->list);
 		io_req_tw_post_queue(req, status, 0);
 	}
 }
@@ -1925,10 +1927,11 @@ static __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
 	__must_hold(&ctx->completion_lock)
 {
 	u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
-	struct io_kiocb *req, *tmp;
+	struct io_timeout *timeout, *tmp;
 
 	spin_lock_irq(&ctx->timeout_lock);
-	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
+	list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
+		struct io_kiocb *req = cmd_to_io_kiocb(timeout);
 		u32 events_needed, events_got;
 
 		if (io_is_timeout_noseq(req))
@@ -1941,7 +1944,7 @@ static __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
 		 * these subtractions won't have wrapped, so we can check if
 		 * target is in [last_seq, current_seq] by comparing the two.
 		 */
-		events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
+		events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
 		events_got = seq - ctx->cq_last_tm_flush;
 		if (events_got < events_needed)
 			break;
@@ -2546,11 +2549,12 @@ static struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req)
 
 	if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
 		struct io_timeout_data *io = link->async_data;
+		struct io_timeout *timeout = io_kiocb_to_cmd(link);
 
 		io_remove_next_linked(req);
-		link->timeout.head = NULL;
+		timeout->head = NULL;
 		if (hrtimer_try_to_cancel(&io->timer) != -1) {
-			list_del(&link->timeout.list);
+			list_del(&timeout->list);
 			return link;
 		}
 	}
@@ -7302,11 +7306,12 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
 	struct io_timeout_data *data = container_of(timer,
 						struct io_timeout_data, timer);
 	struct io_kiocb *req = data->req;
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_ring_ctx *ctx = req->ctx;
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctx->timeout_lock, flags);
-	list_del_init(&req->timeout.list);
+	list_del_init(&timeout->list);
 	atomic_set(&req->ctx->cq_timeouts,
 		atomic_read(&req->ctx->cq_timeouts) + 1);
 	spin_unlock_irqrestore(&ctx->timeout_lock, flags);
@@ -7324,29 +7329,32 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
 					   struct io_cancel_data *cd)
 	__must_hold(&ctx->timeout_lock)
 {
+	struct io_timeout *timeout;
 	struct io_timeout_data *io;
-	struct io_kiocb *req;
-	bool found = false;
+	struct io_kiocb *req = NULL;
+
+	list_for_each_entry(timeout, &ctx->timeout_list, list) {
+		struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
 
-	list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
 		if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
-		    cd->data != req->cqe.user_data)
+		    cd->data != tmp->cqe.user_data)
 			continue;
 		if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
-			if (cd->seq == req->work.cancel_seq)
+			if (cd->seq == tmp->work.cancel_seq)
 				continue;
-			req->work.cancel_seq = cd->seq;
+			tmp->work.cancel_seq = cd->seq;
 		}
-		found = true;
+		req = tmp;
 		break;
 	}
-	if (!found)
+	if (!req)
 		return ERR_PTR(-ENOENT);
 
 	io = req->async_data;
 	if (hrtimer_try_to_cancel(&io->timer) == -1)
 		return ERR_PTR(-EALREADY);
-	list_del_init(&req->timeout.list);
+	timeout = io_kiocb_to_cmd(req);
+	list_del_init(&timeout->list);
 	return req;
 }
 
@@ -7386,15 +7394,18 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
 	__must_hold(&ctx->timeout_lock)
 {
 	struct io_timeout_data *io;
-	struct io_kiocb *req;
-	bool found = false;
+	struct io_timeout *timeout;
+	struct io_kiocb *req = NULL;
+
+	list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
+		struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
 
-	list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {
-		found = user_data == req->cqe.user_data;
-		if (found)
+		if (user_data == tmp->cqe.user_data) {
+			req = tmp;
 			break;
+		}
 	}
-	if (!found)
+	if (!req)
 		return -ENOENT;
 
 	io = req->async_data;
@@ -7412,14 +7423,15 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
 {
 	struct io_cancel_data cd = { .data = user_data, };
 	struct io_kiocb *req = io_timeout_extract(ctx, &cd);
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_timeout_data *data;
 
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
-	req->timeout.off = 0; /* noseq */
+	timeout->off = 0; /* noseq */
 	data = req->async_data;
-	list_add_tail(&req->timeout.list, &ctx->timeout_list);
+	list_add_tail(&timeout->list, &ctx->timeout_list);
 	hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
 	data->timer.function = io_timeout_fn;
 	hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
@@ -7429,7 +7441,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
 static int io_timeout_remove_prep(struct io_kiocb *req,
 				  const struct io_uring_sqe *sqe)
 {
-	struct io_timeout_rem *tr = &req->timeout_rem;
+	struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
 
 	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
 		return -EINVAL;
@@ -7469,11 +7481,11 @@ static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
  */
 static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
 {
-	struct io_timeout_rem *tr = &req->timeout_rem;
+	struct io_timeout_rem *tr = io_kiocb_to_cmd(req);
 	struct io_ring_ctx *ctx = req->ctx;
 	int ret;
 
-	if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
+	if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
 		struct io_cancel_data cd = { .data = tr->addr, };
 
 		spin_lock(&ctx->completion_lock);
@@ -7500,6 +7512,7 @@ static int __io_timeout_prep(struct io_kiocb *req,
 			     const struct io_uring_sqe *sqe,
 			     bool is_timeout_link)
 {
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_timeout_data *data;
 	unsigned flags;
 	u32 off = READ_ONCE(sqe->off);
@@ -7516,8 +7529,8 @@ static int __io_timeout_prep(struct io_kiocb *req,
 	if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
 		return -EINVAL;
 
-	INIT_LIST_HEAD(&req->timeout.list);
-	req->timeout.off = off;
+	INIT_LIST_HEAD(&timeout->list);
+	timeout->off = off;
 	if (unlikely(off && !req->ctx->off_timeout_used))
 		req->ctx->off_timeout_used = true;
 
@@ -7536,7 +7549,7 @@ static int __io_timeout_prep(struct io_kiocb *req,
 	if (data->ts.tv_sec < 0 || data->ts.tv_nsec < 0)
 		return -EINVAL;
 
-	INIT_LIST_HEAD(&req->timeout.list);
+	INIT_LIST_HEAD(&timeout->list);
 	data->mode = io_translate_timeout_mode(flags);
 	hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
 
@@ -7547,7 +7560,7 @@ static int __io_timeout_prep(struct io_kiocb *req,
 			return -EINVAL;
 		if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
 			return -EINVAL;
-		req->timeout.head = link->last;
+		timeout->head = link->last;
 		link->last->flags |= REQ_F_ARM_LTIMEOUT;
 	}
 	return 0;
@@ -7567,10 +7580,11 @@ static int io_link_timeout_prep(struct io_kiocb *req,
 
 static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
 {
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_timeout_data *data = req->async_data;
 	struct list_head *entry;
-	u32 tail, off = req->timeout.off;
+	u32 tail, off = timeout->off;
 
 	spin_lock_irq(&ctx->timeout_lock);
 
@@ -7585,7 +7599,7 @@ static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
 	}
 
 	tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
-	req->timeout.target_seq = tail + off;
+	timeout->target_seq = tail + off;
 
 	/* Update the last seq here in case io_flush_timeouts() hasn't.
 	 * This is safe because ->completion_lock is held, and submissions
@@ -7598,17 +7612,17 @@ static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
 	 * the one we need first.
 	 */
 	list_for_each_prev(entry, &ctx->timeout_list) {
-		struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
-						  timeout.list);
+		struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
+		struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
 
 		if (io_is_timeout_noseq(nxt))
 			continue;
 		/* nxt.seq is behind @tail, otherwise would've been completed */
-		if (off >= nxt->timeout.target_seq - tail)
+		if (off >= nextt->target_seq - tail)
 			break;
 	}
 add:
-	list_add(&req->timeout.list, entry);
+	list_add(&timeout->list, entry);
 	data->timer.function = io_timeout_fn;
 	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
 	spin_unlock_irq(&ctx->timeout_lock);
@@ -8198,7 +8212,8 @@ static struct file *io_file_get_normal(struct io_kiocb *req, int fd)
 
 static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
 {
-	struct io_kiocb *prev = req->timeout.prev;
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
+	struct io_kiocb *prev = timeout->prev;
 	int ret = -ENOENT;
 
 	if (prev) {
@@ -8222,12 +8237,13 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
 	struct io_timeout_data *data = container_of(timer,
 						struct io_timeout_data, timer);
 	struct io_kiocb *prev, *req = data->req;
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_ring_ctx *ctx = req->ctx;
 	unsigned long flags;
 
 	spin_lock_irqsave(&ctx->timeout_lock, flags);
-	prev = req->timeout.head;
-	req->timeout.head = NULL;
+	prev = timeout->head;
+	timeout->head = NULL;
 
 	/*
 	 * We don't expect the list to be empty, that will only happen if we
@@ -8238,8 +8254,8 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
 		if (!req_ref_inc_not_zero(prev))
 			prev = NULL;
 	}
-	list_del(&req->timeout.list);
-	req->timeout.prev = prev;
+	list_del(&timeout->list);
+	timeout->prev = prev;
 	spin_unlock_irqrestore(&ctx->timeout_lock, flags);
 
 	req->io_task_work.func = io_req_task_link_timeout;
@@ -8249,6 +8265,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
 
 static void io_queue_linked_timeout(struct io_kiocb *req)
 {
+	struct io_timeout *timeout = io_kiocb_to_cmd(req);
 	struct io_ring_ctx *ctx = req->ctx;
 
 	spin_lock_irq(&ctx->timeout_lock);
@@ -8256,13 +8273,13 @@ static void io_queue_linked_timeout(struct io_kiocb *req)
 	 * If the back reference is NULL, then our linked request finished
 	 * before we got a chance to setup the timer
 	 */
-	if (req->timeout.head) {
+	if (timeout->head) {
 		struct io_timeout_data *data = req->async_data;
 
 		data->timer.function = io_link_timeout_fn;
 		hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
 				data->mode);
-		list_add_tail(&req->timeout.list, &ctx->ltimeout_list);
+		list_add_tail(&timeout->list, &ctx->ltimeout_list);
 	}
 	spin_unlock_irq(&ctx->timeout_lock);
 	/* drop submission reference */
@@ -10930,12 +10947,14 @@ static __cold void io_ring_exit_work(struct work_struct *work)
 static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx,
 				    struct task_struct *tsk, bool cancel_all)
 {
-	struct io_kiocb *req, *tmp;
+	struct io_timeout *timeout, *tmp;
 	int canceled = 0;
 
 	spin_lock(&ctx->completion_lock);
 	spin_lock_irq(&ctx->timeout_lock);
-	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
+	list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
+		struct io_kiocb *req = cmd_to_io_kiocb(timeout);
+
 		if (io_match_task(req, tsk, cancel_all)) {
 			io_kill_timeout(req, -ECANCELED);
 			canceled++;