diff options
| author | Linus Torvalds | 2021-08-30 19:30:30 -0700 |
|---|---|---|
| committer | Linus Torvalds | 2021-08-30 19:30:30 -0700 |
| commit | 3b629f8d6dc04d3af94429c18fe17239d6fbe2c3 (patch) | |
| tree | f67dd6ef5c662927ee4c6533450c7b2c5724f69e /fs | |
| parent | c547d89a9a445f6bb757b93247de43d312e722da (diff) | |
| parent | 3d5b3fbedad65088ec079a4c4d1a2f47e11ae1e7 (diff) | |
Merge tag 'io_uring-bio-cache.5-2021-08-30' of git://git.kernel.dk/linux-block
Pull support for struct bio recycling from Jens Axboe:
"This adds bio recycling support for polled IO, allowing quick reuse of
a bio for high IOPS scenarios via a percpu bio_set list.
It's good for almost a 10% improvement in performance, bumping our
per-core IO limit from ~3.2M IOPS to ~3.5M IOPS"
* tag 'io_uring-bio-cache.5-2021-08-30' of git://git.kernel.dk/linux-block:
bio: improve kerneldoc documentation for bio_alloc_kiocb()
block: provide bio_clear_hipri() helper
block: use the percpu bio cache in __blkdev_direct_IO
io_uring: enable use of bio alloc cache
block: clear BIO_PERCPU_CACHE flag if polling isn't supported
bio: add allocation cache abstraction
fs: add kiocb alloc cache flag
bio: optimize initialization of a bio
Diffstat (limited to 'fs')
| -rw-r--r-- | fs/block_dev.c | 6 | ||||
| -rw-r--r-- | fs/io_uring.c | 2 |
2 files changed, 5 insertions, 3 deletions
diff --git a/fs/block_dev.c b/fs/block_dev.c index 1f21ac984253..45df6cbccf12 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -386,7 +386,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, (bdev_logical_block_size(bdev) - 1)) return -EINVAL; - bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool); + bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); dio->is_sync = is_sync = is_sync_kiocb(iocb); @@ -514,7 +514,9 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) static __init int blkdev_init(void) { - return bioset_init(&blkdev_dio_pool, 4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS); + return bioset_init(&blkdev_dio_pool, 4, + offsetof(struct blkdev_dio, bio), + BIOSET_NEED_BVECS|BIOSET_PERCPU_CACHE); } module_init(blkdev_init); diff --git a/fs/io_uring.c b/fs/io_uring.c index 7cc458e0b636..73928d957691 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2835,7 +2835,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) !kiocb->ki_filp->f_op->iopoll) return -EOPNOTSUPP; - kiocb->ki_flags |= IOCB_HIPRI; + kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE; kiocb->ki_complete = io_complete_rw_iopoll; req->iopoll_completed = 0; } else { |