From f31c21e4365c02ccf7226c33ea978cd5dbfc351e Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 22 Nov 2017 14:25:18 +1100 Subject: dm: remove unused 'num_write_bios' target interface No DM target provides num_write_bios and none has since dm-cache's brief use in 2013. Having the possibility of num_write_bios > 1 complicates bio allocation. So remove the interface and assume there is only one bio needed. If a target ever needs more, it must provide a suitable bioset and allocate itself based on its particular needs. Signed-off-by: NeilBrown Signed-off-by: Mike Snitzer --- include/linux/device-mapper.h | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'include') diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index a5538433c927..5a68b366e664 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -220,14 +220,6 @@ struct target_type { #define DM_TARGET_WILDCARD 0x00000008 #define dm_target_is_wildcard(type) ((type)->features & DM_TARGET_WILDCARD) -/* - * Some targets need to be sent the same WRITE bio severals times so - * that they can send copies of it to different devices. This function - * examines any supplied bio and returns the number of copies of it the - * target requires. - */ -typedef unsigned (*dm_num_write_bios_fn) (struct dm_target *ti, struct bio *bio); - /* * A target implements own bio data integrity. */ @@ -291,13 +283,6 @@ struct dm_target { */ unsigned per_io_data_size; - /* - * If defined, this function is called to find out how many - * duplicate bios should be sent to the target when writing - * data. - */ - dm_num_write_bios_fn num_write_bios; - /* target specific data */ void *private; -- cgit v1.2.3 From 64f52b0e31489b46465cff2e61ab2e1f60a3b4eb Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 11 Dec 2017 23:17:47 -0500 Subject: dm: improve performance by moving dm_io structure to per-bio-data Eliminates need for a separate mempool to allocate 'struct dm_io' objects from. As such, it saves an extra mempool allocation for each original bio that DM core is issued. This complicates the per-bio-data accessor functions by needing to conditonally add extra padding to get to a target's per-bio-data. But in the end this provides a decent performance improvement for all bio-based DM devices. On an NVMe-loop based testbed to a ramdisk (~3100 MB/s): bio-based DM linear performance improved by 2% (went from 2665 to 2777 MB/s). Signed-off-by: Mike Snitzer --- drivers/md/dm-core.h | 1 + drivers/md/dm.c | 169 ++++++++++++++++++++++++++++++++---------- include/linux/device-mapper.h | 32 +------- 3 files changed, 133 insertions(+), 69 deletions(-) (limited to 'include') diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index 6a14f945783c..8a7dc8f9e40f 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -91,6 +91,7 @@ struct mapped_device { /* * io objects are allocated from here. */ + struct bio_set *io_bs; mempool_t *io_pool; struct bio_set *bs; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 3e3fbc6f708f..01d0f9c410fb 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -59,10 +59,39 @@ void dm_issue_global_event(void) wake_up(&dm_global_eventq); } +/* + * One of these is allocated (on-stack) per original bio. + */ +struct clone_info { + struct mapped_device *md; + struct dm_table *map; + struct bio *bio; + struct dm_io *io; + sector_t sector; + unsigned sector_count; +}; + +/* + * One of these is allocated per clone bio. + */ +#define DM_TIO_MAGIC 7282014 +struct dm_target_io { + unsigned magic; + struct dm_io *io; + struct dm_target *ti; + unsigned target_bio_nr; + unsigned *len_ptr; + bool inside_dm_io; + struct bio clone; +}; + /* * One of these is allocated per original bio. + * It contains the first clone used for that original. */ +#define DM_IO_MAGIC 5191977 struct dm_io { + unsigned magic; struct mapped_device *md; blk_status_t status; atomic_t io_count; @@ -70,8 +99,35 @@ struct dm_io { unsigned long start_time; spinlock_t endio_lock; struct dm_stats_aux stats_aux; + /* last member of dm_target_io is 'struct bio' */ + struct dm_target_io tio; }; +void *dm_per_bio_data(struct bio *bio, size_t data_size) +{ + struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); + if (!tio->inside_dm_io) + return (char *)bio - offsetof(struct dm_target_io, clone) - data_size; + return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size; +} +EXPORT_SYMBOL_GPL(dm_per_bio_data); + +struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) +{ + struct dm_io *io = (struct dm_io *)((char *)data + data_size); + if (io->magic == DM_IO_MAGIC) + return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone)); + BUG_ON(io->magic != DM_TIO_MAGIC); + return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone)); +} +EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); + +unsigned dm_bio_get_target_bio_nr(const struct bio *bio) +{ + return container_of(bio, struct dm_target_io, clone)->target_bio_nr; +} +EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); + #define MINOR_ALLOCED ((void *)-1) /* @@ -95,6 +151,7 @@ static int dm_numa_node = DM_NUMA_NODE; struct dm_md_mempools { mempool_t *io_pool; struct bio_set *bs; + struct bio_set *io_bs; }; struct table_device { @@ -488,16 +545,58 @@ out: static struct dm_io *alloc_io(struct mapped_device *md) { - return mempool_alloc(md->io_pool, GFP_NOIO); + struct dm_io *io; + struct dm_target_io *tio; + struct bio *clone; + + clone = bio_alloc_bioset(GFP_NOIO, 0, md->io_bs); + if (!clone) + return NULL; + + tio = container_of(clone, struct dm_target_io, clone); + tio->inside_dm_io = true; + tio->io = NULL; + + io = container_of(tio, struct dm_io, tio); + io->magic = DM_IO_MAGIC; + + return io; } static void free_io(struct mapped_device *md, struct dm_io *io) { - mempool_free(io, md->io_pool); + bio_put(&io->tio.clone); +} + +static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti, + unsigned target_bio_nr, gfp_t gfp_mask) +{ + struct dm_target_io *tio; + + if (!ci->io->tio.io) { + /* the dm_target_io embedded in ci->io is available */ + tio = &ci->io->tio; + } else { + struct bio *clone = bio_alloc_bioset(gfp_mask, 0, ci->md->bs); + if (!clone) + return NULL; + + tio = container_of(clone, struct dm_target_io, clone); + tio->inside_dm_io = false; + } + + tio->magic = DM_TIO_MAGIC; + tio->io = ci->io; + tio->ti = ti; + tio->target_bio_nr = target_bio_nr; + + return tio; } static void free_tio(struct dm_target_io *tio) { + if (tio->inside_dm_io) + return; bio_put(&tio->clone); } @@ -1110,6 +1209,7 @@ static void __map_bio(struct dm_target_io *tio) int r; sector_t sector; struct bio *clone = &tio->clone; + struct dm_io *io = tio->io; struct dm_target *ti = tio->ti; clone->bi_end_io = clone_endio; @@ -1119,7 +1219,7 @@ static void __map_bio(struct dm_target_io *tio) * anything, the target has assumed ownership of * this io. */ - atomic_inc(&tio->io->io_count); + atomic_inc(&io->io_count); sector = clone->bi_iter.bi_sector; r = ti->type->map(ti, clone); @@ -1129,16 +1229,16 @@ static void __map_bio(struct dm_target_io *tio) case DM_MAPIO_REMAPPED: /* the bio has been remapped so dispatch it */ trace_block_bio_remap(clone->bi_disk->queue, clone, - bio_dev(tio->io->orig_bio), sector); + bio_dev(io->orig_bio), sector); generic_make_request(clone); break; case DM_MAPIO_KILL: - dec_pending(tio->io, BLK_STS_IOERR); free_tio(tio); + dec_pending(io, BLK_STS_IOERR); break; case DM_MAPIO_REQUEUE: - dec_pending(tio->io, BLK_STS_DM_REQUEUE); free_tio(tio); + dec_pending(io, BLK_STS_DM_REQUEUE); break; default: DMWARN("unimplemented target map return value: %d", r); @@ -1146,15 +1246,6 @@ static void __map_bio(struct dm_target_io *tio) } } -struct clone_info { - struct mapped_device *md; - struct dm_table *map; - struct bio *bio; - struct dm_io *io; - sector_t sector; - unsigned sector_count; -}; - static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) { bio->bi_iter.bi_sector = sector; @@ -1197,24 +1288,6 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, return 0; } -static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti, - unsigned target_bio_nr, gfp_t gfp_mask) -{ - struct dm_target_io *tio; - struct bio *clone; - - clone = bio_alloc_bioset(gfp_mask, 0, ci->md->bs); - if (!clone) - return NULL; - - tio = container_of(clone, struct dm_target_io, clone); - tio->io = ci->io; - tio->ti = ti; - tio->target_bio_nr = target_bio_nr; - - return tio; -} - static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, struct dm_target *ti, unsigned num_bios) { @@ -1628,6 +1701,8 @@ static void cleanup_mapped_device(struct mapped_device *md) mempool_destroy(md->io_pool); if (md->bs) bioset_free(md->bs); + if (md->io_bs) + bioset_free(md->io_bs); if (md->dax_dev) { kill_dax(md->dax_dev); @@ -1793,15 +1868,19 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t) struct dm_md_mempools *p = dm_table_get_md_mempools(t); if (dm_table_bio_based(t)) { - /* The md may already have mempools that need changing. */ + /* + * The md may already have mempools that need changing. + * If so, reload bioset because front_pad may have changed + * because a different table was loaded. + */ if (md->bs) { - /* - * Reload bioset because front_pad may have changed - * because a different table was loaded. - */ bioset_free(md->bs); md->bs = NULL; } + if (md->io_bs) { + bioset_free(md->io_bs); + md->io_bs = NULL; + } if (md->io_pool) { /* * Reload io_pool because pool_size may have changed @@ -1823,12 +1902,14 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t) goto out; } - BUG_ON(!p || md->io_pool || md->bs); + BUG_ON(!p || md->io_pool || md->bs || md->io_bs); md->io_pool = p->io_pool; p->io_pool = NULL; md->bs = p->bs; p->bs = NULL; + md->io_bs = p->io_bs; + p->io_bs = NULL; out: /* mempool bind completed, no longer need any mempools in the table */ dm_table_free_md_mempools(t); @@ -2719,7 +2800,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu { struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); unsigned int pool_size = 0; - unsigned int front_pad; + unsigned int front_pad, io_front_pad; if (!pools) return NULL; @@ -2729,6 +2810,12 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu case DM_TYPE_DAX_BIO_BASED: pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); + io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio); + pools->io_bs = bioset_create(pool_size, io_front_pad, 0); + if (!pools->io_bs) + goto out; + if (integrity && bioset_integrity_create(pools->io_bs, pool_size)) + goto out; pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache); if (!pools->io_pool) goto out; @@ -2767,6 +2854,8 @@ void dm_free_md_mempools(struct dm_md_mempools *pools) if (pools->bs) bioset_free(pools->bs); + if (pools->io_bs) + bioset_free(pools->io_bs); kfree(pools); } diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 5a68b366e664..0e518d2ee280 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -314,35 +314,9 @@ struct dm_target_callbacks { int (*congested_fn) (struct dm_target_callbacks *, int); }; -/* - * For bio-based dm. - * One of these is allocated for each bio. - * This structure shouldn't be touched directly by target drivers. - * It is here so that we can inline dm_per_bio_data and - * dm_bio_from_per_bio_data - */ -struct dm_target_io { - struct dm_io *io; - struct dm_target *ti; - unsigned target_bio_nr; - unsigned *len_ptr; - struct bio clone; -}; - -static inline void *dm_per_bio_data(struct bio *bio, size_t data_size) -{ - return (char *)bio - offsetof(struct dm_target_io, clone) - data_size; -} - -static inline struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) -{ - return (struct bio *)((char *)data + data_size + offsetof(struct dm_target_io, clone)); -} - -static inline unsigned dm_bio_get_target_bio_nr(const struct bio *bio) -{ - return container_of(bio, struct dm_target_io, clone)->target_bio_nr; -} +void *dm_per_bio_data(struct bio *bio, size_t data_size); +struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size); +unsigned dm_bio_get_target_bio_nr(const struct bio *bio); int dm_register_target(struct target_type *t); void dm_unregister_target(struct target_type *t); -- cgit v1.2.3 From 22c11858e8002592c59ebb762e4e42dc634bf84f Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 4 Dec 2017 21:07:37 -0500 Subject: dm: introduce DM_TYPE_NVME_BIO_BASED If dm_table_determine_type() establishes DM_TYPE_NVME_BIO_BASED then all devices in the DM table do not support partial completions. Also, the table has a single immutable target that doesn't require DM core to split bios. This will enable adding NVMe optimizations to bio-based DM. Signed-off-by: Mike Snitzer --- drivers/md/dm-table.c | 54 ++++++++++++++++++++++++++++++++++++++----- drivers/md/dm.c | 2 ++ include/linux/device-mapper.h | 1 + 3 files changed, 51 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 504e79bc3a55..ad4ac294dd57 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -866,7 +866,8 @@ EXPORT_SYMBOL(dm_consume_args); static bool __table_type_bio_based(enum dm_queue_mode table_type) { return (table_type == DM_TYPE_BIO_BASED || - table_type == DM_TYPE_DAX_BIO_BASED); + table_type == DM_TYPE_DAX_BIO_BASED || + table_type == DM_TYPE_NVME_BIO_BASED); } static bool __table_type_request_based(enum dm_queue_mode table_type) @@ -909,6 +910,8 @@ static bool dm_table_supports_dax(struct dm_table *t) return true; } +static bool dm_table_does_not_support_partial_completion(struct dm_table *t); + static int dm_table_determine_type(struct dm_table *t) { unsigned i; @@ -923,6 +926,14 @@ static int dm_table_determine_type(struct dm_table *t) /* target already set the table's type */ if (t->type == DM_TYPE_BIO_BASED) return 0; + else if (t->type == DM_TYPE_NVME_BIO_BASED) { + if (!dm_table_does_not_support_partial_completion(t)) { + DMERR("nvme bio-based is only possible with devices" + " that don't support partial completion"); + return -EINVAL; + } + /* Fallthru, also verify all devices are blk-mq */ + } BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED); goto verify_rq_based; } @@ -937,8 +948,8 @@ static int dm_table_determine_type(struct dm_table *t) bio_based = 1; if (bio_based && request_based) { - DMWARN("Inconsistent table: different target types" - " can't be mixed up"); + DMERR("Inconsistent table: different target types" + " can't be mixed up"); return -EINVAL; } } @@ -959,8 +970,14 @@ static int dm_table_determine_type(struct dm_table *t) /* We must use this table as bio-based */ t->type = DM_TYPE_BIO_BASED; if (dm_table_supports_dax(t) || - (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) + (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) { t->type = DM_TYPE_DAX_BIO_BASED; + } else if ((dm_table_get_immutable_target(t) && + dm_table_does_not_support_partial_completion(t)) || + (list_empty(devices) && live_md_type == DM_TYPE_NVME_BIO_BASED)) { + t->type = DM_TYPE_NVME_BIO_BASED; + goto verify_rq_based; + } return 0; } @@ -980,7 +997,8 @@ verify_rq_based: * (e.g. request completion process for partial completion.) */ if (t->num_targets > 1) { - DMWARN("Request-based dm doesn't support multiple targets yet"); + DMERR("%s DM doesn't support multiple targets", + t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based"); return -EINVAL; } @@ -997,6 +1015,15 @@ verify_rq_based: return 0; } + tgt = dm_table_get_immutable_target(t); + if (!tgt) { + DMERR("table load rejected: immutable target is required"); + return -EINVAL; + } else if (tgt->max_io_len) { + DMERR("table load rejected: immutable target that splits IO is not supported"); + return -EINVAL; + } + /* Non-request-stackable devices can't be used for request-based dm */ list_for_each_entry(dd, devices, list) { struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev); @@ -1018,7 +1045,8 @@ verify_rq_based: } t->all_blk_mq = mq_count > 0; - if (t->type == DM_TYPE_MQ_REQUEST_BASED && !t->all_blk_mq) { + if (!t->all_blk_mq && + (t->type == DM_TYPE_MQ_REQUEST_BASED || t->type == DM_TYPE_NVME_BIO_BASED)) { DMERR("table load rejected: all devices are not blk-mq request-stackable"); return -EINVAL; } @@ -1708,6 +1736,20 @@ static bool dm_table_all_devices_attribute(struct dm_table *t, return true; } +static int device_no_partial_completion(struct dm_target *ti, struct dm_dev *dev, + sector_t start, sector_t len, void *data) +{ + char b[BDEVNAME_SIZE]; + + /* For now, NVMe devices are the only devices of this class */ + return (strncmp(bdevname(dev->bdev, b), "nvme", 3) == 0); +} + +static bool dm_table_does_not_support_partial_completion(struct dm_table *t) +{ + return dm_table_all_devices_attribute(t, device_no_partial_completion); +} + static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { diff --git a/drivers/md/dm.c b/drivers/md/dm.c index cbb4ae5051fc..a1bd7a6ff522 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2073,6 +2073,7 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) break; case DM_TYPE_BIO_BASED: case DM_TYPE_DAX_BIO_BASED: + case DM_TYPE_NVME_BIO_BASED: dm_init_normal_md_queue(md); blk_queue_make_request(md->queue, dm_make_request); break; @@ -2780,6 +2781,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu switch (type) { case DM_TYPE_BIO_BASED: case DM_TYPE_DAX_BIO_BASED: + case DM_TYPE_NVME_BIO_BASED: pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 0e518d2ee280..41ec228b02a6 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -28,6 +28,7 @@ enum dm_queue_mode { DM_TYPE_REQUEST_BASED = 2, DM_TYPE_MQ_REQUEST_BASED = 3, DM_TYPE_DAX_BIO_BASED = 4, + DM_TYPE_NVME_BIO_BASED = 5, }; typedef enum { STATUSTYPE_INFO, STATUSTYPE_TABLE } status_type_t; -- cgit v1.2.3 From f6e7baadd96bd746d3fb584959d3f152189d05e1 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Tue, 28 Mar 2017 11:31:02 -0700 Subject: dm: move dm_table_destroy() to same header as dm_table_create() If anyone is going to use dm_table_create(), they probably should be able to use dm_table_destroy() too. Move the dm_table_destroy() definition outside the private header, near dm_table_create() Signed-off-by: Brian Norris Signed-off-by: Mike Snitzer --- drivers/md/dm.h | 1 - include/linux/device-mapper.h | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 7c66c316add3..114a81b27c37 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -49,7 +49,6 @@ struct dm_md_mempools; /*----------------------------------------------------------------- * Internal table functions. *---------------------------------------------------------------*/ -void dm_table_destroy(struct dm_table *t); void dm_table_event_callback(struct dm_table *t, void (*fn)(void *), void *context); struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 41ec228b02a6..9ba84532947d 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -459,6 +459,11 @@ void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type); */ int dm_table_complete(struct dm_table *t); +/* + * Destroy the table when finished. + */ +void dm_table_destroy(struct dm_table *t); + /* * Target may require that it is never sent I/O larger than len. */ -- cgit v1.2.3 From ac514ffc968bf14649dd0e048447dc966ee49555 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 12 Jan 2018 19:53:40 -0500 Subject: dm mpath: delay the retry of a request if the target responded as busy Add DM_ENDIO_DELAY_REQUEUE to allow request-based multipath's multipath_end_io() to instruct dm-rq.c:dm_done() to delay a requeue. This is beneficial to do if BLK_STS_RESOURCE is returned from the target (because target is busy). Relative to blk-mq: kick the hw queues via blk_mq_requeue_work(), indirectly from dm-rq.c:__dm_mq_kick_requeue_list(), after a delay. For old .request_fn: use blk_delay_queue(). bio-based multipath doesn't have feature parity with request-based for retryable error requeues; that is something that'll need fixing in the future. Suggested-by: Bart Van Assche Signed-off-by: Mike Snitzer Acked-by: Bart Van Assche [as interpreted from Bart's "... patch looks fine to me."] --- drivers/md/dm-mpath.c | 5 ++++- drivers/md/dm-rq.c | 4 ++++ include/linux/device-mapper.h | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 815de2b091a5..a8b1ffc0cb3d 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -1585,7 +1585,10 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone, if (error && !noretry_error(error)) { struct multipath *m = ti->private; - r = DM_ENDIO_REQUEUE; + if (error == BLK_STS_RESOURCE) + r = DM_ENDIO_DELAY_REQUEUE; + else + r = DM_ENDIO_REQUEUE; if (pgpath) fail_path(pgpath); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 9d32f25489c2..b78ff6921cfb 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -315,6 +315,10 @@ static void dm_done(struct request *clone, blk_status_t error, bool mapped) /* The target wants to requeue the I/O */ dm_requeue_original_request(tio, false); break; + case DM_ENDIO_DELAY_REQUEUE: + /* The target wants to requeue the I/O after a delay */ + dm_requeue_original_request(tio, true); + break; default: DMWARN("unimplemented target endio return value: %d", r); BUG(); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 9ba84532947d..da83f64952e7 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -550,6 +550,7 @@ do { \ #define DM_ENDIO_DONE 0 #define DM_ENDIO_INCOMPLETE 1 #define DM_ENDIO_REQUEUE 2 +#define DM_ENDIO_DELAY_REQUEUE 3 /* * Definitions of return values from target map function. @@ -557,7 +558,7 @@ do { \ #define DM_MAPIO_SUBMITTED 0 #define DM_MAPIO_REMAPPED 1 #define DM_MAPIO_REQUEUE DM_ENDIO_REQUEUE -#define DM_MAPIO_DELAY_REQUEUE 3 +#define DM_MAPIO_DELAY_REQUEUE DM_ENDIO_DELAY_REQUEUE #define DM_MAPIO_KILL 4 #define dm_sector_div64(x, y)( \ -- cgit v1.2.3