summaryrefslogtreecommitdiff
path: root/block/blk-mq-tag.c
diff options
context:
space:
mode:
authorMing Lei <tom.leiming@gmail.com>2014-06-01 00:43:37 +0800
committerJens Axboe <axboe@fb.com>2014-06-03 21:04:39 -0600
commitcb96a42cc1f50ba1c7b1e9b2343bec80b926107f (patch)
treec69f953fcd7d04288903abe73a0f92f155882bfd /block/blk-mq-tag.c
parent1aecfe4887713838c79bc52f774609a57db4f988 (diff)
downloadkernel-cb96a42cc1f50ba1c7b1e9b2343bec80b926107f.tar.gz
blk-mq: fix schedule from atomic context
blk_mq_put_ctx() has to be called before io_schedule() in bt_get(). This patch fixes the problem by taking similar approach from percpu_ida allocation for the situation. Signed-off-by: Ming Lei <tom.leiming@gmail.com> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block/blk-mq-tag.c')
-rw-r--r--block/blk-mq-tag.c48
1 files changed, 31 insertions, 17 deletions
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index d90c4aeb7dd3..1aab39f71d95 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -221,8 +221,10 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt,
return bs;
}
-static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,
- unsigned int *last_tag, gfp_t gfp)
+static int bt_get(struct blk_mq_alloc_data *data,
+ struct blk_mq_bitmap_tags *bt,
+ struct blk_mq_hw_ctx *hctx,
+ unsigned int *last_tag)
{
struct bt_wait_state *bs;
DEFINE_WAIT(wait);
@@ -232,7 +234,7 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,
if (tag != -1)
return tag;
- if (!(gfp & __GFP_WAIT))
+ if (!(data->gfp & __GFP_WAIT))
return -1;
bs = bt_wait_ptr(bt, hctx);
@@ -249,50 +251,62 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx,
if (was_empty)
atomic_set(&bs->wait_cnt, bt->wake_cnt);
+ blk_mq_put_ctx(data->ctx);
+
io_schedule();
+
+ data->ctx = blk_mq_get_ctx(data->q);
+ data->hctx = data->q->mq_ops->map_queue(data->q,
+ data->ctx->cpu);
+ if (data->reserved) {
+ bt = &data->hctx->tags->breserved_tags;
+ } else {
+ last_tag = &data->ctx->last_tag;
+ hctx = data->hctx;
+ bt = &hctx->tags->bitmap_tags;
+ }
+ finish_wait(&bs->wait, &wait);
+ bs = bt_wait_ptr(bt, hctx);
} while (1);
finish_wait(&bs->wait, &wait);
return tag;
}
-static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags,
- struct blk_mq_hw_ctx *hctx,
- unsigned int *last_tag, gfp_t gfp)
+static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
int tag;
- tag = bt_get(&tags->bitmap_tags, hctx, last_tag, gfp);
+ tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
+ &data->ctx->last_tag);
if (tag >= 0)
- return tag + tags->nr_reserved_tags;
+ return tag + data->hctx->tags->nr_reserved_tags;
return BLK_MQ_TAG_FAIL;
}
-static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags,
- gfp_t gfp)
+static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
{
int tag, zero = 0;
- if (unlikely(!tags->nr_reserved_tags)) {
+ if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
WARN_ON_ONCE(1);
return BLK_MQ_TAG_FAIL;
}
- tag = bt_get(&tags->breserved_tags, NULL, &zero, gfp);
+ tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero);
if (tag < 0)
return BLK_MQ_TAG_FAIL;
return tag;
}
-unsigned int blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, unsigned int *last_tag,
- gfp_t gfp, bool reserved)
+unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
- if (!reserved)
- return __blk_mq_get_tag(hctx->tags, hctx, last_tag, gfp);
+ if (!data->reserved)
+ return __blk_mq_get_tag(data);
- return __blk_mq_get_reserved_tag(hctx->tags, gfp);
+ return __blk_mq_get_reserved_tag(data);
}
static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt)