diff options
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r-- | block/blk-mq.c | 46 |
1 files changed, 33 insertions, 13 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c index 849479debac3..067b5ad3e047 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -85,7 +85,7 @@ static int blk_mq_queue_enter(struct request_queue *q) if (percpu_ref_tryget_live(&q->mq_usage_counter)) return 0; - ret = wait_event_interruptible(q->mq_freeze_wq, + ret = swait_event_interruptible(q->mq_freeze_wq, !q->mq_freeze_depth || blk_queue_dying(q)); if (blk_queue_dying(q)) return -ENODEV; @@ -104,7 +104,7 @@ static void blk_mq_usage_counter_release(struct percpu_ref *ref) struct request_queue *q = container_of(ref, struct request_queue, mq_usage_counter); - wake_up_all(&q->mq_freeze_wq); + swait_wake_all(&q->mq_freeze_wq); } static void blk_mq_freeze_queue_start(struct request_queue *q) @@ -123,7 +123,7 @@ static void blk_mq_freeze_queue_start(struct request_queue *q) static void blk_mq_freeze_queue_wait(struct request_queue *q) { - wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter)); + swait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter)); } /* @@ -146,7 +146,7 @@ static void blk_mq_unfreeze_queue(struct request_queue *q) spin_unlock_irq(q->queue_lock); if (wake) { percpu_ref_reinit(&q->mq_usage_counter); - wake_up_all(&q->mq_freeze_wq); + swait_wake_all(&q->mq_freeze_wq); } } @@ -194,6 +194,9 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, rq->resid_len = 0; rq->sense = NULL; +#ifdef CONFIG_PREEMPT_RT_FULL + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work); +#endif INIT_LIST_HEAD(&rq->timeout_list); rq->timeout = 0; @@ -313,6 +316,17 @@ void blk_mq_end_request(struct request *rq, int error) } EXPORT_SYMBOL(blk_mq_end_request); +#ifdef CONFIG_PREEMPT_RT_FULL + +void __blk_mq_complete_request_remote_work(struct work_struct *work) +{ + struct request *rq = container_of(work, struct request, work); + + rq->q->softirq_done_fn(rq); +} + +#else + static void __blk_mq_complete_request_remote(void *data) { struct request *rq = data; @@ -320,6 +334,8 @@ static void __blk_mq_complete_request_remote(void *data) rq->q->softirq_done_fn(rq); } +#endif + static void blk_mq_ipi_complete_request(struct request *rq) { struct blk_mq_ctx *ctx = rq->mq_ctx; @@ -331,19 +347,23 @@ static void blk_mq_ipi_complete_request(struct request *rq) return; } - cpu = get_cpu(); + cpu = get_cpu_light(); if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) shared = cpus_share_cache(cpu, ctx->cpu); if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { +#ifdef CONFIG_PREEMPT_RT_FULL + schedule_work_on(ctx->cpu, &rq->work); +#else rq->csd.func = __blk_mq_complete_request_remote; rq->csd.info = rq; rq->csd.flags = 0; smp_call_function_single_async(ctx->cpu, &rq->csd); +#endif } else { rq->q->softirq_done_fn(rq); } - put_cpu(); + put_cpu_light(); } void __blk_mq_complete_request(struct request *rq) @@ -814,9 +834,9 @@ void blk_mq_run_queues(struct request_queue *q, bool async) test_bit(BLK_MQ_S_STOPPED, &hctx->state)) continue; - preempt_disable(); + migrate_disable(); blk_mq_run_hw_queue(hctx, async); - preempt_enable(); + migrate_enable(); } } EXPORT_SYMBOL(blk_mq_run_queues); @@ -843,9 +863,9 @@ void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) { clear_bit(BLK_MQ_S_STOPPED, &hctx->state); - preempt_disable(); + migrate_disable(); blk_mq_run_hw_queue(hctx, false); - preempt_enable(); + migrate_enable(); } EXPORT_SYMBOL(blk_mq_start_hw_queue); @@ -870,9 +890,9 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) continue; clear_bit(BLK_MQ_S_STOPPED, &hctx->state); - preempt_disable(); + migrate_disable(); blk_mq_run_hw_queue(hctx, async); - preempt_enable(); + migrate_enable(); } } EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); @@ -1494,7 +1514,7 @@ static int blk_mq_hctx_notify(void *data, unsigned long action, { struct blk_mq_hw_ctx *hctx = data; - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) + if (action == CPU_POST_DEAD) return blk_mq_hctx_cpu_offline(hctx, cpu); else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) return blk_mq_hctx_cpu_online(hctx, cpu); |