block: only allocate poll_stats if there's a user of them

This is essentially never used, yet it's about 1/3rd of the total
queue size. Allocate it when needed, and don't embed it in the queue.

Kill the queue flag for this while at it, since we can just check the
assigned pointer now.

Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index 4f2cf8399f3de31aae7a26e19c3ab35b98b9626b..f4022b1985805032de2469fb98b64eecc8aa7a85 100644 (file)
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -122,7 +122,6 @@ static const char *const blk_queue_flag_name[] = {
        QUEUE_FLAG_NAME(FUA),
        QUEUE_FLAG_NAME(DAX),
        QUEUE_FLAG_NAME(STATS),
-       QUEUE_FLAG_NAME(POLL_STATS),
        QUEUE_FLAG_NAME(REGISTERED),
        QUEUE_FLAG_NAME(QUIESCED),
        QUEUE_FLAG_NAME(PCI_P2PDMA),

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 3af88ffc9e2c61505dd8719b1fc86474dbfb6705..7cd408408a373d04e976348f70161127fd5d58f0 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -4581,11 +4581,10 @@ EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
 /* Enable polling stats and return whether they were already enabled. */
 static bool blk_poll_stats_enable(struct request_queue *q)
 {
-       if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
-           blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q))
+       if (q->poll_stat)
                return true;
-       blk_stat_add_callback(q, q->poll_cb);
-       return false;
+
+       return blk_stats_alloc_enable(q);
 }
 
 static void blk_mq_poll_stats_start(struct request_queue *q)
@@ -4594,8 +4593,7 @@ static void blk_mq_poll_stats_start(struct request_queue *q)
         * We don't arm the callback if polling stats are not enabled or the
         * callback is already active.
         */
-       if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
-           blk_stat_is_active(q->poll_cb))
+       if (!q->poll_stat || blk_stat_is_active(q->poll_cb))
                return;
 
        blk_stat_activate_msecs(q->poll_cb, 100);

diff --git a/block/blk-stat.c b/block/blk-stat.c
index ae3dd1fb8e61d7b88c581d22dc6682bdb0ca994f..efb2a80db90616bbcd3130dd6fc5c7fc0c85f6fb 100644 (file)
--- a/block/blk-stat.c
+++ b/block/blk-stat.c
@@ -219,3 +219,21 @@ void blk_free_queue_stats(struct blk_queue_stats *stats)
 
        kfree(stats);
 }
+
+bool blk_stats_alloc_enable(struct request_queue *q)
+{
+       struct blk_rq_stat *poll_stat;
+
+       poll_stat = kcalloc(BLK_MQ_POLL_STATS_BKTS, sizeof(*poll_stat),
+                               GFP_ATOMIC);
+       if (!poll_stat)
+               return false;
+
+       if (cmpxchg(&q->poll_stat, NULL, poll_stat) != NULL) {
+               kfree(poll_stat);
+               return true;
+       }
+
+       blk_stat_add_callback(q, q->poll_cb);
+       return false;
+}

diff --git a/block/blk-stat.h b/block/blk-stat.h
index 17b47a86eefb331c5d2266243183c2c71b10e2bc..58f029af49e5bf2d00a8d20316c9266e905945d9 100644 (file)
--- a/block/blk-stat.h
+++ b/block/blk-stat.h
@@ -64,6 +64,7 @@ struct blk_stat_callback {
 
 struct blk_queue_stats *blk_alloc_queue_stats(void);
 void blk_free_queue_stats(struct blk_queue_stats *);
+bool blk_stats_alloc_enable(struct request_queue *q);
 
 void blk_stat_add(struct request *rq, u64 now);
 

diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index cd75b0f73dc6fc32eb62e91c919a176f96042dc2..c079be1c58a37822b19600a0eaa83068ec69707a 100644 (file)
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -785,11 +785,12 @@ static void blk_release_queue(struct kobject *kobj)
 
        might_sleep();
 
-       if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
+       if (q->poll_stat)
                blk_stat_remove_callback(q, q->poll_cb);
        blk_stat_free_callback(q->poll_cb);
 
        blk_free_queue_stats(q->stats);
+       kfree(q->poll_stat);
 
        blk_exit_queue(q);
 

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index bd4370baccca31ace90c0d3bbb23d08e7a14bcff..74118e67f64904af7d6d1cf57893ecdaf42aad08 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -267,7 +267,7 @@ struct request_queue {
        int                     poll_nsec;
 
        struct blk_stat_callback        *poll_cb;
-       struct blk_rq_stat      poll_stat[BLK_MQ_POLL_STATS_BKTS];
+       struct blk_rq_stat      *poll_stat;
 
        struct timer_list       timeout;
        struct work_struct      timeout_work;
@@ -397,7 +397,6 @@ struct request_queue {
 #define QUEUE_FLAG_FUA         18      /* device supports FUA writes */
 #define QUEUE_FLAG_DAX         19      /* device supports DAX */
 #define QUEUE_FLAG_STATS       20      /* track IO start and completion times */
-#define QUEUE_FLAG_POLL_STATS  21      /* collecting stats for hybrid polling */
 #define QUEUE_FLAG_REGISTERED  22      /* queue has been registered to a disk */
 #define QUEUE_FLAG_QUIESCED    24      /* queue has been quiesced */
 #define QUEUE_FLAG_PCI_P2PDMA  25      /* device supports PCI p2p requests */