mm/filemap: Convert page wait queues to be folios

Reinforce that page flags are actually in the head page by changing the
type from page to folio.  Increases the size of cachefiles by two bytes,
but the kernel core is unchanged in size.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Acked-by: Jeff Layton <jlayton@kernel.org>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Reviewed-by: David Howells <dhowells@redhat.com>

diff --git a/fs/cachefiles/rdwr.c b/fs/cachefiles/rdwr.c
index 8ffc40e84a59442312295536a3ef44733b356f66..fcf4f3b72923350e3838afcf9020516662b1b3a2 100644 (file)
--- a/fs/cachefiles/rdwr.c
+++ b/fs/cachefiles/rdwr.c
@@ -25,20 +25,20 @@ static int cachefiles_read_waiter(wait_queue_entry_t *wait, unsigned mode,
        struct cachefiles_object *object;
        struct fscache_retrieval *op = monitor->op;
        struct wait_page_key *key = _key;
-       struct page *page = wait->private;
+       struct folio *folio = wait->private;
 
        ASSERT(key);
 
        _enter("{%lu},%u,%d,{%p,%u}",
               monitor->netfs_page->index, mode, sync,
-              key->page, key->bit_nr);
+              key->folio, key->bit_nr);
 
-       if (key->page != page || key->bit_nr != PG_locked)
+       if (key->folio != folio || key->bit_nr != PG_locked)
                return 0;
 
-       _debug("--- monitor %p %lx ---", page, page->flags);
+       _debug("--- monitor %p %lx ---", folio, folio->flags);
 
-       if (!PageUptodate(page) && !PageError(page)) {
+       if (!folio_test_uptodate(folio) && !folio_test_error(folio)) {
                /* unlocked, not uptodate and not erronous? */
                _debug("page probably truncated");
        }
@@ -107,7 +107,7 @@ static int cachefiles_read_reissue(struct cachefiles_object *object,
        put_page(backpage2);
 
        INIT_LIST_HEAD(&monitor->op_link);
-       add_page_wait_queue(backpage, &monitor->monitor);
+       folio_add_wait_queue(page_folio(backpage), &monitor->monitor);
 
        if (trylock_page(backpage)) {
                ret = -EIO;
@@ -294,7 +294,7 @@ monitor_backing_page:
        get_page(backpage);
        monitor->back_page = backpage;
        monitor->monitor.private = backpage;
-       add_page_wait_queue(backpage, &monitor->monitor);
+       folio_add_wait_queue(page_folio(backpage), &monitor->monitor);
        monitor = NULL;
 
        /* but the page may have been read before the monitor was installed, so
@@ -548,7 +548,7 @@ static int cachefiles_read_backing_file(struct cachefiles_object *object,
                get_page(backpage);
                monitor->back_page = backpage;
                monitor->monitor.private = backpage;
-               add_page_wait_queue(backpage, &monitor->monitor);
+               folio_add_wait_queue(page_folio(backpage), &monitor->monitor);
                monitor = NULL;
 
                /* but the page may have been read before the monitor was

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 2f481327dee8fff86c07b35e73e43d73141f7a10..ebc62e9e453b84bb816d9d4ee35cd2339daf50a5 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -628,13 +628,13 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
 }
 
 struct wait_page_key {
-       struct page *page;
+       struct folio *folio;
        int bit_nr;
        int page_match;
 };
 
 struct wait_page_queue {
-       struct page *page;
+       struct folio *folio;
        int bit_nr;
        wait_queue_entry_t wait;
 };
@@ -642,7 +642,7 @@ struct wait_page_queue {
 static inline bool wake_page_match(struct wait_page_queue *wait_page,
                                  struct wait_page_key *key)
 {
-       if (wait_page->page != key->page)
+       if (wait_page->folio != key->folio)
               return false;
        key->page_match = 1;
 
@@ -802,7 +802,7 @@ int wait_on_page_private_2_killable(struct page *page);
 /*
  * Add an arbitrary waiter to a page's wait queue
  */
-extern void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter);
+void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter);
 
 /*
  * Fault everything in given userspace address range in.

diff --git a/mm/filemap.c b/mm/filemap.c
index a896348ff4de7ad56498dce7f0d12b549ed2b15c..1f8c00c2a4b747613d153303950b03cc682bfbea 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1074,11 +1074,11 @@ EXPORT_SYMBOL(filemap_invalidate_unlock_two);
  */
 #define PAGE_WAIT_TABLE_BITS 8
 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
-static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
+static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
 
-static wait_queue_head_t *page_waitqueue(struct page *page)
+static wait_queue_head_t *folio_waitqueue(struct folio *folio)
 {
-       return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
+       return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)];
 }
 
 void __init pagecache_init(void)
@@ -1086,7 +1086,7 @@ void __init pagecache_init(void)
        int i;
 
        for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
-               init_waitqueue_head(&page_wait_table[i]);
+               init_waitqueue_head(&folio_wait_table[i]);
 
        page_writeback_init();
 }
@@ -1141,10 +1141,10 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync,
         */
        flags = wait->flags;
        if (flags & WQ_FLAG_EXCLUSIVE) {
-               if (test_bit(key->bit_nr, &key->page->flags))
+               if (test_bit(key->bit_nr, &key->folio->flags))
                        return -1;
                if (flags & WQ_FLAG_CUSTOM) {
-                       if (test_and_set_bit(key->bit_nr, &key->page->flags))
+                       if (test_and_set_bit(key->bit_nr, &key->folio->flags))
                                return -1;
                        flags |= WQ_FLAG_DONE;
                }
@@ -1178,12 +1178,12 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync,
 
 static void folio_wake_bit(struct folio *folio, int bit_nr)
 {
-       wait_queue_head_t *q = page_waitqueue(&folio->page);
+       wait_queue_head_t *q = folio_waitqueue(folio);
        struct wait_page_key key;
        unsigned long flags;
        wait_queue_entry_t bookmark;
 
-       key.page = &folio->page;
+       key.folio = folio;
        key.bit_nr = bit_nr;
        key.page_match = 0;
 
@@ -1275,7 +1275,7 @@ int sysctl_page_lock_unfairness = 5;
 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
                int state, enum behavior behavior)
 {
-       wait_queue_head_t *q = page_waitqueue(&folio->page);
+       wait_queue_head_t *q = folio_waitqueue(folio);
        int unfairness = sysctl_page_lock_unfairness;
        struct wait_page_queue wait_page;
        wait_queue_entry_t *wait = &wait_page.wait;
@@ -1295,7 +1295,7 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
 
        init_wait(wait);
        wait->func = wake_page_function;
-       wait_page.page = &folio->page;
+       wait_page.folio = folio;
        wait_page.bit_nr = bit_nr;
 
 repeat:
@@ -1444,23 +1444,23 @@ int put_and_wait_on_page_locked(struct page *page, int state)
 }
 
 /**
- * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
- * @page: Page defining the wait queue of interest
+ * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue
+ * @folio: Folio defining the wait queue of interest
  * @waiter: Waiter to add to the queue
  *
- * Add an arbitrary @waiter to the wait queue for the nominated @page.
+ * Add an arbitrary @waiter to the wait queue for the nominated @folio.
  */
-void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
+void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter)
 {
-       wait_queue_head_t *q = page_waitqueue(page);
+       wait_queue_head_t *q = folio_waitqueue(folio);
        unsigned long flags;
 
        spin_lock_irqsave(&q->lock, flags);
        __add_wait_queue_entry_tail(q, waiter);
-       SetPageWaiters(page);
+       folio_set_waiters(folio);
        spin_unlock_irqrestore(&q->lock, flags);
 }
-EXPORT_SYMBOL_GPL(add_page_wait_queue);
+EXPORT_SYMBOL_GPL(folio_add_wait_queue);
 
 #ifndef clear_bit_unlock_is_negative_byte
 
@@ -1650,10 +1650,10 @@ EXPORT_SYMBOL_GPL(__folio_lock_killable);
 
 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait)
 {
-       struct wait_queue_head *q = page_waitqueue(&folio->page);
+       struct wait_queue_head *q = folio_waitqueue(folio);
        int ret = 0;
 
-       wait->page = &folio->page;
+       wait->folio = folio;
        wait->bit_nr = PG_locked;
 
        spin_lock_irq(&q->lock);