mm: zswap: function ordering: compress & decompress functions

Writeback needs to decompress. Move the (de)compression API above what
will be the consolidated shrinking/writeback code.

Link: https://lkml.kernel.org/r/20240130014208.565554-18-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Cc: Chengming Zhou <zhouchengming@bytedance.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/zswap.c b/mm/zswap.c
index 19d3482fd035b1a6289bdbea139741c403a26bf5..6d1e5843c4dd0a1216b4365a908f06ad3fe6ca58 100644 (file)
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -992,6 +992,111 @@ static void zswap_invalidate_entry(struct zswap_tree *tree,
                zswap_entry_put(entry);
 }
 
+/*********************************
+* compressed storage functions
+**********************************/
+static bool zswap_compress(struct folio *folio, struct zswap_entry *entry)
+{
+       struct crypto_acomp_ctx *acomp_ctx;
+       struct scatterlist input, output;
+       unsigned int dlen = PAGE_SIZE;
+       unsigned long handle;
+       struct zpool *zpool;
+       char *buf;
+       gfp_t gfp;
+       int ret;
+       u8 *dst;
+
+       acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
+
+       mutex_lock(&acomp_ctx->mutex);
+
+       dst = acomp_ctx->buffer;
+       sg_init_table(&input, 1);
+       sg_set_page(&input, &folio->page, PAGE_SIZE, 0);
+
+       /*
+        * We need PAGE_SIZE * 2 here since there maybe over-compression case,
+        * and hardware-accelerators may won't check the dst buffer size, so
+        * giving the dst buffer with enough length to avoid buffer overflow.
+        */
+       sg_init_one(&output, dst, PAGE_SIZE * 2);
+       acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
+
+       /*
+        * it maybe looks a little bit silly that we send an asynchronous request,
+        * then wait for its completion synchronously. This makes the process look
+        * synchronous in fact.
+        * Theoretically, acomp supports users send multiple acomp requests in one
+        * acomp instance, then get those requests done simultaneously. but in this
+        * case, zswap actually does store and load page by page, there is no
+        * existing method to send the second page before the first page is done
+        * in one thread doing zwap.
+        * but in different threads running on different cpu, we have different
+        * acomp instance, so multiple threads can do (de)compression in parallel.
+        */
+       ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
+       dlen = acomp_ctx->req->dlen;
+       if (ret) {
+               zswap_reject_compress_fail++;
+               goto unlock;
+       }
+
+       zpool = zswap_find_zpool(entry);
+       gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
+       if (zpool_malloc_support_movable(zpool))
+               gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
+       ret = zpool_malloc(zpool, dlen, gfp, &handle);
+       if (ret == -ENOSPC) {
+               zswap_reject_compress_poor++;
+               goto unlock;
+       }
+       if (ret) {
+               zswap_reject_alloc_fail++;
+               goto unlock;
+       }
+
+       buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
+       memcpy(buf, dst, dlen);
+       zpool_unmap_handle(zpool, handle);
+
+       entry->handle = handle;
+       entry->length = dlen;
+
+unlock:
+       mutex_unlock(&acomp_ctx->mutex);
+       return ret == 0;
+}
+
+static void zswap_decompress(struct zswap_entry *entry, struct page *page)
+{
+       struct zpool *zpool = zswap_find_zpool(entry);
+       struct scatterlist input, output;
+       struct crypto_acomp_ctx *acomp_ctx;
+       u8 *src;
+
+       acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
+       mutex_lock(&acomp_ctx->mutex);
+
+       src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
+       if (!zpool_can_sleep_mapped(zpool)) {
+               memcpy(acomp_ctx->buffer, src, entry->length);
+               src = acomp_ctx->buffer;
+               zpool_unmap_handle(zpool, entry->handle);
+       }
+
+       sg_init_one(&input, src, entry->length);
+       sg_init_table(&output, 1);
+       sg_set_page(&output, page, PAGE_SIZE, 0);
+       acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
+       BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
+       BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
+       mutex_unlock(&acomp_ctx->mutex);
+
+       if (zpool_can_sleep_mapped(zpool))
+               zpool_unmap_handle(zpool, entry->handle);
+}
+
 /*********************************
 * shrinker functions
 **********************************/
@@ -1317,108 +1422,6 @@ resched:
        zswap_pool_put(pool);
 }
 
-static bool zswap_compress(struct folio *folio, struct zswap_entry *entry)
-{
-       struct crypto_acomp_ctx *acomp_ctx;
-       struct scatterlist input, output;
-       unsigned int dlen = PAGE_SIZE;
-       unsigned long handle;
-       struct zpool *zpool;
-       char *buf;
-       gfp_t gfp;
-       int ret;
-       u8 *dst;
-
-       acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
-
-       mutex_lock(&acomp_ctx->mutex);
-
-       dst = acomp_ctx->buffer;
-       sg_init_table(&input, 1);
-       sg_set_page(&input, &folio->page, PAGE_SIZE, 0);
-
-       /*
-        * We need PAGE_SIZE * 2 here since there maybe over-compression case,
-        * and hardware-accelerators may won't check the dst buffer size, so
-        * giving the dst buffer with enough length to avoid buffer overflow.
-        */
-       sg_init_one(&output, dst, PAGE_SIZE * 2);
-       acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
-
-       /*
-        * it maybe looks a little bit silly that we send an asynchronous request,
-        * then wait for its completion synchronously. This makes the process look
-        * synchronous in fact.
-        * Theoretically, acomp supports users send multiple acomp requests in one
-        * acomp instance, then get those requests done simultaneously. but in this
-        * case, zswap actually does store and load page by page, there is no
-        * existing method to send the second page before the first page is done
-        * in one thread doing zwap.
-        * but in different threads running on different cpu, we have different
-        * acomp instance, so multiple threads can do (de)compression in parallel.
-        */
-       ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
-       dlen = acomp_ctx->req->dlen;
-       if (ret) {
-               zswap_reject_compress_fail++;
-               goto unlock;
-       }
-
-       zpool = zswap_find_zpool(entry);
-       gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
-       if (zpool_malloc_support_movable(zpool))
-               gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
-       ret = zpool_malloc(zpool, dlen, gfp, &handle);
-       if (ret == -ENOSPC) {
-               zswap_reject_compress_poor++;
-               goto unlock;
-       }
-       if (ret) {
-               zswap_reject_alloc_fail++;
-               goto unlock;
-       }
-
-       buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
-       memcpy(buf, dst, dlen);
-       zpool_unmap_handle(zpool, handle);
-
-       entry->handle = handle;
-       entry->length = dlen;
-
-unlock:
-       mutex_unlock(&acomp_ctx->mutex);
-       return ret == 0;
-}
-
-static void zswap_decompress(struct zswap_entry *entry, struct page *page)
-{
-       struct zpool *zpool = zswap_find_zpool(entry);
-       struct scatterlist input, output;
-       struct crypto_acomp_ctx *acomp_ctx;
-       u8 *src;
-
-       acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
-       mutex_lock(&acomp_ctx->mutex);
-
-       src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
-       if (!zpool_can_sleep_mapped(zpool)) {
-               memcpy(acomp_ctx->buffer, src, entry->length);
-               src = acomp_ctx->buffer;
-               zpool_unmap_handle(zpool, entry->handle);
-       }
-
-       sg_init_one(&input, src, entry->length);
-       sg_init_table(&output, 1);
-       sg_set_page(&output, page, PAGE_SIZE, 0);
-       acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
-       BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
-       BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
-       mutex_unlock(&acomp_ctx->mutex);
-
-       if (zpool_can_sleep_mapped(zpool))
-               zpool_unmap_handle(zpool, entry->handle);
-}
-
 /*********************************
 * writeback code
 **********************************/