From 6aaa7b0664e6886f6154070edbc24435d6e1f86b Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 8 Dec 2020 17:58:32 +0800 Subject: erofs: get rid of magical Z_EROFS_MAPPING_STAGING Previously, we played around with magical page->mapping for short-lived temporary pages since we need to identify different types of pages in the same pcluster but both invalidated and short-lived temporary pages can have page->mapping == NULL. It was considered as safe because that temporary pages are all non-LRU / non-movable pages. This patch tends to use specific page->private to identify short-lived pages instead so it won't rely on page->mapping anymore. Details are described in "compress.h" as well. Link: https://lore.kernel.org/r/20201208095834.3133565-1-hsiangkao@redhat.com Reviewed-by: Chao Yu Signed-off-by: Gao Xiang --- fs/erofs/compress.h | 51 ++++++++++++++++++++++++++++++++----------- fs/erofs/decompressor.c | 2 +- fs/erofs/zdata.c | 57 +++++++++++++++++++++++++++---------------------- fs/erofs/zdata.h | 1 + 4 files changed, 71 insertions(+), 40 deletions(-) (limited to 'fs/erofs') diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h index 3d452443c545..4dadde18cdf1 100644 --- a/fs/erofs/compress.h +++ b/fs/erofs/compress.h @@ -26,30 +26,55 @@ struct z_erofs_decompress_req { bool inplace_io, partial_decoding; }; +/* some special page->private (unsigned long, see below) */ +#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2) + /* - * - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) - - * used to mark temporary allocated pages from other - * file/cached pages and NULL mapping pages. + * For all pages in a pcluster, page->private should be one of + * Type Last 2bits page->private + * short-lived page 00 Z_EROFS_SHORTLIVED_PAGE + * cached/managed page 00 pointer to z_erofs_pcluster + * online page (file-backed, 01/10/11 sub-index << 2 | count + * some pages can be used for inplace I/O) + * + * page->mapping should be one of + * Type page->mapping + * short-lived page NULL + * cached/managed page non-NULL or NULL (invalidated/truncated page) + * online page non-NULL + * + * For all managed pages, PG_private should be set with 1 extra refcount, + * which is used for page reclaim / migration. */ -#define Z_EROFS_MAPPING_STAGING ((void *)0x5A110C8D) -/* check if a page is marked as staging */ -static inline bool z_erofs_page_is_staging(struct page *page) +/* + * short-lived pages are pages directly from buddy system with specific + * page->private (no need to set PagePrivate since these are non-LRU / + * non-movable pages and bypass reclaim / migration code). + */ +static inline bool z_erofs_is_shortlived_page(struct page *page) { - return page->mapping == Z_EROFS_MAPPING_STAGING; + if (page->private != Z_EROFS_SHORTLIVED_PAGE) + return false; + + DBG_BUGON(page->mapping); + return true; } -static inline bool z_erofs_put_stagingpage(struct list_head *pagepool, - struct page *page) +static inline bool z_erofs_put_shortlivedpage(struct list_head *pagepool, + struct page *page) { - if (!z_erofs_page_is_staging(page)) + if (!z_erofs_is_shortlived_page(page)) return false; - /* staging pages should not be used by others at the same time */ - if (page_ref_count(page) > 1) + /* short-lived pages should not be used by others at the same time */ + if (page_ref_count(page) > 1) { put_page(page); - else + } else { + /* follow the pcluster rule above. */ + set_page_private(page, 0); list_add(&page->lru, pagepool); + } return true; } diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index cbadbf55c6c2..1cb1ffd10569 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -76,7 +76,7 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, victim = erofs_allocpage(pagepool, GFP_KERNEL); if (!victim) return -ENOMEM; - victim->mapping = Z_EROFS_MAPPING_STAGING; + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); } rq->out[i] = victim; } diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 86fd3bf62af6..cfb0d11f893b 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -226,11 +226,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, /* barrier is implied in the following 'unlock_page' */ WRITE_ONCE(pcl->compressed_pages[i], NULL); - set_page_private(page, 0); - ClearPagePrivate(page); - + detach_page_private(page); unlock_page(page); - put_page(page); } return 0; } @@ -254,10 +251,8 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, } erofs_workgroup_unfreeze(&pcl->obj, 1); - if (ret) { - ClearPagePrivate(page); - put_page(page); - } + if (ret) + detach_page_private(page); } return ret; } @@ -648,12 +643,12 @@ hitted: retry: err = z_erofs_attach_page(clt, page, page_type); - /* should allocate an additional staging page for pagevec */ + /* should allocate an additional short-lived page for pagevec */ if (err == -EAGAIN) { struct page *const newpage = alloc_page(GFP_NOFS | __GFP_NOFAIL); - newpage->mapping = Z_EROFS_MAPPING_STAGING; + set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE); err = z_erofs_attach_page(clt, newpage, Z_EROFS_PAGE_TYPE_EXCLUSIVE); if (!err) @@ -710,6 +705,11 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, queue_work(z_erofs_workqueue, &io->u.work); } +static bool z_erofs_page_is_invalidated(struct page *page) +{ + return !page->mapping && !z_erofs_is_shortlived_page(page); +} + static void z_erofs_decompressqueue_endio(struct bio *bio) { tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private); @@ -722,7 +722,7 @@ static void z_erofs_decompressqueue_endio(struct bio *bio) struct page *page = bvec->bv_page; DBG_BUGON(PageUptodate(page)); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); if (err) SetPageError(page); @@ -795,9 +795,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, /* all pages in pagevec ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (z_erofs_put_stagingpage(pagepool, page)) + if (z_erofs_put_shortlivedpage(pagepool, page)) continue; if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) @@ -831,9 +831,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, /* all compressed pages ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (!z_erofs_page_is_staging(page)) { + if (!z_erofs_is_shortlived_page(page)) { if (erofs_page_is_managed(sbi, page)) { if (!PageUptodate(page)) err = -EIO; @@ -858,7 +858,7 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, overlapped = true; } - /* PG_error needs checking for inplaced and staging pages */ + /* PG_error needs checking for all non-managed pages */ if (PageError(page)) { DBG_BUGON(PageUptodate(page)); err = -EIO; @@ -897,8 +897,8 @@ out: if (erofs_page_is_managed(sbi, page)) continue; - /* recycle all individual staging pages */ - (void)z_erofs_put_stagingpage(pagepool, page); + /* recycle all individual short-lived pages */ + (void)z_erofs_put_shortlivedpage(pagepool, page); WRITE_ONCE(compressed_pages[i], NULL); } @@ -908,10 +908,10 @@ out: if (!page) continue; - DBG_BUGON(!page->mapping); + DBG_BUGON(z_erofs_page_is_invalidated(page)); - /* recycle all individual staging pages */ - if (z_erofs_put_stagingpage(pagepool, page)) + /* recycle all individual short-lived pages */ + if (z_erofs_put_shortlivedpage(pagepool, page)) continue; if (err < 0) @@ -1011,13 +1011,17 @@ repeat: mapping = READ_ONCE(page->mapping); /* - * unmanaged (file) pages are all locked solidly, + * file-backed online pages in plcuster are all locked steady, * therefore it is impossible for `mapping' to be NULL. */ if (mapping && mapping != mc) /* ought to be unmanaged pages */ goto out; + /* directly return for shortlived page as well */ + if (z_erofs_is_shortlived_page(page)) + goto out; + lock_page(page); /* only true if page reclaim goes wrong, should never happen */ @@ -1062,8 +1066,8 @@ repeat: out_allocpage: page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { - /* non-LRU / non-movable temporary page is needed */ - page->mapping = Z_EROFS_MAPPING_STAGING; + /* turn into temporary page if fails */ + set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); tocache = false; } @@ -1080,8 +1084,9 @@ out_allocpage: } if (tocache) { - set_page_private(page, (unsigned long)pcl); - SetPagePrivate(page); + attach_page_private(page, pcl); + /* drop a ref added by allocpage (then we have 2 refs here) */ + put_page(page); } out: /* the only exit (for tracing and debugging) */ return page; diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h index 68c9b29fc0ca..b503b353d4ab 100644 --- a/fs/erofs/zdata.h +++ b/fs/erofs/zdata.h @@ -173,6 +173,7 @@ static inline void z_erofs_onlinepage_endio(struct page *page) v = atomic_dec_return(u.o); if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) { + set_page_private(page, 0); ClearPagePrivate(page); if (!PageError(page)) SetPageUptodate(page); -- cgit v1.2.3