niansa-misc/linux
1
0
Fork 0
mirror of synced 2025-03-06 20:59:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

zsmalloc: add a LRU to zs_pool to keep track of zspages in LRU order

Browse source 
This helps determines the coldest zspages as candidates for writeback.

Link: https://lkml.kernel.org/r/20221128191616.1261026-5-nphamcs@gmail.com
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Sergey Senozhatsky <senozhatsky@chromium.org>
Cc: Dan Streetman <ddstreet@ieee.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Nitin Gupta <ngupta@vflare.org>
Cc: Seth Jennings <sjenning@redhat.com>
Cc: Vitaly Wool <vitaly.wool@konsulko.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Nhat Pham 2022-11-28 11:16:13 -08:00 • committed by Andrew Morton
parent c0547d0b6a
commit 64f768c6b3
1 changed files with 50 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
mm/zsmalloc.c
View file

@ -239,6 +239,11 @@ struct zs_pool {
/* Compact classes */ /* Compact classes */
struct shrinker shrinker; struct shrinker shrinker;
#ifdef CONFIG_ZPOOL
/* List tracking the zspages in LRU order by most recently added object */
struct list_head lru;
#endif
#ifdef CONFIG_ZSMALLOC_STAT #ifdef CONFIG_ZSMALLOC_STAT
struct dentry *stat_dentry; struct dentry *stat_dentry;
#endif #endif
@ -260,6 +265,12 @@ struct zspage {
unsigned int freeobj; unsigned int freeobj;
struct page *first_page; struct page *first_page;
struct list_head list; /* fullness list */ struct list_head list; /* fullness list */
#ifdef CONFIG_ZPOOL
/* links the zspage to the lru list in the pool */
struct list_head lru;
#endif
struct zs_pool *pool; struct zs_pool *pool;
#ifdef CONFIG_COMPACTION #ifdef CONFIG_COMPACTION
rwlock_t lock; rwlock_t lock;
@ -953,6 +964,9 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
} }
remove_zspage(class, zspage, ZS_EMPTY); remove_zspage(class, zspage, ZS_EMPTY);
#ifdef CONFIG_ZPOOL
list_del(&zspage->lru);
#endif
__free_zspage(pool, class, zspage); __free_zspage(pool, class, zspage);
} }
@ -998,6 +1012,10 @@ static void init_zspage(struct size_class *class, struct zspage *zspage)
off %= PAGE_SIZE; off %= PAGE_SIZE;
} }
#ifdef CONFIG_ZPOOL
INIT_LIST_HEAD(&zspage->lru);
#endif
set_freeobj(zspage, 0); set_freeobj(zspage, 0);
} }
@ -1270,6 +1288,31 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
obj_to_location(obj, &page, &obj_idx); obj_to_location(obj, &page, &obj_idx);
zspage = get_zspage(page); zspage = get_zspage(page);
#ifdef CONFIG_ZPOOL
/*
* Move the zspage to front of pool's LRU.
*
* Note that this is swap-specific, so by definition there are no ongoing
* accesses to the memory while the page is swapped out that would make
* it "hot". A new entry is hot, then ages to the tail until it gets either
* written back or swaps back in.
*
* Furthermore, map is also called during writeback. We must not put an
* isolated page on the LRU mid-reclaim.
*
* As a result, only update the LRU when the page is mapped for write
* when it's first instantiated.
*
* This is a deviation from the other backends, which perform this update
* in the allocation function (zbud_alloc, z3fold_alloc).
*/
if (mm == ZS_MM_WO) {
if (!list_empty(&zspage->lru))
list_del(&zspage->lru);
list_add(&zspage->lru, &pool->lru);
}
#endif
/* /*
* migration cannot move any zpages in this zspage. Here, pool->lock * migration cannot move any zpages in this zspage. Here, pool->lock
* is too heavy since callers would take some time until they calls * is too heavy since callers would take some time until they calls
@ -1988,6 +2031,9 @@ static void async_free_zspage(struct work_struct *work)
VM_BUG_ON(fullness != ZS_EMPTY); VM_BUG_ON(fullness != ZS_EMPTY);
class = pool->size_class[class_idx]; class = pool->size_class[class_idx];
spin_lock(&pool->lock); spin_lock(&pool->lock);
#ifdef CONFIG_ZPOOL
list_del(&zspage->lru);
#endif
__free_zspage(pool, class, zspage); __free_zspage(pool, class, zspage);
spin_unlock(&pool->lock); spin_unlock(&pool->lock);
} }
@ -2299,6 +2345,10 @@ struct zs_pool *zs_create_pool(const char *name)
*/ */
zs_register_shrinker(pool); zs_register_shrinker(pool);
#ifdef CONFIG_ZPOOL
INIT_LIST_HEAD(&pool->lru);
#endif
return pool; return pool;
err: err: