FROMLIST: mm: multi-gen LRU: groundwork
Evictable pages are divided into multiple generations for each lruvec. The youngest generation number is stored in lrugen->max_seq for both anon and file types as they are aged on an equal footing. The oldest generation numbers are stored in lrugen->min_seq[] separately for anon and file types as clean file pages can be evicted regardless of swap constraints. These three variables are monotonically increasing. Generation numbers are truncated into order_base_2(MAX_NR_GENS+1) bits in order to fit into the gen counter in page->flags. Each truncated generation number is an index to lrugen->lists[]. The sliding window technique is used to track at least MIN_NR_GENS and at most MAX_NR_GENS generations. The gen counter stores a value within [1, MAX_NR_GENS] while a page is on one of lrugen->lists[]. Otherwise it stores 0. There are two conceptually independent procedures: "the aging", which produces young generations, and "the eviction", which consumes old generations. They form a closed-loop system, i.e., "the page reclaim". Both procedures can be invoked from userspace for the purposes of working set estimation and proactive reclaim. These features are required to optimize job scheduling (bin packing) in data centers. The variable size of the sliding window is designed for such use cases [1][2]. To avoid confusion, the terms "hot" and "cold" will be applied to the multi-gen LRU, as a new convention; the terms "active" and "inactive" will be applied to the active/inactive LRU, as usual. The protection of hot pages and the selection of cold pages are based on page access channels and patterns. There are two access channels: one through page tables and the other through file descriptors. The protection of the former channel is by design stronger because: 1. The uncertainty in determining the access patterns of the former channel is higher due to the approximation of the accessed bit. 2. The cost of evicting the former channel is higher due to the TLB flushes required and the likelihood of encountering the dirty bit. 3. The penalty of underprotecting the former channel is higher because applications usually do not prepare themselves for major page faults like they do for blocked I/O. E.g., GUI applications commonly use dedicated I/O threads to avoid blocking the rendering threads. There are also two access patterns: one with temporal locality and the other without. For the reasons listed above, the former channel is assumed to follow the former pattern unless VM_SEQ_READ or VM_RAND_READ is present; the latter channel is assumed to follow the latter pattern unless outlying refaults have been observed [3][4]. The next patch will address the "outlying refaults". Three macros, i.e., LRU_REFS_WIDTH, LRU_REFS_PGOFF and LRU_REFS_MASK, used later are added in this patch to make the entire patchset less diffy. A page is added to the youngest generation on faulting. The aging needs to check the accessed bit at least twice before handing this page over to the eviction. The first check takes care of the accessed bit set on the initial fault; the second check makes sure this page has not been used since then. This protocol, AKA second chance, requires a minimum of two generations, hence MIN_NR_GENS. [1] https://dl.acm.org/doi/10.1145/3297858.3304053 [2] https://dl.acm.org/doi/10.1145/3503222.3507731 [3] https://lwn.net/Articles/495543/ [4] https://lwn.net/Articles/815342/ Link: https://lore.kernel.org/r/20220309021230.721028-6-yuzhao@google.com/ Signed-off-by: Yu Zhao <yuzhao@google.com> Acked-by: Brian Geffon <bgeffon@google.com> Acked-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org> Acked-by: Oleksandr Natalenko <oleksandr@natalenko.name> Acked-by: Steven Barrett <steven@liquorix.net> Acked-by: Suleiman Souhlal <suleiman@google.com> Tested-by: Daniel Byrne <djbyrne@mtu.edu> Tested-by: Donald Carr <d@chaos-reins.com> Tested-by: Holger Hoffstätte <holger@applied-asynchrony.com> Tested-by: Konstantin Kharlamov <Hi-Angel@yandex.ru> Tested-by: Shuang Zhai <szhai2@cs.rochester.edu> Tested-by: Sofia Trinh <sofia.trinh@edi.works> Tested-by: Vaibhav Jain <vaibhav@linux.ibm.com> Bug: 228114874 Change-Id: I333ec6a1d2abfa60d93d6adc190ed3eefe441512
This commit is contained in:
Yu Zhao
spakkkk
parent
1d85493d1e
commit
c5b899bcd2
@ -845,7 +845,8 @@ static int fuse_check_page(struct page *page)
|
||||
1 << PG_active |
|
||||
1 << PG_workingset |
|
||||
1 << PG_reclaim |
|
||||
1 << PG_waiters))) {
|
||||
1 << PG_waiters |
|
||||
LRU_GEN_MASK | LRU_REFS_MASK))) {
|
||||
printk(KERN_WARNING "fuse: trying to steal weird page\n");
|
||||
printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
|
||||
return 1;
|
||||
|
||||
@ -830,6 +830,8 @@ vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf);
|
||||
#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
|
||||
#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
|
||||
#define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
|
||||
#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
|
||||
#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH)
|
||||
|
||||
/*
|
||||
* Define the bit shifts to access each section. For non-existent
|
||||
|
||||
@ -29,6 +29,8 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec,
|
||||
{
|
||||
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
|
||||
|
||||
lockdep_assert_held(&pgdat->lru_lock);
|
||||
|
||||
__mod_node_page_state(pgdat, NR_LRU_BASE + lru, nr_pages);
|
||||
__mod_zone_page_state(&pgdat->node_zones[zid],
|
||||
NR_ZONE_LRU_BASE + lru, nr_pages);
|
||||
@ -87,11 +89,177 @@ static __always_inline enum lru_list page_lru(struct page *page)
|
||||
|
||||
#define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
static inline bool lru_gen_enabled(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_in_fault(void)
|
||||
{
|
||||
return current->in_lru_fault;
|
||||
}
|
||||
|
||||
static inline int lru_gen_from_seq(unsigned long seq)
|
||||
{
|
||||
return seq % MAX_NR_GENS;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
|
||||
{
|
||||
unsigned long max_seq = lruvec->lrugen.max_seq;
|
||||
|
||||
VM_BUG_ON(gen >= MAX_NR_GENS);
|
||||
|
||||
/* see the comment on MIN_NR_GENS */
|
||||
return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
|
||||
}
|
||||
|
||||
static inline void lru_gen_update_size(struct lruvec *lruvec, struct page *page,
|
||||
int old_gen, int new_gen)
|
||||
{
|
||||
int type = page_is_file_cache(page);
|
||||
int zone = page_zonenum(page);
|
||||
int delta = hpage_nr_pages(page);
|
||||
enum lru_list lru = type * LRU_INACTIVE_FILE;
|
||||
struct lru_gen_struct *lrugen = &lruvec->lrugen;
|
||||
|
||||
VM_BUG_ON(old_gen != -1 && old_gen >= MAX_NR_GENS);
|
||||
VM_BUG_ON(new_gen != -1 && new_gen >= MAX_NR_GENS);
|
||||
VM_BUG_ON(old_gen == -1 && new_gen == -1);
|
||||
|
||||
if (old_gen >= 0)
|
||||
WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
|
||||
lrugen->nr_pages[old_gen][type][zone] - delta);
|
||||
if (new_gen >= 0)
|
||||
WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
|
||||
lrugen->nr_pages[new_gen][type][zone] + delta);
|
||||
|
||||
/* addition */
|
||||
if (old_gen < 0) {
|
||||
if (lru_gen_is_active(lruvec, new_gen))
|
||||
lru += LRU_ACTIVE;
|
||||
update_lru_size(lruvec, lru, zone, delta);
|
||||
return;
|
||||
}
|
||||
|
||||
/* deletion */
|
||||
if (new_gen < 0) {
|
||||
if (lru_gen_is_active(lruvec, old_gen))
|
||||
lru += LRU_ACTIVE;
|
||||
update_lru_size(lruvec, lru, zone, -delta);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
|
||||
{
|
||||
int gen;
|
||||
unsigned long old_flags, new_flags;
|
||||
int type = page_is_file_cache(page);
|
||||
int zone = page_zonenum(page);
|
||||
struct lru_gen_struct *lrugen = &lruvec->lrugen;
|
||||
|
||||
if (PageUnevictable(page))
|
||||
return false;
|
||||
/*
|
||||
* There are three common cases for this page:
|
||||
* 1. If it's hot, e.g., freshly faulted in or previously hot and
|
||||
* migrated, add it to the youngest generation.
|
||||
* 2. If it's cold but can't be evicted immediately, i.e., an anon page
|
||||
* not in swapcache or a dirty page pending writeback, add it to the
|
||||
* second oldest generation.
|
||||
* 3. Everything else (clean, cold) is added to the oldest generation.
|
||||
*/
|
||||
if (PageActive(page))
|
||||
gen = lru_gen_from_seq(lrugen->max_seq);
|
||||
else if ((type == LRU_GEN_ANON && !PageSwapCache(page)) ||
|
||||
(PageReclaim(page) && (PageDirty(page) || PageWriteback(page))))
|
||||
gen = lru_gen_from_seq(lrugen->min_seq[type] + 1);
|
||||
else
|
||||
gen = lru_gen_from_seq(lrugen->min_seq[type]);
|
||||
|
||||
do {
|
||||
new_flags = old_flags = READ_ONCE(page->flags);
|
||||
VM_BUG_ON_PAGE(new_flags & LRU_GEN_MASK, page);
|
||||
|
||||
/* see the comment on MIN_NR_GENS */
|
||||
new_flags &= ~(LRU_GEN_MASK | BIT(PG_active));
|
||||
new_flags |= (gen + 1UL) << LRU_GEN_PGOFF;
|
||||
} while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags);
|
||||
|
||||
lru_gen_update_size(lruvec, page, -1, gen);
|
||||
/* for rotate_reclaimable_page() */
|
||||
if (reclaiming)
|
||||
list_add_tail(&page->lru, &lrugen->lists[gen][type][zone]);
|
||||
else
|
||||
list_add(&page->lru, &lrugen->lists[gen][type][zone]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_del_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
|
||||
{
|
||||
int gen;
|
||||
unsigned long old_flags, new_flags;
|
||||
|
||||
do {
|
||||
new_flags = old_flags = READ_ONCE(page->flags);
|
||||
if (!(new_flags & LRU_GEN_MASK))
|
||||
return false;
|
||||
|
||||
VM_BUG_ON_PAGE(PageActive(page), page);
|
||||
VM_BUG_ON_PAGE(PageUnevictable(page), page);
|
||||
|
||||
gen = ((new_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
|
||||
|
||||
new_flags &= ~LRU_GEN_MASK;
|
||||
/* for shrink_page_list() */
|
||||
if (reclaiming)
|
||||
new_flags &= ~(BIT(PG_referenced) | BIT(PG_reclaim));
|
||||
else if (lru_gen_is_active(lruvec, gen))
|
||||
new_flags |= BIT(PG_active);
|
||||
} while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags);
|
||||
|
||||
lru_gen_update_size(lruvec, page, gen, -1);
|
||||
list_del(&page->lru);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
static inline bool lru_gen_enabled(void)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_in_fault(void)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool lru_gen_del_page(struct lruvec *lruvec, struct page *page, bool reclaiming)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
static __always_inline void add_page_to_lru_list(struct page *page,
|
||||
struct lruvec *lruvec)
|
||||
{
|
||||
enum lru_list lru = page_lru(page);
|
||||
|
||||
if (lru_gen_add_page(lruvec, page, false))
|
||||
return;
|
||||
|
||||
update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page));
|
||||
list_add(&page->lru, &lruvec->lists[lru]);
|
||||
}
|
||||
@ -101,6 +269,9 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page,
|
||||
{
|
||||
enum lru_list lru = page_lru(page);
|
||||
|
||||
if (lru_gen_add_page(lruvec, page, true))
|
||||
return;
|
||||
|
||||
update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page));
|
||||
list_add_tail(&page->lru, &lruvec->lists[lru]);
|
||||
}
|
||||
@ -108,6 +279,9 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page,
|
||||
static __always_inline void del_page_from_lru_list(struct page *page,
|
||||
struct lruvec *lruvec)
|
||||
{
|
||||
if (lru_gen_del_page(lruvec, page, false))
|
||||
return;
|
||||
|
||||
list_del(&page->lru);
|
||||
update_lru_size(lruvec, page_lru(page), page_zonenum(page),
|
||||
-hpage_nr_pages(page));
|
||||
|
||||
@ -244,6 +244,97 @@ struct zone_reclaim_stat {
|
||||
unsigned long recent_scanned[ANON_AND_FILE];
|
||||
};
|
||||
|
||||
#endif /* !__GENERATING_BOUNDS_H */
|
||||
|
||||
/*
|
||||
* Evictable pages are divided into multiple generations. The youngest and the
|
||||
* oldest generation numbers, max_seq and min_seq, are monotonically increasing.
|
||||
* They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
|
||||
* offset within MAX_NR_GENS, gen, indexes the LRU list of the corresponding
|
||||
* generation. The gen counter in page->flags stores gen+1 while a page is on
|
||||
* one of lrugen->lists[]. Otherwise it stores 0.
|
||||
*
|
||||
* A page is added to the youngest generation on faulting. The aging needs to
|
||||
* check the accessed bit at least twice before handing this page over to the
|
||||
* eviction. The first check takes care of the accessed bit set on the initial
|
||||
* fault; the second check makes sure this page hasn't been used since then.
|
||||
* This process, AKA second chance, requires a minimum of two generations,
|
||||
* hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive
|
||||
* LRU, these two generations are considered active; the rest of generations, if
|
||||
* they exist, are considered inactive. See lru_gen_is_active(). PG_active is
|
||||
* always cleared while a page is on one of lrugen->lists[] so that the aging
|
||||
* needs not to worry about it. And it's set again when a page considered active
|
||||
* is isolated for non-reclaiming purposes, e.g., migration. See
|
||||
* lru_gen_add_page() and lru_gen_del_page().
|
||||
*
|
||||
* MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice of the
|
||||
* categories of the active/inactive LRU when keeping track of accesses through
|
||||
* page tables. It requires order_base_2(MAX_NR_GENS+1) bits in page->flags.
|
||||
*/
|
||||
#define MIN_NR_GENS 2U
|
||||
#define MAX_NR_GENS 4U
|
||||
|
||||
#ifndef __GENERATING_BOUNDS_H
|
||||
|
||||
struct lruvec;
|
||||
struct mem_cgroup;
|
||||
|
||||
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
|
||||
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
enum {
|
||||
LRU_GEN_ANON,
|
||||
LRU_GEN_FILE,
|
||||
};
|
||||
|
||||
/*
|
||||
* The youngest generation number is stored in max_seq for both anon and file
|
||||
* types as they are aged on an equal footing. The oldest generation numbers are
|
||||
* stored in min_seq[] separately for anon and file types as clean file pages
|
||||
* can be evicted regardless of swap constraints.
|
||||
*
|
||||
* Normally anon and file min_seq are in sync. But if swapping is constrained,
|
||||
* e.g., out of swap space, file min_seq is allowed to advance and leave anon
|
||||
* min_seq behind.
|
||||
*/
|
||||
struct lru_gen_struct {
|
||||
/* the aging increments the youngest generation number */
|
||||
unsigned long max_seq;
|
||||
/* the eviction increments the oldest generation numbers */
|
||||
unsigned long min_seq[ANON_AND_FILE];
|
||||
/* the multi-gen LRU lists */
|
||||
struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
|
||||
/* the sizes of the above lists */
|
||||
unsigned long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
|
||||
};
|
||||
|
||||
void lru_gen_init_lruvec(struct lruvec *lruvec);
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
void lru_gen_init_memcg(struct mem_cgroup *memcg);
|
||||
void lru_gen_exit_memcg(struct mem_cgroup *memcg);
|
||||
#endif
|
||||
|
||||
#else /* !CONFIG_LRU_GEN */
|
||||
|
||||
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
|
||||
{
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
static inline void lru_gen_init_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
struct lruvec {
|
||||
struct list_head lists[NR_LRU_LISTS];
|
||||
struct zone_reclaim_stat reclaim_stat;
|
||||
@ -251,6 +342,10 @@ struct lruvec {
|
||||
atomic_long_t inactive_age;
|
||||
/* Refaults at the time of last reclaim cycle */
|
||||
unsigned long refaults;
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* evictable pages divided into generations */
|
||||
struct lru_gen_struct lrugen;
|
||||
#endif
|
||||
#ifdef CONFIG_MEMCG
|
||||
struct pglist_data *pgdat;
|
||||
#endif
|
||||
|
||||
@ -56,7 +56,8 @@
|
||||
|
||||
#define ZONES_WIDTH ZONES_SHIFT
|
||||
|
||||
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LRU_GEN_WIDTH+LRU_REFS_WIDTH \
|
||||
<= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#define NODES_WIDTH NODES_SHIFT
|
||||
#else
|
||||
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
||||
@ -83,15 +84,15 @@
|
||||
#define KASAN_TAG_WIDTH 0
|
||||
#endif
|
||||
|
||||
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_CPUPID_SHIFT+KASAN_TAG_WIDTH \
|
||||
<= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LRU_GEN_WIDTH+LRU_REFS_WIDTH+ \
|
||||
KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
|
||||
#else
|
||||
#define LAST_CPUPID_WIDTH 0
|
||||
#endif
|
||||
|
||||
#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH+LAST_CPUPID_WIDTH+KASAN_TAG_WIDTH \
|
||||
> BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_WIDTH+LRU_GEN_WIDTH+LRU_REFS_WIDTH+ \
|
||||
KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
|
||||
#error "Not enough bits in page flags"
|
||||
#endif
|
||||
|
||||
|
||||
@ -770,7 +770,7 @@ static inline void ClearPageSlabPfmemalloc(struct page *page)
|
||||
1UL << PG_private | 1UL << PG_private_2 | \
|
||||
1UL << PG_writeback | 1UL << PG_reserved | \
|
||||
1UL << PG_slab | 1UL << PG_active | \
|
||||
1UL << PG_unevictable | __PG_MLOCKED)
|
||||
1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
|
||||
|
||||
/*
|
||||
* Flags checked when a page is prepped for return by the page allocator.
|
||||
@ -781,7 +781,7 @@ static inline void ClearPageSlabPfmemalloc(struct page *page)
|
||||
* alloc-free cycle to prevent from reusing the page.
|
||||
*/
|
||||
#define PAGE_FLAGS_CHECK_AT_PREP \
|
||||
(((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
|
||||
((((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
|
||||
|
||||
#define PAGE_FLAGS_PRIVATE \
|
||||
(1UL << PG_private | 1UL << PG_private_2)
|
||||
|
||||
@ -980,6 +980,10 @@ struct task_struct {
|
||||
unsigned memcg_kmem_skip_account:1;
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
/* whether the LRU algorithm may apply to this access */
|
||||
unsigned in_lru_fault:1;
|
||||
#endif
|
||||
#ifdef CONFIG_COMPAT_BRK
|
||||
unsigned brk_randomized:1;
|
||||
#endif
|
||||
|
||||
@ -22,6 +22,13 @@ int main(void)
|
||||
DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
|
||||
#endif
|
||||
DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
|
||||
DEFINE(LRU_REFS_WIDTH, 0);
|
||||
#else
|
||||
DEFINE(LRU_GEN_WIDTH, 0);
|
||||
DEFINE(LRU_REFS_WIDTH, 0);
|
||||
#endif
|
||||
/* End of constants */
|
||||
|
||||
return 0;
|
||||
|
||||
@ -843,4 +843,12 @@ config GUP_BENCHMARK
|
||||
config ARCH_HAS_PTE_SPECIAL
|
||||
bool
|
||||
|
||||
config LRU_GEN
|
||||
bool "Multi-Gen LRU"
|
||||
depends on MMU
|
||||
# the following options can use up the spare bits in page flags
|
||||
depends on !MAXSMP && (64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP)
|
||||
help
|
||||
A high performance LRU implementation to overcommit memory.
|
||||
|
||||
endmenu
|
||||
|
||||
@ -2475,7 +2475,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
|
||||
(1L << PG_workingset) |
|
||||
(1L << PG_locked) |
|
||||
(1L << PG_unevictable) |
|
||||
(1L << PG_dirty)));
|
||||
(1L << PG_dirty) |
|
||||
LRU_GEN_MASK | LRU_REFS_MASK));
|
||||
|
||||
/* ->mapping in first tail page is compound_mapcount */
|
||||
VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
|
||||
|
||||
@ -1184,12 +1184,14 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
|
||||
*lru_size += nr_pages;
|
||||
|
||||
size = *lru_size;
|
||||
#ifndef CONFIG_LRU_GEN
|
||||
if (WARN_ONCE(size < 0,
|
||||
"%s(%p, %d, %d): lru_size %ld\n",
|
||||
__func__, lruvec, lru, nr_pages, size)) {
|
||||
VM_BUG_ON(1);
|
||||
*lru_size = 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (nr_pages > 0)
|
||||
*lru_size += nr_pages;
|
||||
@ -4501,6 +4503,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
|
||||
|
||||
static void mem_cgroup_free(struct mem_cgroup *memcg)
|
||||
{
|
||||
lru_gen_exit_memcg(memcg);
|
||||
memcg_wb_domain_exit(memcg);
|
||||
__mem_cgroup_free(memcg);
|
||||
}
|
||||
@ -4551,6 +4554,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
|
||||
INIT_LIST_HEAD(&memcg->cgwb_list);
|
||||
#endif
|
||||
idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
|
||||
lru_gen_init_memcg(memcg);
|
||||
return memcg;
|
||||
fail:
|
||||
mem_cgroup_id_remove(memcg);
|
||||
|
||||
27
mm/memory.c
27
mm/memory.c
@ -40,6 +40,7 @@
|
||||
|
||||
#include <linux/kernel_stat.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/mm_inline.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/sched/coredump.h>
|
||||
#include <linux/sched/numa_balancing.h>
|
||||
@ -3088,6 +3089,27 @@ void unmap_mapping_range(struct address_space *mapping,
|
||||
}
|
||||
EXPORT_SYMBOL(unmap_mapping_range);
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
||||
{
|
||||
/* the LRU algorithm doesn't apply to sequential or random reads */
|
||||
current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ));
|
||||
}
|
||||
|
||||
static void lru_gen_exit_fault(void)
|
||||
{
|
||||
current->in_lru_fault = false;
|
||||
}
|
||||
#else
|
||||
static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
||||
{
|
||||
}
|
||||
|
||||
static void lru_gen_exit_fault(void)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
/*
|
||||
* We enter with non-exclusive mmap_sem (to exclude vma changes,
|
||||
* but allow concurrent faults), and pte mapped but not yet locked.
|
||||
@ -3263,6 +3285,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
|
||||
} else {
|
||||
do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
|
||||
mem_cgroup_commit_charge(page, memcg, true, false);
|
||||
if (!lru_gen_enabled())
|
||||
activate_page(page);
|
||||
}
|
||||
|
||||
@ -4410,11 +4433,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
|
||||
if (flags & FAULT_FLAG_USER)
|
||||
mem_cgroup_enter_user_fault();
|
||||
|
||||
lru_gen_enter_fault(vma);
|
||||
|
||||
if (unlikely(is_vm_hugetlb_page(vma)))
|
||||
ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
|
||||
else
|
||||
ret = __handle_mm_fault(vma, address, flags);
|
||||
|
||||
lru_gen_exit_fault();
|
||||
|
||||
if (flags & FAULT_FLAG_USER) {
|
||||
mem_cgroup_exit_user_fault();
|
||||
/*
|
||||
|
||||
@ -66,13 +66,17 @@ void __init mminit_verify_pageflags_layout(void)
|
||||
unsigned long or_mask, add_mask;
|
||||
|
||||
shift = 8 * sizeof(unsigned long);
|
||||
width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - LAST_CPUPID_SHIFT;
|
||||
width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
|
||||
- LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
|
||||
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d Flags %d\n",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
|
||||
SECTIONS_WIDTH,
|
||||
NODES_WIDTH,
|
||||
ZONES_WIDTH,
|
||||
LAST_CPUPID_WIDTH,
|
||||
KASAN_TAG_WIDTH,
|
||||
LRU_GEN_WIDTH,
|
||||
LRU_REFS_WIDTH,
|
||||
NR_PAGEFLAGS);
|
||||
mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
|
||||
"Section %d Node %d Zone %d Lastcpupid %d\n",
|
||||
|
||||
@ -95,6 +95,8 @@ void lruvec_init(struct lruvec *lruvec)
|
||||
|
||||
for_each_lru(lru)
|
||||
INIT_LIST_HEAD(&lruvec->lists[lru]);
|
||||
|
||||
lru_gen_init_lruvec(lruvec);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
|
||||
|
||||
@ -398,6 +398,11 @@ static void __lru_cache_add(struct page *page)
|
||||
{
|
||||
struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
|
||||
|
||||
/* see the comment in lru_gen_add_page() */
|
||||
if (lru_gen_enabled() && !PageUnevictable(page) && !PageActive(page) &&
|
||||
lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
|
||||
SetPageActive(page);
|
||||
|
||||
get_page(page);
|
||||
if (!pagevec_add(pvec, page) || PageCompound(page))
|
||||
__pagevec_lru_add(pvec);
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
#include <linux/mm.h>
|
||||
#include <linux/mm_inline.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/sched/task.h>
|
||||
#include <linux/hugetlb.h>
|
||||
@ -1832,6 +1833,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
|
||||
* Move the page to the active list so it is not
|
||||
* immediately swapped out again after swapon.
|
||||
*/
|
||||
if (!lru_gen_enabled())
|
||||
activate_page(page);
|
||||
out:
|
||||
pte_unmap_unlock(pte, ptl);
|
||||
@ -1981,6 +1983,7 @@ static int unuse_mm(struct mm_struct *mm,
|
||||
* Activate page so shrink_inactive_list is unlikely to unmap
|
||||
* its ptes while lock is dropped, so swapoff can make progress.
|
||||
*/
|
||||
if (!lru_gen_enabled())
|
||||
activate_page(page);
|
||||
unlock_page(page);
|
||||
down_read(&mm->mmap_sem);
|
||||
|
||||
75
mm/vmscan.c
75
mm/vmscan.c
@ -2532,6 +2532,81 @@ out:
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LRU_GEN
|
||||
|
||||
/******************************************************************************
|
||||
* shorthand helpers
|
||||
******************************************************************************/
|
||||
|
||||
#define for_each_gen_type_zone(gen, type, zone) \
|
||||
for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \
|
||||
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
|
||||
for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++)
|
||||
|
||||
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
|
||||
{
|
||||
struct pglist_data *pgdat = NODE_DATA(nid);
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
if (memcg) {
|
||||
struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
|
||||
|
||||
/* for hotadd_new_pgdat() */
|
||||
if (!lruvec->pgdat)
|
||||
lruvec->pgdat = pgdat;
|
||||
|
||||
return lruvec;
|
||||
}
|
||||
#endif
|
||||
VM_BUG_ON(!mem_cgroup_disabled());
|
||||
|
||||
return pgdat ? &pgdat->lruvec : NULL;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
* initialization
|
||||
******************************************************************************/
|
||||
|
||||
void lru_gen_init_lruvec(struct lruvec *lruvec)
|
||||
{
|
||||
int gen, type, zone;
|
||||
struct lru_gen_struct *lrugen = &lruvec->lrugen;
|
||||
|
||||
lrugen->max_seq = MIN_NR_GENS + 1;
|
||||
|
||||
for_each_gen_type_zone(gen, type, zone)
|
||||
INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
void lru_gen_init_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
}
|
||||
|
||||
void lru_gen_exit_memcg(struct mem_cgroup *memcg)
|
||||
{
|
||||
int nid;
|
||||
|
||||
for_each_node(nid) {
|
||||
struct lruvec *lruvec = get_lruvec(memcg, nid);
|
||||
|
||||
VM_BUG_ON(memchr_inv(lruvec->lrugen.nr_pages, 0,
|
||||
sizeof(lruvec->lrugen.nr_pages)));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static int __init init_lru_gen(void)
|
||||
{
|
||||
BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS);
|
||||
BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS);
|
||||
|
||||
return 0;
|
||||
};
|
||||
late_initcall(init_lru_gen);
|
||||
|
||||
#endif /* CONFIG_LRU_GEN */
|
||||
|
||||
/*
|
||||
* This is a basic per-node page freer. Used by both kswapd and direct reclaim.
|
||||
*/
|
||||
|
||||
Loading…
Reference in New Issue
Block a user