diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index a8fbffd47fa0..019d70dfb983 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1564,7 +1564,7 @@ cont: break; } pte_unmap_unlock(pte - 1, ptl); - reclaim_pages_from_list(&page_list); + reclaim_pages_from_list(&page_list, vma); if (addr != end) goto cont; diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 519b7101a8a7..71fd2b33ddd1 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -12,7 +12,8 @@ extern int isolate_lru_page(struct page *page); extern void putback_lru_page(struct page *page); -extern unsigned long reclaim_pages_from_list(struct list_head *page_list); +extern unsigned long reclaim_pages_from_list(struct list_head *page_list, + struct vm_area_struct *vma); /* * The anon_vma heads a list of private "related" vmas, to scan if @@ -190,7 +191,8 @@ int page_referenced(struct page *, int is_locked, #define TTU_ACTION(x) ((x) & TTU_ACTION_MASK) -int try_to_unmap(struct page *, enum ttu_flags flags); +int try_to_unmap(struct page *, enum ttu_flags flags, + struct vm_area_struct *vma); /* * Used by uprobes to replace a userspace page safely @@ -267,6 +269,7 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma); */ struct rmap_walk_control { void *arg; + struct vm_area_struct *target_vma; int (*rmap_one)(struct page *page, struct vm_area_struct *vma, unsigned long addr, void *arg); int (*done)(struct page *page); @@ -291,7 +294,7 @@ static inline int page_referenced(struct page *page, int is_locked, return 0; } -#define try_to_unmap(page, refs) SWAP_FAIL +#define try_to_unmap(page, refs, vma) SWAP_FAIL static inline int page_mkclean(struct page *page) { diff --git a/mm/ksm.c b/mm/ksm.c index 5f1855b7a840..1b1a8f759779 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1964,6 +1964,12 @@ int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc) stable_node = page_stable_node(page); if (!stable_node) return ret; + + if (rwc->target_vma) { + unsigned long address = vma_address(page, rwc->target_vma); + ret = rwc->rmap_one(page, rwc->target_vma, address, rwc->arg); + goto out; + } again: hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { struct anon_vma *anon_vma = rmap_item->anon_vma; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index ce7d416edab7..b335423bb257 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -972,7 +972,7 @@ static int hwpoison_user_mappings(struct page *p, unsigned long pfn, if (kill) collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); - ret = try_to_unmap(hpage, ttu); + ret = try_to_unmap(hpage, ttu, NULL); if (ret != SWAP_SUCCESS) pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n", pfn, page_mapcount(hpage)); diff --git a/mm/migrate.c b/mm/migrate.c index 435f67424cda..f0b786db0f54 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1020,7 +1020,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, page); try_to_unmap(page, - TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); + TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS, NULL); page_was_mapped = 1; } @@ -1238,7 +1238,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, if (page_mapped(hpage)) { try_to_unmap(hpage, - TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); + TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS, NULL); page_was_mapped = 1; } diff --git a/mm/rmap.c b/mm/rmap.c index cd37c1c7e21b..dfb19f0ccf9b 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1616,9 +1616,12 @@ static int page_mapcount_is_zero(struct page *page) * try_to_unmap - try to remove all page table mappings to a page * @page: the page to get unmapped * @flags: action and flags + * @vma : target vma for reclaim * * Tries to remove all the page table entries which are mapping this * page, used in the pageout path. Caller must hold the page lock. + * If @vma is not NULL, this function try to remove @page from only @vma + * without peeking all mapped vma for @page. * Return values are: * * SWAP_SUCCESS - we succeeded in removing all mappings @@ -1626,7 +1629,8 @@ static int page_mapcount_is_zero(struct page *page) * SWAP_FAIL - the page is unswappable * SWAP_MLOCK - page is mlocked. */ -int try_to_unmap(struct page *page, enum ttu_flags flags) +int try_to_unmap(struct page *page, enum ttu_flags flags, + struct vm_area_struct *vma) { int ret; struct rmap_private rp = { @@ -1639,6 +1643,7 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) .arg = &rp, .done = page_mapcount_is_zero, .anon_lock = page_lock_anon_vma_read, + .target_vma = vma, }; /* @@ -1698,6 +1703,7 @@ int try_to_munlock(struct page *page) .arg = &rp, .done = page_not_mapped, .anon_lock = page_lock_anon_vma_read, + .target_vma = NULL, }; @@ -1760,6 +1766,11 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, struct anon_vma_chain *avc; int ret = SWAP_AGAIN; + if (rwc->target_vma) { + unsigned long address = vma_address(page, rwc->target_vma); + return rwc->rmap_one(page, rwc->target_vma, address, rwc->arg); + } + if (locked) { anon_vma = page_anon_vma(page); /* anon_vma disappear under us? */ @@ -1767,6 +1778,7 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, } else { anon_vma = rmap_walk_anon_lock(page, rwc); } + if (!anon_vma) return ret; @@ -1811,6 +1823,7 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, struct address_space *mapping = page_mapping(page); pgoff_t pgoff; struct vm_area_struct *vma; + unsigned long address; int ret = SWAP_AGAIN; /* @@ -1827,6 +1840,13 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, pgoff = page_to_pgoff(page); if (!locked) i_mmap_lock_read(mapping); + + if (rwc->target_vma) { + address = vma_address(page, rwc->target_vma); + ret = rwc->rmap_one(page, rwc->target_vma, address, rwc->arg); + goto done; + } + vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { unsigned long address = vma_address(page, vma); diff --git a/mm/vmscan.c b/mm/vmscan.c index af2e116ded40..838bd5340f9f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -108,6 +108,13 @@ struct scan_control { /* Number of pages freed so far during a call to shrink_zones() */ unsigned long nr_reclaimed; + + /* + * Reclaim pages from a vma. If the page is shared by other tasks + * it is zapped from a vma without reclaim so it ends up remaining + * on memory until last task zap it. + */ + struct vm_area_struct *target_vma; }; #ifdef ARCH_HAS_PREFETCH @@ -1131,7 +1138,8 @@ static unsigned long shrink_page_list(struct list_head *page_list, if (page_mapped(page) && mapping) { switch (ret = try_to_unmap(page, lazyfree ? (ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) : - (ttu_flags | TTU_BATCH_FLUSH))) { + (ttu_flags | TTU_BATCH_FLUSH), + sc->target_vma)) { case SWAP_FAIL: goto activate_locked; case SWAP_AGAIN: @@ -1346,7 +1354,8 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone, } #ifdef CONFIG_PROCESS_RECLAIM -unsigned long reclaim_pages_from_list(struct list_head *page_list) +unsigned long reclaim_pages_from_list(struct list_head *page_list, + struct vm_area_struct *vma) { struct scan_control sc = { .gfp_mask = GFP_KERNEL, @@ -1354,6 +1363,7 @@ unsigned long reclaim_pages_from_list(struct list_head *page_list) .may_writepage = 1, .may_unmap = 1, .may_swap = 1, + .target_vma = vma, }; unsigned long nr_reclaimed;