niansa-misc/linux
1
0
Fork 0
mirror of synced 2025-03-06 20:59:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

mm: convert do_set_pte() to set_pte_range()

Browse source 
set_pte_range() allows to setup page table entries for a specific
range.  It takes advantage of batched rmap update for large folio.
It now takes care of calling update_mmu_cache_range().

Link: https://lkml.kernel.org/r/20230802151406.3735276-37-willy@infradead.org
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Yin Fengwei 2023-08-02 16:14:04 +01:00 • committed by Andrew Morton
parent 86f35f69db
commit 3bd786f76d
4 changed files with 28 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Documentation/filesystems/locking.rst
View file

@ -661,7 +661,7 @@ locked. The VM will unlock the page.
Filesystem should find and map pages associated with offsets from "start_pgoff" Filesystem should find and map pages associated with offsets from "start_pgoff"
till "end_pgoff". ->map_pages() is called with the RCU lock held and must till "end_pgoff". ->map_pages() is called with the RCU lock held and must
not block. If it's not possible to reach a page without blocking, not block. If it's not possible to reach a page without blocking,
filesystem should skip it. Filesystem should use do_set_pte() to setup filesystem should skip it. Filesystem should use set_pte_range() to setup
page table entry. Pointer to entry associated with the page is passed in page table entry. Pointer to entry associated with the page is passed in
"pte" field in vm_fault structure. Pointers to entries for other offsets "pte" field in vm_fault structure. Pointers to entries for other offsets
should be calculated relative to "pte". should be calculated relative to "pte".

3
include/linux/mm.h
View file

@ -1322,7 +1322,8 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
} }
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr); void set_pte_range(struct vm_fault *vmf, struct folio *folio,
struct page *page, unsigned int nr, unsigned long addr);
vm_fault_t finish_fault(struct vm_fault *vmf); vm_fault_t finish_fault(struct vm_fault *vmf);
vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf);

3
mm/filemap.c
View file

@ -3501,8 +3501,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
ret = VM_FAULT_NOPAGE; ret = VM_FAULT_NOPAGE;
ref_count++; ref_count++;
do_set_pte(vmf, page, addr); set_pte_range(vmf, folio, page, 1, addr);
update_mmu_cache(vma, addr, vmf->pte);
} while (vmf->pte++, page++, addr += PAGE_SIZE, ++count < nr_pages); } while (vmf->pte++, page++, addr += PAGE_SIZE, ++count < nr_pages);
/* Restore the vmf->pte */ /* Restore the vmf->pte */

37
mm/memory.c
View file

@ -4330,15 +4330,24 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
} }
#endif #endif
void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr) /**
* set_pte_range - Set a range of PTEs to point to pages in a folio.
* @vmf: Fault decription.
* @folio: The folio that contains @page.
* @page: The first page to create a PTE for.
* @nr: The number of PTEs to create.
* @addr: The first address to create a PTE for.
*/
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
struct page *page, unsigned int nr, unsigned long addr)
{ {
struct vm_area_struct *vma = vmf->vma; struct vm_area_struct *vma = vmf->vma;
bool uffd_wp = vmf_orig_pte_uffd_wp(vmf); bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
bool write = vmf->flags & FAULT_FLAG_WRITE; bool write = vmf->flags & FAULT_FLAG_WRITE;
bool prefault = vmf->address != addr; bool prefault = in_range(vmf->address, addr, nr * PAGE_SIZE);
pte_t entry; pte_t entry;
flush_icache_page(vma, page); flush_icache_pages(vma, page, nr);
entry = mk_pte(page, vma->vm_page_prot); entry = mk_pte(page, vma->vm_page_prot);
if (prefault && arch_wants_old_prefaulted_pte()) if (prefault && arch_wants_old_prefaulted_pte())
@ -4352,14 +4361,18 @@ void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr)
entry = pte_mkuffd_wp(entry); entry = pte_mkuffd_wp(entry);
/* copy-on-write page */ /* copy-on-write page */
if (write && !(vma->vm_flags & VM_SHARED)) { if (write && !(vma->vm_flags & VM_SHARED)) {
inc_mm_counter(vma->vm_mm, MM_ANONPAGES); add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr);
page_add_new_anon_rmap(page, vma, addr); VM_BUG_ON_FOLIO(nr != 1, folio);
lru_cache_add_inactive_or_unevictable(page, vma); folio_add_new_anon_rmap(folio, vma, addr);
folio_add_lru_vma(folio, vma);
} else { } else {
inc_mm_counter(vma->vm_mm, mm_counter_file(page)); add_mm_counter(vma->vm_mm, mm_counter_file(page), nr);
page_add_file_rmap(page, vma, false); folio_add_file_rmap_range(folio, page, nr, vma, false);
} }
set_pte_at(vma->vm_mm, addr, vmf->pte, entry); set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
/* no need to invalidate: a not-present page won't be cached */
update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
} }
static bool vmf_pte_changed(struct vm_fault *vmf) static bool vmf_pte_changed(struct vm_fault *vmf)
@ -4427,11 +4440,9 @@ vm_fault_t finish_fault(struct vm_fault *vmf)
/* Re-check under ptl */ /* Re-check under ptl */
if (likely(!vmf_pte_changed(vmf))) { if (likely(!vmf_pte_changed(vmf))) {
do_set_pte(vmf, page, vmf->address); struct folio *folio = page_folio(page);
/* no need to invalidate: a not-present page won't be cached */
update_mmu_cache(vma, vmf->address, vmf->pte);
set_pte_range(vmf, folio, page, 1, vmf->address);
ret = 0; ret = 0;
} else { } else {
update_mmu_tlb(vma, vmf->address, vmf->pte); update_mmu_tlb(vma, vmf->address, vmf->pte);