diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 91828599c9f8..96d420b8fd40 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1232,11 +1232,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, goto out_mm; } for (vma = mm->mmap; vma; vma = vma->vm_next) { - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, - vma->vm_flags & ~VM_SOFTDIRTY); + vma->vm_flags &= ~VM_SOFTDIRTY; vma_set_page_prot(vma); - vm_write_end(vma); } downgrade_write(&mm->mmap_sem); break; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 3ada0736144f..0c6e16edd476 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -677,11 +677,8 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) octx = vma->vm_userfaultfd_ctx.ctx; if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) { - vm_write_begin(vma); vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - WRITE_ONCE(vma->vm_flags, - vma->vm_flags & ~(VM_UFFD_WP | VM_UFFD_MISSING)); - vm_write_end(vma); + vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING); return 0; } @@ -923,10 +920,8 @@ static int userfaultfd_release(struct inode *inode, struct file *file) else prev = vma; } - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, new_flags); + vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vm_write_end(vma); } up_write(&mm->mmap_sem); mmput(mm); @@ -1488,10 +1483,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, * the next vma was merged into the current one and * the current one has not been updated yet. */ - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, new_flags); + vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx.ctx = ctx; - vm_write_end(vma); skip: prev = vma; @@ -1653,10 +1646,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, * the next vma was merged into the current one and * the current one has not been updated yet. */ - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, new_flags); + vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; - vm_write_end(vma); skip: prev = vma; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 5270a38b7e5b..5dd14ef2e1de 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1027,7 +1027,6 @@ static void collapse_huge_page(struct mm_struct *mm, if (mm_find_pmd(mm, address) != pmd) goto out; - vm_write_begin(vma); anon_vma_lock_write(vma->anon_vma); pte = pte_offset_map(pmd, address); @@ -1063,7 +1062,6 @@ static void collapse_huge_page(struct mm_struct *mm, pmd_populate(mm, pmd, pmd_pgtable(_pmd)); spin_unlock(pmd_ptl); anon_vma_unlock_write(vma->anon_vma); - vm_write_end(vma); result = SCAN_FAIL; goto out; } @@ -1098,7 +1096,6 @@ static void collapse_huge_page(struct mm_struct *mm, set_pmd_at(mm, address, pmd, _pmd); update_mmu_cache_pmd(vma, address, pmd); spin_unlock(pmd_ptl); - vm_write_end(vma); *hpage = NULL; diff --git a/mm/madvise.c b/mm/madvise.c index 31b2a86d126d..68a9626ef8f2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -184,9 +184,7 @@ success: /* * vm_flags is protected by the mmap_sem held in write mode. */ - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, new_flags); - vm_write_end(vma); + vma->vm_flags = new_flags; out: return error; } @@ -452,11 +450,9 @@ static void madvise_free_page_range(struct mmu_gather *tlb, .private = tlb, }; - vm_write_begin(vma); tlb_start_vma(tlb, vma); walk_page_range(addr, end, &free_walk); tlb_end_vma(tlb, vma); - vm_write_end(vma); } static int madvise_free_single_vma(struct vm_area_struct *vma, diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 80a975644644..7270a1d6844d 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -380,11 +380,8 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) struct vm_area_struct *vma; down_write(&mm->mmap_sem); - for (vma = mm->mmap; vma; vma = vma->vm_next) { - vm_write_begin(vma); + for (vma = mm->mmap; vma; vma = vma->vm_next) mpol_rebind_policy(vma->vm_policy, new); - vm_write_end(vma); - } up_write(&mm->mmap_sem); } @@ -599,11 +596,9 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, { int nr_updated; - vm_write_begin(vma); nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1); if (nr_updated) count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated); - vm_write_end(vma); return nr_updated; } @@ -717,7 +712,6 @@ static int vma_replace_policy(struct vm_area_struct *vma, if (IS_ERR(new)) return PTR_ERR(new); - vm_write_begin(vma); if (vma->vm_ops && vma->vm_ops->set_policy) { err = vma->vm_ops->set_policy(vma, new); if (err) @@ -725,17 +719,11 @@ static int vma_replace_policy(struct vm_area_struct *vma, } old = vma->vm_policy; - /* - * The speculative page fault handler accesses this field without - * hodling the mmap_sem. - */ - WRITE_ONCE(vma->vm_policy, new); - vm_write_end(vma); + vma->vm_policy = new; /* protected by mmap_sem */ mpol_put(old); return 0; err_out: - vm_write_end(vma); mpol_put(new); return err; } @@ -1700,28 +1688,23 @@ COMPAT_SYSCALL_DEFINE4(migrate_pages, compat_pid_t, pid, struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, unsigned long addr) { - struct mempolicy *pol; + struct mempolicy *pol = NULL; - if (!vma) - return NULL; + if (vma) { + if (vma->vm_ops && vma->vm_ops->get_policy) { + pol = vma->vm_ops->get_policy(vma, addr); + } else if (vma->vm_policy) { + pol = vma->vm_policy; - if (vma->vm_ops && vma->vm_ops->get_policy) - return vma->vm_ops->get_policy(vma, addr); - - /* - * This could be called without holding the mmap_sem in the - * speculative page fault handler's path. - */ - pol = READ_ONCE(vma->vm_policy); - if (pol) { - /* - * shmem_alloc_page() passes MPOL_F_SHARED policy with - * a pseudo vma whose vma->vm_ops=NULL. Take a reference - * count on these policies which will be dropped by - * mpol_cond_put() later - */ - if (mpol_needs_cond_ref(pol)) - mpol_get(pol); + /* + * shmem_alloc_page() passes MPOL_F_SHARED policy with + * a pseudo vma whose vma->vm_ops=NULL. Take a reference + * count on these policies which will be dropped by + * mpol_cond_put() later + */ + if (mpol_needs_cond_ref(pol)) + mpol_get(pol); + } } return pol; diff --git a/mm/mlock.c b/mm/mlock.c index 622c845a3ab8..ef2abafbc18f 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -445,9 +445,7 @@ static unsigned long __munlock_pagevec_fill(struct pagevec *pvec, void munlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, vma->vm_flags & VM_LOCKED_CLEAR_MASK); - vm_write_end(vma); + vma->vm_flags &= VM_LOCKED_CLEAR_MASK; while (start < end) { struct page *page; @@ -571,11 +569,10 @@ success: * It's okay if try_to_unmap_one unmaps a page just after we * set VM_LOCKED, populate_vma_page_range will bring it back. */ - if (lock) { - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, newflags); - vm_write_end(vma); - } else + + if (lock) + vma->vm_flags = newflags; + else munlock_vma_pages_range(vma, start, end); out: diff --git a/mm/mmap.c b/mm/mmap.c index 531f27caec7b..c5c3f98d1e15 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -827,18 +827,17 @@ again: } if (start != vma->vm_start) { - WRITE_ONCE(vma->vm_start, start); + vma->vm_start = start; start_changed = true; } if (end != vma->vm_end) { - WRITE_ONCE(vma->vm_end, end); + vma->vm_end = end; end_changed = true; } - WRITE_ONCE(vma->vm_pgoff, pgoff); + vma->vm_pgoff = pgoff; if (adjust_next) { - WRITE_ONCE(next->vm_start, - next->vm_start + (adjust_next << PAGE_SHIFT)); - WRITE_ONCE(next->vm_pgoff, next->vm_pgoff + adjust_next); + next->vm_start += adjust_next << PAGE_SHIFT; + next->vm_pgoff += adjust_next; } if (root) { @@ -1818,14 +1817,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr, out: perf_event_mmap(vma); - vm_write_begin(vma); vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); if (vm_flags & VM_LOCKED) { if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm)) - WRITE_ONCE(vma->vm_flags, - vma->vm_flags & VM_LOCKED_CLEAR_MASK); + vma->vm_flags &= VM_LOCKED_CLEAR_MASK; else mm->locked_vm += (len >> PAGE_SHIFT); } @@ -1840,10 +1837,9 @@ out: * then new mapped in-place (which must be aimed as * a completely new data area). */ - WRITE_ONCE(vma->vm_flags, vma->vm_flags | VM_SOFTDIRTY); + vma->vm_flags |= VM_SOFTDIRTY; vma_set_page_prot(vma); - vm_write_end(vma); return addr; @@ -2473,8 +2469,8 @@ int expand_downwards(struct vm_area_struct *vma, mm->locked_vm += grow; vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); - WRITE_ONCE(vma->vm_start, address); - WRITE_ONCE(vma->vm_pgoff, vma->vm_pgoff - grow); + vma->vm_start = address; + vma->vm_pgoff -= grow; anon_vma_interval_tree_post_update_vma(vma); vma_gap_update(vma); spin_unlock(&mm->page_table_lock); diff --git a/mm/mprotect.c b/mm/mprotect.c index 836a7586d4bf..5c175d46d4cb 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -458,14 +458,12 @@ success: * vm_flags and vm_page_prot are protected by the mmap_sem * held in write mode. */ - vm_write_begin(vma); - WRITE_ONCE(vma->vm_flags, newflags); + vma->vm_flags = newflags; dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); vma_set_page_prot(vma); change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable, 0); - vm_write_end(vma); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major diff --git a/mm/swap_state.c b/mm/swap_state.c index 565e3464a364..b99e8fd93cd3 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -565,10 +565,6 @@ static unsigned long swapin_nr_pages(unsigned long offset) * the readahead. * * Caller must hold down_read on the vma->vm_mm if vmf->vma is not NULL. - * This is needed to ensure the VMA will not be freed in our back. In the case - * of the speculative page fault handler, this cannot happen, even if we don't - * hold the mmap_sem. Callees are assumed to take care of reading VMA's fields - * using READ_ONCE() to read consistent values. */ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, struct vm_fault *vmf) @@ -662,9 +658,9 @@ static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { - *start = max3(lpfn, PFN_DOWN(READ_ONCE(vma->vm_start)), + *start = max3(lpfn, PFN_DOWN(vma->vm_start), PFN_DOWN(faddr & PMD_MASK)); - *end = min3(rpfn, PFN_DOWN(READ_ONCE(vma->vm_end)), + *end = min3(rpfn, PFN_DOWN(vma->vm_end), PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE)); }