Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

page cache: use xa_lock

Remove the address_space ->tree_lock and use the xa_lock newly added to
the radix_tree_root. Rename the address_space ->page_tree to ->i_pages,
since we don't really care that it's a tree.

[willy@infradead.org: fix nds32, fs/dax.c]
Link: http://lkml.kernel.org/r/20180406145415.GB20605@bombadil.infradead.orgLink: http://lkml.kernel.org/r/20180313132639.17387-9-willy@infradead.org
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Acked-by: Jeff Layton <jlayton@redhat.com>
Cc: Darrick J. Wong <darrick.wong@oracle.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

authored by

Matthew Wilcox and committed by
Linus Torvalds
b93b0163 f6bb2a2c

+345 -366
+1 -1
Documentation/cgroup-v1/memory.txt
··· 262 262 2.6 Locking 263 263 264 264 lock_page_cgroup()/unlock_page_cgroup() should not be called under 265 - mapping->tree_lock. 265 + the i_pages lock. 266 266 267 267 Other lock order is following: 268 268 PG_locked.
+7 -7
Documentation/vm/page_migration
··· 90 90 91 91 1. Lock the page to be migrated 92 92 93 - 2. Insure that writeback is complete. 93 + 2. Ensure that writeback is complete. 94 94 95 95 3. Lock the new page that we want to move to. It is locked so that accesses to 96 96 this (not yet uptodate) page immediately lock while the move is in progress. ··· 100 100 mapcount is not zero then we do not migrate the page. All user space 101 101 processes that attempt to access the page will now wait on the page lock. 102 102 103 - 5. The radix tree lock is taken. This will cause all processes trying 104 - to access the page via the mapping to block on the radix tree spinlock. 103 + 5. The i_pages lock is taken. This will cause all processes trying 104 + to access the page via the mapping to block on the spinlock. 105 105 106 106 6. The refcount of the page is examined and we back out if references remain 107 107 otherwise we know that we are the only one referencing this page. ··· 114 114 115 115 9. The radix tree is changed to point to the new page. 116 116 117 - 10. The reference count of the old page is dropped because the radix tree 117 + 10. The reference count of the old page is dropped because the address space 118 118 reference is gone. A reference to the new page is established because 119 - the new page is referenced to by the radix tree. 119 + the new page is referenced by the address space. 120 120 121 - 11. The radix tree lock is dropped. With that lookups in the mapping 122 - become possible again. Processes will move from spinning on the tree_lock 121 + 11. The i_pages lock is dropped. With that lookups in the mapping 122 + become possible again. Processes will move from spinning on the lock 123 123 to sleeping on the locked new page. 124 124 125 125 12. The page contents are copied to the new page.
+2 -4
arch/arm/include/asm/cacheflush.h
··· 318 318 #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE 319 319 extern void flush_kernel_dcache_page(struct page *); 320 320 321 - #define flush_dcache_mmap_lock(mapping) \ 322 - spin_lock_irq(&(mapping)->tree_lock) 323 - #define flush_dcache_mmap_unlock(mapping) \ 324 - spin_unlock_irq(&(mapping)->tree_lock) 321 + #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) 322 + #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) 325 323 326 324 #define flush_icache_user_range(vma,page,addr,len) \ 327 325 flush_dcache_page(page)
+2 -2
arch/nds32/include/asm/cacheflush.h
··· 34 34 void flush_kernel_dcache_page(struct page *page); 35 35 void flush_icache_range(unsigned long start, unsigned long end); 36 36 void flush_icache_page(struct vm_area_struct *vma, struct page *page); 37 - #define flush_dcache_mmap_lock(mapping) spin_lock_irq(&(mapping)->tree_lock) 38 - #define flush_dcache_mmap_unlock(mapping) spin_unlock_irq(&(mapping)->tree_lock) 37 + #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&(mapping)->i_pages) 38 + #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&(mapping)->i_pages) 39 39 40 40 #else 41 41 #include <asm-generic/cacheflush.h>
+2 -4
arch/nios2/include/asm/cacheflush.h
··· 46 46 extern void flush_dcache_range(unsigned long start, unsigned long end); 47 47 extern void invalidate_dcache_range(unsigned long start, unsigned long end); 48 48 49 - #define flush_dcache_mmap_lock(mapping) \ 50 - spin_lock_irq(&(mapping)->tree_lock) 51 - #define flush_dcache_mmap_unlock(mapping) \ 52 - spin_unlock_irq(&(mapping)->tree_lock) 49 + #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) 50 + #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) 53 51 54 52 #endif /* _ASM_NIOS2_CACHEFLUSH_H */
+2 -4
arch/parisc/include/asm/cacheflush.h
··· 55 55 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 56 56 extern void flush_dcache_page(struct page *page); 57 57 58 - #define flush_dcache_mmap_lock(mapping) \ 59 - spin_lock_irq(&(mapping)->tree_lock) 60 - #define flush_dcache_mmap_unlock(mapping) \ 61 - spin_unlock_irq(&(mapping)->tree_lock) 58 + #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) 59 + #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) 62 60 63 61 #define flush_icache_page(vma,page) do { \ 64 62 flush_kernel_dcache_page(page); \
+1 -1
drivers/staging/lustre/lustre/llite/glimpse.c
··· 69 69 void *results[1]; 70 70 71 71 if (inode->i_mapping) 72 - cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->page_tree, 72 + cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->i_pages, 73 73 results, 0, 1, 74 74 PAGECACHE_TAG_DIRTY); 75 75 if (cnt == 0 && atomic_read(&vob->vob_mmap_cnt) > 0)
+4 -4
drivers/staging/lustre/lustre/mdc/mdc_request.c
··· 934 934 struct page *page; 935 935 int found; 936 936 937 - spin_lock_irq(&mapping->tree_lock); 938 - found = radix_tree_gang_lookup(&mapping->page_tree, 937 + xa_lock_irq(&mapping->i_pages); 938 + found = radix_tree_gang_lookup(&mapping->i_pages, 939 939 (void **)&page, offset, 1); 940 940 if (found > 0 && !radix_tree_exceptional_entry(page)) { 941 941 struct lu_dirpage *dp; 942 942 943 943 get_page(page); 944 - spin_unlock_irq(&mapping->tree_lock); 944 + xa_unlock_irq(&mapping->i_pages); 945 945 /* 946 946 * In contrast to find_lock_page() we are sure that directory 947 947 * page cannot be truncated (while DLM lock is held) and, ··· 989 989 page = ERR_PTR(-EIO); 990 990 } 991 991 } else { 992 - spin_unlock_irq(&mapping->tree_lock); 992 + xa_unlock_irq(&mapping->i_pages); 993 993 page = NULL; 994 994 } 995 995 return page;
+5 -4
fs/afs/write.c
··· 570 570 571 571 _debug("wback %lx", page->index); 572 572 573 - /* at this point we hold neither mapping->tree_lock nor lock on 574 - * the page itself: the page may be truncated or invalidated 575 - * (changing page->mapping to NULL), or even swizzled back from 576 - * swapper_space to tmpfs file mapping 573 + /* 574 + * at this point we hold neither the i_pages lock nor the 575 + * page lock: the page may be truncated or invalidated 576 + * (changing page->mapping to NULL), or even swizzled 577 + * back from swapper_space to tmpfs file mapping 577 578 */ 578 579 ret = lock_page_killable(page); 579 580 if (ret < 0) {
+1 -1
fs/btrfs/compression.c
··· 458 458 break; 459 459 460 460 rcu_read_lock(); 461 - page = radix_tree_lookup(&mapping->page_tree, pg_index); 461 + page = radix_tree_lookup(&mapping->i_pages, pg_index); 462 462 rcu_read_unlock(); 463 463 if (page && !radix_tree_exceptional_entry(page)) { 464 464 misses++;
+8 -8
fs/btrfs/extent_io.c
··· 3963 3963 3964 3964 done_index = page->index; 3965 3965 /* 3966 - * At this point we hold neither mapping->tree_lock nor 3967 - * lock on the page itself: the page may be truncated or 3968 - * invalidated (changing page->mapping to NULL), or even 3969 - * swizzled back from swapper_space to tmpfs file 3970 - * mapping 3966 + * At this point we hold neither the i_pages lock nor 3967 + * the page lock: the page may be truncated or 3968 + * invalidated (changing page->mapping to NULL), 3969 + * or even swizzled back from swapper_space to 3970 + * tmpfs file mapping 3971 3971 */ 3972 3972 if (!trylock_page(page)) { 3973 3973 flush_write_bio(epd); ··· 5174 5174 WARN_ON(!PagePrivate(page)); 5175 5175 5176 5176 clear_page_dirty_for_io(page); 5177 - spin_lock_irq(&page->mapping->tree_lock); 5177 + xa_lock_irq(&page->mapping->i_pages); 5178 5178 if (!PageDirty(page)) { 5179 - radix_tree_tag_clear(&page->mapping->page_tree, 5179 + radix_tree_tag_clear(&page->mapping->i_pages, 5180 5180 page_index(page), 5181 5181 PAGECACHE_TAG_DIRTY); 5182 5182 } 5183 - spin_unlock_irq(&page->mapping->tree_lock); 5183 + xa_unlock_irq(&page->mapping->i_pages); 5184 5184 ClearPageError(page); 5185 5185 unlock_page(page); 5186 5186 }
+6 -7
fs/buffer.c
··· 185 185 * we get exclusion from try_to_free_buffers with the blockdev mapping's 186 186 * private_lock. 187 187 * 188 - * Hack idea: for the blockdev mapping, i_bufferlist_lock contention 188 + * Hack idea: for the blockdev mapping, private_lock contention 189 189 * may be quite high. This code could TryLock the page, and if that 190 - * succeeds, there is no need to take private_lock. (But if 191 - * private_lock is contended then so is mapping->tree_lock). 190 + * succeeds, there is no need to take private_lock. 192 191 */ 193 192 static struct buffer_head * 194 193 __find_get_block_slow(struct block_device *bdev, sector_t block) ··· 598 599 { 599 600 unsigned long flags; 600 601 601 - spin_lock_irqsave(&mapping->tree_lock, flags); 602 + xa_lock_irqsave(&mapping->i_pages, flags); 602 603 if (page->mapping) { /* Race with truncate? */ 603 604 WARN_ON_ONCE(warn && !PageUptodate(page)); 604 605 account_page_dirtied(page, mapping); 605 - radix_tree_tag_set(&mapping->page_tree, 606 + radix_tree_tag_set(&mapping->i_pages, 606 607 page_index(page), PAGECACHE_TAG_DIRTY); 607 608 } 608 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 609 + xa_unlock_irqrestore(&mapping->i_pages, flags); 609 610 } 610 611 EXPORT_SYMBOL_GPL(__set_page_dirty); 611 612 ··· 1095 1096 * inode list. 1096 1097 * 1097 1098 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, 1098 - * mapping->tree_lock and mapping->host->i_lock. 1099 + * i_pages lock and mapping->host->i_lock. 1099 1100 */ 1100 1101 void mark_buffer_dirty(struct buffer_head *bh) 1101 1102 {
+4 -5
fs/cifs/file.c
··· 1987 1987 for (i = 0; i < found_pages; i++) { 1988 1988 page = wdata->pages[i]; 1989 1989 /* 1990 - * At this point we hold neither mapping->tree_lock nor 1991 - * lock on the page itself: the page may be truncated or 1992 - * invalidated (changing page->mapping to NULL), or even 1993 - * swizzled back from swapper_space to tmpfs file 1994 - * mapping 1990 + * At this point we hold neither the i_pages lock nor the 1991 + * page lock: the page may be truncated or invalidated 1992 + * (changing page->mapping to NULL), or even swizzled 1993 + * back from swapper_space to tmpfs file mapping 1995 1994 */ 1996 1995 1997 1996 if (nr_pages == 0)
+60 -64
fs/dax.c
··· 158 158 } 159 159 160 160 /* 161 - * We do not necessarily hold the mapping->tree_lock when we call this 162 - * function so it is possible that 'entry' is no longer a valid item in the 163 - * radix tree. This is okay because all we really need to do is to find the 164 - * correct waitqueue where tasks might be waiting for that old 'entry' and 165 - * wake them. 161 + * @entry may no longer be the entry at the index in the mapping. 162 + * The important information it's conveying is whether the entry at 163 + * this index used to be a PMD entry. 166 164 */ 167 165 static void dax_wake_mapping_entry_waiter(struct address_space *mapping, 168 166 pgoff_t index, void *entry, bool wake_all) ··· 172 174 173 175 /* 174 176 * Checking for locked entry and prepare_to_wait_exclusive() happens 175 - * under mapping->tree_lock, ditto for entry handling in our callers. 177 + * under the i_pages lock, ditto for entry handling in our callers. 176 178 * So at this point all tasks that could have seen our entry locked 177 179 * must be in the waitqueue and the following check will see them. 178 180 */ ··· 181 183 } 182 184 183 185 /* 184 - * Check whether the given slot is locked. The function must be called with 185 - * mapping->tree_lock held 186 + * Check whether the given slot is locked. Must be called with the i_pages 187 + * lock held. 186 188 */ 187 189 static inline int slot_locked(struct address_space *mapping, void **slot) 188 190 { 189 191 unsigned long entry = (unsigned long) 190 - radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 192 + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 191 193 return entry & RADIX_DAX_ENTRY_LOCK; 192 194 } 193 195 194 196 /* 195 - * Mark the given slot is locked. The function must be called with 196 - * mapping->tree_lock held 197 + * Mark the given slot as locked. Must be called with the i_pages lock held. 197 198 */ 198 199 static inline void *lock_slot(struct address_space *mapping, void **slot) 199 200 { 200 201 unsigned long entry = (unsigned long) 201 - radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 202 + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 202 203 203 204 entry |= RADIX_DAX_ENTRY_LOCK; 204 - radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); 205 + radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); 205 206 return (void *)entry; 206 207 } 207 208 208 209 /* 209 - * Mark the given slot is unlocked. The function must be called with 210 - * mapping->tree_lock held 210 + * Mark the given slot as unlocked. Must be called with the i_pages lock held. 211 211 */ 212 212 static inline void *unlock_slot(struct address_space *mapping, void **slot) 213 213 { 214 214 unsigned long entry = (unsigned long) 215 - radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 215 + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 216 216 217 217 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; 218 - radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); 218 + radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); 219 219 return (void *)entry; 220 220 } 221 221 ··· 224 228 * put_locked_mapping_entry() when he locked the entry and now wants to 225 229 * unlock it. 226 230 * 227 - * The function must be called with mapping->tree_lock held. 231 + * Must be called with the i_pages lock held. 228 232 */ 229 233 static void *get_unlocked_mapping_entry(struct address_space *mapping, 230 234 pgoff_t index, void ***slotp) ··· 237 241 ewait.wait.func = wake_exceptional_entry_func; 238 242 239 243 for (;;) { 240 - entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, 244 + entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, 241 245 &slot); 242 246 if (!entry || 243 247 WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || ··· 250 254 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); 251 255 prepare_to_wait_exclusive(wq, &ewait.wait, 252 256 TASK_UNINTERRUPTIBLE); 253 - spin_unlock_irq(&mapping->tree_lock); 257 + xa_unlock_irq(&mapping->i_pages); 254 258 schedule(); 255 259 finish_wait(wq, &ewait.wait); 256 - spin_lock_irq(&mapping->tree_lock); 260 + xa_lock_irq(&mapping->i_pages); 257 261 } 258 262 } 259 263 ··· 262 266 { 263 267 void *entry, **slot; 264 268 265 - spin_lock_irq(&mapping->tree_lock); 266 - entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); 269 + xa_lock_irq(&mapping->i_pages); 270 + entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); 267 271 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || 268 272 !slot_locked(mapping, slot))) { 269 - spin_unlock_irq(&mapping->tree_lock); 273 + xa_unlock_irq(&mapping->i_pages); 270 274 return; 271 275 } 272 276 unlock_slot(mapping, slot); 273 - spin_unlock_irq(&mapping->tree_lock); 277 + xa_unlock_irq(&mapping->i_pages); 274 278 dax_wake_mapping_entry_waiter(mapping, index, entry, false); 275 279 } 276 280 ··· 384 388 void *entry, **slot; 385 389 386 390 restart: 387 - spin_lock_irq(&mapping->tree_lock); 391 + xa_lock_irq(&mapping->i_pages); 388 392 entry = get_unlocked_mapping_entry(mapping, index, &slot); 389 393 390 394 if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { ··· 416 420 if (pmd_downgrade) { 417 421 /* 418 422 * Make sure 'entry' remains valid while we drop 419 - * mapping->tree_lock. 423 + * the i_pages lock. 420 424 */ 421 425 entry = lock_slot(mapping, slot); 422 426 } 423 427 424 - spin_unlock_irq(&mapping->tree_lock); 428 + xa_unlock_irq(&mapping->i_pages); 425 429 /* 426 430 * Besides huge zero pages the only other thing that gets 427 431 * downgraded are empty entries which don't need to be ··· 438 442 put_locked_mapping_entry(mapping, index); 439 443 return ERR_PTR(err); 440 444 } 441 - spin_lock_irq(&mapping->tree_lock); 445 + xa_lock_irq(&mapping->i_pages); 442 446 443 447 if (!entry) { 444 448 /* 445 - * We needed to drop the page_tree lock while calling 449 + * We needed to drop the i_pages lock while calling 446 450 * radix_tree_preload() and we didn't have an entry to 447 451 * lock. See if another thread inserted an entry at 448 452 * our index during this time. 449 453 */ 450 - entry = __radix_tree_lookup(&mapping->page_tree, index, 454 + entry = __radix_tree_lookup(&mapping->i_pages, index, 451 455 NULL, &slot); 452 456 if (entry) { 453 457 radix_tree_preload_end(); 454 - spin_unlock_irq(&mapping->tree_lock); 458 + xa_unlock_irq(&mapping->i_pages); 455 459 goto restart; 456 460 } 457 461 } 458 462 459 463 if (pmd_downgrade) { 460 464 dax_disassociate_entry(entry, mapping, false); 461 - radix_tree_delete(&mapping->page_tree, index); 465 + radix_tree_delete(&mapping->i_pages, index); 462 466 mapping->nrexceptional--; 463 467 dax_wake_mapping_entry_waiter(mapping, index, entry, 464 468 true); ··· 466 470 467 471 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); 468 472 469 - err = __radix_tree_insert(&mapping->page_tree, index, 473 + err = __radix_tree_insert(&mapping->i_pages, index, 470 474 dax_radix_order(entry), entry); 471 475 radix_tree_preload_end(); 472 476 if (err) { 473 - spin_unlock_irq(&mapping->tree_lock); 477 + xa_unlock_irq(&mapping->i_pages); 474 478 /* 475 479 * Our insertion of a DAX entry failed, most likely 476 480 * because we were inserting a PMD entry and it ··· 483 487 } 484 488 /* Good, we have inserted empty locked entry into the tree. */ 485 489 mapping->nrexceptional++; 486 - spin_unlock_irq(&mapping->tree_lock); 490 + xa_unlock_irq(&mapping->i_pages); 487 491 return entry; 488 492 } 489 493 entry = lock_slot(mapping, slot); 490 494 out_unlock: 491 - spin_unlock_irq(&mapping->tree_lock); 495 + xa_unlock_irq(&mapping->i_pages); 492 496 return entry; 493 497 } 494 498 ··· 497 501 { 498 502 int ret = 0; 499 503 void *entry; 500 - struct radix_tree_root *page_tree = &mapping->page_tree; 504 + struct radix_tree_root *pages = &mapping->i_pages; 501 505 502 - spin_lock_irq(&mapping->tree_lock); 506 + xa_lock_irq(pages); 503 507 entry = get_unlocked_mapping_entry(mapping, index, NULL); 504 508 if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) 505 509 goto out; 506 510 if (!trunc && 507 - (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || 508 - radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))) 511 + (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) || 512 + radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))) 509 513 goto out; 510 514 dax_disassociate_entry(entry, mapping, trunc); 511 - radix_tree_delete(page_tree, index); 515 + radix_tree_delete(pages, index); 512 516 mapping->nrexceptional--; 513 517 ret = 1; 514 518 out: 515 519 put_unlocked_mapping_entry(mapping, index, entry); 516 - spin_unlock_irq(&mapping->tree_lock); 520 + xa_unlock_irq(pages); 517 521 return ret; 518 522 } 519 523 /* ··· 583 587 void *entry, pfn_t pfn_t, 584 588 unsigned long flags, bool dirty) 585 589 { 586 - struct radix_tree_root *page_tree = &mapping->page_tree; 590 + struct radix_tree_root *pages = &mapping->i_pages; 587 591 unsigned long pfn = pfn_t_to_pfn(pfn_t); 588 592 pgoff_t index = vmf->pgoff; 589 593 void *new_entry; ··· 600 604 unmap_mapping_pages(mapping, vmf->pgoff, 1, false); 601 605 } 602 606 603 - spin_lock_irq(&mapping->tree_lock); 607 + xa_lock_irq(pages); 604 608 new_entry = dax_radix_locked_entry(pfn, flags); 605 609 if (dax_entry_size(entry) != dax_entry_size(new_entry)) { 606 610 dax_disassociate_entry(entry, mapping, false); ··· 620 624 void **slot; 621 625 void *ret; 622 626 623 - ret = __radix_tree_lookup(page_tree, index, &node, &slot); 627 + ret = __radix_tree_lookup(pages, index, &node, &slot); 624 628 WARN_ON_ONCE(ret != entry); 625 - __radix_tree_replace(page_tree, node, slot, 629 + __radix_tree_replace(pages, node, slot, 626 630 new_entry, NULL); 627 631 entry = new_entry; 628 632 } 629 633 630 634 if (dirty) 631 - radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY); 635 + radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY); 632 636 633 - spin_unlock_irq(&mapping->tree_lock); 637 + xa_unlock_irq(pages); 634 638 return entry; 635 639 } 636 640 ··· 719 723 static int dax_writeback_one(struct dax_device *dax_dev, 720 724 struct address_space *mapping, pgoff_t index, void *entry) 721 725 { 722 - struct radix_tree_root *page_tree = &mapping->page_tree; 726 + struct radix_tree_root *pages = &mapping->i_pages; 723 727 void *entry2, **slot; 724 728 unsigned long pfn; 725 729 long ret = 0; ··· 732 736 if (WARN_ON(!radix_tree_exceptional_entry(entry))) 733 737 return -EIO; 734 738 735 - spin_lock_irq(&mapping->tree_lock); 739 + xa_lock_irq(pages); 736 740 entry2 = get_unlocked_mapping_entry(mapping, index, &slot); 737 741 /* Entry got punched out / reallocated? */ 738 742 if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) ··· 751 755 } 752 756 753 757 /* Another fsync thread may have already written back this entry */ 754 - if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) 758 + if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)) 755 759 goto put_unlocked; 756 760 /* Lock the entry to serialize with page faults */ 757 761 entry = lock_slot(mapping, slot); ··· 759 763 * We can clear the tag now but we have to be careful so that concurrent 760 764 * dax_writeback_one() calls for the same index cannot finish before we 761 765 * actually flush the caches. This is achieved as the calls will look 762 - * at the entry only under tree_lock and once they do that they will 763 - * see the entry locked and wait for it to unlock. 766 + * at the entry only under the i_pages lock and once they do that 767 + * they will see the entry locked and wait for it to unlock. 764 768 */ 765 - radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE); 766 - spin_unlock_irq(&mapping->tree_lock); 769 + radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE); 770 + xa_unlock_irq(pages); 767 771 768 772 /* 769 773 * Even if dax_writeback_mapping_range() was given a wbc->range_start ··· 783 787 * the pfn mappings are writeprotected and fault waits for mapping 784 788 * entry lock. 785 789 */ 786 - spin_lock_irq(&mapping->tree_lock); 787 - radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY); 788 - spin_unlock_irq(&mapping->tree_lock); 790 + xa_lock_irq(pages); 791 + radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY); 792 + xa_unlock_irq(pages); 789 793 trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); 790 794 put_locked_mapping_entry(mapping, index); 791 795 return ret; 792 796 793 797 put_unlocked: 794 798 put_unlocked_mapping_entry(mapping, index, entry2); 795 - spin_unlock_irq(&mapping->tree_lock); 799 + xa_unlock_irq(pages); 796 800 return ret; 797 801 } 798 802 ··· 1562 1566 pgoff_t index = vmf->pgoff; 1563 1567 int vmf_ret, error; 1564 1568 1565 - spin_lock_irq(&mapping->tree_lock); 1569 + xa_lock_irq(&mapping->i_pages); 1566 1570 entry = get_unlocked_mapping_entry(mapping, index, &slot); 1567 1571 /* Did we race with someone splitting entry or so? */ 1568 1572 if (!entry || 1569 1573 (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || 1570 1574 (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { 1571 1575 put_unlocked_mapping_entry(mapping, index, entry); 1572 - spin_unlock_irq(&mapping->tree_lock); 1576 + xa_unlock_irq(&mapping->i_pages); 1573 1577 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, 1574 1578 VM_FAULT_NOPAGE); 1575 1579 return VM_FAULT_NOPAGE; 1576 1580 } 1577 - radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY); 1581 + radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY); 1578 1582 entry = lock_slot(mapping, slot); 1579 - spin_unlock_irq(&mapping->tree_lock); 1583 + xa_unlock_irq(&mapping->i_pages); 1580 1584 switch (pe_size) { 1581 1585 case PE_SIZE_PTE: 1582 1586 error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
+3 -3
fs/f2fs/data.c
··· 2424 2424 SetPageDirty(page); 2425 2425 spin_unlock(&mapping->private_lock); 2426 2426 2427 - spin_lock_irqsave(&mapping->tree_lock, flags); 2427 + xa_lock_irqsave(&mapping->i_pages, flags); 2428 2428 WARN_ON_ONCE(!PageUptodate(page)); 2429 2429 account_page_dirtied(page, mapping); 2430 - radix_tree_tag_set(&mapping->page_tree, 2430 + radix_tree_tag_set(&mapping->i_pages, 2431 2431 page_index(page), PAGECACHE_TAG_DIRTY); 2432 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 2432 + xa_unlock_irqrestore(&mapping->i_pages, flags); 2433 2433 unlock_page_memcg(page); 2434 2434 2435 2435 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
+3 -3
fs/f2fs/dir.c
··· 732 732 733 733 if (bit_pos == NR_DENTRY_IN_BLOCK && 734 734 !truncate_hole(dir, page->index, page->index + 1)) { 735 - spin_lock_irqsave(&mapping->tree_lock, flags); 736 - radix_tree_tag_clear(&mapping->page_tree, page_index(page), 735 + xa_lock_irqsave(&mapping->i_pages, flags); 736 + radix_tree_tag_clear(&mapping->i_pages, page_index(page), 737 737 PAGECACHE_TAG_DIRTY); 738 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 738 + xa_unlock_irqrestore(&mapping->i_pages, flags); 739 739 740 740 clear_page_dirty_for_io(page); 741 741 ClearPagePrivate(page);
+3 -3
fs/f2fs/inline.c
··· 226 226 kunmap_atomic(src_addr); 227 227 set_page_dirty(dn.inode_page); 228 228 229 - spin_lock_irqsave(&mapping->tree_lock, flags); 230 - radix_tree_tag_clear(&mapping->page_tree, page_index(page), 229 + xa_lock_irqsave(&mapping->i_pages, flags); 230 + radix_tree_tag_clear(&mapping->i_pages, page_index(page), 231 231 PAGECACHE_TAG_DIRTY); 232 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 232 + xa_unlock_irqrestore(&mapping->i_pages, flags); 233 233 234 234 set_inode_flag(inode, FI_APPEND_WRITE); 235 235 set_inode_flag(inode, FI_DATA_EXIST);
+4 -4
fs/f2fs/node.c
··· 91 91 unsigned int long flags; 92 92 93 93 if (PageDirty(page)) { 94 - spin_lock_irqsave(&mapping->tree_lock, flags); 95 - radix_tree_tag_clear(&mapping->page_tree, 94 + xa_lock_irqsave(&mapping->i_pages, flags); 95 + radix_tree_tag_clear(&mapping->i_pages, 96 96 page_index(page), 97 97 PAGECACHE_TAG_DIRTY); 98 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 98 + xa_unlock_irqrestore(&mapping->i_pages, flags); 99 99 100 100 clear_page_dirty_for_io(page); 101 101 dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); ··· 1161 1161 f2fs_bug_on(sbi, check_nid_range(sbi, nid)); 1162 1162 1163 1163 rcu_read_lock(); 1164 - apage = radix_tree_lookup(&NODE_MAPPING(sbi)->page_tree, nid); 1164 + apage = radix_tree_lookup(&NODE_MAPPING(sbi)->i_pages, nid); 1165 1165 rcu_read_unlock(); 1166 1166 if (apage) 1167 1167 return;
+11 -11
fs/fs-writeback.c
··· 347 347 * By the time control reaches here, RCU grace period has passed 348 348 * since I_WB_SWITCH assertion and all wb stat update transactions 349 349 * between unlocked_inode_to_wb_begin/end() are guaranteed to be 350 - * synchronizing against mapping->tree_lock. 350 + * synchronizing against the i_pages lock. 351 351 * 352 - * Grabbing old_wb->list_lock, inode->i_lock and mapping->tree_lock 352 + * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock 353 353 * gives us exclusion against all wb related operations on @inode 354 354 * including IO list manipulations and stat updates. 355 355 */ ··· 361 361 spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); 362 362 } 363 363 spin_lock(&inode->i_lock); 364 - spin_lock_irq(&mapping->tree_lock); 364 + xa_lock_irq(&mapping->i_pages); 365 365 366 366 /* 367 367 * Once I_FREEING is visible under i_lock, the eviction path owns ··· 373 373 /* 374 374 * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points 375 375 * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to 376 - * pages actually under underwriteback. 376 + * pages actually under writeback. 377 377 */ 378 - radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, 378 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, 379 379 PAGECACHE_TAG_DIRTY) { 380 380 struct page *page = radix_tree_deref_slot_protected(slot, 381 - &mapping->tree_lock); 381 + &mapping->i_pages.xa_lock); 382 382 if (likely(page) && PageDirty(page)) { 383 383 dec_wb_stat(old_wb, WB_RECLAIMABLE); 384 384 inc_wb_stat(new_wb, WB_RECLAIMABLE); 385 385 } 386 386 } 387 387 388 - radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, 388 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, 389 389 PAGECACHE_TAG_WRITEBACK) { 390 390 struct page *page = radix_tree_deref_slot_protected(slot, 391 - &mapping->tree_lock); 391 + &mapping->i_pages.xa_lock); 392 392 if (likely(page)) { 393 393 WARN_ON_ONCE(!PageWriteback(page)); 394 394 dec_wb_stat(old_wb, WB_WRITEBACK); ··· 430 430 */ 431 431 smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); 432 432 433 - spin_unlock_irq(&mapping->tree_lock); 433 + xa_unlock_irq(&mapping->i_pages); 434 434 spin_unlock(&inode->i_lock); 435 435 spin_unlock(&new_wb->list_lock); 436 436 spin_unlock(&old_wb->list_lock); ··· 506 506 507 507 /* 508 508 * In addition to synchronizing among switchers, I_WB_SWITCH tells 509 - * the RCU protected stat update paths to grab the mapping's 510 - * tree_lock so that stat transfer can synchronize against them. 509 + * the RCU protected stat update paths to grab the i_page 510 + * lock so that stat transfer can synchronize against them. 511 511 * Let's continue after I_WB_SWITCH is guaranteed to be visible. 512 512 */ 513 513 call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
+5 -6
fs/inode.c
··· 348 348 349 349 static void __address_space_init_once(struct address_space *mapping) 350 350 { 351 - INIT_RADIX_TREE(&mapping->page_tree, GFP_ATOMIC | __GFP_ACCOUNT); 352 - spin_lock_init(&mapping->tree_lock); 351 + INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT); 353 352 init_rwsem(&mapping->i_mmap_rwsem); 354 353 INIT_LIST_HEAD(&mapping->private_list); 355 354 spin_lock_init(&mapping->private_lock); ··· 503 504 void clear_inode(struct inode *inode) 504 505 { 505 506 /* 506 - * We have to cycle tree_lock here because reclaim can be still in the 507 + * We have to cycle the i_pages lock here because reclaim can be in the 507 508 * process of removing the last page (in __delete_from_page_cache()) 508 - * and we must not free mapping under it. 509 + * and we must not free the mapping under it. 509 510 */ 510 - spin_lock_irq(&inode->i_data.tree_lock); 511 + xa_lock_irq(&inode->i_data.i_pages); 511 512 BUG_ON(inode->i_data.nrpages); 512 513 BUG_ON(inode->i_data.nrexceptional); 513 - spin_unlock_irq(&inode->i_data.tree_lock); 514 + xa_unlock_irq(&inode->i_data.i_pages); 514 515 BUG_ON(!list_empty(&inode->i_data.private_list)); 515 516 BUG_ON(!(inode->i_state & I_FREEING)); 516 517 BUG_ON(inode->i_state & I_CLEAR);
+10 -10
fs/nilfs2/btnode.c
··· 193 193 (unsigned long long)oldkey, 194 194 (unsigned long long)newkey); 195 195 196 - spin_lock_irq(&btnc->tree_lock); 197 - err = radix_tree_insert(&btnc->page_tree, newkey, obh->b_page); 198 - spin_unlock_irq(&btnc->tree_lock); 196 + xa_lock_irq(&btnc->i_pages); 197 + err = radix_tree_insert(&btnc->i_pages, newkey, obh->b_page); 198 + xa_unlock_irq(&btnc->i_pages); 199 199 /* 200 200 * Note: page->index will not change to newkey until 201 201 * nilfs_btnode_commit_change_key() will be called. ··· 251 251 (unsigned long long)newkey); 252 252 mark_buffer_dirty(obh); 253 253 254 - spin_lock_irq(&btnc->tree_lock); 255 - radix_tree_delete(&btnc->page_tree, oldkey); 256 - radix_tree_tag_set(&btnc->page_tree, newkey, 254 + xa_lock_irq(&btnc->i_pages); 255 + radix_tree_delete(&btnc->i_pages, oldkey); 256 + radix_tree_tag_set(&btnc->i_pages, newkey, 257 257 PAGECACHE_TAG_DIRTY); 258 - spin_unlock_irq(&btnc->tree_lock); 258 + xa_unlock_irq(&btnc->i_pages); 259 259 260 260 opage->index = obh->b_blocknr = newkey; 261 261 unlock_page(opage); ··· 283 283 return; 284 284 285 285 if (nbh == NULL) { /* blocksize == pagesize */ 286 - spin_lock_irq(&btnc->tree_lock); 287 - radix_tree_delete(&btnc->page_tree, newkey); 288 - spin_unlock_irq(&btnc->tree_lock); 286 + xa_lock_irq(&btnc->i_pages); 287 + radix_tree_delete(&btnc->i_pages, newkey); 288 + xa_unlock_irq(&btnc->i_pages); 289 289 unlock_page(ctxt->bh->b_page); 290 290 } else 291 291 brelse(nbh);
+11 -11
fs/nilfs2/page.c
··· 331 331 struct page *page2; 332 332 333 333 /* move the page to the destination cache */ 334 - spin_lock_irq(&smap->tree_lock); 335 - page2 = radix_tree_delete(&smap->page_tree, offset); 334 + xa_lock_irq(&smap->i_pages); 335 + page2 = radix_tree_delete(&smap->i_pages, offset); 336 336 WARN_ON(page2 != page); 337 337 338 338 smap->nrpages--; 339 - spin_unlock_irq(&smap->tree_lock); 339 + xa_unlock_irq(&smap->i_pages); 340 340 341 - spin_lock_irq(&dmap->tree_lock); 342 - err = radix_tree_insert(&dmap->page_tree, offset, page); 341 + xa_lock_irq(&dmap->i_pages); 342 + err = radix_tree_insert(&dmap->i_pages, offset, page); 343 343 if (unlikely(err < 0)) { 344 344 WARN_ON(err == -EEXIST); 345 345 page->mapping = NULL; ··· 348 348 page->mapping = dmap; 349 349 dmap->nrpages++; 350 350 if (PageDirty(page)) 351 - radix_tree_tag_set(&dmap->page_tree, 351 + radix_tree_tag_set(&dmap->i_pages, 352 352 offset, 353 353 PAGECACHE_TAG_DIRTY); 354 354 } 355 - spin_unlock_irq(&dmap->tree_lock); 355 + xa_unlock_irq(&dmap->i_pages); 356 356 } 357 357 unlock_page(page); 358 358 } ··· 474 474 struct address_space *mapping = page->mapping; 475 475 476 476 if (mapping) { 477 - spin_lock_irq(&mapping->tree_lock); 477 + xa_lock_irq(&mapping->i_pages); 478 478 if (test_bit(PG_dirty, &page->flags)) { 479 - radix_tree_tag_clear(&mapping->page_tree, 479 + radix_tree_tag_clear(&mapping->i_pages, 480 480 page_index(page), 481 481 PAGECACHE_TAG_DIRTY); 482 - spin_unlock_irq(&mapping->tree_lock); 482 + xa_unlock_irq(&mapping->i_pages); 483 483 return clear_page_dirty_for_io(page); 484 484 } 485 - spin_unlock_irq(&mapping->tree_lock); 485 + xa_unlock_irq(&mapping->i_pages); 486 486 return 0; 487 487 } 488 488 return TestClearPageDirty(page);
+7 -7
include/linux/backing-dev.h
··· 329 329 * @inode: inode of interest 330 330 * 331 331 * Returns the wb @inode is currently associated with. The caller must be 332 - * holding either @inode->i_lock, @inode->i_mapping->tree_lock, or the 332 + * holding either @inode->i_lock, the i_pages lock, or the 333 333 * associated wb's list_lock. 334 334 */ 335 335 static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) ··· 337 337 #ifdef CONFIG_LOCKDEP 338 338 WARN_ON_ONCE(debug_locks && 339 339 (!lockdep_is_held(&inode->i_lock) && 340 - !lockdep_is_held(&inode->i_mapping->tree_lock) && 340 + !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) && 341 341 !lockdep_is_held(&inode->i_wb->list_lock))); 342 342 #endif 343 343 return inode->i_wb; ··· 349 349 * @lockedp: temp bool output param, to be passed to the end function 350 350 * 351 351 * The caller wants to access the wb associated with @inode but isn't 352 - * holding inode->i_lock, mapping->tree_lock or wb->list_lock. This 352 + * holding inode->i_lock, the i_pages lock or wb->list_lock. This 353 353 * function determines the wb associated with @inode and ensures that the 354 354 * association doesn't change until the transaction is finished with 355 355 * unlocked_inode_to_wb_end(). ··· 370 370 *lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; 371 371 372 372 if (unlikely(*lockedp)) 373 - spin_lock_irq(&inode->i_mapping->tree_lock); 373 + xa_lock_irq(&inode->i_mapping->i_pages); 374 374 375 375 /* 376 - * Protected by either !I_WB_SWITCH + rcu_read_lock() or tree_lock. 377 - * inode_to_wb() will bark. Deref directly. 376 + * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages 377 + * lock. inode_to_wb() will bark. Deref directly. 378 378 */ 379 379 return inode->i_wb; 380 380 } ··· 387 387 static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) 388 388 { 389 389 if (unlikely(locked)) 390 - spin_unlock_irq(&inode->i_mapping->tree_lock); 390 + xa_unlock_irq(&inode->i_mapping->i_pages); 391 391 392 392 rcu_read_unlock(); 393 393 }
+4 -4
include/linux/fs.h
··· 13 13 #include <linux/list_lru.h> 14 14 #include <linux/llist.h> 15 15 #include <linux/radix-tree.h> 16 + #include <linux/xarray.h> 16 17 #include <linux/rbtree.h> 17 18 #include <linux/init.h> 18 19 #include <linux/pid.h> ··· 391 390 392 391 struct address_space { 393 392 struct inode *host; /* owner: inode, block_device */ 394 - struct radix_tree_root page_tree; /* radix tree of all pages */ 395 - spinlock_t tree_lock; /* and lock protecting it */ 393 + struct radix_tree_root i_pages; /* cached pages */ 396 394 atomic_t i_mmap_writable;/* count VM_SHARED mappings */ 397 395 struct rb_root_cached i_mmap; /* tree of private and shared mappings */ 398 396 struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */ 399 - /* Protected by tree_lock together with the radix tree */ 397 + /* Protected by the i_pages lock */ 400 398 unsigned long nrpages; /* number of total pages */ 401 399 /* number of shadow or DAX exceptional entries */ 402 400 unsigned long nrexceptional; ··· 1989 1989 * 1990 1990 * I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to 1991 1991 * synchronize competing switching instances and to tell 1992 - * wb stat updates to grab mapping->tree_lock. See 1992 + * wb stat updates to grab the i_pages lock. See 1993 1993 * inode_switch_wb_work_fn() for details. 1994 1994 * 1995 1995 * I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper
+1 -1
include/linux/mm.h
··· 747 747 * refcount. The each user mapping also has a reference to the page. 748 748 * 749 749 * The pagecache pages are stored in a per-mapping radix tree, which is 750 - * rooted at mapping->page_tree, and indexed by offset. 750 + * rooted at mapping->i_pages, and indexed by offset. 751 751 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space 752 752 * lists, we instead now tag pages as dirty/writeback in the radix tree. 753 753 *
+2 -2
include/linux/pagemap.h
··· 144 144 * 3. check the page is still in pagecache (if no, goto 1) 145 145 * 146 146 * Remove-side that cares about stability of _refcount (eg. reclaim) has the 147 - * following (with tree_lock held for write): 147 + * following (with the i_pages lock held): 148 148 * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg) 149 149 * B. remove page from pagecache 150 150 * C. free the page ··· 157 157 * 158 158 * It is possible that between 1 and 2, the page is removed then the exact same 159 159 * page is inserted into the same position in pagecache. That's OK: the 160 - * old find_get_page using tree_lock could equally have run before or after 160 + * old find_get_page using a lock could equally have run before or after 161 161 * such a re-insertion, depending on order that locks are granted. 162 162 * 163 163 * Lookups racing against pagecache insertion isn't a big problem: either 1
+41 -43
mm/filemap.c
··· 66 66 * ->i_mmap_rwsem (truncate_pagecache) 67 67 * ->private_lock (__free_pte->__set_page_dirty_buffers) 68 68 * ->swap_lock (exclusive_swap_page, others) 69 - * ->mapping->tree_lock 69 + * ->i_pages lock 70 70 * 71 71 * ->i_mutex 72 72 * ->i_mmap_rwsem (truncate->unmap_mapping_range) ··· 74 74 * ->mmap_sem 75 75 * ->i_mmap_rwsem 76 76 * ->page_table_lock or pte_lock (various, mainly in memory.c) 77 - * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 77 + * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) 78 78 * 79 79 * ->mmap_sem 80 80 * ->lock_page (access_process_vm) ··· 84 84 * 85 85 * bdi->wb.list_lock 86 86 * sb_lock (fs/fs-writeback.c) 87 - * ->mapping->tree_lock (__sync_single_inode) 87 + * ->i_pages lock (__sync_single_inode) 88 88 * 89 89 * ->i_mmap_rwsem 90 90 * ->anon_vma.lock (vma_adjust) ··· 95 95 * ->page_table_lock or pte_lock 96 96 * ->swap_lock (try_to_unmap_one) 97 97 * ->private_lock (try_to_unmap_one) 98 - * ->tree_lock (try_to_unmap_one) 98 + * ->i_pages lock (try_to_unmap_one) 99 99 * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) 100 100 * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) 101 101 * ->private_lock (page_remove_rmap->set_page_dirty) 102 - * ->tree_lock (page_remove_rmap->set_page_dirty) 102 + * ->i_pages lock (page_remove_rmap->set_page_dirty) 103 103 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 104 104 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 105 105 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) ··· 118 118 void **slot; 119 119 int error; 120 120 121 - error = __radix_tree_create(&mapping->page_tree, page->index, 0, 121 + error = __radix_tree_create(&mapping->i_pages, page->index, 0, 122 122 &node, &slot); 123 123 if (error) 124 124 return error; 125 125 if (*slot) { 126 126 void *p; 127 127 128 - p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 128 + p = radix_tree_deref_slot_protected(slot, 129 + &mapping->i_pages.xa_lock); 129 130 if (!radix_tree_exceptional_entry(p)) 130 131 return -EEXIST; 131 132 ··· 134 133 if (shadowp) 135 134 *shadowp = p; 136 135 } 137 - __radix_tree_replace(&mapping->page_tree, node, slot, page, 136 + __radix_tree_replace(&mapping->i_pages, node, slot, page, 138 137 workingset_lookup_update(mapping)); 139 138 mapping->nrpages++; 140 139 return 0; ··· 156 155 struct radix_tree_node *node; 157 156 void **slot; 158 157 159 - __radix_tree_lookup(&mapping->page_tree, page->index + i, 158 + __radix_tree_lookup(&mapping->i_pages, page->index + i, 160 159 &node, &slot); 161 160 162 161 VM_BUG_ON_PAGE(!node && nr != 1, page); 163 162 164 - radix_tree_clear_tags(&mapping->page_tree, node, slot); 165 - __radix_tree_replace(&mapping->page_tree, node, slot, shadow, 163 + radix_tree_clear_tags(&mapping->i_pages, node, slot); 164 + __radix_tree_replace(&mapping->i_pages, node, slot, shadow, 166 165 workingset_lookup_update(mapping)); 167 166 } 168 167 ··· 254 253 /* 255 254 * Delete a page from the page cache and free it. Caller has to make 256 255 * sure the page is locked and that nobody else uses it - or that usage 257 - * is safe. The caller must hold the mapping's tree_lock. 256 + * is safe. The caller must hold the i_pages lock. 258 257 */ 259 258 void __delete_from_page_cache(struct page *page, void *shadow) 260 259 { ··· 297 296 unsigned long flags; 298 297 299 298 BUG_ON(!PageLocked(page)); 300 - spin_lock_irqsave(&mapping->tree_lock, flags); 299 + xa_lock_irqsave(&mapping->i_pages, flags); 301 300 __delete_from_page_cache(page, NULL); 302 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 301 + xa_unlock_irqrestore(&mapping->i_pages, flags); 303 302 304 303 page_cache_free_page(mapping, page); 305 304 } ··· 310 309 * @mapping: the mapping to which pages belong 311 310 * @pvec: pagevec with pages to delete 312 311 * 313 - * The function walks over mapping->page_tree and removes pages passed in @pvec 314 - * from the radix tree. The function expects @pvec to be sorted by page index. 315 - * It tolerates holes in @pvec (radix tree entries at those indices are not 312 + * The function walks over mapping->i_pages and removes pages passed in @pvec 313 + * from the mapping. The function expects @pvec to be sorted by page index. 314 + * It tolerates holes in @pvec (mapping entries at those indices are not 316 315 * modified). The function expects only THP head pages to be present in the 317 - * @pvec and takes care to delete all corresponding tail pages from the radix 318 - * tree as well. 316 + * @pvec and takes care to delete all corresponding tail pages from the 317 + * mapping as well. 319 318 * 320 - * The function expects mapping->tree_lock to be held. 319 + * The function expects the i_pages lock to be held. 321 320 */ 322 321 static void 323 322 page_cache_tree_delete_batch(struct address_space *mapping, ··· 331 330 pgoff_t start; 332 331 333 332 start = pvec->pages[0]->index; 334 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 333 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 335 334 if (i >= pagevec_count(pvec) && !tail_pages) 336 335 break; 337 336 page = radix_tree_deref_slot_protected(slot, 338 - &mapping->tree_lock); 337 + &mapping->i_pages.xa_lock); 339 338 if (radix_tree_exceptional_entry(page)) 340 339 continue; 341 340 if (!tail_pages) { ··· 358 357 } else { 359 358 tail_pages--; 360 359 } 361 - radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); 362 - __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, 360 + radix_tree_clear_tags(&mapping->i_pages, iter.node, slot); 361 + __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL, 363 362 workingset_lookup_update(mapping)); 364 363 total_pages++; 365 364 } ··· 375 374 if (!pagevec_count(pvec)) 376 375 return; 377 376 378 - spin_lock_irqsave(&mapping->tree_lock, flags); 377 + xa_lock_irqsave(&mapping->i_pages, flags); 379 378 for (i = 0; i < pagevec_count(pvec); i++) { 380 379 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); 381 380 382 381 unaccount_page_cache_page(mapping, pvec->pages[i]); 383 382 } 384 383 page_cache_tree_delete_batch(mapping, pvec); 385 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 384 + xa_unlock_irqrestore(&mapping->i_pages, flags); 386 385 387 386 for (i = 0; i < pagevec_count(pvec); i++) 388 387 page_cache_free_page(mapping, pvec->pages[i]); ··· 799 798 new->mapping = mapping; 800 799 new->index = offset; 801 800 802 - spin_lock_irqsave(&mapping->tree_lock, flags); 801 + xa_lock_irqsave(&mapping->i_pages, flags); 803 802 __delete_from_page_cache(old, NULL); 804 803 error = page_cache_tree_insert(mapping, new, NULL); 805 804 BUG_ON(error); ··· 811 810 __inc_node_page_state(new, NR_FILE_PAGES); 812 811 if (PageSwapBacked(new)) 813 812 __inc_node_page_state(new, NR_SHMEM); 814 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 813 + xa_unlock_irqrestore(&mapping->i_pages, flags); 815 814 mem_cgroup_migrate(old, new); 816 815 radix_tree_preload_end(); 817 816 if (freepage) ··· 853 852 page->mapping = mapping; 854 853 page->index = offset; 855 854 856 - spin_lock_irq(&mapping->tree_lock); 855 + xa_lock_irq(&mapping->i_pages); 857 856 error = page_cache_tree_insert(mapping, page, shadowp); 858 857 radix_tree_preload_end(); 859 858 if (unlikely(error)) ··· 862 861 /* hugetlb pages do not participate in page cache accounting. */ 863 862 if (!huge) 864 863 __inc_node_page_state(page, NR_FILE_PAGES); 865 - spin_unlock_irq(&mapping->tree_lock); 864 + xa_unlock_irq(&mapping->i_pages); 866 865 if (!huge) 867 866 mem_cgroup_commit_charge(page, memcg, false, false); 868 867 trace_mm_filemap_add_to_page_cache(page); ··· 870 869 err_insert: 871 870 page->mapping = NULL; 872 871 /* Leave page->index set: truncation relies upon it */ 873 - spin_unlock_irq(&mapping->tree_lock); 872 + xa_unlock_irq(&mapping->i_pages); 874 873 if (!huge) 875 874 mem_cgroup_cancel_charge(page, memcg, false); 876 875 put_page(page); ··· 1354 1353 for (i = 0; i < max_scan; i++) { 1355 1354 struct page *page; 1356 1355 1357 - page = radix_tree_lookup(&mapping->page_tree, index); 1356 + page = radix_tree_lookup(&mapping->i_pages, index); 1358 1357 if (!page || radix_tree_exceptional_entry(page)) 1359 1358 break; 1360 1359 index++; ··· 1395 1394 for (i = 0; i < max_scan; i++) { 1396 1395 struct page *page; 1397 1396 1398 - page = radix_tree_lookup(&mapping->page_tree, index); 1397 + page = radix_tree_lookup(&mapping->i_pages, index); 1399 1398 if (!page || radix_tree_exceptional_entry(page)) 1400 1399 break; 1401 1400 index--; ··· 1428 1427 rcu_read_lock(); 1429 1428 repeat: 1430 1429 page = NULL; 1431 - pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); 1430 + pagep = radix_tree_lookup_slot(&mapping->i_pages, offset); 1432 1431 if (pagep) { 1433 1432 page = radix_tree_deref_slot(pagep); 1434 1433 if (unlikely(!page)) ··· 1634 1633 return 0; 1635 1634 1636 1635 rcu_read_lock(); 1637 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1636 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 1638 1637 struct page *head, *page; 1639 1638 repeat: 1640 1639 page = radix_tree_deref_slot(slot); ··· 1711 1710 return 0; 1712 1711 1713 1712 rcu_read_lock(); 1714 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) { 1713 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) { 1715 1714 struct page *head, *page; 1716 1715 1717 1716 if (iter.index > end) ··· 1796 1795 return 0; 1797 1796 1798 1797 rcu_read_lock(); 1799 - radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) { 1798 + radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) { 1800 1799 struct page *head, *page; 1801 1800 repeat: 1802 1801 page = radix_tree_deref_slot(slot); ··· 1876 1875 return 0; 1877 1876 1878 1877 rcu_read_lock(); 1879 - radix_tree_for_each_tagged(slot, &mapping->page_tree, 1880 - &iter, *index, tag) { 1878 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) { 1881 1879 struct page *head, *page; 1882 1880 1883 1881 if (iter.index > end) ··· 1969 1969 return 0; 1970 1970 1971 1971 rcu_read_lock(); 1972 - radix_tree_for_each_tagged(slot, &mapping->page_tree, 1973 - &iter, start, tag) { 1972 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) { 1974 1973 struct page *head, *page; 1975 1974 repeat: 1976 1975 page = radix_tree_deref_slot(slot); ··· 2623 2624 struct page *head, *page; 2624 2625 2625 2626 rcu_read_lock(); 2626 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, 2627 - start_pgoff) { 2627 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) { 2628 2628 if (iter.index > end_pgoff) 2629 2629 break; 2630 2630 repeat:
+5 -5
mm/huge_memory.c
··· 2450 2450 } else { 2451 2451 /* Additional pin to radix tree */ 2452 2452 page_ref_add(head, 2); 2453 - spin_unlock(&head->mapping->tree_lock); 2453 + xa_unlock(&head->mapping->i_pages); 2454 2454 } 2455 2455 2456 2456 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); ··· 2658 2658 if (mapping) { 2659 2659 void **pslot; 2660 2660 2661 - spin_lock(&mapping->tree_lock); 2662 - pslot = radix_tree_lookup_slot(&mapping->page_tree, 2661 + xa_lock(&mapping->i_pages); 2662 + pslot = radix_tree_lookup_slot(&mapping->i_pages, 2663 2663 page_index(head)); 2664 2664 /* 2665 2665 * Check if the head page is present in radix tree. 2666 2666 * We assume all tail are present too, if head is there. 2667 2667 */ 2668 2668 if (radix_tree_deref_slot_protected(pslot, 2669 - &mapping->tree_lock) != head) 2669 + &mapping->i_pages.xa_lock) != head) 2670 2670 goto fail; 2671 2671 } 2672 2672 ··· 2700 2700 } 2701 2701 spin_unlock(&pgdata->split_queue_lock); 2702 2702 fail: if (mapping) 2703 - spin_unlock(&mapping->tree_lock); 2703 + xa_unlock(&mapping->i_pages); 2704 2704 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); 2705 2705 unfreeze_page(head); 2706 2706 ret = -EBUSY;
+23 -26
mm/khugepaged.c
··· 1344 1344 */ 1345 1345 1346 1346 index = start; 1347 - spin_lock_irq(&mapping->tree_lock); 1348 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1347 + xa_lock_irq(&mapping->i_pages); 1348 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 1349 1349 int n = min(iter.index, end) - index; 1350 1350 1351 1351 /* ··· 1358 1358 } 1359 1359 nr_none += n; 1360 1360 for (; index < min(iter.index, end); index++) { 1361 - radix_tree_insert(&mapping->page_tree, index, 1361 + radix_tree_insert(&mapping->i_pages, index, 1362 1362 new_page + (index % HPAGE_PMD_NR)); 1363 1363 } 1364 1364 ··· 1367 1367 break; 1368 1368 1369 1369 page = radix_tree_deref_slot_protected(slot, 1370 - &mapping->tree_lock); 1370 + &mapping->i_pages.xa_lock); 1371 1371 if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { 1372 - spin_unlock_irq(&mapping->tree_lock); 1372 + xa_unlock_irq(&mapping->i_pages); 1373 1373 /* swap in or instantiate fallocated page */ 1374 1374 if (shmem_getpage(mapping->host, index, &page, 1375 1375 SGP_NOHUGE)) { 1376 1376 result = SCAN_FAIL; 1377 1377 goto tree_unlocked; 1378 1378 } 1379 - spin_lock_irq(&mapping->tree_lock); 1379 + xa_lock_irq(&mapping->i_pages); 1380 1380 } else if (trylock_page(page)) { 1381 1381 get_page(page); 1382 1382 } else { ··· 1385 1385 } 1386 1386 1387 1387 /* 1388 - * The page must be locked, so we can drop the tree_lock 1388 + * The page must be locked, so we can drop the i_pages lock 1389 1389 * without racing with truncate. 1390 1390 */ 1391 1391 VM_BUG_ON_PAGE(!PageLocked(page), page); ··· 1396 1396 result = SCAN_TRUNCATED; 1397 1397 goto out_unlock; 1398 1398 } 1399 - spin_unlock_irq(&mapping->tree_lock); 1399 + xa_unlock_irq(&mapping->i_pages); 1400 1400 1401 1401 if (isolate_lru_page(page)) { 1402 1402 result = SCAN_DEL_PAGE_LRU; ··· 1406 1406 if (page_mapped(page)) 1407 1407 unmap_mapping_pages(mapping, index, 1, false); 1408 1408 1409 - spin_lock_irq(&mapping->tree_lock); 1409 + xa_lock_irq(&mapping->i_pages); 1410 1410 1411 - slot = radix_tree_lookup_slot(&mapping->page_tree, index); 1411 + slot = radix_tree_lookup_slot(&mapping->i_pages, index); 1412 1412 VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, 1413 - &mapping->tree_lock), page); 1413 + &mapping->i_pages.xa_lock), page); 1414 1414 VM_BUG_ON_PAGE(page_mapped(page), page); 1415 1415 1416 1416 /* ··· 1431 1431 list_add_tail(&page->lru, &pagelist); 1432 1432 1433 1433 /* Finally, replace with the new page. */ 1434 - radix_tree_replace_slot(&mapping->page_tree, slot, 1434 + radix_tree_replace_slot(&mapping->i_pages, slot, 1435 1435 new_page + (index % HPAGE_PMD_NR)); 1436 1436 1437 1437 slot = radix_tree_iter_resume(slot, &iter); 1438 1438 index++; 1439 1439 continue; 1440 1440 out_lru: 1441 - spin_unlock_irq(&mapping->tree_lock); 1441 + xa_unlock_irq(&mapping->i_pages); 1442 1442 putback_lru_page(page); 1443 1443 out_isolate_failed: 1444 1444 unlock_page(page); ··· 1464 1464 } 1465 1465 1466 1466 for (; index < end; index++) { 1467 - radix_tree_insert(&mapping->page_tree, index, 1467 + radix_tree_insert(&mapping->i_pages, index, 1468 1468 new_page + (index % HPAGE_PMD_NR)); 1469 1469 } 1470 1470 nr_none += n; 1471 1471 } 1472 1472 1473 1473 tree_locked: 1474 - spin_unlock_irq(&mapping->tree_lock); 1474 + xa_unlock_irq(&mapping->i_pages); 1475 1475 tree_unlocked: 1476 1476 1477 1477 if (result == SCAN_SUCCEED) { ··· 1520 1520 } else { 1521 1521 /* Something went wrong: rollback changes to the radix-tree */ 1522 1522 shmem_uncharge(mapping->host, nr_none); 1523 - spin_lock_irq(&mapping->tree_lock); 1524 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, 1525 - start) { 1523 + xa_lock_irq(&mapping->i_pages); 1524 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 1526 1525 if (iter.index >= end) 1527 1526 break; 1528 1527 page = list_first_entry_or_null(&pagelist, ··· 1531 1532 break; 1532 1533 nr_none--; 1533 1534 /* Put holes back where they were */ 1534 - radix_tree_delete(&mapping->page_tree, 1535 - iter.index); 1535 + radix_tree_delete(&mapping->i_pages, iter.index); 1536 1536 continue; 1537 1537 } 1538 1538 ··· 1540 1542 /* Unfreeze the page. */ 1541 1543 list_del(&page->lru); 1542 1544 page_ref_unfreeze(page, 2); 1543 - radix_tree_replace_slot(&mapping->page_tree, 1544 - slot, page); 1545 + radix_tree_replace_slot(&mapping->i_pages, slot, page); 1545 1546 slot = radix_tree_iter_resume(slot, &iter); 1546 - spin_unlock_irq(&mapping->tree_lock); 1547 + xa_unlock_irq(&mapping->i_pages); 1547 1548 putback_lru_page(page); 1548 1549 unlock_page(page); 1549 - spin_lock_irq(&mapping->tree_lock); 1550 + xa_lock_irq(&mapping->i_pages); 1550 1551 } 1551 1552 VM_BUG_ON(nr_none); 1552 - spin_unlock_irq(&mapping->tree_lock); 1553 + xa_unlock_irq(&mapping->i_pages); 1553 1554 1554 1555 /* Unfreeze new_page, caller would take care about freeing it */ 1555 1556 page_ref_unfreeze(new_page, 1); ··· 1576 1579 swap = 0; 1577 1580 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 1578 1581 rcu_read_lock(); 1579 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1582 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 1580 1583 if (iter.index >= start + HPAGE_PMD_NR) 1581 1584 break; 1582 1585
+2 -2
mm/memcontrol.c
··· 5974 5974 5975 5975 /* 5976 5976 * Interrupts should be disabled here because the caller holds the 5977 - * mapping->tree_lock lock which is taken with interrupts-off. It is 5977 + * i_pages lock which is taken with interrupts-off. It is 5978 5978 * important here to have the interrupts disabled because it is the 5979 - * only synchronisation we have for udpating the per-CPU variables. 5979 + * only synchronisation we have for updating the per-CPU variables. 5980 5980 */ 5981 5981 VM_BUG_ON(!irqs_disabled()); 5982 5982 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
+16 -16
mm/migrate.c
··· 467 467 oldzone = page_zone(page); 468 468 newzone = page_zone(newpage); 469 469 470 - spin_lock_irq(&mapping->tree_lock); 470 + xa_lock_irq(&mapping->i_pages); 471 471 472 - pslot = radix_tree_lookup_slot(&mapping->page_tree, 472 + pslot = radix_tree_lookup_slot(&mapping->i_pages, 473 473 page_index(page)); 474 474 475 475 expected_count += 1 + page_has_private(page); 476 476 if (page_count(page) != expected_count || 477 - radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 478 - spin_unlock_irq(&mapping->tree_lock); 477 + radix_tree_deref_slot_protected(pslot, 478 + &mapping->i_pages.xa_lock) != page) { 479 + xa_unlock_irq(&mapping->i_pages); 479 480 return -EAGAIN; 480 481 } 481 482 482 483 if (!page_ref_freeze(page, expected_count)) { 483 - spin_unlock_irq(&mapping->tree_lock); 484 + xa_unlock_irq(&mapping->i_pages); 484 485 return -EAGAIN; 485 486 } 486 487 ··· 495 494 if (mode == MIGRATE_ASYNC && head && 496 495 !buffer_migrate_lock_buffers(head, mode)) { 497 496 page_ref_unfreeze(page, expected_count); 498 - spin_unlock_irq(&mapping->tree_lock); 497 + xa_unlock_irq(&mapping->i_pages); 499 498 return -EAGAIN; 500 499 } 501 500 ··· 523 522 SetPageDirty(newpage); 524 523 } 525 524 526 - radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 525 + radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); 527 526 528 527 /* 529 528 * Drop cache reference from old page by unfreezing ··· 532 531 */ 533 532 page_ref_unfreeze(page, expected_count - 1); 534 533 535 - spin_unlock(&mapping->tree_lock); 534 + xa_unlock(&mapping->i_pages); 536 535 /* Leave irq disabled to prevent preemption while updating stats */ 537 536 538 537 /* ··· 575 574 int expected_count; 576 575 void **pslot; 577 576 578 - spin_lock_irq(&mapping->tree_lock); 577 + xa_lock_irq(&mapping->i_pages); 579 578 580 - pslot = radix_tree_lookup_slot(&mapping->page_tree, 581 - page_index(page)); 579 + pslot = radix_tree_lookup_slot(&mapping->i_pages, page_index(page)); 582 580 583 581 expected_count = 2 + page_has_private(page); 584 582 if (page_count(page) != expected_count || 585 - radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 586 - spin_unlock_irq(&mapping->tree_lock); 583 + radix_tree_deref_slot_protected(pslot, &mapping->i_pages.xa_lock) != page) { 584 + xa_unlock_irq(&mapping->i_pages); 587 585 return -EAGAIN; 588 586 } 589 587 590 588 if (!page_ref_freeze(page, expected_count)) { 591 - spin_unlock_irq(&mapping->tree_lock); 589 + xa_unlock_irq(&mapping->i_pages); 592 590 return -EAGAIN; 593 591 } 594 592 ··· 596 596 597 597 get_page(newpage); 598 598 599 - radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 599 + radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); 600 600 601 601 page_ref_unfreeze(page, expected_count - 1); 602 602 603 - spin_unlock_irq(&mapping->tree_lock); 603 + xa_unlock_irq(&mapping->i_pages); 604 604 605 605 return MIGRATEPAGE_SUCCESS; 606 606 }
+20 -23
mm/page-writeback.c
··· 2099 2099 * so that it can tag pages faster than a dirtying process can create them). 2100 2100 */ 2101 2101 /* 2102 - * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. 2102 + * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock 2103 + * latency. 2103 2104 */ 2104 2105 void tag_pages_for_writeback(struct address_space *mapping, 2105 2106 pgoff_t start, pgoff_t end) ··· 2110 2109 struct radix_tree_iter iter; 2111 2110 void **slot; 2112 2111 2113 - spin_lock_irq(&mapping->tree_lock); 2114 - radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start, 2112 + xa_lock_irq(&mapping->i_pages); 2113 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, 2115 2114 PAGECACHE_TAG_DIRTY) { 2116 2115 if (iter.index > end) 2117 2116 break; 2118 - radix_tree_iter_tag_set(&mapping->page_tree, &iter, 2117 + radix_tree_iter_tag_set(&mapping->i_pages, &iter, 2119 2118 PAGECACHE_TAG_TOWRITE); 2120 2119 tagged++; 2121 2120 if ((tagged % WRITEBACK_TAG_BATCH) != 0) 2122 2121 continue; 2123 2122 slot = radix_tree_iter_resume(slot, &iter); 2124 - spin_unlock_irq(&mapping->tree_lock); 2123 + xa_unlock_irq(&mapping->i_pages); 2125 2124 cond_resched(); 2126 - spin_lock_irq(&mapping->tree_lock); 2125 + xa_lock_irq(&mapping->i_pages); 2127 2126 } 2128 - spin_unlock_irq(&mapping->tree_lock); 2127 + xa_unlock_irq(&mapping->i_pages); 2129 2128 } 2130 2129 EXPORT_SYMBOL(tag_pages_for_writeback); 2131 2130 ··· 2468 2467 return 1; 2469 2468 } 2470 2469 2471 - spin_lock_irqsave(&mapping->tree_lock, flags); 2470 + xa_lock_irqsave(&mapping->i_pages, flags); 2472 2471 BUG_ON(page_mapping(page) != mapping); 2473 2472 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 2474 2473 account_page_dirtied(page, mapping); 2475 - radix_tree_tag_set(&mapping->page_tree, page_index(page), 2474 + radix_tree_tag_set(&mapping->i_pages, page_index(page), 2476 2475 PAGECACHE_TAG_DIRTY); 2477 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 2476 + xa_unlock_irqrestore(&mapping->i_pages, flags); 2478 2477 unlock_page_memcg(page); 2479 2478 2480 2479 if (mapping->host) { ··· 2719 2718 struct backing_dev_info *bdi = inode_to_bdi(inode); 2720 2719 unsigned long flags; 2721 2720 2722 - spin_lock_irqsave(&mapping->tree_lock, flags); 2721 + xa_lock_irqsave(&mapping->i_pages, flags); 2723 2722 ret = TestClearPageWriteback(page); 2724 2723 if (ret) { 2725 - radix_tree_tag_clear(&mapping->page_tree, 2726 - page_index(page), 2724 + radix_tree_tag_clear(&mapping->i_pages, page_index(page), 2727 2725 PAGECACHE_TAG_WRITEBACK); 2728 2726 if (bdi_cap_account_writeback(bdi)) { 2729 2727 struct bdi_writeback *wb = inode_to_wb(inode); ··· 2736 2736 PAGECACHE_TAG_WRITEBACK)) 2737 2737 sb_clear_inode_writeback(mapping->host); 2738 2738 2739 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 2739 + xa_unlock_irqrestore(&mapping->i_pages, flags); 2740 2740 } else { 2741 2741 ret = TestClearPageWriteback(page); 2742 2742 } ··· 2766 2766 struct backing_dev_info *bdi = inode_to_bdi(inode); 2767 2767 unsigned long flags; 2768 2768 2769 - spin_lock_irqsave(&mapping->tree_lock, flags); 2769 + xa_lock_irqsave(&mapping->i_pages, flags); 2770 2770 ret = TestSetPageWriteback(page); 2771 2771 if (!ret) { 2772 2772 bool on_wblist; ··· 2774 2774 on_wblist = mapping_tagged(mapping, 2775 2775 PAGECACHE_TAG_WRITEBACK); 2776 2776 2777 - radix_tree_tag_set(&mapping->page_tree, 2778 - page_index(page), 2777 + radix_tree_tag_set(&mapping->i_pages, page_index(page), 2779 2778 PAGECACHE_TAG_WRITEBACK); 2780 2779 if (bdi_cap_account_writeback(bdi)) 2781 2780 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); ··· 2788 2789 sb_mark_inode_writeback(mapping->host); 2789 2790 } 2790 2791 if (!PageDirty(page)) 2791 - radix_tree_tag_clear(&mapping->page_tree, 2792 - page_index(page), 2792 + radix_tree_tag_clear(&mapping->i_pages, page_index(page), 2793 2793 PAGECACHE_TAG_DIRTY); 2794 2794 if (!keep_write) 2795 - radix_tree_tag_clear(&mapping->page_tree, 2796 - page_index(page), 2795 + radix_tree_tag_clear(&mapping->i_pages, page_index(page), 2797 2796 PAGECACHE_TAG_TOWRITE); 2798 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 2797 + xa_unlock_irqrestore(&mapping->i_pages, flags); 2799 2798 } else { 2800 2799 ret = TestSetPageWriteback(page); 2801 2800 } ··· 2813 2816 */ 2814 2817 int mapping_tagged(struct address_space *mapping, int tag) 2815 2818 { 2816 - return radix_tree_tagged(&mapping->page_tree, tag); 2819 + return radix_tree_tagged(&mapping->i_pages, tag); 2817 2820 } 2818 2821 EXPORT_SYMBOL(mapping_tagged); 2819 2822
+1 -1
mm/readahead.c
··· 175 175 break; 176 176 177 177 rcu_read_lock(); 178 - page = radix_tree_lookup(&mapping->page_tree, page_offset); 178 + page = radix_tree_lookup(&mapping->i_pages, page_offset); 179 179 rcu_read_unlock(); 180 180 if (page && !radix_tree_exceptional_entry(page)) 181 181 continue;
+2 -2
mm/rmap.c
··· 32 32 * mmlist_lock (in mmput, drain_mmlist and others) 33 33 * mapping->private_lock (in __set_page_dirty_buffers) 34 34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) 35 - * mapping->tree_lock (widely used) 35 + * i_pages lock (widely used) 36 36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 37 37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 38 38 * sb_lock (within inode_lock in fs/fs-writeback.c) 39 - * mapping->tree_lock (widely used, in set_page_dirty, 39 + * i_pages lock (widely used, in set_page_dirty, 40 40 * in arch-dependent flush_dcache_mmap_lock, 41 41 * within bdi.wb->list_lock in __sync_single_inode) 42 42 *
+30 -30
mm/shmem.c
··· 332 332 333 333 VM_BUG_ON(!expected); 334 334 VM_BUG_ON(!replacement); 335 - item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 335 + item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot); 336 336 if (!item) 337 337 return -ENOENT; 338 338 if (item != expected) 339 339 return -ENOENT; 340 - __radix_tree_replace(&mapping->page_tree, node, pslot, 340 + __radix_tree_replace(&mapping->i_pages, node, pslot, 341 341 replacement, NULL); 342 342 return 0; 343 343 } ··· 355 355 void *item; 356 356 357 357 rcu_read_lock(); 358 - item = radix_tree_lookup(&mapping->page_tree, index); 358 + item = radix_tree_lookup(&mapping->i_pages, index); 359 359 rcu_read_unlock(); 360 360 return item == swp_to_radix_entry(swap); 361 361 } ··· 590 590 page->mapping = mapping; 591 591 page->index = index; 592 592 593 - spin_lock_irq(&mapping->tree_lock); 593 + xa_lock_irq(&mapping->i_pages); 594 594 if (PageTransHuge(page)) { 595 595 void __rcu **results; 596 596 pgoff_t idx; 597 597 int i; 598 598 599 599 error = 0; 600 - if (radix_tree_gang_lookup_slot(&mapping->page_tree, 600 + if (radix_tree_gang_lookup_slot(&mapping->i_pages, 601 601 &results, &idx, index, 1) && 602 602 idx < index + HPAGE_PMD_NR) { 603 603 error = -EEXIST; ··· 605 605 606 606 if (!error) { 607 607 for (i = 0; i < HPAGE_PMD_NR; i++) { 608 - error = radix_tree_insert(&mapping->page_tree, 608 + error = radix_tree_insert(&mapping->i_pages, 609 609 index + i, page + i); 610 610 VM_BUG_ON(error); 611 611 } 612 612 count_vm_event(THP_FILE_ALLOC); 613 613 } 614 614 } else if (!expected) { 615 - error = radix_tree_insert(&mapping->page_tree, index, page); 615 + error = radix_tree_insert(&mapping->i_pages, index, page); 616 616 } else { 617 617 error = shmem_radix_tree_replace(mapping, index, expected, 618 618 page); ··· 624 624 __inc_node_page_state(page, NR_SHMEM_THPS); 625 625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 626 626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 627 - spin_unlock_irq(&mapping->tree_lock); 627 + xa_unlock_irq(&mapping->i_pages); 628 628 } else { 629 629 page->mapping = NULL; 630 - spin_unlock_irq(&mapping->tree_lock); 630 + xa_unlock_irq(&mapping->i_pages); 631 631 page_ref_sub(page, nr); 632 632 } 633 633 return error; ··· 643 643 644 644 VM_BUG_ON_PAGE(PageCompound(page), page); 645 645 646 - spin_lock_irq(&mapping->tree_lock); 646 + xa_lock_irq(&mapping->i_pages); 647 647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 648 648 page->mapping = NULL; 649 649 mapping->nrpages--; 650 650 __dec_node_page_state(page, NR_FILE_PAGES); 651 651 __dec_node_page_state(page, NR_SHMEM); 652 - spin_unlock_irq(&mapping->tree_lock); 652 + xa_unlock_irq(&mapping->i_pages); 653 653 put_page(page); 654 654 BUG_ON(error); 655 655 } ··· 662 662 { 663 663 void *old; 664 664 665 - spin_lock_irq(&mapping->tree_lock); 666 - old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 667 - spin_unlock_irq(&mapping->tree_lock); 665 + xa_lock_irq(&mapping->i_pages); 666 + old = radix_tree_delete_item(&mapping->i_pages, index, radswap); 667 + xa_unlock_irq(&mapping->i_pages); 668 668 if (old != radswap) 669 669 return -ENOENT; 670 670 free_swap_and_cache(radix_to_swp_entry(radswap)); ··· 675 675 * Determine (in bytes) how many of the shmem object's pages mapped by the 676 676 * given offsets are swapped out. 677 677 * 678 - * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 678 + * This is safe to call without i_mutex or the i_pages lock thanks to RCU, 679 679 * as long as the inode doesn't go away and racy results are not a problem. 680 680 */ 681 681 unsigned long shmem_partial_swap_usage(struct address_space *mapping, ··· 688 688 689 689 rcu_read_lock(); 690 690 691 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 691 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 692 692 if (iter.index >= end) 693 693 break; 694 694 ··· 717 717 * Determine (in bytes) how many of the shmem object's pages mapped by the 718 718 * given vma is swapped out. 719 719 * 720 - * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 720 + * This is safe to call without i_mutex or the i_pages lock thanks to RCU, 721 721 * as long as the inode doesn't go away and racy results are not a problem. 722 722 */ 723 723 unsigned long shmem_swap_usage(struct vm_area_struct *vma) ··· 1132 1132 int error = 0; 1133 1133 1134 1134 radswap = swp_to_radix_entry(swap); 1135 - index = find_swap_entry(&mapping->page_tree, radswap); 1135 + index = find_swap_entry(&mapping->i_pages, radswap); 1136 1136 if (index == -1) 1137 1137 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1138 1138 ··· 1448 1448 1449 1449 hindex = round_down(index, HPAGE_PMD_NR); 1450 1450 rcu_read_lock(); 1451 - if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1451 + if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx, 1452 1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1453 1453 rcu_read_unlock(); 1454 1454 return NULL; ··· 1561 1561 * Our caller will very soon move newpage out of swapcache, but it's 1562 1562 * a nice clean interface for us to replace oldpage by newpage there. 1563 1563 */ 1564 - spin_lock_irq(&swap_mapping->tree_lock); 1564 + xa_lock_irq(&swap_mapping->i_pages); 1565 1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1566 1566 newpage); 1567 1567 if (!error) { 1568 1568 __inc_node_page_state(newpage, NR_FILE_PAGES); 1569 1569 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1570 1570 } 1571 - spin_unlock_irq(&swap_mapping->tree_lock); 1571 + xa_unlock_irq(&swap_mapping->i_pages); 1572 1572 1573 1573 if (unlikely(error)) { 1574 1574 /* ··· 2634 2634 start = 0; 2635 2635 rcu_read_lock(); 2636 2636 2637 - radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2637 + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { 2638 2638 page = radix_tree_deref_slot(slot); 2639 2639 if (!page || radix_tree_exception(page)) { 2640 2640 if (radix_tree_deref_retry(page)) { ··· 2642 2642 continue; 2643 2643 } 2644 2644 } else if (page_count(page) - page_mapcount(page) > 1) { 2645 - spin_lock_irq(&mapping->tree_lock); 2646 - radix_tree_tag_set(&mapping->page_tree, iter.index, 2645 + xa_lock_irq(&mapping->i_pages); 2646 + radix_tree_tag_set(&mapping->i_pages, iter.index, 2647 2647 SHMEM_TAG_PINNED); 2648 - spin_unlock_irq(&mapping->tree_lock); 2648 + xa_unlock_irq(&mapping->i_pages); 2649 2649 } 2650 2650 2651 2651 if (need_resched()) { ··· 2677 2677 2678 2678 error = 0; 2679 2679 for (scan = 0; scan <= LAST_SCAN; scan++) { 2680 - if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2680 + if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED)) 2681 2681 break; 2682 2682 2683 2683 if (!scan) ··· 2687 2687 2688 2688 start = 0; 2689 2689 rcu_read_lock(); 2690 - radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2690 + radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 2691 2691 start, SHMEM_TAG_PINNED) { 2692 2692 2693 2693 page = radix_tree_deref_slot(slot); ··· 2713 2713 error = -EBUSY; 2714 2714 } 2715 2715 2716 - spin_lock_irq(&mapping->tree_lock); 2717 - radix_tree_tag_clear(&mapping->page_tree, 2716 + xa_lock_irq(&mapping->i_pages); 2717 + radix_tree_tag_clear(&mapping->i_pages, 2718 2718 iter.index, SHMEM_TAG_PINNED); 2719 - spin_unlock_irq(&mapping->tree_lock); 2719 + xa_unlock_irq(&mapping->i_pages); 2720 2720 continue_resched: 2721 2721 if (need_resched()) { 2722 2722 slot = radix_tree_iter_resume(slot, &iter);
+8 -9
mm/swap_state.c
··· 124 124 SetPageSwapCache(page); 125 125 126 126 address_space = swap_address_space(entry); 127 - spin_lock_irq(&address_space->tree_lock); 127 + xa_lock_irq(&address_space->i_pages); 128 128 for (i = 0; i < nr; i++) { 129 129 set_page_private(page + i, entry.val + i); 130 - error = radix_tree_insert(&address_space->page_tree, 130 + error = radix_tree_insert(&address_space->i_pages, 131 131 idx + i, page + i); 132 132 if (unlikely(error)) 133 133 break; ··· 145 145 VM_BUG_ON(error == -EEXIST); 146 146 set_page_private(page + i, 0UL); 147 147 while (i--) { 148 - radix_tree_delete(&address_space->page_tree, idx + i); 148 + radix_tree_delete(&address_space->i_pages, idx + i); 149 149 set_page_private(page + i, 0UL); 150 150 } 151 151 ClearPageSwapCache(page); 152 152 page_ref_sub(page, nr); 153 153 } 154 - spin_unlock_irq(&address_space->tree_lock); 154 + xa_unlock_irq(&address_space->i_pages); 155 155 156 156 return error; 157 157 } ··· 188 188 address_space = swap_address_space(entry); 189 189 idx = swp_offset(entry); 190 190 for (i = 0; i < nr; i++) { 191 - radix_tree_delete(&address_space->page_tree, idx + i); 191 + radix_tree_delete(&address_space->i_pages, idx + i); 192 192 set_page_private(page + i, 0); 193 193 } 194 194 ClearPageSwapCache(page); ··· 272 272 entry.val = page_private(page); 273 273 274 274 address_space = swap_address_space(entry); 275 - spin_lock_irq(&address_space->tree_lock); 275 + xa_lock_irq(&address_space->i_pages); 276 276 __delete_from_swap_cache(page); 277 - spin_unlock_irq(&address_space->tree_lock); 277 + xa_unlock_irq(&address_space->i_pages); 278 278 279 279 put_swap_page(page, entry); 280 280 page_ref_sub(page, hpage_nr_pages(page)); ··· 628 628 return -ENOMEM; 629 629 for (i = 0; i < nr; i++) { 630 630 space = spaces + i; 631 - INIT_RADIX_TREE(&space->page_tree, GFP_ATOMIC|__GFP_NOWARN); 631 + INIT_RADIX_TREE(&space->i_pages, GFP_ATOMIC|__GFP_NOWARN); 632 632 atomic_set(&space->i_mmap_writable, 0); 633 633 space->a_ops = &swap_aops; 634 634 /* swap cache doesn't use writeback related tags */ 635 635 mapping_set_no_writeback_tags(space); 636 - spin_lock_init(&space->tree_lock); 637 636 } 638 637 nr_swapper_spaces[type] = nr; 639 638 rcu_assign_pointer(swapper_spaces[type], spaces);
+11 -11
mm/truncate.c
··· 36 36 struct radix_tree_node *node; 37 37 void **slot; 38 38 39 - if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot)) 39 + if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot)) 40 40 return; 41 41 if (*slot != entry) 42 42 return; 43 - __radix_tree_replace(&mapping->page_tree, node, slot, NULL, 43 + __radix_tree_replace(&mapping->i_pages, node, slot, NULL, 44 44 workingset_update_node); 45 45 mapping->nrexceptional--; 46 46 } ··· 48 48 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, 49 49 void *entry) 50 50 { 51 - spin_lock_irq(&mapping->tree_lock); 51 + xa_lock_irq(&mapping->i_pages); 52 52 __clear_shadow_entry(mapping, index, entry); 53 - spin_unlock_irq(&mapping->tree_lock); 53 + xa_unlock_irq(&mapping->i_pages); 54 54 } 55 55 56 56 /* ··· 79 79 dax = dax_mapping(mapping); 80 80 lock = !dax && indices[j] < end; 81 81 if (lock) 82 - spin_lock_irq(&mapping->tree_lock); 82 + xa_lock_irq(&mapping->i_pages); 83 83 84 84 for (i = j; i < pagevec_count(pvec); i++) { 85 85 struct page *page = pvec->pages[i]; ··· 102 102 } 103 103 104 104 if (lock) 105 - spin_unlock_irq(&mapping->tree_lock); 105 + xa_unlock_irq(&mapping->i_pages); 106 106 pvec->nr = j; 107 107 } 108 108 ··· 518 518 * modification that does not see AS_EXITING is 519 519 * completed before starting the final truncate. 520 520 */ 521 - spin_lock_irq(&mapping->tree_lock); 522 - spin_unlock_irq(&mapping->tree_lock); 521 + xa_lock_irq(&mapping->i_pages); 522 + xa_unlock_irq(&mapping->i_pages); 523 523 524 524 truncate_inode_pages(mapping, 0); 525 525 } ··· 627 627 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) 628 628 return 0; 629 629 630 - spin_lock_irqsave(&mapping->tree_lock, flags); 630 + xa_lock_irqsave(&mapping->i_pages, flags); 631 631 if (PageDirty(page)) 632 632 goto failed; 633 633 634 634 BUG_ON(page_has_private(page)); 635 635 __delete_from_page_cache(page, NULL); 636 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 636 + xa_unlock_irqrestore(&mapping->i_pages, flags); 637 637 638 638 if (mapping->a_ops->freepage) 639 639 mapping->a_ops->freepage(page); ··· 641 641 put_page(page); /* pagecache ref */ 642 642 return 1; 643 643 failed: 644 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 644 + xa_unlock_irqrestore(&mapping->i_pages, flags); 645 645 return 0; 646 646 } 647 647
+6 -6
mm/vmscan.c
··· 693 693 BUG_ON(!PageLocked(page)); 694 694 BUG_ON(mapping != page_mapping(page)); 695 695 696 - spin_lock_irqsave(&mapping->tree_lock, flags); 696 + xa_lock_irqsave(&mapping->i_pages, flags); 697 697 /* 698 698 * The non racy check for a busy page. 699 699 * ··· 717 717 * load is not satisfied before that of page->_refcount. 718 718 * 719 719 * Note that if SetPageDirty is always performed via set_page_dirty, 720 - * and thus under tree_lock, then this ordering is not required. 720 + * and thus under the i_pages lock, then this ordering is not required. 721 721 */ 722 722 if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) 723 723 refcount = 1 + HPAGE_PMD_NR; ··· 735 735 swp_entry_t swap = { .val = page_private(page) }; 736 736 mem_cgroup_swapout(page, swap); 737 737 __delete_from_swap_cache(page); 738 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 738 + xa_unlock_irqrestore(&mapping->i_pages, flags); 739 739 put_swap_page(page, swap); 740 740 } else { 741 741 void (*freepage)(struct page *); ··· 756 756 * only page cache pages found in these are zero pages 757 757 * covering holes, and because we don't want to mix DAX 758 758 * exceptional entries and shadow exceptional entries in the 759 - * same page_tree. 759 + * same address_space. 760 760 */ 761 761 if (reclaimed && page_is_file_cache(page) && 762 762 !mapping_exiting(mapping) && !dax_mapping(mapping)) 763 763 shadow = workingset_eviction(mapping, page); 764 764 __delete_from_page_cache(page, shadow); 765 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 765 + xa_unlock_irqrestore(&mapping->i_pages, flags); 766 766 767 767 if (freepage != NULL) 768 768 freepage(page); ··· 771 771 return 1; 772 772 773 773 cannot_free: 774 - spin_unlock_irqrestore(&mapping->tree_lock, flags); 774 + xa_unlock_irqrestore(&mapping->i_pages, flags); 775 775 return 0; 776 776 } 777 777
+11 -11
mm/workingset.c
··· 202 202 * @mapping: address space the page was backing 203 203 * @page: the page being evicted 204 204 * 205 - * Returns a shadow entry to be stored in @mapping->page_tree in place 205 + * Returns a shadow entry to be stored in @mapping->i_pages in place 206 206 * of the evicted @page so that a later refault can be detected. 207 207 */ 208 208 void *workingset_eviction(struct address_space *mapping, struct page *page) ··· 348 348 * 349 349 * Avoid acquiring the list_lru lock when the nodes are 350 350 * already where they should be. The list_empty() test is safe 351 - * as node->private_list is protected by &mapping->tree_lock. 351 + * as node->private_list is protected by the i_pages lock. 352 352 */ 353 353 if (node->count && node->count == node->exceptional) { 354 354 if (list_empty(&node->private_list)) ··· 366 366 unsigned long nodes; 367 367 unsigned long cache; 368 368 369 - /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ 369 + /* list_lru lock nests inside the IRQ-safe i_pages lock */ 370 370 local_irq_disable(); 371 371 nodes = list_lru_shrink_count(&shadow_nodes, sc); 372 372 local_irq_enable(); ··· 419 419 420 420 /* 421 421 * Page cache insertions and deletions synchroneously maintain 422 - * the shadow node LRU under the mapping->tree_lock and the 422 + * the shadow node LRU under the i_pages lock and the 423 423 * lru_lock. Because the page cache tree is emptied before 424 424 * the inode can be destroyed, holding the lru_lock pins any 425 425 * address_space that has radix tree nodes on the LRU. 426 426 * 427 - * We can then safely transition to the mapping->tree_lock to 427 + * We can then safely transition to the i_pages lock to 428 428 * pin only the address_space of the particular node we want 429 429 * to reclaim, take the node off-LRU, and drop the lru_lock. 430 430 */ 431 431 432 432 node = container_of(item, struct radix_tree_node, private_list); 433 - mapping = container_of(node->root, struct address_space, page_tree); 433 + mapping = container_of(node->root, struct address_space, i_pages); 434 434 435 435 /* Coming from the list, invert the lock order */ 436 - if (!spin_trylock(&mapping->tree_lock)) { 436 + if (!xa_trylock(&mapping->i_pages)) { 437 437 spin_unlock(lru_lock); 438 438 ret = LRU_RETRY; 439 439 goto out; ··· 468 468 if (WARN_ON_ONCE(node->exceptional)) 469 469 goto out_invalid; 470 470 inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); 471 - __radix_tree_delete_node(&mapping->page_tree, node, 471 + __radix_tree_delete_node(&mapping->i_pages, node, 472 472 workingset_lookup_update(mapping)); 473 473 474 474 out_invalid: 475 - spin_unlock(&mapping->tree_lock); 475 + xa_unlock(&mapping->i_pages); 476 476 ret = LRU_REMOVED_RETRY; 477 477 out: 478 478 local_irq_enable(); ··· 487 487 { 488 488 unsigned long ret; 489 489 490 - /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ 490 + /* list_lru lock nests inside the IRQ-safe i_pages lock */ 491 491 local_irq_disable(); 492 492 ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); 493 493 local_irq_enable(); ··· 503 503 504 504 /* 505 505 * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe 506 - * mapping->tree_lock. 506 + * i_pages lock. 507 507 */ 508 508 static struct lock_class_key shadow_nodes_key; 509 509