diff -urN oldtree/include/linux/page-flags.h newtree/include/linux/page-flags.h --- oldtree/include/linux/page-flags.h 2006-10-05 15:26:55.000000000 -0400 +++ newtree/include/linux/page-flags.h 2006-10-07 08:58:49.000000000 -0400 @@ -91,8 +91,9 @@ #define PG_nosave_free 18 /* Used for system suspend/resume */ #define PG_buddy 19 /* Page is free, on buddy lists */ -#define PG_readahead 20 /* Reminder to do readahead */ - +#define PG_nonewrefs 20 /* Block concurrent pagecache lookups + * while testing refcount */ +#define PG_readahead 21 /* Reminder to do readahead */ #if (BITS_PER_LONG > 32) /* @@ -253,6 +254,11 @@ #define SetPageReadahead(page) set_bit(PG_readahead, &(page)->flags) #define TestClearPageReadahead(page) test_and_clear_bit(PG_readahead, &(page)->flags) +#define PageNoNewRefs(page) test_bit(PG_nonewrefs, &(page)->flags) +#define SetPageNoNewRefs(page) set_bit(PG_nonewrefs, &(page)->flags) +#define ClearPageNoNewRefs(page) clear_bit(PG_nonewrefs, &(page)->flags) +#define __ClearPageNoNewRefs(page) __clear_bit(PG_nonewrefs, &(page)->flags) + struct page; /* forward declaration */ int test_clear_page_dirty(struct page *page); diff -urN oldtree/include/linux/pagemap.h newtree/include/linux/pagemap.h --- oldtree/include/linux/pagemap.h 2006-10-05 15:26:55.000000000 -0400 +++ newtree/include/linux/pagemap.h 2006-10-07 08:58:49.000000000 -0400 @@ -11,6 +11,8 @@ #include #include #include +#include +#include /* for in_interrupt() */ /* * Bits in mapping->flags. The lower __GFP_BITS_SHIFT bits are the page @@ -51,6 +53,109 @@ #define page_cache_release(page) put_page(page) void release_pages(struct page **pages, int nr, int cold); +/* + * speculatively take a reference to a page. + * If the page is free (_count == 0), then _count is untouched, and 0 + * is returned. Otherwise, _count is incremented by 1 and 1 is returned. + * + * This function must be run in the same rcu_read_lock() section as has + * been used to lookup the page in the pagecache radix-tree: this allows + * allocators to use a synchronize_rcu() to stabilize _count. + * + * Unless an RCU grace period has passed, the count of all pages coming out + * of the allocator must be considered unstable. page_count may return higher + * than expected, and put_page must be able to do the right thing when the + * page has been finished with (because put_page is what is used to drop an + * invalid speculative reference). + * + * After incrementing the refcount, this function spins until PageNoNewRefs + * is clear, then a read memory barrier is issued. + * + * This forms the core of the lockless pagecache locking protocol, where + * the lookup-side (eg. find_get_page) has the following pattern: + * 1. find page in radix tree + * 2. conditionally increment refcount + * 3. wait for PageNoNewRefs + * 4. check the page is still in pagecache + * + * Remove-side (that cares about _count, eg. reclaim) has the following: + * A. SetPageNoNewRefs + * B. check refcount is correct + * C. remove page + * D. ClearPageNoNewRefs + * + * There are 2 critical interleavings that matter: + * - 2 runs before B: in this case, B sees elevated refcount and bails out + * - B runs before 2: in this case, 3 ensures 4 will not run until *after* C + * (after D, even). In which case, 4 will notice C and lookup side can retry + * + * It is possible that between 1 and 2, the page is removed then the exact same + * page is inserted into the same position in pagecache. That's OK: the + * old find_get_page using tree_lock could equally have run before or after + * the write-side, depending on timing. + * + * Pagecache insertion isn't a big problem: either 1 will find the page or + * it will not. Likewise, the old find_get_page could run either before the + * insertion or afterwards, depending on timing. + */ +static inline int page_cache_get_speculative(struct page *page) +{ + VM_BUG_ON(in_interrupt()); + +#ifndef CONFIG_SMP +# ifdef CONFIG_PREEMPT + VM_BUG_ON(!in_atomic()); +# endif + /* + * Preempt must be disabled here - we rely on rcu_read_lock doing + * this for us. + * + * Pagecache won't be truncated from interrupt context, so if we have + * found a page in the radix tree here, we have pinned its refcount by + * disabling preempt, and hence no need for the "speculative get" that + * SMP requires. + */ + VM_BUG_ON(page_count(page) == 0); + atomic_inc(&page->_count); + +#else + if (unlikely(!get_page_unless_zero(page))) + return 0; /* page has been freed */ + + /* + * Note that get_page_unless_zero provides a memory barrier. + * This is needed to ensure PageNoNewRefs is evaluated after the + * page refcount has been raised. See below comment. + */ + + while (unlikely(PageNoNewRefs(page))) + cpu_relax(); + + /* + * smp_rmb is to ensure the load of page->flags (for PageNoNewRefs()) + * is performed before a future load used to ensure the page is + * the correct on (usually: page->mapping and page->index). + * + * Those places that set PageNoNewRefs have the following pattern: + * SetPageNoNewRefs(page) + * wmb(); + * if (page_count(page) == X) + * remove page from pagecache + * wmb(); + * ClearPageNoNewRefs(page) + * + * If the load was out of order, page->mapping might be loaded before + * the page is removed from pagecache but PageNoNewRefs evaluated + * after the ClearPageNoNewRefs(). + */ + smp_rmb(); + +#endif + VM_BUG_ON(PageCompound(page) && (struct page *)page_private(page) != page); + + return 1; +} + #ifdef CONFIG_NUMA extern struct page *page_cache_alloc(struct address_space *x); extern struct page *page_cache_alloc_cold(struct address_space *x); diff -urN oldtree/mm/filemap.c newtree/mm/filemap.c --- oldtree/mm/filemap.c 2006-10-06 19:12:07.000000000 -0400 +++ newtree/mm/filemap.c 2006-10-07 08:58:49.000000000 -0400 @@ -450,6 +450,8 @@ int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); if (error == 0) { + SetPageNoNewRefs(page); + smp_wmb(); write_lock_irq(&mapping->tree_lock); error = radix_tree_insert(&mapping->page_tree, offset, page); if (!error) { @@ -461,6 +463,8 @@ __inc_zone_page_state(page, NR_FILE_PAGES); } write_unlock_irq(&mapping->tree_lock); + smp_wmb(); + ClearPageNoNewRefs(page); radix_tree_preload_end(); } return error; diff -urN oldtree/mm/migrate.c newtree/mm/migrate.c --- oldtree/mm/migrate.c 2006-10-05 15:26:55.000000000 -0400 +++ newtree/mm/migrate.c 2006-10-07 08:58:49.000000000 -0400 @@ -303,6 +303,8 @@ return 0; } + SetPageNoNewRefs(page); + smp_wmb(); write_lock_irq(&mapping->tree_lock); pslot = radix_tree_lookup_slot(&mapping->page_tree, @@ -311,6 +313,7 @@ if (page_count(page) != 2 + !!PagePrivate(page) || (struct page *)radix_tree_deref_slot(pslot) != page) { write_unlock_irq(&mapping->tree_lock); + ClearPageNoNewRefs(page); return -EAGAIN; } @@ -326,6 +329,10 @@ #endif radix_tree_replace_slot(pslot, newpage); + page->mapping = NULL; + write_unlock_irq(&mapping->tree_lock); + smp_wmb(); + ClearPageNoNewRefs(page); /* * Drop cache reference from old page. @@ -333,8 +340,6 @@ */ __put_page(page); - write_unlock_irq(&mapping->tree_lock); - return 0; } diff -urN oldtree/mm/swap_state.c newtree/mm/swap_state.c --- oldtree/mm/swap_state.c 2006-10-05 15:26:55.000000000 -0400 +++ newtree/mm/swap_state.c 2006-10-07 08:58:49.000000000 -0400 @@ -79,6 +79,8 @@ BUG_ON(PagePrivate(page)); error = radix_tree_preload(gfp_mask); if (!error) { + SetPageNoNewRefs(page); + smp_wmb(); write_lock_irq(&swapper_space.tree_lock); error = radix_tree_insert(&swapper_space.page_tree, entry.val, page); @@ -92,6 +94,8 @@ __inc_zone_page_state(page, NR_FILE_PAGES); } write_unlock_irq(&swapper_space.tree_lock); + smp_wmb(); + ClearPageNoNewRefs(page); radix_tree_preload_end(); } return error; diff -urN oldtree/mm/vmscan.c newtree/mm/vmscan.c --- oldtree/mm/vmscan.c 2006-10-06 19:12:07.000000000 -0400 +++ newtree/mm/vmscan.c 2006-10-07 08:58:49.000000000 -0400 @@ -416,6 +416,8 @@ BUG_ON(!PageLocked(page)); BUG_ON(mapping != page_mapping(page)); + SetPageNoNewRefs(page); + smp_wmb(); write_lock_irq(&mapping->tree_lock); /* * The non racy check for a busy page. @@ -454,17 +456,21 @@ __delete_from_swap_cache(page); write_unlock_irq(&mapping->tree_lock); swap_free(swap); - __put_page(page); /* The pagecache ref */ - return 1; + goto free_it; } __remove_from_page_cache(page); write_unlock_irq(&mapping->tree_lock); - __put_page(page); + +free_it: + smp_wmb(); + __ClearPageNoNewRefs(page); + __put_page(page); /* The pagecache ref */ return 1; cannot_free: write_unlock_irq(&mapping->tree_lock); + ClearPageNoNewRefs(page); return 0; }