diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Makefile | 20 | ||||
| -rw-r--r-- | mm/bootmem.c | 400 | ||||
| -rw-r--r-- | mm/fadvise.c | 111 | ||||
| -rw-r--r-- | mm/filemap.c | 2306 | ||||
| -rw-r--r-- | mm/fremap.c | 256 | ||||
| -rw-r--r-- | mm/highmem.c | 607 | ||||
| -rw-r--r-- | mm/hugetlb.c | 260 | ||||
| -rw-r--r-- | mm/internal.h | 13 | ||||
| -rw-r--r-- | mm/madvise.c | 242 | ||||
| -rw-r--r-- | mm/memory.c | 2165 | ||||
| -rw-r--r-- | mm/mempolicy.c | 1138 | ||||
| -rw-r--r-- | mm/mempool.c | 290 | ||||
| -rw-r--r-- | mm/mincore.c | 191 | ||||
| -rw-r--r-- | mm/mlock.c | 253 | ||||
| -rw-r--r-- | mm/mmap.c | 2082 | ||||
| -rw-r--r-- | mm/mprotect.c | 282 | ||||
| -rw-r--r-- | mm/mremap.c | 426 | ||||
| -rw-r--r-- | mm/msync.c | 236 | ||||
| -rw-r--r-- | mm/nommu.c | 1180 | ||||
| -rw-r--r-- | mm/oom_kill.c | 292 | ||||
| -rw-r--r-- | mm/page-writeback.c | 819 | ||||
| -rw-r--r-- | mm/page_alloc.c | 2220 | ||||
| -rw-r--r-- | mm/page_io.c | 160 | ||||
| -rw-r--r-- | mm/pdflush.c | 228 | ||||
| -rw-r--r-- | mm/prio_tree.c | 207 | ||||
| -rw-r--r-- | mm/readahead.c | 557 | ||||
| -rw-r--r-- | mm/rmap.c | 862 | ||||
| -rw-r--r-- | mm/shmem.c | 2326 | ||||
| -rw-r--r-- | mm/slab.c | 3060 | ||||
| -rw-r--r-- | mm/swap.c | 485 | ||||
| -rw-r--r-- | mm/swap_state.c | 382 | ||||
| -rw-r--r-- | mm/swapfile.c | 1672 | ||||
| -rw-r--r-- | mm/thrash.c | 102 | ||||
| -rw-r--r-- | mm/tiny-shmem.c | 122 | ||||
| -rw-r--r-- | mm/truncate.c | 336 | ||||
| -rw-r--r-- | mm/vmalloc.c | 588 | ||||
| -rw-r--r-- | mm/vmscan.c | 1311 |
37 files changed, 28187 insertions, 0 deletions
diff --git a/mm/Makefile b/mm/Makefile new file mode 100644 index 000000000000..097408064f6a --- /dev/null +++ b/mm/Makefile @@ -0,0 +1,20 @@ +# +# Makefile for the linux memory manager. +# + +mmu-y := nommu.o +mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \ + mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \ + vmalloc.o + +obj-y := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \ + page_alloc.o page-writeback.o pdflush.o \ + readahead.o slab.o swap.o truncate.o vmscan.o \ + prio_tree.o $(mmu-y) + +obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o thrash.o +obj-$(CONFIG_HUGETLBFS) += hugetlb.o +obj-$(CONFIG_NUMA) += mempolicy.o +obj-$(CONFIG_SHMEM) += shmem.o +obj-$(CONFIG_TINY_SHMEM) += tiny-shmem.o + diff --git a/mm/bootmem.c b/mm/bootmem.c new file mode 100644 index 000000000000..260e703850d8 --- /dev/null +++ b/mm/bootmem.c @@ -0,0 +1,400 @@ +/* + * linux/mm/bootmem.c + * + * Copyright (C) 1999 Ingo Molnar + * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 + * + * simple boot-time physical memory area allocator and + * free memory collector. It's used to deal with reserved + * system memory and memory holes as well. + */ + +#include <linux/mm.h> +#include <linux/kernel_stat.h> +#include <linux/swap.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/bootmem.h> +#include <linux/mmzone.h> +#include <linux/module.h> +#include <asm/dma.h> +#include <asm/io.h> +#include "internal.h" + +/* + * Access to this subsystem has to be serialized externally. (this is + * true for the boot process anyway) + */ +unsigned long max_low_pfn; +unsigned long min_low_pfn; +unsigned long max_pfn; + +EXPORT_SYMBOL(max_pfn); /* This is exported so + * dma_get_required_mask(), which uses + * it, can be an inline function */ + +/* return the number of _pages_ that will be allocated for the boot bitmap */ +unsigned long __init bootmem_bootmap_pages (unsigned long pages) +{ + unsigned long mapsize; + + mapsize = (pages+7)/8; + mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK; + mapsize >>= PAGE_SHIFT; + + return mapsize; +} + +/* + * Called once to set up the allocator itself. + */ +static unsigned long __init init_bootmem_core (pg_data_t *pgdat, + unsigned long mapstart, unsigned long start, unsigned long end) +{ + bootmem_data_t *bdata = pgdat->bdata; + unsigned long mapsize = ((end - start)+7)/8; + + pgdat->pgdat_next = pgdat_list; + pgdat_list = pgdat; + + mapsize = (mapsize + (sizeof(long) - 1UL)) & ~(sizeof(long) - 1UL); + bdata->node_bootmem_map = phys_to_virt(mapstart << PAGE_SHIFT); + bdata->node_boot_start = (start << PAGE_SHIFT); + bdata->node_low_pfn = end; + + /* + * Initially all pages are reserved - setup_arch() has to + * register free RAM areas explicitly. + */ + memset(bdata->node_bootmem_map, 0xff, mapsize); + + return mapsize; +} + +/* + * Marks a particular physical memory range as unallocatable. Usable RAM + * might be used for boot-time allocations - or it might get added + * to the free page pool later on. + */ +static void __init reserve_bootmem_core(bootmem_data_t *bdata, unsigned long addr, unsigned long size) +{ + unsigned long i; + /* + * round up, partially reserved pages are considered + * fully reserved. + */ + unsigned long sidx = (addr - bdata->node_boot_start)/PAGE_SIZE; + unsigned long eidx = (addr + size - bdata->node_boot_start + + PAGE_SIZE-1)/PAGE_SIZE; + unsigned long end = (addr + size + PAGE_SIZE-1)/PAGE_SIZE; + + BUG_ON(!size); + BUG_ON(sidx >= eidx); + BUG_ON((addr >> PAGE_SHIFT) >= bdata->node_low_pfn); + BUG_ON(end > bdata->node_low_pfn); + + for (i = sidx; i < eidx; i++) + if (test_and_set_bit(i, bdata->node_bootmem_map)) { +#ifdef CONFIG_DEBUG_BOOTMEM + printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE); +#endif + } +} + +static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, unsigned long size) +{ + unsigned long i; + unsigned long start; + /* + * round down end of usable mem, partially free pages are + * considered reserved. + */ + unsigned long sidx; + unsigned long eidx = (addr + size - bdata->node_boot_start)/PAGE_SIZE; + unsigned long end = (addr + size)/PAGE_SIZE; + + BUG_ON(!size); + BUG_ON(end > bdata->node_low_pfn); + + if (addr < bdata->last_success) + bdata->last_success = addr; + + /* + * Round up the beginning of the address. + */ + start = (addr + PAGE_SIZE-1) / PAGE_SIZE; + sidx = start - (bdata->node_boot_start/PAGE_SIZE); + + for (i = sidx; i < eidx; i++) { + if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map))) + BUG(); + } +} + +/* + * We 'merge' subsequent allocations to save space. We might 'lose' + * some fraction of a page if allocations cannot be satisfied due to + * size constraints on boxes where there is physical RAM space + * fragmentation - in these cases (mostly large memory boxes) this + * is not a problem. + * + * On low memory boxes we get it right in 100% of the cases. + * + * alignment has to be a power of 2 value. + * + * NOTE: This function is _not_ reentrant. + */ +static void * __init +__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, + unsigned long align, unsigned long goal) +{ + unsigned long offset, remaining_size, areasize, preferred; + unsigned long i, start = 0, incr, eidx; + void *ret; + + if(!size) { + printk("__alloc_bootmem_core(): zero-sized request\n"); + BUG(); + } + BUG_ON(align & (align-1)); + + eidx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT); + offset = 0; + if (align && + (bdata->node_boot_start & (align - 1UL)) != 0) + offset = (align - (bdata->node_boot_start & (align - 1UL))); + offset >>= PAGE_SHIFT; + + /* + * We try to allocate bootmem pages above 'goal' + * first, then we try to allocate lower pages. + */ + if (goal && (goal >= bdata->node_boot_start) && + ((goal >> PAGE_SHIFT) < bdata->node_low_pfn)) { + preferred = goal - bdata->node_boot_start; + + if (bdata->last_success >= preferred) + preferred = bdata->last_success; + } else + preferred = 0; + + preferred = ((preferred + align - 1) & ~(align - 1)) >> PAGE_SHIFT; + preferred += offset; + areasize = (size+PAGE_SIZE-1)/PAGE_SIZE; + incr = align >> PAGE_SHIFT ? : 1; + +restart_scan: + for (i = preferred; i < eidx; i += incr) { + unsigned long j; + i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i); + i = ALIGN(i, incr); + if (test_bit(i, bdata->node_bootmem_map)) + continue; + for (j = i + 1; j < i + areasize; ++j) { + if (j >= eidx) + goto fail_block; + if (test_bit (j, bdata->node_bootmem_map)) + goto fail_block; + } + start = i; + goto found; + fail_block: + i = ALIGN(j, incr); + } + + if (preferred > offset) { + preferred = offset; + goto restart_scan; + } + return NULL; + +found: + bdata->last_success = start << PAGE_SHIFT; + BUG_ON(start >= eidx); + + /* + * Is the next page of the previous allocation-end the start + * of this allocation's buffer? If yes then we can 'merge' + * the previous partial page with this allocation. + */ + if (align < PAGE_SIZE && + bdata->last_offset && bdata->last_pos+1 == start) { + offset = (bdata->last_offset+align-1) & ~(align-1); + BUG_ON(offset > PAGE_SIZE); + remaining_size = PAGE_SIZE-offset; + if (size < remaining_size) { + areasize = 0; + /* last_pos unchanged */ + bdata->last_offset = offset+size; + ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset + + bdata->node_boot_start); + } else { + remaining_size = size - remaining_size; + areasize = (remaining_size+PAGE_SIZE-1)/PAGE_SIZE; + ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset + + bdata->node_boot_start); + bdata->last_pos = start+areasize-1; + bdata->last_offset = remaining_size; + } + bdata->last_offset &= ~PAGE_MASK; + } else { + bdata->last_pos = start + areasize - 1; + bdata->last_offset = size & ~PAGE_MASK; + ret = phys_to_virt(start * PAGE_SIZE + bdata->node_boot_start); + } + + /* + * Reserve the area now: + */ + for (i = start; i < start+areasize; i++) + if (unlikely(test_and_set_bit(i, bdata->node_bootmem_map))) + BUG(); + memset(ret, 0, size); + return ret; +} + +static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat) +{ + struct page *page; + bootmem_data_t *bdata = pgdat->bdata; + unsigned long i, count, total = 0; + unsigned long idx; + unsigned long *map; + int gofast = 0; + + BUG_ON(!bdata->node_bootmem_map); + + count = 0; + /* first extant page of the node */ + page = virt_to_page(phys_to_virt(bdata->node_boot_start)); + idx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT); + map = bdata->node_bootmem_map; + /* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */ + if (bdata->node_boot_start == 0 || + ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG)) + gofast = 1; + for (i = 0; i < idx; ) { + unsigned long v = ~map[i / BITS_PER_LONG]; + if (gofast && v == ~0UL) { + int j, order; + + count += BITS_PER_LONG; + __ClearPageReserved(page); + order = ffs(BITS_PER_LONG) - 1; + set_page_refs(page, order); + for (j = 1; j < BITS_PER_LONG; j++) { + if (j + 16 < BITS_PER_LONG) + prefetchw(page + j + 16); + __ClearPageReserved(page + j); + } + __free_pages(page, order); + i += BITS_PER_LONG; + page += BITS_PER_LONG; + } else if (v) { + unsigned long m; + for (m = 1; m && i < idx; m<<=1, page++, i++) { + if (v & m) { + count++; + __ClearPageReserved(page); + set_page_refs(page, 0); + __free_page(page); + } + } + } else { + i+=BITS_PER_LONG; + page += BITS_PER_LONG; + } + } + total += count; + + /* + * Now free the allocator bitmap itself, it's not + * needed anymore: + */ + page = virt_to_page(bdata->node_bootmem_map); + count = 0; + for (i = 0; i < ((bdata->node_low_pfn-(bdata->node_boot_start >> PAGE_SHIFT))/8 + PAGE_SIZE-1)/PAGE_SIZE; i++,page++) { + count++; + __ClearPageReserved(page); + set_page_count(page, 1); + __free_page(page); + } + total += count; + bdata->node_bootmem_map = NULL; + + return total; +} + +unsigned long __init init_bootmem_node (pg_data_t *pgdat, unsigned long freepfn, unsigned long startpfn, unsigned long endpfn) +{ + return(init_bootmem_core(pgdat, freepfn, startpfn, endpfn)); +} + +void __init reserve_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size) +{ + reserve_bootmem_core(pgdat->bdata, physaddr, size); +} + +void __init free_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size) +{ + free_bootmem_core(pgdat->bdata, physaddr, size); +} + +unsigned long __init free_all_bootmem_node (pg_data_t *pgdat) +{ + return(free_all_bootmem_core(pgdat)); +} + +unsigned long __init init_bootmem (unsigned long start, unsigned long pages) +{ + max_low_pfn = pages; + min_low_pfn = start; + return(init_bootmem_core(NODE_DATA(0), start, 0, pages)); +} + +#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE +void __init reserve_bootmem (unsigned long addr, unsigned long size) +{ + reserve_bootmem_core(NODE_DATA(0)->bdata, addr, size); +} +#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ + +void __init free_bootmem (unsigned long addr, unsigned long size) +{ + free_bootmem_core(NODE_DATA(0)->bdata, addr, size); +} + +unsigned long __init free_all_bootmem (void) +{ + return(free_all_bootmem_core(NODE_DATA(0))); +} + +void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned long goal) +{ + pg_data_t *pgdat = pgdat_list; + void *ptr; + + for_each_pgdat(pgdat) + if ((ptr = __alloc_bootmem_core(pgdat->bdata, size, + align, goal))) + return(ptr); + + /* + * Whoops, we cannot satisfy the allocation request. + */ + printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); + panic("Out of memory"); + return NULL; +} + +void * __init __alloc_bootmem_node (pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) +{ + void *ptr; + + ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal); + if (ptr) + return (ptr); + + return __alloc_bootmem(size, align, goal); +} + diff --git a/mm/fadvise.c b/mm/fadvise.c new file mode 100644 index 000000000000..57264d74b8bf --- /dev/null +++ b/mm/fadvise.c @@ -0,0 +1,111 @@ +/* + * mm/fadvise.c + * + * Copyright (C) 2002, Linus Torvalds + * + * 11Jan2003 akpm@digeo.com + * Initial version. + */ + +#include <linux/kernel.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/backing-dev.h> +#include <linux/pagevec.h> +#include <linux/fadvise.h> +#include <linux/syscalls.h> + +#include <asm/unistd.h> + +/* + * POSIX_FADV_WILLNEED could set PG_Referenced, and POSIX_FADV_NOREUSE could + * deactivate the pages and clear PG_Referenced. + */ +asmlinkage long sys_fadvise64_64(int fd, loff_t offset, loff_t len, int advice) +{ + struct file *file = fget(fd); + struct address_space *mapping; + struct backing_dev_info *bdi; + loff_t endbyte; + pgoff_t start_index; + pgoff_t end_index; + unsigned long nrpages; + int ret = 0; + + if (!file) + return -EBADF; + + mapping = file->f_mapping; + if (!mapping || len < 0) { + ret = -EINVAL; + goto out; + } + + /* Careful about overflows. Len == 0 means "as much as possible" */ + endbyte = offset + len; + if (!len || endbyte < len) + endbyte = -1; + + bdi = mapping->backing_dev_info; + + switch (advice) { + case POSIX_FADV_NORMAL: + file->f_ra.ra_pages = bdi->ra_pages; + break; + case POSIX_FADV_RANDOM: + file->f_ra.ra_pages = 0; + break; + case POSIX_FADV_SEQUENTIAL: + file->f_ra.ra_pages = bdi->ra_pages * 2; + break; + case POSIX_FADV_WILLNEED: + case POSIX_FADV_NOREUSE: + if (!mapping->a_ops->readpage) { + ret = -EINVAL; + break; + } + + /* First and last PARTIAL page! */ + start_index = offset >> PAGE_CACHE_SHIFT; + end_index = (endbyte-1) >> PAGE_CACHE_SHIFT; + + /* Careful about overflow on the "+1" */ + nrpages = end_index - start_index + 1; + if (!nrpages) + nrpages = ~0UL; + + ret = force_page_cache_readahead(mapping, file, + start_index, + max_sane_readahead(nrpages)); + if (ret > 0) + ret = 0; + break; + case POSIX_FADV_DONTNEED: + if (!bdi_write_congested(mapping->backing_dev_info)) + filemap_flush(mapping); + + /* First and last FULL page! */ + start_index = (offset + (PAGE_CACHE_SIZE-1)) >> PAGE_CACHE_SHIFT; + end_index = (endbyte >> PAGE_CACHE_SHIFT); + + if (end_index > start_index) + invalidate_mapping_pages(mapping, start_index, end_index-1); + break; + default: + ret = -EINVAL; + } +out: + fput(file); + return ret; +} + +#ifdef __ARCH_WANT_SYS_FADVISE64 + +asmlinkage long sys_fadvise64(int fd, loff_t offset, size_t len, int advice) +{ + return sys_fadvise64_64(fd, offset, len, advice); +} + +#endif diff --git a/mm/filemap.c b/mm/filemap.c new file mode 100644 index 000000000000..439b2bea8e34 --- /dev/null +++ b/mm/filemap.c @@ -0,0 +1,2306 @@ +/* + * linux/mm/filemap.c + * + * Copyright (C) 1994-1999 Linus Torvalds + */ + +/* + * This file handles the generic file mmap semantics used by + * most "normal" filesystems (but you don't /have/ to use this: + * the NFS filesystem used to do this differently, for example) + */ +#include <linux/config.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/compiler.h> +#include <linux/fs.h> +#include <linux/aio.h> +#include <linux/kernel_stat.h> +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/mman.h> +#include <linux/pagemap.h> +#include <linux/file.h> +#include <linux/uio.h> +#include <linux/hash.h> +#include <linux/writeback.h> +#include <linux/pagevec.h> +#include <linux/blkdev.h> +#include <linux/security.h> +#include <linux/syscalls.h> +/* + * This is needed for the following functions: + * - try_to_release_page + * - block_invalidatepage + * - generic_osync_inode + * + * FIXME: remove all knowledge of the buffer layer from the core VM + */ +#include <linux/buffer_head.h> /* for generic_osync_inode */ + +#include <asm/uaccess.h> +#include <asm/mman.h> + +/* + * Shared mappings implemented 30.11.1994. It's not fully working yet, + * though. + * + * Shared mappings now work. 15.8.1995 Bruno. + * + * finished 'unifying' the page and buffer cache and SMP-threaded the + * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> + * + * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> + */ + +/* + * Lock ordering: + * + * ->i_mmap_lock (vmtruncate) + * ->private_lock (__free_pte->__set_page_dirty_buffers) + * ->swap_list_lock + * ->swap_device_lock (exclusive_swap_page, others) + * ->mapping->tree_lock + * + * ->i_sem + * ->i_mmap_lock (truncate->unmap_mapping_range) + * + * ->mmap_sem + * ->i_mmap_lock + * ->page_table_lock (various places, mainly in mmap.c) + * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) + * + * ->mmap_sem + * ->lock_page (access_process_vm) + * + * ->mmap_sem + * ->i_sem (msync) + * + * ->i_sem + * ->i_alloc_sem (various) + * + * ->inode_lock + * ->sb_lock (fs/fs-writeback.c) + * ->mapping->tree_lock (__sync_single_inode) + * + * ->i_mmap_lock + * ->anon_vma.lock (vma_adjust) + * + * ->anon_vma.lock + * ->page_table_lock (anon_vma_prepare and various) + * + * ->page_table_lock + * ->swap_device_lock (try_to_unmap_one) + * ->private_lock (try_to_unmap_one) + * ->tree_lock (try_to_unmap_one) + * ->zone.lru_lock (follow_page->mark_page_accessed) + * ->private_lock (page_remove_rmap->set_page_dirty) + * ->tree_lock (page_remove_rmap->set_page_dirty) + * ->inode_lock (page_remove_rmap->set_page_dirty) + * ->inode_lock (zap_pte_range->set_page_dirty) + * ->private_lock (zap_pte_range->__set_page_dirty_buffers) + * + * ->task->proc_lock + * ->dcache_lock (proc_pid_lookup) + */ + +/* + * Remove a page from the page cache and free it. Caller has to make + * sure the page is locked and that nobody else uses it - or that usage + * is safe. The caller must hold a write_lock on the mapping's tree_lock. + */ +void __remove_from_page_cache(struct page *page) +{ + struct address_space *mapping = page->mapping; + + radix_tree_delete(&mapping->page_tree, page->index); + page->mapping = NULL; + mapping->nrpages--; + pagecache_acct(-1); +} + +void remove_from_page_cache(struct page *page) +{ + struct address_space *mapping = page->mapping; + + if (unlikely(!PageLocked(page))) + PAGE_BUG(page); + + write_lock_irq(&mapping->tree_lock); + __remove_from_page_cache(page); + write_unlock_irq(&mapping->tree_lock); +} + +static int sync_page(void *word) +{ + struct address_space *mapping; + struct page *page; + + page = container_of((page_flags_t *)word, struct page, flags); + + /* + * FIXME, fercrissake. What is this barrier here for? + */ + smp_mb(); + mapping = page_mapping(page); + if (mapping && mapping->a_ops && mapping->a_ops->sync_page) + mapping->a_ops->sync_page(page); + io_schedule(); + return 0; +} + +/** + * filemap_fdatawrite_range - start writeback against all of a mapping's + * dirty pages that lie within the byte offsets <start, end> + * @mapping: address space structure to write + * @start: offset in bytes where the range starts + * @end : offset in bytes where the range ends + * + * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as + * opposed to a regular memory * cleansing writeback. The difference between + * these two operations is that if a dirty page/buffer is encountered, it must + * be waited upon, and not just skipped over. + */ +static int __filemap_fdatawrite_range(struct address_space *mapping, + loff_t start, loff_t end, int sync_mode) +{ + int ret; + struct writeback_control wbc = { + .sync_mode = sync_mode, + .nr_to_write = mapping->nrpages * 2, + .start = start, + .end = end, + }; + + if (!mapping_cap_writeback_dirty(mapping)) + return 0; + + ret = do_writepages(mapping, &wbc); + return ret; +} + +static inline int __filemap_fdatawrite(struct address_space *mapping, + int sync_mode) +{ + return __filemap_fdatawrite_range(mapping, 0, 0, sync_mode); +} + +int filemap_fdatawrite(struct address_space *mapping) +{ + return __filemap_fdatawrite(mapping, WB_SYNC_ALL); +} +EXPORT_SYMBOL(filemap_fdatawrite); + +static int filemap_fdatawrite_range(struct address_space *mapping, + loff_t start, loff_t end) +{ + return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); +} + +/* + * This is a mostly non-blocking flush. Not suitable for data-integrity + * purposes - I/O may not be started against all dirty pages. + */ +int filemap_flush(struct address_space *mapping) +{ + return __filemap_fdatawrite(mapping, WB_SYNC_NONE); +} +EXPORT_SYMBOL(filemap_flush); + +/* + * Wait for writeback to complete against pages indexed by start->end + * inclusive + */ +static int wait_on_page_writeback_range(struct address_space *mapping, + pgoff_t start, pgoff_t end) +{ + struct pagevec pvec; + int nr_pages; + int ret = 0; + pgoff_t index; + + if (end < start) + return 0; + + pagevec_init(&pvec, 0); + index = start; + while ((index <= end) && + (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, + PAGECACHE_TAG_WRITEBACK, + min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { + unsigned i; + + for (i = 0; i < nr_pages; i++) { + struct page *page = pvec.pages[i]; + + /* until radix tree lookup accepts end_index */ + if (page->index > end) + continue; + + wait_on_page_writeback(page); + if (PageError(page)) + ret = -EIO; + } + pagevec_release(&pvec); + cond_resched(); + } + + /* Check for outstanding write errors */ + if (test_and_clear_bit(AS_ENOSPC, &mapping->flags)) + ret = -ENOSPC; + if (test_and_clear_bit(AS_EIO, &mapping->flags)) + ret = -EIO; + + return ret; +} + +/* + * Write and wait upon all the pages in the passed range. This is a "data + * integrity" operation. It waits upon in-flight writeout before starting and + * waiting upon new writeout. If there was an IO error, return it. + * + * We need to re-take i_sem during the generic_osync_inode list walk because + * it is otherwise livelockable. + */ +int sync_page_range(struct inode *inode, struct address_space *mapping, + loff_t pos, size_t count) +{ + pgoff_t start = pos >> PAGE_CACHE_SHIFT; + pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; + int ret; + + if (!mapping_cap_writeback_dirty(mapping) || !count) + return 0; + ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); + if (ret == 0) { + down(&inode->i_sem); + ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); + up(&inode->i_sem); + } + if (ret == 0) + ret = wait_on_page_writeback_range(mapping, start, end); + return ret; +} +EXPORT_SYMBOL(sync_page_range); + +/* + * Note: Holding i_sem across sync_page_range_nolock is not a good idea + * as it forces O_SYNC writers to different parts of the same file + * to be serialised right until io completion. + */ +int sync_page_range_nolock(struct inode *inode, struct address_space *mapping, + loff_t pos, size_t count) +{ + pgoff_t start = pos >> PAGE_CACHE_SHIFT; + pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; + int ret; + + if (!mapping_cap_writeback_dirty(mapping) || !count) + return 0; + ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); + if (ret == 0) + ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); + if (ret == 0) + ret = wait_on_page_writeback_range(mapping, start, end); + return ret; +} +EXPORT_SYMBOL(sync_page_range_nolock); + +/** + * filemap_fdatawait - walk the list of under-writeback pages of the given + * address space and wait for all of them. + * + * @mapping: address space structure to wait for + */ +int filemap_fdatawait(struct address_space *mapping) +{ + loff_t i_size = i_size_read(mapping->host); + + if (i_size == 0) + return 0; + + return wait_on_page_writeback_range(mapping, 0, + (i_size - 1) >> PAGE_CACHE_SHIFT); +} +EXPORT_SYMBOL(filemap_fdatawait); + +int filemap_write_and_wait(struct address_space *mapping) +{ + int retval = 0; + + if (mapping->nrpages) { + retval = filemap_fdatawrite(mapping); + if (retval == 0) + retval = filemap_fdatawait(mapping); + } + return retval; +} + +int filemap_write_and_wait_range(struct address_space *mapping, + loff_t lstart, loff_t lend) +{ + int retval = 0; + + if (mapping->nrpages) { + retval = __filemap_fdatawrite_range(mapping, lstart, lend, + WB_SYNC_ALL); + if (retval == 0) + retval = wait_on_page_writeback_range(mapping, + lstart >> PAGE_CACHE_SHIFT, + lend >> PAGE_CACHE_SHIFT); + } + return retval; +} + +/* + * This function is used to add newly allocated pagecache pages: + * the page is new, so we can just run SetPageLocked() against it. + * The other page state flags were set by rmqueue(). + * + * This function does not add the page to the LRU. The caller must do that. + */ +int add_to_page_cache(struct page *page, struct address_space *mapping, + pgoff_t offset, int gfp_mask) +{ + int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); + + if (error == 0) { + write_lock_irq(&mapping->tree_lock); + error = radix_tree_insert(&mapping->page_tree, offset, page); + if (!error) { + page_cache_get(page); + SetPageLocked(page); + page->mapping = mapping; + page->index = offset; + mapping->nrpages++; + pagecache_acct(1); + } + write_unlock_irq(&mapping->tree_lock); + radix_tree_preload_end(); + } + return error; +} + +EXPORT_SYMBOL(add_to_page_cache); + +int add_to_page_cache_lru(struct page *page, struct address_space *mapping, + pgoff_t offset, int gfp_mask) +{ + int ret = add_to_page_cache(page, mapping, offset, gfp_mask); + if (ret == 0) + lru_cache_add(page); + return ret; +} + +/* + * In order to wait for pages to become available there must be + * waitqueues associated with pages. By using a hash table of + * waitqueues where the bucket discipline is to maintain all + * waiters on the same queue and wake all when any of the pages + * become available, and for the woken contexts to check to be + * sure the appropriate page became available, this saves space + * at a cost of "thundering herd" phenomena during rare hash + * collisions. + */ +static wait_queue_head_t *page_waitqueue(struct page *page) +{ + const struct zone *zone = page_zone(page); + + return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; +} + +static inline void wake_up_page(struct page *page, int bit) +{ + __wake_up_bit(page_waitqueue(page), &page->flags, bit); +} + +void fastcall wait_on_page_bit(struct page *page, int bit_nr) +{ + DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); + + if (test_bit(bit_nr, &page->flags)) + __wait_on_bit(page_waitqueue(page), &wait, sync_page, + TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(wait_on_page_bit); + +/** + * unlock_page() - unlock a locked page + * + * @page: the page + * + * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). + * Also wakes sleepers in wait_on_page_writeback() because the wakeup + * mechananism between PageLocked pages and PageWriteback pages is shared. + * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. + * + * The first mb is necessary to safely close the critical section opened by the + * TestSetPageLocked(), the second mb is necessary to enforce ordering between + * the clear_bit and the read of the waitqueue (to avoid SMP races with a + * parallel wait_on_page_locked()). + */ +void fastcall unlock_page(struct page *page) +{ + smp_mb__before_clear_bit(); + if (!TestClearPageLocked(page)) + BUG(); + smp_mb__after_clear_bit(); + wake_up_page(page, PG_locked); +} +EXPORT_SYMBOL(unlock_page); + +/* + * End writeback against a page. + */ +void end_page_writeback(struct page *page) +{ + if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) { + if (!test_clear_page_writeback(page)) + BUG(); + } + smp_mb__after_clear_bit(); + wake_up_page(page, PG_writeback); +} +EXPORT_SYMBOL(end_page_writeback); + +/* + * Get a lock on the page, assuming we need to sleep to get it. + * + * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some + * random driver's requestfn sets TASK_RUNNING, we could busywait. However + * chances are that on the second loop, the block layer's plug list is empty, + * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. + */ +void fastcall __lock_page(struct page *page) +{ + DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); + + __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, + TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(__lock_page); + +/* + * a rather lightweight function, finding and getting a reference to a + * hashed page atomically. + */ +struct page * find_get_page(struct address_space *mapping, unsigned long offset) +{ + struct page *page; + + read_lock_irq(&mapping->tree_lock); + page = radix_tree_lookup(&mapping->page_tree, offset); + if (page) + page_cache_get(page); + read_unlock_irq(&mapping->tree_lock); + return page; +} + +EXPORT_SYMBOL(find_get_page); + +/* + * Same as above, but trylock it instead of incrementing the count. + */ +struct page *find_trylock_page(struct address_space *mapping, unsigned long offset) +{ + struct page *page; + + read_lock_irq(&mapping->tree_lock); + page = radix_tree_lookup(&mapping->page_tree, offset); + if (page && TestSetPageLocked(page)) + page = NULL; + read_unlock_irq(&mapping->tree_lock); + return page; +} + +EXPORT_SYMBOL(find_trylock_page); + +/** + * find_lock_page - locate, pin and lock a pagecache page + * + * @mapping - the address_space to search + * @offset - the page index + * + * Locates the desired pagecache page, locks it, increments its reference + * count and returns its address. + * + * Returns zero if the page was not present. find_lock_page() may sleep. + */ +struct page *find_lock_page(struct address_space *mapping, + unsigned long offset) +{ + struct page *page; + + read_lock_irq(&mapping->tree_lock); +repeat: + page = radix_tree_lookup(&mapping->page_tree, offset); + if (page) { + page_cache_get(page); + if (TestSetPageLocked(page)) { + read_unlock_irq(&mapping->tree_lock); + lock_page(page); + read_lock_irq(&mapping->tree_lock); + + /* Has the page been truncated while we slept? */ + if (page->mapping != mapping || page->index != offset) { + unlock_page(page); + page_cache_release(page); + goto repeat; + } + } + } + read_unlock_irq(&mapping->tree_lock); + return page; +} + +EXPORT_SYMBOL(find_lock_page); + +/** + * find_or_create_page - locate or add a pagecache page + * + * @mapping - the page's address_space + * @index - the page's index into the mapping + * @gfp_mask - page allocation mode + * + * Locates a page in the pagecache. If the page is not present, a new page + * is allocated using @gfp_mask and is added to the pagecache and to the VM's + * LRU list. The returned page is locked and has its reference count + * incremented. + * + * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic + * allocation! + * + * find_or_create_page() returns the desired page's address, or zero on + * memory exhaustion. + */ +struct page *find_or_create_page(struct address_space *mapping, + unsigned long index, unsigned int gfp_mask) +{ + struct page *page, *cached_page = NULL; + int err; +repeat: + page = find_lock_page(mapping, index); + if (!page) { + if (!cached_page) { + cached_page = alloc_page(gfp_mask); + if (!cached_page) + return NULL; + } + err = add_to_page_cache_lru(cached_page, mapping, + index, gfp_mask); + if (!err) { + page = cached_page; + cached_page = NULL; + } else if (err == -EEXIST) + goto repeat; + } + if (cached_page) + page_cache_release(cached_page); + return page; +} + +EXPORT_SYMBOL(find_or_create_page); + +/** + * find_get_pages - gang pagecache lookup + * @mapping: The address_space to search + * @start: The starting page index + * @nr_pages: The maximum number of pages + * @pages: Where the resulting pages are placed + * + * find_get_pages() will search for and return a group of up to + * @nr_pages pages in the mapping. The pages are placed at @pages. + * find_get_pages() takes a reference against the returned pages. + * + * The search returns a group of mapping-contiguous pages with ascending + * indexes. There may be holes in the indices due to not-present pages. + * + * find_get_pages() returns the number of pages which were found. + */ +unsigned find_get_pages(struct address_space *mapping, pgoff_t start, + unsigned int nr_pages, struct page **pages) +{ + unsigned int i; + unsigned int ret; + + read_lock_irq(&mapping->tree_lock); + ret = radix_tree_gang_lookup(&mapping->page_tree, + (void **)pages, start, nr_pages); + for (i = 0; i < ret; i++) + page_cache_get(pages[i]); + read_unlock_irq(&mapping->tree_lock); + return ret; +} + +/* + * Like find_get_pages, except we only return pages which are tagged with + * `tag'. We update *index to index the next page for the traversal. + */ +unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, + int tag, unsigned int nr_pages, struct page **pages) +{ + unsigned int i; + unsigned int ret; + + read_lock_irq(&mapping->tree_lock); + ret = radix_tree_gang_lookup_tag(&mapping->page_tree, + (void **)pages, *index, nr_pages, tag); + for (i = 0; i < ret; i++) + page_cache_get(pages[i]); + if (ret) + *index = pages[ret - 1]->index + 1; + read_unlock_irq(&mapping->tree_lock); + return ret; +} + +/* + * Same as grab_cache_page, but do not wait if the page is unavailable. + * This is intended for speculative data generators, where the data can + * be regenerated if the page couldn't be grabbed. This routine should + * be safe to call while holding the lock for another page. + * + * Clear __GFP_FS when allocating the page to avoid recursion into the fs + * and deadlock against the caller's locked page. + */ +struct page * +grab_cache_page_nowait(struct address_space *mapping, unsigned long index) +{ + struct page *page = find_get_page(mapping, index); + unsigned int gfp_mask; + + if (page) { + if (!TestSetPageLocked(page)) + return page; + page_cache_release(page); + return NULL; + } + gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS; + page = alloc_pages(gfp_mask, 0); + if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) { + page_cache_release(page); + page = NULL; + } + return page; +} + +EXPORT_SYMBOL(grab_cache_page_nowait); + +/* + * This is a generic file read routine, and uses the + * mapping->a_ops->readpage() function for the actual low-level + * stuff. + * + * This is really ugly. But the goto's actually try to clarify some + * of the logic when it comes to error handling etc. + * + * Note the struct file* is only passed for the use of readpage. It may be + * NULL. + */ +void do_generic_mapping_read(struct address_space *mapping, + struct file_ra_state *_ra, + struct file *filp, + loff_t *ppos, + read_descriptor_t *desc, + read_actor_t actor) +{ + struct inode *inode = mapping->host; + unsigned long index; + unsigned long end_index; + unsigned long offset; + unsigned long last_index; + unsigned long next_index; + unsigned long prev_index; + loff_t isize; + struct page *cached_page; + int error; + struct file_ra_state ra = *_ra; + + cached_page = NULL; + index = *ppos >> PAGE_CACHE_SHIFT; + next_index = index; + prev_index = ra.prev_page; + last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; + offset = *ppos & ~PAGE_CACHE_MASK; + + isize = i_size_read(inode); + if (!isize) + goto out; + + end_index = (isize - 1) >> PAGE_CACHE_SHIFT; + for (;;) { + struct page *page; + unsigned long nr, ret; + + /* nr is the maximum number of bytes to copy from this page */ + nr = PAGE_CACHE_SIZE; + if (index >= end_index) { + if (index > end_index) + goto out; + nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; + if (nr <= offset) { + goto out; + } + } + nr = nr - offset; + + cond_resched(); + if (index == next_index) + next_index = page_cache_readahead(mapping, &ra, filp, + index, last_index - index); + +find_page: + page = find_get_page(mapping, index); + if (unlikely(page == NULL)) { + handle_ra_miss(mapping, &ra, index); + goto no_cached_page; + } + if (!PageUptodate(page)) + goto page_not_up_to_date; +page_ok: + + /* If users can be writing to this page using arbitrary + * virtual addresses, take care about potential aliasing + * before reading the page on the kernel side. + */ + if (mapping_writably_mapped(mapping)) + flush_dcache_page(page); + + /* + * When (part of) the same page is read multiple times + * in succession, only mark it as accessed the first time. + */ + if (prev_index != index) + mark_page_accessed(page); + prev_index = index; + + /* + * Ok, we have the page, and it's up-to-date, so + * now we can copy it to user space... + * + * The actor routine returns how many bytes were actually used.. + * NOTE! This may not be the same as how much of a user buffer + * we filled up (we may be padding etc), so we can only update + * "pos" here (the actor routine has to update the user buffer + * pointers and the remaining count). + */ + ret = actor(desc, page, offset, nr); + offset += ret; + index += offset >> PAGE_CACHE_SHIFT; + offset &= ~PAGE_CACHE_MASK; + + page_cache_release(page); + if (ret == nr && desc->count) + continue; + goto out; + +page_not_up_to_date: + /* Get exclusive access to the page ... */ + lock_page(page); + + /* Did it get unhashed before we got the lock? */ + if (!page->mapping) { + unlock_page(page); + page_cache_release(page); + continue; + } + + /* Did somebody else fill it already? */ + if (PageUptodate(page)) { + unlock_page(page); + goto page_ok; + } + +readpage: + /* Start the actual read. The read will unlock the page. */ + error = mapping->a_ops->readpage(filp, page); + + if (unlikely(error)) + goto readpage_error; + + if (!PageUptodate(page)) { + lock_page(page); + if (!PageUptodate(page)) { + if (page->mapping == NULL) { + /* + * invalidate_inode_pages got it + */ + unlock_page(page); + page_cache_release(page); + goto find_page; + } + unlock_page(page); + error = -EIO; + goto readpage_error; + } + unlock_page(page); + } + + /* + * i_size must be checked after we have done ->readpage. + * + * Checking i_size after the readpage allows us to calculate + * the correct value for "nr", which means the zero-filled + * part of the page is not copied back to userspace (unless + * another truncate extends the file - this is desired though). + */ + isize = i_size_read(inode); + end_index = (isize - 1) >> PAGE_CACHE_SHIFT; + if (unlikely(!isize || index > end_index)) { + page_cache_release(page); + goto out; + } + + /* nr is the maximum number of bytes to copy from this page */ + nr = PAGE_CACHE_SIZE; + if (index == end_index) { + nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; + if (nr <= offset) { + page_cache_release(page); + goto out; + } + } + nr = nr - offset; + goto page_ok; + +readpage_error: + /* UHHUH! A synchronous read error occurred. Report it */ + desc->error = error; + page_cache_release(page); + goto out; + +no_cached_page: + /* + * Ok, it wasn't cached, so we need to create a new + * page.. + */ + if (!cached_page) { + cached_page = page_cache_alloc_cold(mapping); + if (!cached_page) { + desc->error = -ENOMEM; + goto out; + } + } + error = add_to_page_cache_lru(cached_page, mapping, + index, GFP_KERNEL); + if (error) { + if (error == -EEXIST) + goto find_page; + desc->error = error; + goto out; + } + page = cached_page; + cached_page = NULL; + goto readpage; + } + +out: + *_ra = ra; + + *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; + if (cached_page) + page_cache_release(cached_page); + if (filp) + file_accessed(filp); +} + +EXPORT_SYMBOL(do_generic_mapping_read); + +int file_read_actor(read_descriptor_t *desc, struct page *page, + unsigned long offset, unsigned long size) +{ + char *kaddr; + unsigned long left, count = desc->count; + + if (size > count) + size = count; + + /* + * Faults on the destination of a read are common, so do it before + * taking the kmap. + */ + if (!fault_in_pages_writeable(desc->arg.buf, size)) { + kaddr = kmap_atomic(page, KM_USER0); + left = __copy_to_user_inatomic(desc->arg.buf, + kaddr + offset, size); + kunmap_atomic(kaddr, KM_USER0); + if (left == 0) + goto success; + } + + /* Do it the slow way */ + kaddr = kmap(page); + left = __copy_to_user(desc->arg.buf, kaddr + offset, size); + kunmap(page); + + if (left) { + size -= left; + desc->error = -EFAULT; + } +success: + desc->count = count - size; + desc->written += size; + desc->arg.buf += size; + return size; +} + +/* + * This is the "read()" routine for all filesystems + * that can use the page cache directly. + */ +ssize_t +__generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct file *filp = iocb->ki_filp; + ssize_t retval; + unsigned long seg; + size_t count; + + count = 0; + for (seg = 0; seg < nr_segs; seg++) { + const struct iovec *iv = &iov[seg]; + + /* + * If any segment has a negative length, or the cumulative + * length ever wraps negative then return -EINVAL. + */ + count += iv->iov_len; + if (unlikely((ssize_t)(count|iv->iov_len) < 0)) + return -EINVAL; + if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len)) + continue; + if (seg == 0) + return -EFAULT; + nr_segs = seg; + count -= iv->iov_len; /* This segment is no good */ + break; + } + + /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ + if (filp->f_flags & O_DIRECT) { + loff_t pos = *ppos, size; + struct address_space *mapping; + struct inode *inode; + + mapping = filp->f_mapping; + inode = mapping->host; + retval = 0; + if (!count) + goto out; /* skip atime */ + size = i_size_read(inode); + if (pos < size) { + retval = generic_file_direct_IO(READ, iocb, + iov, pos, nr_segs); + if (retval >= 0 && !is_sync_kiocb(iocb)) + retval = -EIOCBQUEUED; + if (retval > 0) + *ppos = pos + retval; + } + file_accessed(filp); + goto out; + } + + retval = 0; + if (count) { + for (seg = 0; seg < nr_segs; seg++) { + read_descriptor_t desc; + + desc.written = 0; + desc.arg.buf = iov[seg].iov_base; + desc.count = iov[seg].iov_len; + if (desc.count == 0) + continue; + desc.error = 0; + do_generic_file_read(filp,ppos,&desc,file_read_actor); + retval += desc.written; + if (!retval) { + retval = desc.error; + break; + } + } + } +out: + return retval; +} + +EXPORT_SYMBOL(__generic_file_aio_read); + +ssize_t +generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos) +{ + struct iovec local_iov = { .iov_base = buf, .iov_len = count }; + + BUG_ON(iocb->ki_pos != pos); + return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos); +} + +EXPORT_SYMBOL(generic_file_aio_read); + +ssize_t +generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) +{ + struct iovec local_iov = { .iov_base = buf, .iov_len = count }; + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos); + if (-EIOCBQUEUED == ret) + ret = wait_on_sync_kiocb(&kiocb); + return ret; +} + +EXPORT_SYMBOL(generic_file_read); + +int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size) +{ + ssize_t written; + unsigned long count = desc->count; + struct file *file = desc->arg.data; + + if (size > count) + size = count; + + written = file->f_op->sendpage(file, page, offset, + size, &file->f_pos, size<count); + if (written < 0) { + desc->error = written; + written = 0; + } + desc->count = count - written; + desc->written += written; + return written; +} + +ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos, + size_t count, read_actor_t actor, void *target) +{ + read_descriptor_t desc; + + if (!count) + return 0; + + desc.written = 0; + desc.count = count; + desc.arg.data = target; + desc.error = 0; + + do_generic_file_read(in_file, ppos, &desc, actor); + if (desc.written) + return desc.written; + return desc.error; +} + +EXPORT_SYMBOL(generic_file_sendfile); + +static ssize_t +do_readahead(struct address_space *mapping, struct file *filp, + unsigned long index, unsigned long nr) +{ + if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage) + return -EINVAL; + + force_page_cache_readahead(mapping, filp, index, + max_sane_readahead(nr)); + return 0; +} + +asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count) +{ + ssize_t ret; + struct file *file; + + ret = -EBADF; + file = fget(fd); + if (file) { + if (file->f_mode & FMODE_READ) { + struct address_space *mapping = file->f_mapping; + unsigned long start = offset >> PAGE_CACHE_SHIFT; + unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT; + unsigned long len = end - start + 1; + ret = do_readahead(mapping, file, start, len); + } + fput(file); + } + return ret; +} + +#ifdef CONFIG_MMU +/* + * This adds the requested page to the page cache if it isn't already there, + * and schedules an I/O to read in its contents from disk. + */ +static int FASTCALL(page_cache_read(struct file * file, unsigned long offset)); +static int fastcall page_cache_read(struct file * file, unsigned long offset) +{ + struct address_space *mapping = file->f_mapping; + struct page *page; + int error; + + page = page_cache_alloc_cold(mapping); + if (!page) + return -ENOMEM; + + error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL); + if (!error) { + error = mapping->a_ops->readpage(file, page); + page_cache_release(page); + return error; + } + + /* + * We arrive here in the unlikely event that someone + * raced with us and added our page to the cache first + * or we are out of memory for radix-tree nodes. + */ + page_cache_release(page); + return error == -EEXIST ? 0 : error; +} + +#define MMAP_LOTSAMISS (100) + +/* + * filemap_nopage() is invoked via the vma operations vector for a + * mapped memory region to read in file data during a page fault. + * + * The goto's are kind of ugly, but this streamlines the normal case of having + * it in the page cache, and handles the special cases reasonably without + * having a lot of duplicated code. + */ +struct page *filemap_nopage(struct vm_area_struct *area, + unsigned long address, int *type) +{ + int error; + struct file *file = area->vm_file; + struct address_space *mapping = file->f_mapping; + struct file_ra_state *ra = &file->f_ra; + struct inode *inode = mapping->host; + struct page *page; + unsigned long size, pgoff; + int did_readaround = 0, majmin = VM_FAULT_MINOR; + + pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff; + +retry_all: + size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; + if (pgoff >= size) + goto outside_data_content; + + /* If we don't want any read-ahead, don't bother */ + if (VM_RandomReadHint(area)) + goto no_cached_page; + + /* + * The readahead code wants to be told about each and every page + * so it can build and shrink its windows appropriately + * + * For sequential accesses, we use the generic readahead logic. + */ + if (VM_SequentialReadHint(area)) + page_cache_readahead(mapping, ra, file, pgoff, 1); + + /* + * Do we have something in the page cache already? + */ +retry_find: + page = find_get_page(mapping, pgoff); + if (!page) { + unsigned long ra_pages; + + if (VM_SequentialReadHint(area)) { + handle_ra_miss(mapping, ra, pgoff); + goto no_cached_page; + } + ra->mmap_miss++; + + /* + * Do we miss much more than hit in this file? If so, + * stop bothering with read-ahead. It will only hurt. + */ + if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS) + goto no_cached_page; + + /* + * To keep the pgmajfault counter straight, we need to + * check did_readaround, as this is an inner loop. + */ + if (!did_readaround) { + majmin = VM_FAULT_MAJOR; + inc_page_state(pgmajfault); + } + did_readaround = 1; + ra_pages = max_sane_readahead(file->f_ra.ra_pages); + if (ra_pages) { + pgoff_t start = 0; + + if (pgoff > ra_pages / 2) + start = pgoff - ra_pages / 2; + do_page_cache_readahead(mapping, file, start, ra_pages); + } + page = find_get_page(mapping, pgoff); + if (!page) + goto no_cached_page; + } + + if (!did_readaround) + ra->mmap_hit++; + + /* + * Ok, found a page in the page cache, now we need to check + * that it's up-to-date. + */ + if (!PageUptodate(page)) + goto page_not_uptodate; + +success: + /* + * Found the page and have a reference on it. + */ + mark_page_accessed(page); + if (type) + *type = majmin; + return page; + +outside_data_content: + /* + * An external ptracer can access pages that normally aren't + * accessible.. + */ + if (area->vm_mm == current->mm) + return NULL; + /* Fall through to the non-read-ahead case */ +no_cached_page: + /* + * We're only likely to ever get here if MADV_RANDOM is in + * effect. + */ + error = page_cache_read(file, pgoff); + grab_swap_token(); + + /* + * The page we want has now been added to the page cache. + * In the unlikely event that someone removed it in the + * meantime, we'll just come back here and read it again. + */ + if (error >= 0) + goto retry_find; + + /* + * An error return from page_cache_read can result if the + * system is low on memory, or a problem occurs while trying + * to schedule I/O. + */ + if (error == -ENOMEM) + return NOPAGE_OOM; + return NULL; + +page_not_uptodate: + if (!did_readaround) { + majmin = VM_FAULT_MAJOR; + inc_page_state(pgmajfault); + } + lock_page(page); + + /* Did it get unhashed while we waited for it? */ + if (!page->mapping) { + unlock_page(page); + page_cache_release(page); + goto retry_all; + } + + /* Did somebody else get it up-to-date? */ + if (PageUptodate(page)) { + unlock_page(page); + goto success; + } + + if (!mapping->a_ops->readpage(file, page)) { + wait_on_page_locked(page); + if (PageUptodate(page)) + goto success; + } + + /* + * Umm, take care of errors if the page isn't up-to-date. + * Try to re-read it _once_. We do this synchronously, + * because there really aren't any performance issues here + * and we need to check for errors. + */ + lock_page(page); + + /* Somebody truncated the page on us? */ + if (!page->mapping) { + unlock_page(page); + page_cache_release(page); + goto retry_all; + } + + /* Somebody else successfully read it in? */ + if (PageUptodate(page)) { + unlock_page(page); + goto success; + } + ClearPageError(page); + if (!mapping->a_ops->readpage(file, page)) { + wait_on_page_locked(page); + if (PageUptodate(page)) + goto success; + } + + /* + * Things didn't work out. Return zero to tell the + * mm layer so, possibly freeing the page cache page first. + */ + page_cache_release(page); + return NULL; +} + +EXPORT_SYMBOL(filemap_nopage); + +static struct page * filemap_getpage(struct file *file, unsigned long pgoff, + int nonblock) +{ + struct address_space *mapping = file->f_mapping; + struct page *page; + int error; + + /* + * Do we have something in the page cache already? + */ +retry_find: + page = find_get_page(mapping, pgoff); + if (!page) { + if (nonblock) + return NULL; + goto no_cached_page; + } + + /* + * Ok, found a page in the page cache, now we need to check + * that it's up-to-date. + */ + if (!PageUptodate(page)) + goto page_not_uptodate; + +success: + /* + * Found the page and have a reference on it. + */ + mark_page_accessed(page); + return page; + +no_cached_page: + error = page_cache_read(file, pgoff); + + /* + * The page we want has now been added to the page cache. + * In the unlikely event that someone removed it in the + * meantime, we'll just come back here and read it again. + */ + if (error >= 0) + goto retry_find; + + /* + * An error return from page_cache_read can result if the + * system is low on memory, or a problem occurs while trying + * to schedule I/O. + */ + return NULL; + +page_not_uptodate: + lock_page(page); + + /* Did it get unhashed while we waited for it? */ + if (!page->mapping) { + unlock_page(page); + goto err; + } + + /* Did somebody else get it up-to-date? */ + if (PageUptodate(page)) { + unlock_page(page); + goto success; + } + + if (!mapping->a_ops->readpage(file, page)) { + wait_on_page_locked(page); + if (PageUptodate(page)) + goto success; + } + + /* + * Umm, take care of errors if the page isn't up-to-date. + * Try to re-read it _once_. We do this synchronously, + * because there really aren't any performance issues here + * and we need to check for errors. + */ + lock_page(page); + + /* Somebody truncated the page on us? */ + if (!page->mapping) { + unlock_page(page); + goto err; + } + /* Somebody else successfully read it in? */ + if (PageUptodate(page)) { + unlock_page(page); + goto success; + } + + ClearPageError(page); + if (!mapping->a_ops->readpage(file, page)) { + wait_on_page_locked(page); + if (PageUptodate(page)) + goto success; + } + + /* + * Things didn't work out. Return zero to tell the + * mm layer so, possibly freeing the page cache page first. + */ +err: + page_cache_release(page); + + return NULL; +} + +int filemap_populate(struct vm_area_struct *vma, unsigned long addr, + unsigned long len, pgprot_t prot, unsigned long pgoff, + int nonblock) +{ + struct file *file = vma->vm_file; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + unsigned long size; + struct mm_struct *mm = vma->vm_mm; + struct page *page; + int err; + + if (!nonblock) + force_page_cache_readahead(mapping, vma->vm_file, + pgoff, len >> PAGE_CACHE_SHIFT); + +repeat: + size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; + if (pgoff + (len >> PAGE_CACHE_SHIFT) > size) + return -EINVAL; + + page = filemap_getpage(file, pgoff, nonblock); + if (!page && !nonblock) + return -ENOMEM; + if (page) { + err = install_page(mm, vma, addr, page, prot); + if (err) { + page_cache_release(page); + return err; + } + } else { + err = install_file_pte(mm, vma, addr, pgoff, prot); + if (err) + return err; + } + + len -= PAGE_SIZE; + addr += PAGE_SIZE; + pgoff++; + if (len) + goto repeat; + + return 0; +} + +struct vm_operations_struct generic_file_vm_ops = { + .nopage = filemap_nopage, + .populate = filemap_populate, +}; + +/* This is used for a general mmap of a disk file */ + +int generic_file_mmap(struct file * file, struct vm_area_struct * vma) +{ + struct address_space *mapping = file->f_mapping; + + if (!mapping->a_ops->readpage) + return -ENOEXEC; + file_accessed(file); + vma->vm_ops = &generic_file_vm_ops; + return 0; +} +EXPORT_SYMBOL(filemap_populate); + +/* + * This is for filesystems which do not implement ->writepage. + */ +int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) +{ + if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + return -EINVAL; + return generic_file_mmap(file, vma); +} +#else +int generic_file_mmap(struct file * file, struct vm_area_struct * vma) +{ + return -ENOSYS; +} +int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) +{ + return -ENOSYS; +} +#endif /* CONFIG_MMU */ + +EXPORT_SYMBOL(generic_file_mmap); +EXPORT_SYMBOL(generic_file_readonly_mmap); + +static inline struct page *__read_cache_page(struct address_space *mapping, + unsigned long index, + int (*filler)(void *,struct page*), + void *data) +{ + struct page *page, *cached_page = NULL; + int err; +repeat: + page = find_get_page(mapping, index); + if (!page) { + if (!cached_page) { + cached_page = page_cache_alloc_cold(mapping); + if (!cached_page) + return ERR_PTR(-ENOMEM); + } + err = add_to_page_cache_lru(cached_page, mapping, + index, GFP_KERNEL); + if (err == -EEXIST) + goto repeat; + if (err < 0) { + /* Presumably ENOMEM for radix tree node */ + page_cache_release(cached_page); + return ERR_PTR(err); + } + page = cached_page; + cached_page = NULL; + err = filler(data, page); + if (err < 0) { + page_cache_release(page); + page = ERR_PTR(err); + } + } + if (cached_page) + page_cache_release(cached_page); + return page; +} + +/* + * Read into the page cache. If a page already exists, + * and PageUptodate() is not set, try to fill the page. + */ +struct page *read_cache_page(struct address_space *mapping, + unsigned long index, + int (*filler)(void *,struct page*), + void *data) +{ + struct page *page; + int err; + +retry: + page = __read_cache_page(mapping, index, filler, data); + if (IS_ERR(page)) + goto out; + mark_page_accessed(page); + if (PageUptodate(page)) + goto out; + + lock_page(page); + if (!page->mapping) { + unlock_page(page); + page_cache_release(page); + goto retry; + } + if (PageUptodate(page)) { + unlock_page(page); + goto out; + } + err = filler(data, page); + if (err < 0) { + page_cache_release(page); + page = ERR_PTR(err); + } + out: + return page; +} + +EXPORT_SYMBOL(read_cache_page); + +/* + * If the page was newly created, increment its refcount and add it to the + * caller's lru-buffering pagevec. This function is specifically for + * generic_file_write(). + */ +static inline struct page * +__grab_cache_page(struct address_space *mapping, unsigned long index, + struct page **cached_page, struct pagevec *lru_pvec) +{ + int err; + struct page *page; +repeat: + page = find_lock_page(mapping, index); + if (!page) { + if (!*cached_page) { + *cached_page = page_cache_alloc(mapping); + if (!*cached_page) + return NULL; + } + err = add_to_page_cache(*cached_page, mapping, + index, GFP_KERNEL); + if (err == -EEXIST) + goto repeat; + if (err == 0) { + page = *cached_page; + page_cache_get(page); + if (!pagevec_add(lru_pvec, page)) + __pagevec_lru_add(lru_pvec); + *cached_page = NULL; + } + } + return page; +} + +/* + * The logic we want is + * + * if suid or (sgid and xgrp) + * remove privs + */ +int remove_suid(struct dentry *dentry) +{ + mode_t mode = dentry->d_inode->i_mode; + int kill = 0; + int result = 0; + + /* suid always must be killed */ + if (unlikely(mode & S_ISUID)) + kill = ATTR_KILL_SUID; + + /* + * sgid without any exec bits is just a mandatory locking mark; leave + * it alone. If some exec bits are set, it's a real sgid; kill it. + */ + if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) + kill |= ATTR_KILL_SGID; + + if (unlikely(kill && !capable(CAP_FSETID))) { + struct iattr newattrs; + + newattrs.ia_valid = ATTR_FORCE | kill; + result = notify_change(dentry, &newattrs); + } + return result; +} +EXPORT_SYMBOL(remove_suid); + +/* + * Copy as much as we can into the page and return the number of bytes which + * were sucessfully copied. If a fault is encountered then clear the page + * out to (offset+bytes) and return the number of bytes which were copied. + */ +static inline size_t +filemap_copy_from_user(struct page *page, unsigned long offset, + const char __user *buf, unsigned bytes) +{ + char *kaddr; + int left; + + kaddr = kmap_atomic(page, KM_USER0); + left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); + kunmap_atomic(kaddr, KM_USER0); + + if (left != 0) { + /* Do it the slow way */ + kaddr = kmap(page); + left = __copy_from_user(kaddr + offset, buf, bytes); + kunmap(page); + } + return bytes - left; +} + +static size_t +__filemap_copy_from_user_iovec(char *vaddr, + const struct iovec *iov, size_t base, size_t bytes) +{ + size_t copied = 0, left = 0; + + while (bytes) { + char __user *buf = iov->iov_base + base; + int copy = min(bytes, iov->iov_len - base); + + base = 0; + left = __copy_from_user_inatomic(vaddr, buf, copy); + copied += copy; + bytes -= copy; + vaddr += copy; + iov++; + + if (unlikely(left)) { + /* zero the rest of the target like __copy_from_user */ + if (bytes) + memset(vaddr, 0, bytes); + break; + } + } + return copied - left; +} + +/* + * This has the same sideeffects and return value as filemap_copy_from_user(). + * The difference is that on a fault we need to memset the remainder of the + * page (out to offset+bytes), to emulate filemap_copy_from_user()'s + * single-segment behaviour. + */ +static inline size_t +filemap_copy_from_user_iovec(struct page *page, unsigned long offset, + const struct iovec *iov, size_t base, size_t bytes) +{ + char *kaddr; + size_t copied; + + kaddr = kmap_atomic(page, KM_USER0); + copied = __filemap_copy_from_user_iovec(kaddr + offset, iov, + base, bytes); + kunmap_atomic(kaddr, KM_USER0); + if (copied != bytes) { + kaddr = kmap(page); + copied = __filemap_copy_from_user_iovec(kaddr + offset, iov, + base, bytes); + kunmap(page); + } + return copied; +} + +static inline void +filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes) +{ + const struct iovec *iov = *iovp; + size_t base = *basep; + + while (bytes) { + int copy = min(bytes, iov->iov_len - base); + + bytes -= copy; + base += copy; + if (iov->iov_len == base) { + iov++; + base = 0; + } + } + *iovp = iov; + *basep = base; +} + +/* + * Performs necessary checks before doing a write + * + * Can adjust writing position aor amount of bytes to write. + * Returns appropriate error code that caller should return or + * zero in case that write should be allowed. + */ +inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk) +{ + struct inode *inode = file->f_mapping->host; + unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; + + if (unlikely(*pos < 0)) + return -EINVAL; + + if (unlikely(file->f_error)) { + int err = file->f_error; + file->f_error = 0; + return err; + } + + if (!isblk) { + /* FIXME: this is for backwards compatibility with 2.4 */ + if (file->f_flags & O_APPEND) + *pos = i_size_read(inode); + + if (limit != RLIM_INFINITY) { + if (*pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + if (*count > limit - (typeof(limit))*pos) { + *count = limit - (typeof(limit))*pos; + } + } + } + + /* + * LFS rule + */ + if (unlikely(*pos + *count > MAX_NON_LFS && + !(file->f_flags & O_LARGEFILE))) { + if (*pos >= MAX_NON_LFS) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + if (*count > MAX_NON_LFS - (unsigned long)*pos) { + *count = MAX_NON_LFS - (unsigned long)*pos; + } + } + + /* + * Are we about to exceed the fs block limit ? + * + * If we have written data it becomes a short write. If we have + * exceeded without writing data we send a signal and return EFBIG. + * Linus frestrict idea will clean these up nicely.. + */ + if (likely(!isblk)) { + if (unlikely(*pos >= inode->i_sb->s_maxbytes)) { + if (*count || *pos > inode->i_sb->s_maxbytes) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + /* zero-length writes at ->s_maxbytes are OK */ + } + + if (unlikely(*pos + *count > inode->i_sb->s_maxbytes)) + *count = inode->i_sb->s_maxbytes - *pos; + } else { + loff_t isize; + if (bdev_read_only(I_BDEV(inode))) + return -EPERM; + isize = i_size_read(inode); + if (*pos >= isize) { + if (*count || *pos > isize) + return -ENOSPC; + } + + if (*pos + *count > isize) + *count = isize - *pos; + } + return 0; +} +EXPORT_SYMBOL(generic_write_checks); + +ssize_t +generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long *nr_segs, loff_t pos, loff_t *ppos, + size_t count, size_t ocount) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t written; + + if (count != ocount) + *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count); + + written = generic_file_direct_IO(WRITE, iocb, iov, pos, *nr_segs); + if (written > 0) { + loff_t end = pos + written; + if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { + i_size_write(inode, end); + mark_inode_dirty(inode); + } + *ppos = end; + } + + /* + * Sync the fs metadata but not the minor inode changes and + * of course not the data as we did direct DMA for the IO. + * i_sem is held, which protects generic_osync_inode() from + * livelocking. + */ + if (written >= 0 && file->f_flags & O_SYNC) + generic_osync_inode(inode, mapping, OSYNC_METADATA); + if (written == count && !is_sync_kiocb(iocb)) + written = -EIOCBQUEUED; + return written; +} +EXPORT_SYMBOL(generic_file_direct_write); + +ssize_t +generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t pos, loff_t *ppos, + size_t count, ssize_t written) +{ + struct file *file = iocb->ki_filp; + struct address_space * mapping = file->f_mapping; + struct address_space_operations *a_ops = mapping->a_ops; + struct inode *inode = mapping->host; + long status = 0; + struct page *page; + struct page *cached_page = NULL; + size_t bytes; + struct pagevec lru_pvec; + const struct iovec *cur_iov = iov; /* current iovec */ + size_t iov_base = 0; /* offset in the current iovec */ + char __user *buf; + + pagevec_init(&lru_pvec, 0); + + /* + * handle partial DIO write. Adjust cur_iov if needed. + */ + if (likely(nr_segs == 1)) + buf = iov->iov_base + written; + else { + filemap_set_next_iovec(&cur_iov, &iov_base, written); + buf = iov->iov_base + iov_base; + } + + do { + unsigned long index; + unsigned long offset; + size_t copied; + + offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ + index = pos >> PAGE_CACHE_SHIFT; + bytes = PAGE_CACHE_SIZE - offset; + if (bytes > count) + bytes = count; + + /* + * Bring in the user page that we will copy from _first_. + * Otherwise there's a nasty deadlock on copying from the + * same page as we're writing to, without it being marked + * up-to-date. + */ + fault_in_pages_readable(buf, bytes); + + page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec); + if (!page) { + status = -ENOMEM; + break; + } + + status = a_ops->prepare_write(file, page, offset, offset+bytes); + if (unlikely(status)) { + loff_t isize = i_size_read(inode); + /* + * prepare_write() may have instantiated a few blocks + * outside i_size. Trim these off again. + */ + unlock_page(page); + page_cache_release(page); + if (pos + bytes > isize) + vmtruncate(inode, isize); + break; + } + if (likely(nr_segs == 1)) + copied = filemap_copy_from_user(page, offset, + buf, bytes); + else + copied = filemap_copy_from_user_iovec(page, offset, + cur_iov, iov_base, bytes); + flush_dcache_page(page); + status = a_ops->commit_write(file, page, offset, offset+bytes); + if (likely(copied > 0)) { + if (!status) + status = copied; + + if (status >= 0) { + written += status; + count -= status; + pos += status; + buf += status; + if (unlikely(nr_segs > 1)) + filemap_set_next_iovec(&cur_iov, + &iov_base, status); + } + } + if (unlikely(copied != bytes)) + if (status >= 0) + status = -EFAULT; + unlock_page(page); + mark_page_accessed(page); + page_cache_release(page); + if (status < 0) + break; + balance_dirty_pages_ratelimited(mapping); + cond_resched(); + } while (count); + *ppos = pos; + + if (cached_page) + page_cache_release(cached_page); + + /* + * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC + */ + if (likely(status >= 0)) { + if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) { + if (!a_ops->writepage || !is_sync_kiocb(iocb)) + status = generic_osync_inode(inode, mapping, + OSYNC_METADATA|OSYNC_DATA); + } + } + + /* + * If we get here for O_DIRECT writes then we must have fallen through + * to buffered writes (block instantiation inside i_size). So we sync + * the file data here, to try to honour O_DIRECT expectations. + */ + if (unlikely(file->f_flags & O_DIRECT) && written) + status = filemap_write_and_wait(mapping); + + pagevec_lru_add(&lru_pvec); + return written ? written : status; +} +EXPORT_SYMBOL(generic_file_buffered_write); + +ssize_t +__generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct file *file = iocb->ki_filp; + struct address_space * mapping = file->f_mapping; + size_t ocount; /* original count */ + size_t count; /* after file limit checks */ + struct inode *inode = mapping->host; + unsigned long seg; + loff_t pos; + ssize_t written; + ssize_t err; + + ocount = 0; + for (seg = 0; seg < nr_segs; seg++) { + const struct iovec *iv = &iov[seg]; + + /* + * If any segment has a negative length, or the cumulative + * length ever wraps negative then return -EINVAL. + */ + ocount += iv->iov_len; + if (unlikely((ssize_t)(ocount|iv->iov_len) < 0)) + return -EINVAL; + if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len)) + continue; + if (seg == 0) + return -EFAULT; + nr_segs = seg; + ocount -= iv->iov_len; /* This segment is no good */ + break; + } + + count = ocount; + pos = *ppos; + + vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); + + /* We can write back this queue in page reclaim */ + current->backing_dev_info = mapping->backing_dev_info; + written = 0; + + err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode)); + if (err) + goto out; + + if (count == 0) + goto out; + + err = remove_suid(file->f_dentry); + if (err) + goto out; + + inode_update_time(inode, 1); + + /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ + if (unlikely(file->f_flags & O_DIRECT)) { + written = generic_file_direct_write(iocb, iov, + &nr_segs, pos, ppos, count, ocount); + if (written < 0 || written == count) + goto out; + /* + * direct-io write to a hole: fall through to buffered I/O + * for completing the rest of the request. + */ + pos += written; + count -= written; + } + + written = generic_file_buffered_write(iocb, iov, nr_segs, + pos, ppos, count, written); +out: + current->backing_dev_info = NULL; + return written ? written : err; +} +EXPORT_SYMBOL(generic_file_aio_write_nolock); + +ssize_t +generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t ret; + loff_t pos = *ppos; + + ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs, ppos); + + if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { + int err; + + err = sync_page_range_nolock(inode, mapping, pos, ret); + if (err < 0) + ret = err; + } + return ret; +} + +ssize_t +__generic_file_write_nolock(struct file *file, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, file); + ret = __generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos); + if (ret == -EIOCBQUEUED) + ret = wait_on_sync_kiocb(&kiocb); + return ret; +} + +ssize_t +generic_file_write_nolock(struct file *file, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, file); + ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos); + if (-EIOCBQUEUED == ret) + ret = wait_on_sync_kiocb(&kiocb); + return ret; +} +EXPORT_SYMBOL(generic_file_write_nolock); + +ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf, + size_t count, loff_t pos) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t ret; + struct iovec local_iov = { .iov_base = (void __user *)buf, + .iov_len = count }; + + BUG_ON(iocb->ki_pos != pos); + + down(&inode->i_sem); + ret = __generic_file_aio_write_nolock(iocb, &local_iov, 1, + &iocb->ki_pos); + up(&inode->i_sem); + + if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { + ssize_t err; + + err = sync_page_range(inode, mapping, pos, ret); + if (err < 0) + ret = err; + } + return ret; +} +EXPORT_SYMBOL(generic_file_aio_write); + +ssize_t generic_file_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t ret; + struct iovec local_iov = { .iov_base = (void __user *)buf, + .iov_len = count }; + + down(&inode->i_sem); + ret = __generic_file_write_nolock(file, &local_iov, 1, ppos); + up(&inode->i_sem); + + if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { + ssize_t err; + + err = sync_page_range(inode, mapping, *ppos - ret, ret); + if (err < 0) + ret = err; + } + return ret; +} +EXPORT_SYMBOL(generic_file_write); + +ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos); + if (-EIOCBQUEUED == ret) + ret = wait_on_sync_kiocb(&kiocb); + return ret; +} +EXPORT_SYMBOL(generic_file_readv); + +ssize_t generic_file_writev(struct file *file, const struct iovec *iov, + unsigned long nr_segs, loff_t *ppos) +{ + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + ssize_t ret; + + down(&inode->i_sem); + ret = __generic_file_write_nolock(file, iov, nr_segs, ppos); + up(&inode->i_sem); + + if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) { + int err; + + err = sync_page_range(inode, mapping, *ppos - ret, ret); + if (err < 0) + ret = err; + } + return ret; +} +EXPORT_SYMBOL(generic_file_writev); + +/* + * Called under i_sem for writes to S_ISREG files. Returns -EIO if something + * went wrong during pagecache shootdown. + */ +ssize_t +generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, + loff_t offset, unsigned long nr_segs) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + ssize_t retval; + size_t write_len = 0; + + /* + * If it's a write, unmap all mmappings of the file up-front. This + * will cause any pte dirty bits to be propagated into the pageframes + * for the subsequent filemap_write_and_wait(). + */ + if (rw == WRITE) { + write_len = iov_length(iov, nr_segs); + if (mapping_mapped(mapping)) + unmap_mapping_range(mapping, offset, write_len, 0); + } + + retval = filemap_write_and_wait(mapping); + if (retval == 0) { + retval = mapping->a_ops->direct_IO(rw, iocb, iov, + offset, nr_segs); + if (rw == WRITE && mapping->nrpages) { + pgoff_t end = (offset + write_len - 1) + >> PAGE_CACHE_SHIFT; + int err = invalidate_inode_pages2_range(mapping, + offset >> PAGE_CACHE_SHIFT, end); + if (err) + retval = err; + } + } + return retval; +} +EXPORT_SYMBOL_GPL(generic_file_direct_IO); diff --git a/mm/fremap.c b/mm/fremap.c new file mode 100644 index 000000000000..3235fb77c133 --- /dev/null +++ b/mm/fremap.c @@ -0,0 +1,256 @@ +/* + * linux/mm/fremap.c + * + * Explicit pagetable population and nonlinear (random) mappings support. + * + * started by Ingo Molnar, Copyright (C) 2002, 2003 + */ + +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/file.h> +#include <linux/mman.h> +#include <linux/pagemap.h> +#include <linux/swapops.h> +#include <linux/rmap.h> +#include <linux/module.h> +#include <linux/syscalls.h> + +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> + +static inline void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pte_t pte = *ptep; + + if (pte_none(pte)) + return; + if (pte_present(pte)) { + unsigned long pfn = pte_pfn(pte); + + flush_cache_page(vma, addr, pfn); + pte = ptep_clear_flush(vma, addr, ptep); + if (pfn_valid(pfn)) { + struct page *page = pfn_to_page(pfn); + if (!PageReserved(page)) { + if (pte_dirty(pte)) + set_page_dirty(page); + page_remove_rmap(page); + page_cache_release(page); + dec_mm_counter(mm, rss); + } + } + } else { + if (!pte_file(pte)) + free_swap_and_cache(pte_to_swp_entry(pte)); + pte_clear(mm, addr, ptep); + } +} + +/* + * Install a file page to a given virtual memory address, release any + * previously existing mapping. + */ +int install_page(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, struct page *page, pgprot_t prot) +{ + struct inode *inode; + pgoff_t size; + int err = -ENOMEM; + pte_t *pte; + pmd_t *pmd; + pud_t *pud; + pgd_t *pgd; + pte_t pte_val; + + pgd = pgd_offset(mm, addr); + spin_lock(&mm->page_table_lock); + + pud = pud_alloc(mm, pgd, addr); + if (!pud) + goto err_unlock; + + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + goto err_unlock; + + pte = pte_alloc_map(mm, pmd, addr); + if (!pte) + goto err_unlock; + + /* + * This page may have been truncated. Tell the + * caller about it. + */ + err = -EINVAL; + inode = vma->vm_file->f_mapping->host; + size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; + if (!page->mapping || page->index >= size) + goto err_unlock; + + zap_pte(mm, vma, addr, pte); + + inc_mm_counter(mm,rss); + flush_icache_page(vma, page); + set_pte_at(mm, addr, pte, mk_pte(page, prot)); + page_add_file_rmap(page); + pte_val = *pte; + pte_unmap(pte); + update_mmu_cache(vma, addr, pte_val); + + err = 0; +err_unlock: + spin_unlock(&mm->page_table_lock); + return err; +} +EXPORT_SYMBOL(install_page); + + +/* + * Install a file pte to a given virtual memory address, release any + * previously existing mapping. + */ +int install_file_pte(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, unsigned long pgoff, pgprot_t prot) +{ + int err = -ENOMEM; + pte_t *pte; + pmd_t *pmd; + pud_t *pud; + pgd_t *pgd; + pte_t pte_val; + + pgd = pgd_offset(mm, addr); + spin_lock(&mm->page_table_lock); + + pud = pud_alloc(mm, pgd, addr); + if (!pud) + goto err_unlock; + + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + goto err_unlock; + + pte = pte_alloc_map(mm, pmd, addr); + if (!pte) + goto err_unlock; + + zap_pte(mm, vma, addr, pte); + + set_pte_at(mm, addr, pte, pgoff_to_pte(pgoff)); + pte_val = *pte; + pte_unmap(pte); + update_mmu_cache(vma, addr, pte_val); + spin_unlock(&mm->page_table_lock); + return 0; + +err_unlock: + spin_unlock(&mm->page_table_lock); + return err; +} + + +/*** + * sys_remap_file_pages - remap arbitrary pages of a shared backing store + * file within an existing vma. + * @start: start of the remapped virtual memory range + * @size: size of the remapped virtual memory range + * @prot: new protection bits of the range + * @pgoff: to be mapped page of the backing store file + * @flags: 0 or MAP_NONBLOCKED - the later will cause no IO. + * + * this syscall works purely via pagetables, so it's the most efficient + * way to map the same (large) file into a given virtual window. Unlike + * mmap()/mremap() it does not create any new vmas. The new mappings are + * also safe across swapout. + * + * NOTE: the 'prot' parameter right now is ignored, and the vma's default + * protection is used. Arbitrary protections might be implemented in the + * future. + */ +asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size, + unsigned long __prot, unsigned long pgoff, unsigned long flags) +{ + struct mm_struct *mm = current->mm; + struct address_space *mapping; + unsigned long end = start + size; + struct vm_area_struct *vma; + int err = -EINVAL; + int has_write_lock = 0; + + if (__prot) + return err; + /* + * Sanitize the syscall parameters: + */ + start = start & PAGE_MASK; + size = size & PAGE_MASK; + + /* Does the address range wrap, or is the span zero-sized? */ + if (start + size <= start) + return err; + + /* Can we represent this offset inside this architecture's pte's? */ +#if PTE_FILE_MAX_BITS < BITS_PER_LONG + if (pgoff + (size >> PAGE_SHIFT) >= (1UL << PTE_FILE_MAX_BITS)) + return err; +#endif + + /* We need down_write() to change vma->vm_flags. */ + down_read(&mm->mmap_sem); + retry: + vma = find_vma(mm, start); + + /* + * Make sure the vma is shared, that it supports prefaulting, + * and that the remapped range is valid and fully within + * the single existing vma. vm_private_data is used as a + * swapout cursor in a VM_NONLINEAR vma (unless VM_RESERVED + * or VM_LOCKED, but VM_LOCKED could be revoked later on). + */ + if (vma && (vma->vm_flags & VM_SHARED) && + (!vma->vm_private_data || + (vma->vm_flags & (VM_NONLINEAR|VM_RESERVED))) && + vma->vm_ops && vma->vm_ops->populate && + end > start && start >= vma->vm_start && + end <= vma->vm_end) { + + /* Must set VM_NONLINEAR before any pages are populated. */ + if (pgoff != linear_page_index(vma, start) && + !(vma->vm_flags & VM_NONLINEAR)) { + if (!has_write_lock) { + up_read(&mm->mmap_sem); + down_write(&mm->mmap_sem); + has_write_lock = 1; + goto retry; + } + mapping = vma->vm_file->f_mapping; + spin_lock(&mapping->i_mmap_lock); + flush_dcache_mmap_lock(mapping); + vma->vm_flags |= VM_NONLINEAR; + vma_prio_tree_remove(vma, &mapping->i_mmap); + vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear); + flush_dcache_mmap_unlock(mapping); + spin_unlock(&mapping->i_mmap_lock); + } + + err = vma->vm_ops->populate(vma, start, size, + vma->vm_page_prot, + pgoff, flags & MAP_NONBLOCK); + + /* + * We can't clear VM_NONLINEAR because we'd have to do + * it after ->populate completes, and that would prevent + * downgrading the lock. (Locks can't be upgraded). + */ + } + if (likely(!has_write_lock)) + up_read(&mm->mmap_sem); + else + up_write(&mm->mmap_sem); + + return err; +} + diff --git a/mm/highmem.c b/mm/highmem.c new file mode 100644 index 000000000000..d01276506b00 --- /dev/null +++ b/mm/highmem.c @@ -0,0 +1,607 @@ +/* + * High memory handling common code and variables. + * + * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de + * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de + * + * + * Redesigned the x86 32-bit VM architecture to deal with + * 64-bit physical space. With current x86 CPUs this + * means up to 64 Gigabytes physical RAM. + * + * Rewrote high memory support to move the page cache into + * high memory. Implemented permanent (schedulable) kmaps + * based on Linus' idea. + * + * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> + */ + +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/swap.h> +#include <linux/bio.h> +#include <linux/pagemap.h> +#include <linux/mempool.h> +#include <linux/blkdev.h> +#include <linux/init.h> +#include <linux/hash.h> +#include <linux/highmem.h> +#include <asm/tlbflush.h> + +static mempool_t *page_pool, *isa_page_pool; + +static void *page_pool_alloc(unsigned int __nocast gfp_mask, void *data) +{ + unsigned int gfp = gfp_mask | (unsigned int) (long) data; + + return alloc_page(gfp); +} + +static void page_pool_free(void *page, void *data) +{ + __free_page(page); +} + +/* + * Virtual_count is not a pure "count". + * 0 means that it is not mapped, and has not been mapped + * since a TLB flush - it is usable. + * 1 means that there are no users, but it has been mapped + * since the last TLB flush - so we can't use it. + * n means that there are (n-1) current users of it. + */ +#ifdef CONFIG_HIGHMEM +static int pkmap_count[LAST_PKMAP]; +static unsigned int last_pkmap_nr; +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); + +pte_t * pkmap_page_table; + +static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); + +static void flush_all_zero_pkmaps(void) +{ + int i; + + flush_cache_kmaps(); + + for (i = 0; i < LAST_PKMAP; i++) { + struct page *page; + + /* + * zero means we don't have anything to do, + * >1 means that it is still in use. Only + * a count of 1 means that it is free but + * needs to be unmapped + */ + if (pkmap_count[i] != 1) + continue; + pkmap_count[i] = 0; + + /* sanity check */ + if (pte_none(pkmap_page_table[i])) + BUG(); + + /* + * Don't need an atomic fetch-and-clear op here; + * no-one has the page mapped, and cannot get at + * its virtual address (and hence PTE) without first + * getting the kmap_lock (which is held here). + * So no dangers, even with speculative execution. + */ + page = pte_page(pkmap_page_table[i]); + pte_clear(&init_mm, (unsigned long)page_address(page), + &pkmap_page_table[i]); + + set_page_address(page, NULL); + } + flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP)); +} + +static inline unsigned long map_new_virtual(struct page *page) +{ + unsigned long vaddr; + int count; + +start: + count = LAST_PKMAP; + /* Find an empty entry */ + for (;;) { + last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; + if (!last_pkmap_nr) { + flush_all_zero_pkmaps(); + count = LAST_PKMAP; + } + if (!pkmap_count[last_pkmap_nr]) + break; /* Found a usable entry */ + if (--count) + continue; + + /* + * Sleep for somebody else to unmap their entries + */ + { + DECLARE_WAITQUEUE(wait, current); + + __set_current_state(TASK_UNINTERRUPTIBLE); + add_wait_queue(&pkmap_map_wait, &wait); + spin_unlock(&kmap_lock); + schedule(); + remove_wait_queue(&pkmap_map_wait, &wait); + spin_lock(&kmap_lock); + + /* Somebody else might have mapped it while we slept */ + if (page_address(page)) + return (unsigned long)page_address(page); + + /* Re-start */ + goto start; + } + } + vaddr = PKMAP_ADDR(last_pkmap_nr); + set_pte_at(&init_mm, vaddr, + &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot)); + + pkmap_count[last_pkmap_nr] = 1; + set_page_address(page, (void *)vaddr); + + return vaddr; +} + +void fastcall *kmap_high(struct page *page) +{ + unsigned long vaddr; + + /* + * For highmem pages, we can't trust "virtual" until + * after we have the lock. + * + * We cannot call this from interrupts, as it may block + */ + spin_lock(&kmap_lock); + vaddr = (unsigned long)page_address(page); + if (!vaddr) + vaddr = map_new_virtual(page); + pkmap_count[PKMAP_NR(vaddr)]++; + if (pkmap_count[PKMAP_NR(vaddr)] < 2) + BUG(); + spin_unlock(&kmap_lock); + return (void*) vaddr; +} + +EXPORT_SYMBOL(kmap_high); + +void fastcall kunmap_high(struct page *page) +{ + unsigned long vaddr; + unsigned long nr; + int need_wakeup; + + spin_lock(&kmap_lock); + vaddr = (unsigned long)page_address(page); + if (!vaddr) + BUG(); + nr = PKMAP_NR(vaddr); + + /* + * A count must never go down to zero + * without a TLB flush! + */ + need_wakeup = 0; + switch (--pkmap_count[nr]) { + case 0: + BUG(); + case 1: + /* + * Avoid an unnecessary wake_up() function call. + * The common case is pkmap_count[] == 1, but + * no waiters. + * The tasks queued in the wait-queue are guarded + * by both the lock in the wait-queue-head and by + * the kmap_lock. As the kmap_lock is held here, + * no need for the wait-queue-head's lock. Simply + * test if the queue is empty. + */ + need_wakeup = waitqueue_active(&pkmap_map_wait); + } + spin_unlock(&kmap_lock); + + /* do wake-up, if needed, race-free outside of the spin lock */ + if (need_wakeup) + wake_up(&pkmap_map_wait); +} + +EXPORT_SYMBOL(kunmap_high); + +#define POOL_SIZE 64 + +static __init int init_emergency_pool(void) +{ + struct sysinfo i; + si_meminfo(&i); + si_swapinfo(&i); + + if (!i.totalhigh) + return 0; + + page_pool = mempool_create(POOL_SIZE, page_pool_alloc, page_pool_free, NULL); + if (!page_pool) + BUG(); + printk("highmem bounce pool size: %d pages\n", POOL_SIZE); + + return 0; +} + +__initcall(init_emergency_pool); + +/* + * highmem version, map in to vec + */ +static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom) +{ + unsigned long flags; + unsigned char *vto; + + local_irq_save(flags); + vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ); + memcpy(vto + to->bv_offset, vfrom, to->bv_len); + kunmap_atomic(vto, KM_BOUNCE_READ); + local_irq_restore(flags); +} + +#else /* CONFIG_HIGHMEM */ + +#define bounce_copy_vec(to, vfrom) \ + memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len) + +#endif + +#define ISA_POOL_SIZE 16 + +/* + * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA + * as the max address, so check if the pool has already been created. + */ +int init_emergency_isa_pool(void) +{ + if (isa_page_pool) + return 0; + + isa_page_pool = mempool_create(ISA_POOL_SIZE, page_pool_alloc, page_pool_free, (void *) __GFP_DMA); + if (!isa_page_pool) + BUG(); + + printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE); + return 0; +} + +/* + * Simple bounce buffer support for highmem pages. Depending on the + * queue gfp mask set, *to may or may not be a highmem page. kmap it + * always, it will do the Right Thing + */ +static void copy_to_high_bio_irq(struct bio *to, struct bio *from) +{ + unsigned char *vfrom; + struct bio_vec *tovec, *fromvec; + int i; + + __bio_for_each_segment(tovec, to, i, 0) { + fromvec = from->bi_io_vec + i; + + /* + * not bounced + */ + if (tovec->bv_page == fromvec->bv_page) + continue; + + /* + * fromvec->bv_offset and fromvec->bv_len might have been + * modified by the block layer, so use the original copy, + * bounce_copy_vec already uses tovec->bv_len + */ + vfrom = page_address(fromvec->bv_page) + tovec->bv_offset; + + flush_dcache_page(tovec->bv_page); + bounce_copy_vec(tovec, vfrom); + } +} + +static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) +{ + struct bio *bio_orig = bio->bi_private; + struct bio_vec *bvec, *org_vec; + int i; + + if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags)) + set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags); + + /* + * free up bounce indirect pages used + */ + __bio_for_each_segment(bvec, bio, i, 0) { + org_vec = bio_orig->bi_io_vec + i; + if (bvec->bv_page == org_vec->bv_page) + continue; + + mempool_free(bvec->bv_page, pool); + } + + bio_endio(bio_orig, bio_orig->bi_size, err); + bio_put(bio); +} + +static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done,int err) +{ + if (bio->bi_size) + return 1; + + bounce_end_io(bio, page_pool, err); + return 0; +} + +static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err) +{ + if (bio->bi_size) + return 1; + + bounce_end_io(bio, isa_page_pool, err); + return 0; +} + +static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) +{ + struct bio *bio_orig = bio->bi_private; + + if (test_bit(BIO_UPTODATE, &bio->bi_flags)) + copy_to_high_bio_irq(bio_orig, bio); + + bounce_end_io(bio, pool, err); +} + +static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err) +{ + if (bio->bi_size) + return 1; + + __bounce_end_io_read(bio, page_pool, err); + return 0; +} + +static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err) +{ + if (bio->bi_size) + return 1; + + __bounce_end_io_read(bio, isa_page_pool, err); + return 0; +} + +static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig, + mempool_t *pool) +{ + struct page *page; + struct bio *bio = NULL; + int i, rw = bio_data_dir(*bio_orig); + struct bio_vec *to, *from; + + bio_for_each_segment(from, *bio_orig, i) { + page = from->bv_page; + + /* + * is destination page below bounce pfn? + */ + if (page_to_pfn(page) < q->bounce_pfn) + continue; + + /* + * irk, bounce it + */ + if (!bio) + bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt); + + to = bio->bi_io_vec + i; + + to->bv_page = mempool_alloc(pool, q->bounce_gfp); + to->bv_len = from->bv_len; + to->bv_offset = from->bv_offset; + + if (rw == WRITE) { + char *vto, *vfrom; + + flush_dcache_page(from->bv_page); + vto = page_address(to->bv_page) + to->bv_offset; + vfrom = kmap(from->bv_page) + from->bv_offset; + memcpy(vto, vfrom, to->bv_len); + kunmap(from->bv_page); + } + } + + /* + * no pages bounced + */ + if (!bio) + return; + + /* + * at least one page was bounced, fill in possible non-highmem + * pages + */ + __bio_for_each_segment(from, *bio_orig, i, 0) { + to = bio_iovec_idx(bio, i); + if (!to->bv_page) { + to->bv_page = from->bv_page; + to->bv_len = from->bv_len; + to->bv_offset = from->bv_offset; + } + } + + bio->bi_bdev = (*bio_orig)->bi_bdev; + bio->bi_flags |= (1 << BIO_BOUNCED); + bio->bi_sector = (*bio_orig)->bi_sector; + bio->bi_rw = (*bio_orig)->bi_rw; + + bio->bi_vcnt = (*bio_orig)->bi_vcnt; + bio->bi_idx = (*bio_orig)->bi_idx; + bio->bi_size = (*bio_orig)->bi_size; + + if (pool == page_pool) { + bio->bi_end_io = bounce_end_io_write; + if (rw == READ) + bio->bi_end_io = bounce_end_io_read; + } else { + bio->bi_end_io = bounce_end_io_write_isa; + if (rw == READ) + bio->bi_end_io = bounce_end_io_read_isa; + } + + bio->bi_private = *bio_orig; + *bio_orig = bio; +} + +void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig) +{ + mempool_t *pool; + + /* + * for non-isa bounce case, just check if the bounce pfn is equal + * to or bigger than the highest pfn in the system -- in that case, + * don't waste time iterating over bio segments + */ + if (!(q->bounce_gfp & GFP_DMA)) { + if (q->bounce_pfn >= blk_max_pfn) + return; + pool = page_pool; + } else { + BUG_ON(!isa_page_pool); + pool = isa_page_pool; + } + + /* + * slow path + */ + __blk_queue_bounce(q, bio_orig, pool); +} + +EXPORT_SYMBOL(blk_queue_bounce); + +#if defined(HASHED_PAGE_VIRTUAL) + +#define PA_HASH_ORDER 7 + +/* + * Describes one page->virtual association + */ +struct page_address_map { + struct page *page; + void *virtual; + struct list_head list; +}; + +/* + * page_address_map freelist, allocated from page_address_maps. + */ +static struct list_head page_address_pool; /* freelist */ +static spinlock_t pool_lock; /* protects page_address_pool */ + +/* + * Hash table bucket + */ +static struct page_address_slot { + struct list_head lh; /* List of page_address_maps */ + spinlock_t lock; /* Protect this bucket's list */ +} ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; + +static struct page_address_slot *page_slot(struct page *page) +{ + return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; +} + +void *page_address(struct page *page) +{ + unsigned long flags; + void *ret; + struct page_address_slot *pas; + + if (!PageHighMem(page)) + return lowmem_page_address(page); + + pas = page_slot(page); + ret = NULL; + spin_lock_irqsave(&pas->lock, flags); + if (!list_empty(&pas->lh)) { + struct page_address_map *pam; + + list_for_each_entry(pam, &pas->lh, list) { + if (pam->page == page) { + ret = pam->virtual; + goto done; + } + } + } +done: + spin_unlock_irqrestore(&pas->lock, flags); + return ret; +} + +EXPORT_SYMBOL(page_address); + +void set_page_address(struct page *page, void *virtual) +{ + unsigned long flags; + struct page_address_slot *pas; + struct page_address_map *pam; + + BUG_ON(!PageHighMem(page)); + + pas = page_slot(page); + if (virtual) { /* Add */ + BUG_ON(list_empty(&page_address_pool)); + + spin_lock_irqsave(&pool_lock, flags); + pam = list_entry(page_address_pool.next, + struct page_address_map, list); + list_del(&pam->list); + spin_unlock_irqrestore(&pool_lock, flags); + + pam->page = page; + pam->virtual = virtual; + + spin_lock_irqsave(&pas->lock, flags); + list_add_tail(&pam->list, &pas->lh); + spin_unlock_irqrestore(&pas->lock, flags); + } else { /* Remove */ + spin_lock_irqsave(&pas->lock, flags); + list_for_each_entry(pam, &pas->lh, list) { + if (pam->page == page) { + list_del(&pam->list); + spin_unlock_irqrestore(&pas->lock, flags); + spin_lock_irqsave(&pool_lock, flags); + list_add_tail(&pam->list, &page_address_pool); + spin_unlock_irqrestore(&pool_lock, flags); + goto done; + } + } + spin_unlock_irqrestore(&pas->lock, flags); + } +done: + return; +} + +static struct page_address_map page_address_maps[LAST_PKMAP]; + +void __init page_address_init(void) +{ + int i; + + INIT_LIST_HEAD(&page_address_pool); + for (i = 0; i < ARRAY_SIZE(page_address_maps); i++) + list_add(&page_address_maps[i].list, &page_address_pool); + for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) { + INIT_LIST_HEAD(&page_address_htable[i].lh); + spin_lock_init(&page_address_htable[i].lock); + } + spin_lock_init(&pool_lock); +} + +#endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c new file mode 100644 index 000000000000..4eb5ae3fbe10 --- /dev/null +++ b/mm/hugetlb.c @@ -0,0 +1,260 @@ +/* + * Generic hugetlb support. + * (C) William Irwin, April 2004 + */ +#include <linux/gfp.h> +#include <linux/list.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/hugetlb.h> +#include <linux/sysctl.h> +#include <linux/highmem.h> +#include <linux/nodemask.h> + +const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL; +static unsigned long nr_huge_pages, free_huge_pages; +unsigned long max_huge_pages; +static struct list_head hugepage_freelists[MAX_NUMNODES]; +static unsigned int nr_huge_pages_node[MAX_NUMNODES]; +static unsigned int free_huge_pages_node[MAX_NUMNODES]; +static DEFINE_SPINLOCK(hugetlb_lock); + +static void enqueue_huge_page(struct page *page) +{ + int nid = page_to_nid(page); + list_add(&page->lru, &hugepage_freelists[nid]); + free_huge_pages++; + free_huge_pages_node[nid]++; +} + +static struct page *dequeue_huge_page(void) +{ + int nid = numa_node_id(); + struct page *page = NULL; + + if (list_empty(&hugepage_freelists[nid])) { + for (nid = 0; nid < MAX_NUMNODES; ++nid) + if (!list_empty(&hugepage_freelists[nid])) + break; + } + if (nid >= 0 && nid < MAX_NUMNODES && + !list_empty(&hugepage_freelists[nid])) { + page = list_entry(hugepage_freelists[nid].next, + struct page, lru); + list_del(&page->lru); + free_huge_pages--; + free_huge_pages_node[nid]--; + } + return page; +} + +static struct page *alloc_fresh_huge_page(void) +{ + static int nid = 0; + struct page *page; + page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN, + HUGETLB_PAGE_ORDER); + nid = (nid + 1) % num_online_nodes(); + if (page) { + nr_huge_pages++; + nr_huge_pages_node[page_to_nid(page)]++; + } + return page; +} + +void free_huge_page(struct page *page) +{ + BUG_ON(page_count(page)); + + INIT_LIST_HEAD(&page->lru); + page[1].mapping = NULL; + + spin_lock(&hugetlb_lock); + enqueue_huge_page(page); + spin_unlock(&hugetlb_lock); +} + +struct page *alloc_huge_page(void) +{ + struct page *page; + int i; + + spin_lock(&hugetlb_lock); + page = dequeue_huge_page(); + if (!page) { + spin_unlock(&hugetlb_lock); + return NULL; + } + spin_unlock(&hugetlb_lock); + set_page_count(page, 1); + page[1].mapping = (void *)free_huge_page; + for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i) + clear_highpage(&page[i]); + return page; +} + +static int __init hugetlb_init(void) +{ + unsigned long i; + struct page *page; + + for (i = 0; i < MAX_NUMNODES; ++i) + INIT_LIST_HEAD(&hugepage_freelists[i]); + + for (i = 0; i < max_huge_pages; ++i) { + page = alloc_fresh_huge_page(); + if (!page) + break; + spin_lock(&hugetlb_lock); + enqueue_huge_page(page); + spin_unlock(&hugetlb_lock); + } + max_huge_pages = free_huge_pages = nr_huge_pages = i; + printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages); + return 0; +} +module_init(hugetlb_init); + +static int __init hugetlb_setup(char *s) +{ + if (sscanf(s, "%lu", &max_huge_pages) <= 0) + max_huge_pages = 0; + return 1; +} +__setup("hugepages=", hugetlb_setup); + +#ifdef CONFIG_SYSCTL +static void update_and_free_page(struct page *page) +{ + int i; + nr_huge_pages--; + nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--; + for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) { + page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | + 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved | + 1 << PG_private | 1<< PG_writeback); + set_page_count(&page[i], 0); + } + set_page_count(page, 1); + __free_pages(page, HUGETLB_PAGE_ORDER); +} + +#ifdef CONFIG_HIGHMEM +static void try_to_free_low(unsigned long count) +{ + int i, nid; + for (i = 0; i < MAX_NUMNODES; ++i) { + struct page *page, *next; + list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) { + if (PageHighMem(page)) + continue; + list_del(&page->lru); + update_and_free_page(page); + nid = page_zone(page)->zone_pgdat->node_id; + free_huge_pages--; + free_huge_pages_node[nid]--; + if (count >= nr_huge_pages) + return; + } + } +} +#else +static inline void try_to_free_low(unsigned long count) +{ +} +#endif + +static unsigned long set_max_huge_pages(unsigned long count) +{ + while (count > nr_huge_pages) { + struct page *page = alloc_fresh_huge_page(); + if (!page) + return nr_huge_pages; + spin_lock(&hugetlb_lock); + enqueue_huge_page(page); + spin_unlock(&hugetlb_lock); + } + if (count >= nr_huge_pages) + return nr_huge_pages; + + spin_lock(&hugetlb_lock); + try_to_free_low(count); + while (count < nr_huge_pages) { + struct page *page = dequeue_huge_page(); + if (!page) + break; + update_and_free_page(page); + } + spin_unlock(&hugetlb_lock); + return nr_huge_pages; +} + +int hugetlb_sysctl_handler(struct ctl_table *table, int write, + struct file *file, void __user *buffer, + size_t *length, loff_t *ppos) +{ + proc_doulongvec_minmax(table, write, file, buffer, length, ppos); + max_huge_pages = set_max_huge_pages(max_huge_pages); + return 0; +} +#endif /* CONFIG_SYSCTL */ + +int hugetlb_report_meminfo(char *buf) +{ + return sprintf(buf, + "HugePages_Total: %5lu\n" + "HugePages_Free: %5lu\n" + "Hugepagesize: %5lu kB\n", + nr_huge_pages, + free_huge_pages, + HPAGE_SIZE/1024); +} + +int hugetlb_report_node_meminfo(int nid, char *buf) +{ + return sprintf(buf, + "Node %d HugePages_Total: %5u\n" + "Node %d HugePages_Free: %5u\n", + nid, nr_huge_pages_node[nid], + nid, free_huge_pages_node[nid]); +} + +int is_hugepage_mem_enough(size_t size) +{ + return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages; +} + +/* Return the number pages of memory we physically have, in PAGE_SIZE units. */ +unsigned long hugetlb_total_pages(void) +{ + return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE); +} +EXPORT_SYMBOL(hugetlb_total_pages); + +/* + * We cannot handle pagefaults against hugetlb pages at all. They cause + * handle_mm_fault() to try to instantiate regular-sized pages in the + * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get + * this far. + */ +static struct page *hugetlb_nopage(struct vm_area_struct *vma, + unsigned long address, int *unused) +{ + BUG(); + return NULL; +} + +struct vm_operations_struct hugetlb_vm_ops = { + .nopage = hugetlb_nopage, +}; + +void zap_hugepage_range(struct vm_area_struct *vma, + unsigned long start, unsigned long length) +{ + struct mm_struct *mm = vma->vm_mm; + + spin_lock(&mm->page_table_lock); + unmap_hugepage_range(vma, start, start + length); + spin_unlock(&mm->page_table_lock); +} diff --git a/mm/internal.h b/mm/internal.h new file mode 100644 index 000000000000..6bf134e8fb3d --- /dev/null +++ b/mm/internal.h @@ -0,0 +1,13 @@ +/* internal.h: mm/ internal definitions + * + * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +/* page_alloc.c */ +extern void set_page_refs(struct page *page, int order); diff --git a/mm/madvise.c b/mm/madvise.c new file mode 100644 index 000000000000..944b5e52d812 --- /dev/null +++ b/mm/madvise.c @@ -0,0 +1,242 @@ +/* + * linux/mm/madvise.c + * + * Copyright (C) 1999 Linus Torvalds + * Copyright (C) 2002 Christoph Hellwig + */ + +#include <linux/mman.h> +#include <linux/pagemap.h> +#include <linux/syscalls.h> +#include <linux/hugetlb.h> + +/* + * We can potentially split a vm area into separate + * areas, each area with its own behavior. + */ +static long madvise_behavior(struct vm_area_struct * vma, unsigned long start, + unsigned long end, int behavior) +{ + struct mm_struct * mm = vma->vm_mm; + int error = 0; + + if (start != vma->vm_start) { + error = split_vma(mm, vma, start, 1); + if (error) + goto out; + } + + if (end != vma->vm_end) { + error = split_vma(mm, vma, end, 0); + if (error) + goto out; + } + + /* + * vm_flags is protected by the mmap_sem held in write mode. + */ + VM_ClearReadHint(vma); + + switch (behavior) { + case MADV_SEQUENTIAL: + vma->vm_flags |= VM_SEQ_READ; + break; + case MADV_RANDOM: + vma->vm_flags |= VM_RAND_READ; + break; + default: + break; + } + +out: + if (error == -ENOMEM) + error = -EAGAIN; + return error; +} + +/* + * Schedule all required I/O operations. Do not wait for completion. + */ +static long madvise_willneed(struct vm_area_struct * vma, + unsigned long start, unsigned long end) +{ + struct file *file = vma->vm_file; + + if (!file) + return -EBADF; + + start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + if (end > vma->vm_end) + end = vma->vm_end; + end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + + force_page_cache_readahead(file->f_mapping, + file, start, max_sane_readahead(end - start)); + return 0; +} + +/* + * Application no longer needs these pages. If the pages are dirty, + * it's OK to just throw them away. The app will be more careful about + * data it wants to keep. Be sure to free swap resources too. The + * zap_page_range call sets things up for refill_inactive to actually free + * these pages later if no one else has touched them in the meantime, + * although we could add these pages to a global reuse list for + * refill_inactive to pick up before reclaiming other pages. + * + * NB: This interface discards data rather than pushes it out to swap, + * as some implementations do. This has performance implications for + * applications like large transactional databases which want to discard + * pages in anonymous maps after committing to backing store the data + * that was kept in them. There is no reason to write this data out to + * the swap area if the application is discarding it. + * + * An interface that causes the system to free clean pages and flush + * dirty pages is already available as msync(MS_INVALIDATE). + */ +static long madvise_dontneed(struct vm_area_struct * vma, + unsigned long start, unsigned long end) +{ + if ((vma->vm_flags & VM_LOCKED) || is_vm_hugetlb_page(vma)) + return -EINVAL; + + if (unlikely(vma->vm_flags & VM_NONLINEAR)) { + struct zap_details details = { + .nonlinear_vma = vma, + .last_index = ULONG_MAX, + }; + zap_page_range(vma, start, end - start, &details); + } else + zap_page_range(vma, start, end - start, NULL); + return 0; +} + +static long madvise_vma(struct vm_area_struct * vma, unsigned long start, + unsigned long end, int behavior) +{ + long error = -EBADF; + + switch (behavior) { + case MADV_NORMAL: + case MADV_SEQUENTIAL: + case MADV_RANDOM: + error = madvise_behavior(vma, start, end, behavior); + break; + + case MADV_WILLNEED: + error = madvise_willneed(vma, start, end); + break; + + case MADV_DONTNEED: + error = madvise_dontneed(vma, start, end); + break; + + default: + error = -EINVAL; + break; + } + + return error; +} + +/* + * The madvise(2) system call. + * + * Applications can use madvise() to advise the kernel how it should + * handle paging I/O in this VM area. The idea is to help the kernel + * use appropriate read-ahead and caching techniques. The information + * provided is advisory only, and can be safely disregarded by the + * kernel without affecting the correct operation of the application. + * + * behavior values: + * MADV_NORMAL - the default behavior is to read clusters. This + * results in some read-ahead and read-behind. + * MADV_RANDOM - the system should read the minimum amount of data + * on any access, since it is unlikely that the appli- + * cation will need more than what it asks for. + * MADV_SEQUENTIAL - pages in the given range will probably be accessed + * once, so they can be aggressively read ahead, and + * can be freed soon after they are accessed. + * MADV_WILLNEED - the application is notifying the system to read + * some pages ahead. + * MADV_DONTNEED - the application is finished with the given range, + * so the kernel can free resources associated with it. + * + * return values: + * zero - success + * -EINVAL - start + len < 0, start is not page-aligned, + * "behavior" is not a valid value, or application + * is attempting to release locked or shared pages. + * -ENOMEM - addresses in the specified range are not currently + * mapped, or are outside the AS of the process. + * -EIO - an I/O error occurred while paging in data. + * -EBADF - map exists, but area maps something that isn't a file. + * -EAGAIN - a kernel resource was temporarily unavailable. + */ +asmlinkage long sys_madvise(unsigned long start, size_t len_in, int behavior) +{ + unsigned long end; + struct vm_area_struct * vma; + int unmapped_error = 0; + int error = -EINVAL; + size_t len; + + down_write(¤t->mm->mmap_sem); + + if (start & ~PAGE_MASK) + goto out; + len = (len_in + ~PAGE_MASK) & PAGE_MASK; + + /* Check to see whether len was rounded up from small -ve to zero */ + if (len_in && !len) + goto out; + + end = start + len; + if (end < start) + goto out; + + error = 0; + if (end == start) + goto out; + + /* + * If the interval [start,end) covers some unmapped address + * ranges, just ignore them, but return -ENOMEM at the end. + */ + vma = find_vma(current->mm, start); + for (;;) { + /* Still start < end. */ + error = -ENOMEM; + if (!vma) + goto out; + + /* Here start < vma->vm_end. */ + if (start < vma->vm_start) { + unmapped_error = -ENOMEM; + start = vma->vm_start; + } + + /* Here vma->vm_start <= start < vma->vm_end. */ + if (end <= vma->vm_end) { + if (start < end) { + error = madvise_vma(vma, start, end, + behavior); + if (error) + goto out; + } + error = unmapped_error; + goto out; + } + + /* Here vma->vm_start <= start < vma->vm_end < end. */ + error = madvise_vma(vma, start, vma->vm_end, behavior); + if (error) + goto out; + start = vma->vm_end; + vma = vma->vm_next; + } + +out: + up_write(¤t->mm->mmap_sem); + return error; +} diff --git a/mm/memory.c b/mm/memory.c new file mode 100644 index 000000000000..fb6e5deb873a --- /dev/null +++ b/mm/memory.c @@ -0,0 +1,2165 @@ +/* + * linux/mm/memory.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + */ + +/* + * demand-loading started 01.12.91 - seems it is high on the list of + * things wanted, and it should be easy to implement. - Linus + */ + +/* + * Ok, demand-loading was easy, shared pages a little bit tricker. Shared + * pages started 02.12.91, seems to work. - Linus. + * + * Tested sharing by executing about 30 /bin/sh: under the old kernel it + * would have taken more than the 6M I have free, but it worked well as + * far as I could see. + * + * Also corrected some "invalidate()"s - I wasn't doing enough of them. + */ + +/* + * Real VM (paging to/from disk) started 18.12.91. Much more work and + * thought has to go into this. Oh, well.. + * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. + * Found it. Everything seems to work now. + * 20.12.91 - Ok, making the swap-device changeable like the root. + */ + +/* + * 05.04.94 - Multi-page memory management added for v1.1. + * Idea by Alex Bligh (alex@cconcepts.co.uk) + * + * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG + * (Gerhard.Wichert@pdb.siemens.de) + * + * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) + */ + +#include <linux/kernel_stat.h> +#include <linux/mm.h> +#include <linux/hugetlb.h> +#include <linux/mman.h> +#include <linux/swap.h> +#include <linux/highmem.h> +#include <linux/pagemap.h> +#include <linux/rmap.h> +#include <linux/module.h> +#include <linux/init.h> + +#include <asm/pgalloc.h> +#include <asm/uaccess.h> +#include <asm/tlb.h> +#include <asm/tlbflush.h> +#include <asm/pgtable.h> + +#include <linux/swapops.h> +#include <linux/elf.h> + +#ifndef CONFIG_DISCONTIGMEM +/* use the per-pgdat data instead for discontigmem - mbligh */ +unsigned long max_mapnr; +struct page *mem_map; + +EXPORT_SYMBOL(max_mapnr); +EXPORT_SYMBOL(mem_map); +#endif + +unsigned long num_physpages; +/* + * A number of key systems in x86 including ioremap() rely on the assumption + * that high_memory defines the upper bound on direct map memory, then end + * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and + * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL + * and ZONE_HIGHMEM. + */ +void * high_memory; +unsigned long vmalloc_earlyreserve; + +EXPORT_SYMBOL(num_physpages); +EXPORT_SYMBOL(high_memory); +EXPORT_SYMBOL(vmalloc_earlyreserve); + +/* + * If a p?d_bad entry is found while walking page tables, report + * the error, before resetting entry to p?d_none. Usually (but + * very seldom) called out from the p?d_none_or_clear_bad macros. + */ + +void pgd_clear_bad(pgd_t *pgd) +{ + pgd_ERROR(*pgd); + pgd_clear(pgd); +} + +void pud_clear_bad(pud_t *pud) +{ + pud_ERROR(*pud); + pud_clear(pud); +} + +void pmd_clear_bad(pmd_t *pmd) +{ + pmd_ERROR(*pmd); + pmd_clear(pmd); +} + +/* + * Note: this doesn't free the actual pages themselves. That + * has been handled earlier when unmapping all the memory regions. + */ +static inline void clear_pte_range(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr, unsigned long end) +{ + if (!((addr | end) & ~PMD_MASK)) { + /* Only free fully aligned ranges */ + struct page *page = pmd_page(*pmd); + pmd_clear(pmd); + dec_page_state(nr_page_table_pages); + tlb->mm->nr_ptes--; + pte_free_tlb(tlb, page); + } +} + +static inline void clear_pmd_range(struct mmu_gather *tlb, pud_t *pud, + unsigned long addr, unsigned long end) +{ + pmd_t *pmd; + unsigned long next; + pmd_t *empty_pmd = NULL; + + pmd = pmd_offset(pud, addr); + + /* Only free fully aligned ranges */ + if (!((addr | end) & ~PUD_MASK)) + empty_pmd = pmd; + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + clear_pte_range(tlb, pmd, addr, next); + } while (pmd++, addr = next, addr != end); + + if (empty_pmd) { + pud_clear(pud); + pmd_free_tlb(tlb, empty_pmd); + } +} + +static inline void clear_pud_range(struct mmu_gather *tlb, pgd_t *pgd, + unsigned long addr, unsigned long end) +{ + pud_t *pud; + unsigned long next; + pud_t *empty_pud = NULL; + + pud = pud_offset(pgd, addr); + + /* Only free fully aligned ranges */ + if (!((addr | end) & ~PGDIR_MASK)) + empty_pud = pud; + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + clear_pmd_range(tlb, pud, addr, next); + } while (pud++, addr = next, addr != end); + + if (empty_pud) { + pgd_clear(pgd); + pud_free_tlb(tlb, empty_pud); + } +} + +/* + * This function clears user-level page tables of a process. + * Unlike other pagetable walks, some memory layouts might give end 0. + * Must be called with pagetable lock held. + */ +void clear_page_range(struct mmu_gather *tlb, + unsigned long addr, unsigned long end) +{ + pgd_t *pgd; + unsigned long next; + + pgd = pgd_offset(tlb->mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + clear_pud_range(tlb, pgd, addr, next); + } while (pgd++, addr = next, addr != end); +} + +pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address) +{ + if (!pmd_present(*pmd)) { + struct page *new; + + spin_unlock(&mm->page_table_lock); + new = pte_alloc_one(mm, address); + spin_lock(&mm->page_table_lock); + if (!new) + return NULL; + /* + * Because we dropped the lock, we should re-check the + * entry, as somebody else could have populated it.. + */ + if (pmd_present(*pmd)) { + pte_free(new); + goto out; + } + mm->nr_ptes++; + inc_page_state(nr_page_table_pages); + pmd_populate(mm, pmd, new); + } +out: + return pte_offset_map(pmd, address); +} + +pte_t fastcall * pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address) +{ + if (!pmd_present(*pmd)) { + pte_t *new; + + spin_unlock(&mm->page_table_lock); + new = pte_alloc_one_kernel(mm, address); + spin_lock(&mm->page_table_lock); + if (!new) + return NULL; + + /* + * Because we dropped the lock, we should re-check the + * entry, as somebody else could have populated it.. + */ + if (pmd_present(*pmd)) { + pte_free_kernel(new); + goto out; + } + pmd_populate_kernel(mm, pmd, new); + } +out: + return pte_offset_kernel(pmd, address); +} + +/* + * copy one vm_area from one task to the other. Assumes the page tables + * already present in the new task to be cleared in the whole range + * covered by this vma. + * + * dst->page_table_lock is held on entry and exit, + * but may be dropped within p[mg]d_alloc() and pte_alloc_map(). + */ + +static inline void +copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, + pte_t *dst_pte, pte_t *src_pte, unsigned long vm_flags, + unsigned long addr) +{ + pte_t pte = *src_pte; + struct page *page; + unsigned long pfn; + + /* pte contains position in swap or file, so copy. */ + if (unlikely(!pte_present(pte))) { + if (!pte_file(pte)) { + swap_duplicate(pte_to_swp_entry(pte)); + /* make sure dst_mm is on swapoff's mmlist. */ + if (unlikely(list_empty(&dst_mm->mmlist))) { + spin_lock(&mmlist_lock); + list_add(&dst_mm->mmlist, &src_mm->mmlist); + spin_unlock(&mmlist_lock); + } + } + set_pte_at(dst_mm, addr, dst_pte, pte); + return; + } + + pfn = pte_pfn(pte); + /* the pte points outside of valid memory, the + * mapping is assumed to be good, meaningful + * and not mapped via rmap - duplicate the + * mapping as is. + */ + page = NULL; + if (pfn_valid(pfn)) + page = pfn_to_page(pfn); + + if (!page || PageReserved(page)) { + set_pte_at(dst_mm, addr, dst_pte, pte); + return; + } + + /* + * If it's a COW mapping, write protect it both + * in the parent and the child + */ + if ((vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE) { + ptep_set_wrprotect(src_mm, addr, src_pte); + pte = *src_pte; + } + + /* + * If it's a shared mapping, mark it clean in + * the child + */ + if (vm_flags & VM_SHARED) + pte = pte_mkclean(pte); + pte = pte_mkold(pte); + get_page(page); + inc_mm_counter(dst_mm, rss); + if (PageAnon(page)) + inc_mm_counter(dst_mm, anon_rss); + set_pte_at(dst_mm, addr, dst_pte, pte); + page_dup_rmap(page); +} + +static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, + pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + pte_t *src_pte, *dst_pte; + unsigned long vm_flags = vma->vm_flags; + int progress; + +again: + dst_pte = pte_alloc_map(dst_mm, dst_pmd, addr); + if (!dst_pte) + return -ENOMEM; + src_pte = pte_offset_map_nested(src_pmd, addr); + + progress = 0; + spin_lock(&src_mm->page_table_lock); + do { + /* + * We are holding two locks at this point - either of them + * could generate latencies in another task on another CPU. + */ + if (progress >= 32 && (need_resched() || + need_lockbreak(&src_mm->page_table_lock) || + need_lockbreak(&dst_mm->page_table_lock))) + break; + if (pte_none(*src_pte)) { + progress++; + continue; + } + copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vm_flags, addr); + progress += 8; + } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); + spin_unlock(&src_mm->page_table_lock); + + pte_unmap_nested(src_pte - 1); + pte_unmap(dst_pte - 1); + cond_resched_lock(&dst_mm->page_table_lock); + if (addr != end) + goto again; + return 0; +} + +static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, + pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + pmd_t *src_pmd, *dst_pmd; + unsigned long next; + + dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); + if (!dst_pmd) + return -ENOMEM; + src_pmd = pmd_offset(src_pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(src_pmd)) + continue; + if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, + vma, addr, next)) + return -ENOMEM; + } while (dst_pmd++, src_pmd++, addr = next, addr != end); + return 0; +} + +static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, + pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + pud_t *src_pud, *dst_pud; + unsigned long next; + + dst_pud = pud_alloc(dst_mm, dst_pgd, addr); + if (!dst_pud) + return -ENOMEM; + src_pud = pud_offset(src_pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(src_pud)) + continue; + if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, + vma, addr, next)) + return -ENOMEM; + } while (dst_pud++, src_pud++, addr = next, addr != end); + return 0; +} + +int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, + struct vm_area_struct *vma) +{ + pgd_t *src_pgd, *dst_pgd; + unsigned long next; + unsigned long addr = vma->vm_start; + unsigned long end = vma->vm_end; + + if (is_vm_hugetlb_page(vma)) + return copy_hugetlb_page_range(dst_mm, src_mm, vma); + + dst_pgd = pgd_offset(dst_mm, addr); + src_pgd = pgd_offset(src_mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(src_pgd)) + continue; + if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, + vma, addr, next)) + return -ENOMEM; + } while (dst_pgd++, src_pgd++, addr = next, addr != end); + return 0; +} + +static void zap_pte_range(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr, unsigned long end, + struct zap_details *details) +{ + pte_t *pte; + + pte = pte_offset_map(pmd, addr); + do { + pte_t ptent = *pte; + if (pte_none(ptent)) + continue; + if (pte_present(ptent)) { + struct page *page = NULL; + unsigned long pfn = pte_pfn(ptent); + if (pfn_valid(pfn)) { + page = pfn_to_page(pfn); + if (PageReserved(page)) + page = NULL; + } + if (unlikely(details) && page) { + /* + * unmap_shared_mapping_pages() wants to + * invalidate cache without truncating: + * unmap shared but keep private pages. + */ + if (details->check_mapping && + details->check_mapping != page->mapping) + continue; + /* + * Each page->index must be checked when + * invalidating or truncating nonlinear. + */ + if (details->nonlinear_vma && + (page->index < details->first_index || + page->index > details->last_index)) + continue; + } + ptent = ptep_get_and_clear(tlb->mm, addr, pte); + tlb_remove_tlb_entry(tlb, pte, addr); + if (unlikely(!page)) + continue; + if (unlikely(details) && details->nonlinear_vma + && linear_page_index(details->nonlinear_vma, + addr) != page->index) + set_pte_at(tlb->mm, addr, pte, + pgoff_to_pte(page->index)); + if (pte_dirty(ptent)) + set_page_dirty(page); + if (PageAnon(page)) + dec_mm_counter(tlb->mm, anon_rss); + else if (pte_young(ptent)) + mark_page_accessed(page); + tlb->freed++; + page_remove_rmap(page); + tlb_remove_page(tlb, page); + continue; + } + /* + * If details->check_mapping, we leave swap entries; + * if details->nonlinear_vma, we leave file entries. + */ + if (unlikely(details)) + continue; + if (!pte_file(ptent)) + free_swap_and_cache(pte_to_swp_entry(ptent)); + pte_clear(tlb->mm, addr, pte); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); +} + +static inline void zap_pmd_range(struct mmu_gather *tlb, pud_t *pud, + unsigned long addr, unsigned long end, + struct zap_details *details) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + zap_pte_range(tlb, pmd, addr, next, details); + } while (pmd++, addr = next, addr != end); +} + +static inline void zap_pud_range(struct mmu_gather *tlb, pgd_t *pgd, + unsigned long addr, unsigned long end, + struct zap_details *details) +{ + pud_t *pud; + unsigned long next; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + zap_pmd_range(tlb, pud, addr, next, details); + } while (pud++, addr = next, addr != end); +} + +static void unmap_page_range(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, unsigned long end, + struct zap_details *details) +{ + pgd_t *pgd; + unsigned long next; + + if (details && !details->check_mapping && !details->nonlinear_vma) + details = NULL; + + BUG_ON(addr >= end); + tlb_start_vma(tlb, vma); + pgd = pgd_offset(vma->vm_mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + zap_pud_range(tlb, pgd, addr, next, details); + } while (pgd++, addr = next, addr != end); + tlb_end_vma(tlb, vma); +} + +#ifdef CONFIG_PREEMPT +# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE) +#else +/* No preempt: go for improved straight-line efficiency */ +# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE) +#endif + +/** + * unmap_vmas - unmap a range of memory covered by a list of vma's + * @tlbp: address of the caller's struct mmu_gather + * @mm: the controlling mm_struct + * @vma: the starting vma + * @start_addr: virtual address at which to start unmapping + * @end_addr: virtual address at which to end unmapping + * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here + * @details: details of nonlinear truncation or shared cache invalidation + * + * Returns the number of vma's which were covered by the unmapping. + * + * Unmap all pages in the vma list. Called under page_table_lock. + * + * We aim to not hold page_table_lock for too long (for scheduling latency + * reasons). So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to + * return the ending mmu_gather to the caller. + * + * Only addresses between `start' and `end' will be unmapped. + * + * The VMA list must be sorted in ascending virtual address order. + * + * unmap_vmas() assumes that the caller will flush the whole unmapped address + * range after unmap_vmas() returns. So the only responsibility here is to + * ensure that any thus-far unmapped pages are flushed before unmap_vmas() + * drops the lock and schedules. + */ +int unmap_vmas(struct mmu_gather **tlbp, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long start_addr, + unsigned long end_addr, unsigned long *nr_accounted, + struct zap_details *details) +{ + unsigned long zap_bytes = ZAP_BLOCK_SIZE; + unsigned long tlb_start = 0; /* For tlb_finish_mmu */ + int tlb_start_valid = 0; + int ret = 0; + spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL; + int fullmm = tlb_is_full_mm(*tlbp); + + for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) { + unsigned long start; + unsigned long end; + + start = max(vma->vm_start, start_addr); + if (start >= vma->vm_end) + continue; + end = min(vma->vm_end, end_addr); + if (end <= vma->vm_start) + continue; + + if (vma->vm_flags & VM_ACCOUNT) + *nr_accounted += (end - start) >> PAGE_SHIFT; + + ret++; + while (start != end) { + unsigned long block; + + if (!tlb_start_valid) { + tlb_start = start; + tlb_start_valid = 1; + } + + if (is_vm_hugetlb_page(vma)) { + block = end - start; + unmap_hugepage_range(vma, start, end); + } else { + block = min(zap_bytes, end - start); + unmap_page_range(*tlbp, vma, start, + start + block, details); + } + + start += block; + zap_bytes -= block; + if ((long)zap_bytes > 0) + continue; + + tlb_finish_mmu(*tlbp, tlb_start, start); + + if (need_resched() || + need_lockbreak(&mm->page_table_lock) || + (i_mmap_lock && need_lockbreak(i_mmap_lock))) { + if (i_mmap_lock) { + /* must reset count of rss freed */ + *tlbp = tlb_gather_mmu(mm, fullmm); + details->break_addr = start; + goto out; + } + spin_unlock(&mm->page_table_lock); + cond_resched(); + spin_lock(&mm->page_table_lock); + } + + *tlbp = tlb_gather_mmu(mm, fullmm); + tlb_start_valid = 0; + zap_bytes = ZAP_BLOCK_SIZE; + } + } +out: + return ret; +} + +/** + * zap_page_range - remove user pages in a given range + * @vma: vm_area_struct holding the applicable pages + * @address: starting address of pages to zap + * @size: number of bytes to zap + * @details: details of nonlinear truncation or shared cache invalidation + */ +void zap_page_range(struct vm_area_struct *vma, unsigned long address, + unsigned long size, struct zap_details *details) +{ + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb; + unsigned long end = address + size; + unsigned long nr_accounted = 0; + + if (is_vm_hugetlb_page(vma)) { + zap_hugepage_range(vma, address, size); + return; + } + + lru_add_drain(); + spin_lock(&mm->page_table_lock); + tlb = tlb_gather_mmu(mm, 0); + unmap_vmas(&tlb, mm, vma, address, end, &nr_accounted, details); + tlb_finish_mmu(tlb, address, end); + spin_unlock(&mm->page_table_lock); +} + +/* + * Do a quick page-table lookup for a single page. + * mm->page_table_lock must be held. + */ +static struct page * +__follow_page(struct mm_struct *mm, unsigned long address, int read, int write) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + unsigned long pfn; + struct page *page; + + page = follow_huge_addr(mm, address, write); + if (! IS_ERR(page)) + return page; + + pgd = pgd_offset(mm, address); + if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) + goto out; + + pud = pud_offset(pgd, address); + if (pud_none(*pud) || unlikely(pud_bad(*pud))) + goto out; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + goto out; + if (pmd_huge(*pmd)) + return follow_huge_pmd(mm, address, pmd, write); + + ptep = pte_offset_map(pmd, address); + if (!ptep) + goto out; + + pte = *ptep; + pte_unmap(ptep); + if (pte_present(pte)) { + if (write && !pte_write(pte)) + goto out; + if (read && !pte_read(pte)) + goto out; + pfn = pte_pfn(pte); + if (pfn_valid(pfn)) { + page = pfn_to_page(pfn); + if (write && !pte_dirty(pte) && !PageDirty(page)) + set_page_dirty(page); + mark_page_accessed(page); + return page; + } + } + +out: + return NULL; +} + +struct page * +follow_page(struct mm_struct *mm, unsigned long address, int write) +{ + return __follow_page(mm, address, /*read*/0, write); +} + +int +check_user_page_readable(struct mm_struct *mm, unsigned long address) +{ + return __follow_page(mm, address, /*read*/1, /*write*/0) != NULL; +} + +EXPORT_SYMBOL(check_user_page_readable); + +/* + * Given a physical address, is there a useful struct page pointing to + * it? This may become more complex in the future if we start dealing + * with IO-aperture pages for direct-IO. + */ + +static inline struct page *get_page_map(struct page *page) +{ + if (!pfn_valid(page_to_pfn(page))) + return NULL; + return page; +} + + +static inline int +untouched_anonymous_page(struct mm_struct* mm, struct vm_area_struct *vma, + unsigned long address) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + + /* Check if the vma is for an anonymous mapping. */ + if (vma->vm_ops && vma->vm_ops->nopage) + return 0; + + /* Check if page directory entry exists. */ + pgd = pgd_offset(mm, address); + if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) + return 1; + + pud = pud_offset(pgd, address); + if (pud_none(*pud) || unlikely(pud_bad(*pud))) + return 1; + + /* Check if page middle directory entry exists. */ + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return 1; + + /* There is a pte slot for 'address' in 'mm'. */ + return 0; +} + + +int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, + unsigned long start, int len, int write, int force, + struct page **pages, struct vm_area_struct **vmas) +{ + int i; + unsigned int flags; + + /* + * Require read or write permissions. + * If 'force' is set, we only require the "MAY" flags. + */ + flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); + flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); + i = 0; + + do { + struct vm_area_struct * vma; + + vma = find_extend_vma(mm, start); + if (!vma && in_gate_area(tsk, start)) { + unsigned long pg = start & PAGE_MASK; + struct vm_area_struct *gate_vma = get_gate_vma(tsk); + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + if (write) /* user gate pages are read-only */ + return i ? : -EFAULT; + if (pg > TASK_SIZE) + pgd = pgd_offset_k(pg); + else + pgd = pgd_offset_gate(mm, pg); + BUG_ON(pgd_none(*pgd)); + pud = pud_offset(pgd, pg); + BUG_ON(pud_none(*pud)); + pmd = pmd_offset(pud, pg); + BUG_ON(pmd_none(*pmd)); + pte = pte_offset_map(pmd, pg); + BUG_ON(pte_none(*pte)); + if (pages) { + pages[i] = pte_page(*pte); + get_page(pages[i]); + } + pte_unmap(pte); + if (vmas) + vmas[i] = gate_vma; + i++; + start += PAGE_SIZE; + len--; + continue; + } + + if (!vma || (vma->vm_flags & VM_IO) + || !(flags & vma->vm_flags)) + return i ? : -EFAULT; + + if (is_vm_hugetlb_page(vma)) { + i = follow_hugetlb_page(mm, vma, pages, vmas, + &start, &len, i); + continue; + } + spin_lock(&mm->page_table_lock); + do { + struct page *map; + int lookup_write = write; + + cond_resched_lock(&mm->page_table_lock); + while (!(map = follow_page(mm, start, lookup_write))) { + /* + * Shortcut for anonymous pages. We don't want + * to force the creation of pages tables for + * insanly big anonymously mapped areas that + * nobody touched so far. This is important + * for doing a core dump for these mappings. + */ + if (!lookup_write && + untouched_anonymous_page(mm,vma,start)) { + map = ZERO_PAGE(start); + break; + } + spin_unlock(&mm->page_table_lock); + switch (handle_mm_fault(mm,vma,start,write)) { + case VM_FAULT_MINOR: + tsk->min_flt++; + break; + case VM_FAULT_MAJOR: + tsk->maj_flt++; + break; + case VM_FAULT_SIGBUS: + return i ? i : -EFAULT; + case VM_FAULT_OOM: + return i ? i : -ENOMEM; + default: + BUG(); + } + /* + * Now that we have performed a write fault + * and surely no longer have a shared page we + * shouldn't write, we shouldn't ignore an + * unwritable page in the page table if + * we are forcing write access. + */ + lookup_write = write && !force; + spin_lock(&mm->page_table_lock); + } + if (pages) { + pages[i] = get_page_map(map); + if (!pages[i]) { + spin_unlock(&mm->page_table_lock); + while (i--) + page_cache_release(pages[i]); + i = -EFAULT; + goto out; + } + flush_dcache_page(pages[i]); + if (!PageReserved(pages[i])) + page_cache_get(pages[i]); + } + if (vmas) + vmas[i] = vma; + i++; + start += PAGE_SIZE; + len--; + } while(len && start < vma->vm_end); + spin_unlock(&mm->page_table_lock); + } while(len); +out: + return i; +} + +EXPORT_SYMBOL(get_user_pages); + +static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, unsigned long end, pgprot_t prot) +{ + pte_t *pte; + + pte = pte_alloc_map(mm, pmd, addr); + if (!pte) + return -ENOMEM; + do { + pte_t zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE(addr), prot)); + BUG_ON(!pte_none(*pte)); + set_pte_at(mm, addr, pte, zero_pte); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); + return 0; +} + +static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, pgprot_t prot) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + do { + next = pmd_addr_end(addr, end); + if (zeromap_pte_range(mm, pmd, addr, next, prot)) + return -ENOMEM; + } while (pmd++, addr = next, addr != end); + return 0; +} + +static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, pgprot_t prot) +{ + pud_t *pud; + unsigned long next; + + pud = pud_alloc(mm, pgd, addr); + if (!pud) + return -ENOMEM; + do { + next = pud_addr_end(addr, end); + if (zeromap_pmd_range(mm, pud, addr, next, prot)) + return -ENOMEM; + } while (pud++, addr = next, addr != end); + return 0; +} + +int zeromap_page_range(struct vm_area_struct *vma, + unsigned long addr, unsigned long size, pgprot_t prot) +{ + pgd_t *pgd; + unsigned long next; + unsigned long end = addr + size; + struct mm_struct *mm = vma->vm_mm; + int err; + + BUG_ON(addr >= end); + pgd = pgd_offset(mm, addr); + flush_cache_range(vma, addr, end); + spin_lock(&mm->page_table_lock); + do { + next = pgd_addr_end(addr, end); + err = zeromap_pud_range(mm, pgd, addr, next, prot); + if (err) + break; + } while (pgd++, addr = next, addr != end); + spin_unlock(&mm->page_table_lock); + return err; +} + +/* + * maps a range of physical memory into the requested pages. the old + * mappings are removed. any references to nonexistent pages results + * in null mappings (currently treated as "copy-on-access") + */ +static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, unsigned long end, + unsigned long pfn, pgprot_t prot) +{ + pte_t *pte; + + pte = pte_alloc_map(mm, pmd, addr); + if (!pte) + return -ENOMEM; + do { + BUG_ON(!pte_none(*pte)); + if (!pfn_valid(pfn) || PageReserved(pfn_to_page(pfn))) + set_pte_at(mm, addr, pte, pfn_pte(pfn, prot)); + pfn++; + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); + return 0; +} + +static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, + unsigned long pfn, pgprot_t prot) +{ + pmd_t *pmd; + unsigned long next; + + pfn -= addr >> PAGE_SHIFT; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + do { + next = pmd_addr_end(addr, end); + if (remap_pte_range(mm, pmd, addr, next, + pfn + (addr >> PAGE_SHIFT), prot)) + return -ENOMEM; + } while (pmd++, addr = next, addr != end); + return 0; +} + +static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, + unsigned long pfn, pgprot_t prot) +{ + pud_t *pud; + unsigned long next; + + pfn -= addr >> PAGE_SHIFT; + pud = pud_alloc(mm, pgd, addr); + if (!pud) + return -ENOMEM; + do { + next = pud_addr_end(addr, end); + if (remap_pmd_range(mm, pud, addr, next, + pfn + (addr >> PAGE_SHIFT), prot)) + return -ENOMEM; + } while (pud++, addr = next, addr != end); + return 0; +} + +/* Note: this is only safe if the mm semaphore is held when called. */ +int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned long size, pgprot_t prot) +{ + pgd_t *pgd; + unsigned long next; + unsigned long end = addr + size; + struct mm_struct *mm = vma->vm_mm; + int err; + + /* + * Physically remapped pages are special. Tell the + * rest of the world about it: + * VM_IO tells people not to look at these pages + * (accesses can have side effects). + * VM_RESERVED tells swapout not to try to touch + * this region. + */ + vma->vm_flags |= VM_IO | VM_RESERVED; + + BUG_ON(addr >= end); + pfn -= addr >> PAGE_SHIFT; + pgd = pgd_offset(mm, addr); + flush_cache_range(vma, addr, end); + spin_lock(&mm->page_table_lock); + do { + next = pgd_addr_end(addr, end); + err = remap_pud_range(mm, pgd, addr, next, + pfn + (addr >> PAGE_SHIFT), prot); + if (err) + break; + } while (pgd++, addr = next, addr != end); + spin_unlock(&mm->page_table_lock); + return err; +} +EXPORT_SYMBOL(remap_pfn_range); + +/* + * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when + * servicing faults for write access. In the normal case, do always want + * pte_mkwrite. But get_user_pages can cause write faults for mappings + * that do not have writing enabled, when used by access_process_vm. + */ +static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) +{ + if (likely(vma->vm_flags & VM_WRITE)) + pte = pte_mkwrite(pte); + return pte; +} + +/* + * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock + */ +static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address, + pte_t *page_table) +{ + pte_t entry; + + entry = maybe_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)), + vma); + ptep_establish(vma, address, page_table, entry); + update_mmu_cache(vma, address, entry); + lazy_mmu_prot_update(entry); +} + +/* + * This routine handles present pages, when users try to write + * to a shared page. It is done by copying the page to a new address + * and decrementing the shared-page counter for the old page. + * + * Goto-purists beware: the only reason for goto's here is that it results + * in better assembly code.. The "default" path will see no jumps at all. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus we can safely just mark it writable once we've done any necessary + * COW. + * + * We also mark the page dirty at this point even though the page will + * change only once the write actually happens. This avoids a few races, + * and potentially makes it more efficient. + * + * We hold the mm semaphore and the page_table_lock on entry and exit + * with the page_table_lock released. + */ +static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma, + unsigned long address, pte_t *page_table, pmd_t *pmd, pte_t pte) +{ + struct page *old_page, *new_page; + unsigned long pfn = pte_pfn(pte); + pte_t entry; + + if (unlikely(!pfn_valid(pfn))) { + /* + * This should really halt the system so it can be debugged or + * at least the kernel stops what it's doing before it corrupts + * data, but for the moment just pretend this is OOM. + */ + pte_unmap(page_table); + printk(KERN_ERR "do_wp_page: bogus page at address %08lx\n", + address); + spin_unlock(&mm->page_table_lock); + return VM_FAULT_OOM; + } + old_page = pfn_to_page(pfn); + + if (!TestSetPageLocked(old_page)) { + int reuse = can_share_swap_page(old_page); + unlock_page(old_page); + if (reuse) { + flush_cache_page(vma, address, pfn); + entry = maybe_mkwrite(pte_mkyoung(pte_mkdirty(pte)), + vma); + ptep_set_access_flags(vma, address, page_table, entry, 1); + update_mmu_cache(vma, address, entry); + lazy_mmu_prot_update(entry); + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + return VM_FAULT_MINOR; + } + } + pte_unmap(page_table); + + /* + * Ok, we need to copy. Oh, well.. + */ + if (!PageReserved(old_page)) + page_cache_get(old_page); + spin_unlock(&mm->page_table_lock); + + if (unlikely(anon_vma_prepare(vma))) + goto no_new_page; + if (old_page == ZERO_PAGE(address)) { + new_page = alloc_zeroed_user_highpage(vma, address); + if (!new_page) + goto no_new_page; + } else { + new_page = alloc_page_vma(GFP_HIGHUSER, vma, address); + if (!new_page) + goto no_new_page; + copy_user_highpage(new_page, old_page, address); + } + /* + * Re-check the pte - we dropped the lock + */ + spin_lock(&mm->page_table_lock); + page_table = pte_offset_map(pmd, address); + if (likely(pte_same(*page_table, pte))) { + if (PageAnon(old_page)) + dec_mm_counter(mm, anon_rss); + if (PageReserved(old_page)) + inc_mm_counter(mm, rss); + else + page_remove_rmap(old_page); + flush_cache_page(vma, address, pfn); + break_cow(vma, new_page, address, page_table); + lru_cache_add_active(new_page); + page_add_anon_rmap(new_page, vma, address); + + /* Free the old page.. */ + new_page = old_page; + } + pte_unmap(page_table); + page_cache_release(new_page); + page_cache_release(old_page); + spin_unlock(&mm->page_table_lock); + return VM_FAULT_MINOR; + +no_new_page: + page_cache_release(old_page); + return VM_FAULT_OOM; +} + +/* + * Helper functions for unmap_mapping_range(). + * + * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __ + * + * We have to restart searching the prio_tree whenever we drop the lock, + * since the iterator is only valid while the lock is held, and anyway + * a later vma might be split and reinserted earlier while lock dropped. + * + * The list of nonlinear vmas could be handled more efficiently, using + * a placeholder, but handle it in the same way until a need is shown. + * It is important to search the prio_tree before nonlinear list: a vma + * may become nonlinear and be shifted from prio_tree to nonlinear list + * while the lock is dropped; but never shifted from list to prio_tree. + * + * In order to make forward progress despite restarting the search, + * vm_truncate_count is used to mark a vma as now dealt with, so we can + * quickly skip it next time around. Since the prio_tree search only + * shows us those vmas affected by unmapping the range in question, we + * can't efficiently keep all vmas in step with mapping->truncate_count: + * so instead reset them all whenever it wraps back to 0 (then go to 1). + * mapping->truncate_count and vma->vm_truncate_count are protected by + * i_mmap_lock. + * + * In order to make forward progress despite repeatedly restarting some + * large vma, note the break_addr set by unmap_vmas when it breaks out: + * and restart from that address when we reach that vma again. It might + * have been split or merged, shrunk or extended, but never shifted: so + * restart_addr remains valid so long as it remains in the vma's range. + * unmap_mapping_range forces truncate_count to leap over page-aligned + * values so we can save vma's restart_addr in its truncate_count field. + */ +#define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK)) + +static void reset_vma_truncate_counts(struct address_space *mapping) +{ + struct vm_area_struct *vma; + struct prio_tree_iter iter; + + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) + vma->vm_truncate_count = 0; + list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) + vma->vm_truncate_count = 0; +} + +static int unmap_mapping_range_vma(struct vm_area_struct *vma, + unsigned long start_addr, unsigned long end_addr, + struct zap_details *details) +{ + unsigned long restart_addr; + int need_break; + +again: + restart_addr = vma->vm_truncate_count; + if (is_restart_addr(restart_addr) && start_addr < restart_addr) { + start_addr = restart_addr; + if (start_addr >= end_addr) { + /* Top of vma has been split off since last time */ + vma->vm_truncate_count = details->truncate_count; + return 0; + } + } + + details->break_addr = end_addr; + zap_page_range(vma, start_addr, end_addr - start_addr, details); + + /* + * We cannot rely on the break test in unmap_vmas: + * on the one hand, we don't want to restart our loop + * just because that broke out for the page_table_lock; + * on the other hand, it does no test when vma is small. + */ + need_break = need_resched() || + need_lockbreak(details->i_mmap_lock); + + if (details->break_addr >= end_addr) { + /* We have now completed this vma: mark it so */ + vma->vm_truncate_count = details->truncate_count; + if (!need_break) + return 0; + } else { + /* Note restart_addr in vma's truncate_count field */ + vma->vm_truncate_count = details->break_addr; + if (!need_break) + goto again; + } + + spin_unlock(details->i_mmap_lock); + cond_resched(); + spin_lock(details->i_mmap_lock); + return -EINTR; +} + +static inline void unmap_mapping_range_tree(struct prio_tree_root *root, + struct zap_details *details) +{ + struct vm_area_struct *vma; + struct prio_tree_iter iter; + pgoff_t vba, vea, zba, zea; + +restart: + vma_prio_tree_foreach(vma, &iter, root, + details->first_index, details->last_index) { + /* Skip quickly over those we have already dealt with */ + if (vma->vm_truncate_count == details->truncate_count) + continue; + + vba = vma->vm_pgoff; + vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; + /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ + zba = details->first_index; + if (zba < vba) + zba = vba; + zea = details->last_index; + if (zea > vea) + zea = vea; + + if (unmap_mapping_range_vma(vma, + ((zba - vba) << PAGE_SHIFT) + vma->vm_start, + ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, + details) < 0) + goto restart; + } +} + +static inline void unmap_mapping_range_list(struct list_head *head, + struct zap_details *details) +{ + struct vm_area_struct *vma; + + /* + * In nonlinear VMAs there is no correspondence between virtual address + * offset and file offset. So we must perform an exhaustive search + * across *all* the pages in each nonlinear VMA, not just the pages + * whose virtual address lies outside the file truncation point. + */ +restart: + list_for_each_entry(vma, head, shared.vm_set.list) { + /* Skip quickly over those we have already dealt with */ + if (vma->vm_truncate_count == details->truncate_count) + continue; + details->nonlinear_vma = vma; + if (unmap_mapping_range_vma(vma, vma->vm_start, + vma->vm_end, details) < 0) + goto restart; + } +} + +/** + * unmap_mapping_range - unmap the portion of all mmaps + * in the specified address_space corresponding to the specified + * page range in the underlying file. + * @address_space: the address space containing mmaps to be unmapped. + * @holebegin: byte in first page to unmap, relative to the start of + * the underlying file. This will be rounded down to a PAGE_SIZE + * boundary. Note that this is different from vmtruncate(), which + * must keep the partial page. In contrast, we must get rid of + * partial pages. + * @holelen: size of prospective hole in bytes. This will be rounded + * up to a PAGE_SIZE boundary. A holelen of zero truncates to the + * end of the file. + * @even_cows: 1 when truncating a file, unmap even private COWed pages; + * but 0 when invalidating pagecache, don't throw away private data. + */ +void unmap_mapping_range(struct address_space *mapping, + loff_t const holebegin, loff_t const holelen, int even_cows) +{ + struct zap_details details; + pgoff_t hba = holebegin >> PAGE_SHIFT; + pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; + + /* Check for overflow. */ + if (sizeof(holelen) > sizeof(hlen)) { + long long holeend = + (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (holeend & ~(long long)ULONG_MAX) + hlen = ULONG_MAX - hba + 1; + } + + details.check_mapping = even_cows? NULL: mapping; + details.nonlinear_vma = NULL; + details.first_index = hba; + details.last_index = hba + hlen - 1; + if (details.last_index < details.first_index) + details.last_index = ULONG_MAX; + details.i_mmap_lock = &mapping->i_mmap_lock; + + spin_lock(&mapping->i_mmap_lock); + + /* serialize i_size write against truncate_count write */ + smp_wmb(); + /* Protect against page faults, and endless unmapping loops */ + mapping->truncate_count++; + /* + * For archs where spin_lock has inclusive semantics like ia64 + * this smp_mb() will prevent to read pagetable contents + * before the truncate_count increment is visible to + * other cpus. + */ + smp_mb(); + if (unlikely(is_restart_addr(mapping->truncate_count))) { + if (mapping->truncate_count == 0) + reset_vma_truncate_counts(mapping); + mapping->truncate_count++; + } + details.truncate_count = mapping->truncate_count; + + if (unlikely(!prio_tree_empty(&mapping->i_mmap))) + unmap_mapping_range_tree(&mapping->i_mmap, &details); + if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) + unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); + spin_unlock(&mapping->i_mmap_lock); +} +EXPORT_SYMBOL(unmap_mapping_range); + +/* + * Handle all mappings that got truncated by a "truncate()" + * system call. + * + * NOTE! We have to be ready to update the memory sharing + * between the file and the memory map for a potential last + * incomplete page. Ugly, but necessary. + */ +int vmtruncate(struct inode * inode, loff_t offset) +{ + struct address_space *mapping = inode->i_mapping; + unsigned long limit; + + if (inode->i_size < offset) + goto do_expand; + /* + * truncation of in-use swapfiles is disallowed - it would cause + * subsequent swapout to scribble on the now-freed blocks. + */ + if (IS_SWAPFILE(inode)) + goto out_busy; + i_size_write(inode, offset); + unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1); + truncate_inode_pages(mapping, offset); + goto out_truncate; + +do_expand: + limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; + if (limit != RLIM_INFINITY && offset > limit) + goto out_sig; + if (offset > inode->i_sb->s_maxbytes) + goto out_big; + i_size_write(inode, offset); + +out_truncate: + if (inode->i_op && inode->i_op->truncate) + inode->i_op->truncate(inode); + return 0; +out_sig: + send_sig(SIGXFSZ, current, 0); +out_big: + return -EFBIG; +out_busy: + return -ETXTBSY; +} + +EXPORT_SYMBOL(vmtruncate); + +/* + * Primitive swap readahead code. We simply read an aligned block of + * (1 << page_cluster) entries in the swap area. This method is chosen + * because it doesn't cost us any seek time. We also make sure to queue + * the 'original' request together with the readahead ones... + * + * This has been extended to use the NUMA policies from the mm triggering + * the readahead. + * + * Caller must hold down_read on the vma->vm_mm if vma is not NULL. + */ +void swapin_readahead(swp_entry_t entry, unsigned long addr,struct vm_area_struct *vma) +{ +#ifdef CONFIG_NUMA + struct vm_area_struct *next_vma = vma ? vma->vm_next : NULL; +#endif + int i, num; + struct page *new_page; + unsigned long offset; + + /* + * Get the number of handles we should do readahead io to. + */ + num = valid_swaphandles(entry, &offset); + for (i = 0; i < num; offset++, i++) { + /* Ok, do the async read-ahead now */ + new_page = read_swap_cache_async(swp_entry(swp_type(entry), + offset), vma, addr); + if (!new_page) + break; + page_cache_release(new_page); +#ifdef CONFIG_NUMA + /* + * Find the next applicable VMA for the NUMA policy. + */ + addr += PAGE_SIZE; + if (addr == 0) + vma = NULL; + if (vma) { + if (addr >= vma->vm_end) { + vma = next_vma; + next_vma = vma ? vma->vm_next : NULL; + } + if (vma && addr < vma->vm_start) + vma = NULL; + } else { + if (next_vma && addr >= next_vma->vm_start) { + vma = next_vma; + next_vma = vma->vm_next; + } + } +#endif + } + lru_add_drain(); /* Push any new pages onto the LRU now */ +} + +/* + * We hold the mm semaphore and the page_table_lock on entry and + * should release the pagetable lock on exit.. + */ +static int do_swap_page(struct mm_struct * mm, + struct vm_area_struct * vma, unsigned long address, + pte_t *page_table, pmd_t *pmd, pte_t orig_pte, int write_access) +{ + struct page *page; + swp_entry_t entry = pte_to_swp_entry(orig_pte); + pte_t pte; + int ret = VM_FAULT_MINOR; + + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + page = lookup_swap_cache(entry); + if (!page) { + swapin_readahead(entry, address, vma); + page = read_swap_cache_async(entry, vma, address); + if (!page) { + /* + * Back out if somebody else faulted in this pte while + * we released the page table lock. + */ + spin_lock(&mm->page_table_lock); + page_table = pte_offset_map(pmd, address); + if (likely(pte_same(*page_table, orig_pte))) + ret = VM_FAULT_OOM; + else + ret = VM_FAULT_MINOR; + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + goto out; + } + + /* Had to read the page from swap area: Major fault */ + ret = VM_FAULT_MAJOR; + inc_page_state(pgmajfault); + grab_swap_token(); + } + + mark_page_accessed(page); + lock_page(page); + + /* + * Back out if somebody else faulted in this pte while we + * released the page table lock. + */ + spin_lock(&mm->page_table_lock); + page_table = pte_offset_map(pmd, address); + if (unlikely(!pte_same(*page_table, orig_pte))) { + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + unlock_page(page); + page_cache_release(page); + ret = VM_FAULT_MINOR; + goto out; + } + + /* The page isn't present yet, go ahead with the fault. */ + + swap_free(entry); + if (vm_swap_full()) + remove_exclusive_swap_page(page); + + inc_mm_counter(mm, rss); + pte = mk_pte(page, vma->vm_page_prot); + if (write_access && can_share_swap_page(page)) { + pte = maybe_mkwrite(pte_mkdirty(pte), vma); + write_access = 0; + } + unlock_page(page); + + flush_icache_page(vma, page); + set_pte_at(mm, address, page_table, pte); + page_add_anon_rmap(page, vma, address); + + if (write_access) { + if (do_wp_page(mm, vma, address, + page_table, pmd, pte) == VM_FAULT_OOM) + ret = VM_FAULT_OOM; + goto out; + } + + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, address, pte); + lazy_mmu_prot_update(pte); + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); +out: + return ret; +} + +/* + * We are called with the MM semaphore and page_table_lock + * spinlock held to protect against concurrent faults in + * multithreaded programs. + */ +static int +do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, + pte_t *page_table, pmd_t *pmd, int write_access, + unsigned long addr) +{ + pte_t entry; + struct page * page = ZERO_PAGE(addr); + + /* Read-only mapping of ZERO_PAGE. */ + entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot)); + + /* ..except if it's a write access */ + if (write_access) { + /* Allocate our own private page. */ + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + + if (unlikely(anon_vma_prepare(vma))) + goto no_mem; + page = alloc_zeroed_user_highpage(vma, addr); + if (!page) + goto no_mem; + + spin_lock(&mm->page_table_lock); + page_table = pte_offset_map(pmd, addr); + + if (!pte_none(*page_table)) { + pte_unmap(page_table); + page_cache_release(page); + spin_unlock(&mm->page_table_lock); + goto out; + } + inc_mm_counter(mm, rss); + entry = maybe_mkwrite(pte_mkdirty(mk_pte(page, + vma->vm_page_prot)), + vma); + lru_cache_add_active(page); + SetPageReferenced(page); + page_add_anon_rmap(page, vma, addr); + } + + set_pte_at(mm, addr, page_table, entry); + pte_unmap(page_table); + + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, addr, entry); + lazy_mmu_prot_update(entry); + spin_unlock(&mm->page_table_lock); +out: + return VM_FAULT_MINOR; +no_mem: + return VM_FAULT_OOM; +} + +/* + * do_no_page() tries to create a new page mapping. It aggressively + * tries to share with existing pages, but makes a separate copy if + * the "write_access" parameter is true in order to avoid the next + * page fault. + * + * As this is called only for pages that do not currently exist, we + * do not need to flush old virtual caches or the TLB. + * + * This is called with the MM semaphore held and the page table + * spinlock held. Exit with the spinlock released. + */ +static int +do_no_page(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long address, int write_access, pte_t *page_table, pmd_t *pmd) +{ + struct page * new_page; + struct address_space *mapping = NULL; + pte_t entry; + unsigned int sequence = 0; + int ret = VM_FAULT_MINOR; + int anon = 0; + + if (!vma->vm_ops || !vma->vm_ops->nopage) + return do_anonymous_page(mm, vma, page_table, + pmd, write_access, address); + pte_unmap(page_table); + spin_unlock(&mm->page_table_lock); + + if (vma->vm_file) { + mapping = vma->vm_file->f_mapping; + sequence = mapping->truncate_count; + smp_rmb(); /* serializes i_size against truncate_count */ + } +retry: + cond_resched(); + new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, &ret); + /* + * No smp_rmb is needed here as long as there's a full + * spin_lock/unlock sequence inside the ->nopage callback + * (for the pagecache lookup) that acts as an implicit + * smp_mb() and prevents the i_size read to happen + * after the next truncate_count read. + */ + + /* no page was available -- either SIGBUS or OOM */ + if (new_page == NOPAGE_SIGBUS) + return VM_FAULT_SIGBUS; + if (new_page == NOPAGE_OOM) + return VM_FAULT_OOM; + + /* + * Should we do an early C-O-W break? + */ + if (write_access && !(vma->vm_flags & VM_SHARED)) { + struct page *page; + + if (unlikely(anon_vma_prepare(vma))) + goto oom; + page = alloc_page_vma(GFP_HIGHUSER, vma, address); + if (!page) + goto oom; + copy_user_highpage(page, new_page, address); + page_cache_release(new_page); + new_page = page; + anon = 1; + } + + spin_lock(&mm->page_table_lock); + /* + * For a file-backed vma, someone could have truncated or otherwise + * invalidated this page. If unmap_mapping_range got called, + * retry getting the page. + */ + if (mapping && unlikely(sequence != mapping->truncate_count)) { + sequence = mapping->truncate_count; + spin_unlock(&mm->page_table_lock); + page_cache_release(new_page); + goto retry; + } + page_table = pte_offset_map(pmd, address); + + /* + * This silly early PAGE_DIRTY setting removes a race + * due to the bad i386 page protection. But it's valid + * for other architectures too. + * + * Note that if write_access is true, we either now have + * an exclusive copy of the page, or this is a shared mapping, + * so we can make it writable and dirty to avoid having to + * handle that later. + */ + /* Only go through if we didn't race with anybody else... */ + if (pte_none(*page_table)) { + if (!PageReserved(new_page)) + inc_mm_counter(mm, rss); + + flush_icache_page(vma, new_page); + entry = mk_pte(new_page, vma->vm_page_prot); + if (write_access) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + set_pte_at(mm, address, page_table, entry); + if (anon) { + lru_cache_add_active(new_page); + page_add_anon_rmap(new_page, vma, address); + } else + page_add_file_rmap(new_page); + pte_unmap(page_table); + } else { + /* One of our sibling threads was faster, back out. */ + pte_unmap(page_table); + page_cache_release(new_page); + spin_unlock(&mm->page_table_lock); + goto out; + } + + /* no need to invalidate: a not-present page shouldn't be cached */ + update_mmu_cache(vma, address, entry); + lazy_mmu_prot_update(entry); + spin_unlock(&mm->page_table_lock); +out: + return ret; +oom: + page_cache_release(new_page); + ret = VM_FAULT_OOM; + goto out; +} + +/* + * Fault of a previously existing named mapping. Repopulate the pte + * from the encoded file_pte if possible. This enables swappable + * nonlinear vmas. + */ +static int do_file_page(struct mm_struct * mm, struct vm_area_struct * vma, + unsigned long address, int write_access, pte_t *pte, pmd_t *pmd) +{ + unsigned long pgoff; + int err; + + BUG_ON(!vma->vm_ops || !vma->vm_ops->nopage); + /* + * Fall back to the linear mapping if the fs does not support + * ->populate: + */ + if (!vma->vm_ops || !vma->vm_ops->populate || + (write_access && !(vma->vm_flags & VM_SHARED))) { + pte_clear(mm, address, pte); + return do_no_page(mm, vma, address, write_access, pte, pmd); + } + + pgoff = pte_to_pgoff(*pte); + + pte_unmap(pte); + spin_unlock(&mm->page_table_lock); + + err = vma->vm_ops->populate(vma, address & PAGE_MASK, PAGE_SIZE, vma->vm_page_prot, pgoff, 0); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err) + return VM_FAULT_SIGBUS; + return VM_FAULT_MAJOR; +} + +/* + * These routines also need to handle stuff like marking pages dirty + * and/or accessed for architectures that don't do it in hardware (most + * RISC architectures). The early dirtying is also good on the i386. + * + * There is also a hook called "update_mmu_cache()" that architectures + * with external mmu caches can use to update those (ie the Sparc or + * PowerPC hashed page tables that act as extended TLBs). + * + * Note the "page_table_lock". It is to protect against kswapd removing + * pages from under us. Note that kswapd only ever _removes_ pages, never + * adds them. As such, once we have noticed that the page is not present, + * we can drop the lock early. + * + * The adding of pages is protected by the MM semaphore (which we hold), + * so we don't need to worry about a page being suddenly been added into + * our VM. + * + * We enter with the pagetable spinlock held, we are supposed to + * release it when done. + */ +static inline int handle_pte_fault(struct mm_struct *mm, + struct vm_area_struct * vma, unsigned long address, + int write_access, pte_t *pte, pmd_t *pmd) +{ + pte_t entry; + + entry = *pte; + if (!pte_present(entry)) { + /* + * If it truly wasn't present, we know that kswapd + * and the PTE updates will not touch it later. So + * drop the lock. + */ + if (pte_none(entry)) + return do_no_page(mm, vma, address, write_access, pte, pmd); + if (pte_file(entry)) + return do_file_page(mm, vma, address, write_access, pte, pmd); + return do_swap_page(mm, vma, address, pte, pmd, entry, write_access); + } + + if (write_access) { + if (!pte_write(entry)) + return do_wp_page(mm, vma, address, pte, pmd, entry); + + entry = pte_mkdirty(entry); + } + entry = pte_mkyoung(entry); + ptep_set_access_flags(vma, address, pte, entry, write_access); + update_mmu_cache(vma, address, entry); + lazy_mmu_prot_update(entry); + pte_unmap(pte); + spin_unlock(&mm->page_table_lock); + return VM_FAULT_MINOR; +} + +/* + * By the time we get here, we already hold the mm semaphore + */ +int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma, + unsigned long address, int write_access) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + __set_current_state(TASK_RUNNING); + + inc_page_state(pgfault); + + if (is_vm_hugetlb_page(vma)) + return VM_FAULT_SIGBUS; /* mapping truncation does this. */ + + /* + * We need the page table lock to synchronize with kswapd + * and the SMP-safe atomic PTE updates. + */ + pgd = pgd_offset(mm, address); + spin_lock(&mm->page_table_lock); + + pud = pud_alloc(mm, pgd, address); + if (!pud) + goto oom; + + pmd = pmd_alloc(mm, pud, address); + if (!pmd) + goto oom; + + pte = pte_alloc_map(mm, pmd, address); + if (!pte) + goto oom; + + return handle_pte_fault(mm, vma, address, write_access, pte, pmd); + + oom: + spin_unlock(&mm->page_table_lock); + return VM_FAULT_OOM; +} + +#ifndef __PAGETABLE_PUD_FOLDED +/* + * Allocate page upper directory. + * + * We've already handled the fast-path in-line, and we own the + * page table lock. + */ +pud_t fastcall *__pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) +{ + pud_t *new; + + spin_unlock(&mm->page_table_lock); + new = pud_alloc_one(mm, address); + spin_lock(&mm->page_table_lock); + if (!new) + return NULL; + + /* + * Because we dropped the lock, we should re-check the + * entry, as somebody else could have populated it.. + */ + if (pgd_present(*pgd)) { + pud_free(new); + goto out; + } + pgd_populate(mm, pgd, new); + out: + return pud_offset(pgd, address); +} +#endif /* __PAGETABLE_PUD_FOLDED */ + +#ifndef __PAGETABLE_PMD_FOLDED +/* + * Allocate page middle directory. + * + * We've already handled the fast-path in-line, and we own the + * page table lock. + */ +pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) +{ + pmd_t *new; + + spin_unlock(&mm->page_table_lock); + new = pmd_alloc_one(mm, address); + spin_lock(&mm->page_table_lock); + if (!new) + return NULL; + + /* + * Because we dropped the lock, we should re-check the + * entry, as somebody else could have populated it.. + */ +#ifndef __ARCH_HAS_4LEVEL_HACK + if (pud_present(*pud)) { + pmd_free(new); + goto out; + } + pud_populate(mm, pud, new); +#else + if (pgd_present(*pud)) { + pmd_free(new); + goto out; + } + pgd_populate(mm, pud, new); +#endif /* __ARCH_HAS_4LEVEL_HACK */ + + out: + return pmd_offset(pud, address); +} +#endif /* __PAGETABLE_PMD_FOLDED */ + +int make_pages_present(unsigned long addr, unsigned long end) +{ + int ret, len, write; + struct vm_area_struct * vma; + + vma = find_vma(current->mm, addr); + if (!vma) + return -1; + write = (vma->vm_flags & VM_WRITE) != 0; + if (addr >= end) + BUG(); + if (end > vma->vm_end) + BUG(); + len = (end+PAGE_SIZE-1)/PAGE_SIZE-addr/PAGE_SIZE; + ret = get_user_pages(current, current->mm, addr, + len, write, 0, NULL, NULL); + if (ret < 0) + return ret; + return ret == len ? 0 : -1; +} + +/* + * Map a vmalloc()-space virtual address to the physical page. + */ +struct page * vmalloc_to_page(void * vmalloc_addr) +{ + unsigned long addr = (unsigned long) vmalloc_addr; + struct page *page = NULL; + pgd_t *pgd = pgd_offset_k(addr); + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + + if (!pgd_none(*pgd)) { + pud = pud_offset(pgd, addr); + if (!pud_none(*pud)) { + pmd = pmd_offset(pud, addr); + if (!pmd_none(*pmd)) { + ptep = pte_offset_map(pmd, addr); + pte = *ptep; + if (pte_present(pte)) + page = pte_page(pte); + pte_unmap(ptep); + } + } + } + return page; +} + +EXPORT_SYMBOL(vmalloc_to_page); + +/* + * Map a vmalloc()-space virtual address to the physical page frame number. + */ +unsigned long vmalloc_to_pfn(void * vmalloc_addr) +{ + return page_to_pfn(vmalloc_to_page(vmalloc_addr)); +} + +EXPORT_SYMBOL(vmalloc_to_pfn); + +/* + * update_mem_hiwater + * - update per process rss and vm high water data + */ +void update_mem_hiwater(struct task_struct *tsk) +{ + if (tsk->mm) { + unsigned long rss = get_mm_counter(tsk->mm, rss); + + if (tsk->mm->hiwater_rss < rss) + tsk->mm->hiwater_rss = rss; + if (tsk->mm->hiwater_vm < tsk->mm->total_vm) + tsk->mm->hiwater_vm = tsk->mm->total_vm; + } +} + +#if !defined(__HAVE_ARCH_GATE_AREA) + +#if defined(AT_SYSINFO_EHDR) +struct vm_area_struct gate_vma; + +static int __init gate_vma_init(void) +{ + gate_vma.vm_mm = NULL; + gate_vma.vm_start = FIXADDR_USER_START; + gate_vma.vm_end = FIXADDR_USER_END; + gate_vma.vm_page_prot = PAGE_READONLY; + gate_vma.vm_flags = 0; + return 0; +} +__initcall(gate_vma_init); +#endif + +struct vm_area_struct *get_gate_vma(struct task_struct *tsk) +{ +#ifdef AT_SYSINFO_EHDR + return &gate_vma; +#else + return NULL; +#endif +} + +int in_gate_area_no_task(unsigned long addr) +{ +#ifdef AT_SYSINFO_EHDR + if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) + return 1; +#endif + return 0; +} + +#endif /* __HAVE_ARCH_GATE_AREA */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c new file mode 100644 index 000000000000..a3b44a671cec --- /dev/null +++ b/mm/mempolicy.c @@ -0,0 +1,1138 @@ +/* + * Simple NUMA memory policy for the Linux kernel. + * + * Copyright 2003,2004 Andi Kleen, SuSE Labs. + * Subject to the GNU Public License, version 2. + * + * NUMA policy allows the user to give hints in which node(s) memory should + * be allocated. + * + * Support four policies per VMA and per process: + * + * The VMA policy has priority over the process policy for a page fault. + * + * interleave Allocate memory interleaved over a set of nodes, + * with normal fallback if it fails. + * For VMA based allocations this interleaves based on the + * offset into the backing object or offset into the mapping + * for anonymous memory. For process policy an process counter + * is used. + * bind Only allocate memory on a specific set of nodes, + * no fallback. + * preferred Try a specific node first before normal fallback. + * As a special case node -1 here means do the allocation + * on the local CPU. This is normally identical to default, + * but useful to set in a VMA when you have a non default + * process policy. + * default Allocate on the local node first, or when on a VMA + * use the process policy. This is what Linux always did + * in a NUMA aware kernel and still does by, ahem, default. + * + * The process policy is applied for most non interrupt memory allocations + * in that process' context. Interrupts ignore the policies and always + * try to allocate on the local CPU. The VMA policy is only applied for memory + * allocations for a VMA in the VM. + * + * Currently there are a few corner cases in swapping where the policy + * is not applied, but the majority should be handled. When process policy + * is used it is not remembered over swap outs/swap ins. + * + * Only the highest zone in the zone hierarchy gets policied. Allocations + * requesting a lower zone just use default policy. This implies that + * on systems with highmem kernel lowmem allocation don't get policied. + * Same with GFP_DMA allocations. + * + * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between + * all users and remembered even when nobody has memory mapped. + */ + +/* Notebook: + fix mmap readahead to honour policy and enable policy for any page cache + object + statistics for bigpages + global policy for page cache? currently it uses process policy. Requires + first item above. + handle mremap for shared memory (currently ignored for the policy) + grows down? + make bind policy root only? It can trigger oom much faster and the + kernel is not always grateful with that. + could replace all the switch()es with a mempolicy_ops structure. +*/ + +#include <linux/mempolicy.h> +#include <linux/mm.h> +#include <linux/highmem.h> +#include <linux/hugetlb.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/nodemask.h> +#include <linux/cpuset.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/compat.h> +#include <linux/mempolicy.h> +#include <asm/tlbflush.h> +#include <asm/uaccess.h> + +static kmem_cache_t *policy_cache; +static kmem_cache_t *sn_cache; + +#define PDprintk(fmt...) + +/* Highest zone. An specific allocation for a zone below that is not + policied. */ +static int policy_zone; + +static struct mempolicy default_policy = { + .refcnt = ATOMIC_INIT(1), /* never free it */ + .policy = MPOL_DEFAULT, +}; + +/* Check if all specified nodes are online */ +static int nodes_online(unsigned long *nodes) +{ + DECLARE_BITMAP(online2, MAX_NUMNODES); + + bitmap_copy(online2, nodes_addr(node_online_map), MAX_NUMNODES); + if (bitmap_empty(online2, MAX_NUMNODES)) + set_bit(0, online2); + if (!bitmap_subset(nodes, online2, MAX_NUMNODES)) + return -EINVAL; + return 0; +} + +/* Do sanity checking on a policy */ +static int mpol_check_policy(int mode, unsigned long *nodes) +{ + int empty = bitmap_empty(nodes, MAX_NUMNODES); + + switch (mode) { + case MPOL_DEFAULT: + if (!empty) + return -EINVAL; + break; + case MPOL_BIND: + case MPOL_INTERLEAVE: + /* Preferred will only use the first bit, but allow + more for now. */ + if (empty) + return -EINVAL; + break; + } + return nodes_online(nodes); +} + +/* Copy a node mask from user space. */ +static int get_nodes(unsigned long *nodes, unsigned long __user *nmask, + unsigned long maxnode, int mode) +{ + unsigned long k; + unsigned long nlongs; + unsigned long endmask; + + --maxnode; + bitmap_zero(nodes, MAX_NUMNODES); + if (maxnode == 0 || !nmask) + return 0; + + nlongs = BITS_TO_LONGS(maxnode); + if ((maxnode % BITS_PER_LONG) == 0) + endmask = ~0UL; + else + endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1; + + /* When the user specified more nodes than supported just check + if the non supported part is all zero. */ + if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) { + if (nlongs > PAGE_SIZE/sizeof(long)) + return -EINVAL; + for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) { + unsigned long t; + if (get_user(t, nmask + k)) + return -EFAULT; + if (k == nlongs - 1) { + if (t & endmask) + return -EINVAL; + } else if (t) + return -EINVAL; + } + nlongs = BITS_TO_LONGS(MAX_NUMNODES); + endmask = ~0UL; + } + + if (copy_from_user(nodes, nmask, nlongs*sizeof(unsigned long))) + return -EFAULT; + nodes[nlongs-1] &= endmask; + /* Update current mems_allowed */ + cpuset_update_current_mems_allowed(); + /* Ignore nodes not set in current->mems_allowed */ + cpuset_restrict_to_mems_allowed(nodes); + return mpol_check_policy(mode, nodes); +} + +/* Generate a custom zonelist for the BIND policy. */ +static struct zonelist *bind_zonelist(unsigned long *nodes) +{ + struct zonelist *zl; + int num, max, nd; + + max = 1 + MAX_NR_ZONES * bitmap_weight(nodes, MAX_NUMNODES); + zl = kmalloc(sizeof(void *) * max, GFP_KERNEL); + if (!zl) + return NULL; + num = 0; + for (nd = find_first_bit(nodes, MAX_NUMNODES); + nd < MAX_NUMNODES; + nd = find_next_bit(nodes, MAX_NUMNODES, 1+nd)) { + int k; + for (k = MAX_NR_ZONES-1; k >= 0; k--) { + struct zone *z = &NODE_DATA(nd)->node_zones[k]; + if (!z->present_pages) + continue; + zl->zones[num++] = z; + if (k > policy_zone) + policy_zone = k; + } + } + BUG_ON(num >= max); + zl->zones[num] = NULL; + return zl; +} + +/* Create a new policy */ +static struct mempolicy *mpol_new(int mode, unsigned long *nodes) +{ + struct mempolicy *policy; + + PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes[0]); + if (mode == MPOL_DEFAULT) + return NULL; + policy = kmem_cache_alloc(policy_cache, GFP_KERNEL); + if (!policy) + return ERR_PTR(-ENOMEM); + atomic_set(&policy->refcnt, 1); + switch (mode) { + case MPOL_INTERLEAVE: + bitmap_copy(policy->v.nodes, nodes, MAX_NUMNODES); + break; + case MPOL_PREFERRED: + policy->v.preferred_node = find_first_bit(nodes, MAX_NUMNODES); + if (policy->v.preferred_node >= MAX_NUMNODES) + policy->v.preferred_node = -1; + break; + case MPOL_BIND: + policy->v.zonelist = bind_zonelist(nodes); + if (policy->v.zonelist == NULL) { + kmem_cache_free(policy_cache, policy); + return ERR_PTR(-ENOMEM); + } + break; + } + policy->policy = mode; + return policy; +} + +/* Ensure all existing pages follow the policy. */ +static int +verify_pages(struct mm_struct *mm, + unsigned long addr, unsigned long end, unsigned long *nodes) +{ + while (addr < end) { + struct page *p; + pte_t *pte; + pmd_t *pmd; + pud_t *pud; + pgd_t *pgd; + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd)) { + unsigned long next = (addr + PGDIR_SIZE) & PGDIR_MASK; + if (next > addr) + break; + addr = next; + continue; + } + pud = pud_offset(pgd, addr); + if (pud_none(*pud)) { + addr = (addr + PUD_SIZE) & PUD_MASK; + continue; + } + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + addr = (addr + PMD_SIZE) & PMD_MASK; + continue; + } + p = NULL; + pte = pte_offset_map(pmd, addr); + if (pte_present(*pte)) + p = pte_page(*pte); + pte_unmap(pte); + if (p) { + unsigned nid = page_to_nid(p); + if (!test_bit(nid, nodes)) + return -EIO; + } + addr += PAGE_SIZE; + } + return 0; +} + +/* Step 1: check the range */ +static struct vm_area_struct * +check_range(struct mm_struct *mm, unsigned long start, unsigned long end, + unsigned long *nodes, unsigned long flags) +{ + int err; + struct vm_area_struct *first, *vma, *prev; + + first = find_vma(mm, start); + if (!first) + return ERR_PTR(-EFAULT); + prev = NULL; + for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) { + if (!vma->vm_next && vma->vm_end < end) + return ERR_PTR(-EFAULT); + if (prev && prev->vm_end < vma->vm_start) + return ERR_PTR(-EFAULT); + if ((flags & MPOL_MF_STRICT) && !is_vm_hugetlb_page(vma)) { + err = verify_pages(vma->vm_mm, + vma->vm_start, vma->vm_end, nodes); + if (err) { + first = ERR_PTR(err); + break; + } + } + prev = vma; + } + return first; +} + +/* Apply policy to a single VMA */ +static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new) +{ + int err = 0; + struct mempolicy *old = vma->vm_policy; + + PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", + vma->vm_start, vma->vm_end, vma->vm_pgoff, + vma->vm_ops, vma->vm_file, + vma->vm_ops ? vma->vm_ops->set_policy : NULL); + + if (vma->vm_ops && vma->vm_ops->set_policy) + err = vma->vm_ops->set_policy(vma, new); + if (!err) { + mpol_get(new); + vma->vm_policy = new; + mpol_free(old); + } + return err; +} + +/* Step 2: apply policy to a range and do splits. */ +static int mbind_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, struct mempolicy *new) +{ + struct vm_area_struct *next; + int err; + + err = 0; + for (; vma && vma->vm_start < end; vma = next) { + next = vma->vm_next; + if (vma->vm_start < start) + err = split_vma(vma->vm_mm, vma, start, 1); + if (!err && vma->vm_end > end) + err = split_vma(vma->vm_mm, vma, end, 0); + if (!err) + err = policy_vma(vma, new); + if (err) + break; + } + return err; +} + +/* Change policy for a memory range */ +asmlinkage long sys_mbind(unsigned long start, unsigned long len, + unsigned long mode, + unsigned long __user *nmask, unsigned long maxnode, + unsigned flags) +{ + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + struct mempolicy *new; + unsigned long end; + DECLARE_BITMAP(nodes, MAX_NUMNODES); + int err; + + if ((flags & ~(unsigned long)(MPOL_MF_STRICT)) || mode > MPOL_MAX) + return -EINVAL; + if (start & ~PAGE_MASK) + return -EINVAL; + if (mode == MPOL_DEFAULT) + flags &= ~MPOL_MF_STRICT; + len = (len + PAGE_SIZE - 1) & PAGE_MASK; + end = start + len; + if (end < start) + return -EINVAL; + if (end == start) + return 0; + + err = get_nodes(nodes, nmask, maxnode, mode); + if (err) + return err; + + new = mpol_new(mode, nodes); + if (IS_ERR(new)) + return PTR_ERR(new); + + PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len, + mode,nodes[0]); + + down_write(&mm->mmap_sem); + vma = check_range(mm, start, end, nodes, flags); + err = PTR_ERR(vma); + if (!IS_ERR(vma)) + err = mbind_range(vma, start, end, new); + up_write(&mm->mmap_sem); + mpol_free(new); + return err; +} + +/* Set the process memory policy */ +asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask, + unsigned long maxnode) +{ + int err; + struct mempolicy *new; + DECLARE_BITMAP(nodes, MAX_NUMNODES); + + if (mode > MPOL_MAX) + return -EINVAL; + err = get_nodes(nodes, nmask, maxnode, mode); + if (err) + return err; + new = mpol_new(mode, nodes); + if (IS_ERR(new)) + return PTR_ERR(new); + mpol_free(current->mempolicy); + current->mempolicy = new; + if (new && new->policy == MPOL_INTERLEAVE) + current->il_next = find_first_bit(new->v.nodes, MAX_NUMNODES); + return 0; +} + +/* Fill a zone bitmap for a policy */ +static void get_zonemask(struct mempolicy *p, unsigned long *nodes) +{ + int i; + + bitmap_zero(nodes, MAX_NUMNODES); + switch (p->policy) { + case MPOL_BIND: + for (i = 0; p->v.zonelist->zones[i]; i++) + __set_bit(p->v.zonelist->zones[i]->zone_pgdat->node_id, nodes); + break; + case MPOL_DEFAULT: + break; + case MPOL_INTERLEAVE: + bitmap_copy(nodes, p->v.nodes, MAX_NUMNODES); + break; + case MPOL_PREFERRED: + /* or use current node instead of online map? */ + if (p->v.preferred_node < 0) + bitmap_copy(nodes, nodes_addr(node_online_map), MAX_NUMNODES); + else + __set_bit(p->v.preferred_node, nodes); + break; + default: + BUG(); + } +} + +static int lookup_node(struct mm_struct *mm, unsigned long addr) +{ + struct page *p; + int err; + + err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL); + if (err >= 0) { + err = page_to_nid(p); + put_page(p); + } + return err; +} + +/* Copy a kernel node mask to user space */ +static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode, + void *nodes, unsigned nbytes) +{ + unsigned long copy = ALIGN(maxnode-1, 64) / 8; + + if (copy > nbytes) { + if (copy > PAGE_SIZE) + return -EINVAL; + if (clear_user((char __user *)mask + nbytes, copy - nbytes)) + return -EFAULT; + copy = nbytes; + } + return copy_to_user(mask, nodes, copy) ? -EFAULT : 0; +} + +/* Retrieve NUMA policy */ +asmlinkage long sys_get_mempolicy(int __user *policy, + unsigned long __user *nmask, + unsigned long maxnode, + unsigned long addr, unsigned long flags) +{ + int err, pval; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = NULL; + struct mempolicy *pol = current->mempolicy; + + if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR)) + return -EINVAL; + if (nmask != NULL && maxnode < MAX_NUMNODES) + return -EINVAL; + if (flags & MPOL_F_ADDR) { + down_read(&mm->mmap_sem); + vma = find_vma_intersection(mm, addr, addr+1); + if (!vma) { + up_read(&mm->mmap_sem); + return -EFAULT; + } + if (vma->vm_ops && vma->vm_ops->get_policy) + pol = vma->vm_ops->get_policy(vma, addr); + else + pol = vma->vm_policy; + } else if (addr) + return -EINVAL; + + if (!pol) + pol = &default_policy; + + if (flags & MPOL_F_NODE) { + if (flags & MPOL_F_ADDR) { + err = lookup_node(mm, addr); + if (err < 0) + goto out; + pval = err; + } else if (pol == current->mempolicy && + pol->policy == MPOL_INTERLEAVE) { + pval = current->il_next; + } else { + err = -EINVAL; + goto out; + } + } else + pval = pol->policy; + + if (vma) { + up_read(¤t->mm->mmap_sem); + vma = NULL; + } + + if (policy && put_user(pval, policy)) + return -EFAULT; + + err = 0; + if (nmask) { + DECLARE_BITMAP(nodes, MAX_NUMNODES); + get_zonemask(pol, nodes); + err = copy_nodes_to_user(nmask, maxnode, nodes, sizeof(nodes)); + } + + out: + if (vma) + up_read(¤t->mm->mmap_sem); + return err; +} + +#ifdef CONFIG_COMPAT + +asmlinkage long compat_sys_get_mempolicy(int __user *policy, + compat_ulong_t __user *nmask, + compat_ulong_t maxnode, + compat_ulong_t addr, compat_ulong_t flags) +{ + long err; + unsigned long __user *nm = NULL; + unsigned long nr_bits, alloc_size; + DECLARE_BITMAP(bm, MAX_NUMNODES); + + nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); + alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; + + if (nmask) + nm = compat_alloc_user_space(alloc_size); + + err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags); + + if (!err && nmask) { + err = copy_from_user(bm, nm, alloc_size); + /* ensure entire bitmap is zeroed */ + err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8); + err |= compat_put_bitmap(nmask, bm, nr_bits); + } + + return err; +} + +asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask, + compat_ulong_t maxnode) +{ + long err = 0; + unsigned long __user *nm = NULL; + unsigned long nr_bits, alloc_size; + DECLARE_BITMAP(bm, MAX_NUMNODES); + + nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); + alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; + + if (nmask) { + err = compat_get_bitmap(bm, nmask, nr_bits); + nm = compat_alloc_user_space(alloc_size); + err |= copy_to_user(nm, bm, alloc_size); + } + + if (err) + return -EFAULT; + + return sys_set_mempolicy(mode, nm, nr_bits+1); +} + +asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len, + compat_ulong_t mode, compat_ulong_t __user *nmask, + compat_ulong_t maxnode, compat_ulong_t flags) +{ + long err = 0; + unsigned long __user *nm = NULL; + unsigned long nr_bits, alloc_size; + DECLARE_BITMAP(bm, MAX_NUMNODES); + + nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); + alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8; + + if (nmask) { + err = compat_get_bitmap(bm, nmask, nr_bits); + nm = compat_alloc_user_space(alloc_size); + err |= copy_to_user(nm, bm, alloc_size); + } + + if (err) + return -EFAULT; + + return sys_mbind(start, len, mode, nm, nr_bits+1, flags); +} + +#endif + +/* Return effective policy for a VMA */ +static struct mempolicy * +get_vma_policy(struct vm_area_struct *vma, unsigned long addr) +{ + struct mempolicy *pol = current->mempolicy; + + if (vma) { + if (vma->vm_ops && vma->vm_ops->get_policy) + pol = vma->vm_ops->get_policy(vma, addr); + else if (vma->vm_policy && + vma->vm_policy->policy != MPOL_DEFAULT) + pol = vma->vm_policy; + } + if (!pol) + pol = &default_policy; + return pol; +} + +/* Return a zonelist representing a mempolicy */ +static struct zonelist *zonelist_policy(unsigned int __nocast gfp, struct mempolicy *policy) +{ + int nd; + + switch (policy->policy) { + case MPOL_PREFERRED: + nd = policy->v.preferred_node; + if (nd < 0) + nd = numa_node_id(); + break; + case MPOL_BIND: + /* Lower zones don't get a policy applied */ + /* Careful: current->mems_allowed might have moved */ + if (gfp >= policy_zone) + if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist)) + return policy->v.zonelist; + /*FALL THROUGH*/ + case MPOL_INTERLEAVE: /* should not happen */ + case MPOL_DEFAULT: + nd = numa_node_id(); + break; + default: + nd = 0; + BUG(); + } + return NODE_DATA(nd)->node_zonelists + (gfp & GFP_ZONEMASK); +} + +/* Do dynamic interleaving for a process */ +static unsigned interleave_nodes(struct mempolicy *policy) +{ + unsigned nid, next; + struct task_struct *me = current; + + nid = me->il_next; + BUG_ON(nid >= MAX_NUMNODES); + next = find_next_bit(policy->v.nodes, MAX_NUMNODES, 1+nid); + if (next >= MAX_NUMNODES) + next = find_first_bit(policy->v.nodes, MAX_NUMNODES); + me->il_next = next; + return nid; +} + +/* Do static interleaving for a VMA with known offset. */ +static unsigned offset_il_node(struct mempolicy *pol, + struct vm_area_struct *vma, unsigned long off) +{ + unsigned nnodes = bitmap_weight(pol->v.nodes, MAX_NUMNODES); + unsigned target = (unsigned)off % nnodes; + int c; + int nid = -1; + + c = 0; + do { + nid = find_next_bit(pol->v.nodes, MAX_NUMNODES, nid+1); + c++; + } while (c <= target); + BUG_ON(nid >= MAX_NUMNODES); + BUG_ON(!test_bit(nid, pol->v.nodes)); + return nid; +} + +/* Allocate a page in interleaved policy. + Own path because it needs to do special accounting. */ +static struct page *alloc_page_interleave(unsigned int __nocast gfp, unsigned order, unsigned nid) +{ + struct zonelist *zl; + struct page *page; + + BUG_ON(!node_online(nid)); + zl = NODE_DATA(nid)->node_zonelists + (gfp & GFP_ZONEMASK); + page = __alloc_pages(gfp, order, zl); + if (page && page_zone(page) == zl->zones[0]) { + zl->zones[0]->pageset[get_cpu()].interleave_hit++; + put_cpu(); + } + return page; +} + +/** + * alloc_page_vma - Allocate a page for a VMA. + * + * @gfp: + * %GFP_USER user allocation. + * %GFP_KERNEL kernel allocations, + * %GFP_HIGHMEM highmem/user allocations, + * %GFP_FS allocation should not call back into a file system. + * %GFP_ATOMIC don't sleep. + * + * @vma: Pointer to VMA or NULL if not available. + * @addr: Virtual Address of the allocation. Must be inside the VMA. + * + * This function allocates a page from the kernel page pool and applies + * a NUMA policy associated with the VMA or the current process. + * When VMA is not NULL caller must hold down_read on the mmap_sem of the + * mm_struct of the VMA to prevent it from going away. Should be used for + * all allocations for pages that will be mapped into + * user space. Returns NULL when no page can be allocated. + * + * Should be called with the mm_sem of the vma hold. + */ +struct page * +alloc_page_vma(unsigned int __nocast gfp, struct vm_area_struct *vma, unsigned long addr) +{ + struct mempolicy *pol = get_vma_policy(vma, addr); + + cpuset_update_current_mems_allowed(); + + if (unlikely(pol->policy == MPOL_INTERLEAVE)) { + unsigned nid; + if (vma) { + unsigned long off; + BUG_ON(addr >= vma->vm_end); + BUG_ON(addr < vma->vm_start); + off = vma->vm_pgoff; + off += (addr - vma->vm_start) >> PAGE_SHIFT; + nid = offset_il_node(pol, vma, off); + } else { + /* fall back to process interleaving */ + nid = interleave_nodes(pol); + } + return alloc_page_interleave(gfp, 0, nid); + } + return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol)); +} + +/** + * alloc_pages_current - Allocate pages. + * + * @gfp: + * %GFP_USER user allocation, + * %GFP_KERNEL kernel allocation, + * %GFP_HIGHMEM highmem allocation, + * %GFP_FS don't call back into a file system. + * %GFP_ATOMIC don't sleep. + * @order: Power of two of allocation size in pages. 0 is a single page. + * + * Allocate a page from the kernel page pool. When not in + * interrupt context and apply the current process NUMA policy. + * Returns NULL when no page can be allocated. + * + * Don't call cpuset_update_current_mems_allowed() unless + * 1) it's ok to take cpuset_sem (can WAIT), and + * 2) allocating for current task (not interrupt). + */ +struct page *alloc_pages_current(unsigned int __nocast gfp, unsigned order) +{ + struct mempolicy *pol = current->mempolicy; + + if ((gfp & __GFP_WAIT) && !in_interrupt()) + cpuset_update_current_mems_allowed(); + if (!pol || in_interrupt()) + pol = &default_policy; + if (pol->policy == MPOL_INTERLEAVE) + return alloc_page_interleave(gfp, order, interleave_nodes(pol)); + return __alloc_pages(gfp, order, zonelist_policy(gfp, pol)); +} +EXPORT_SYMBOL(alloc_pages_current); + +/* Slow path of a mempolicy copy */ +struct mempolicy *__mpol_copy(struct mempolicy *old) +{ + struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL); + + if (!new) + return ERR_PTR(-ENOMEM); + *new = *old; + atomic_set(&new->refcnt, 1); + if (new->policy == MPOL_BIND) { + int sz = ksize(old->v.zonelist); + new->v.zonelist = kmalloc(sz, SLAB_KERNEL); + if (!new->v.zonelist) { + kmem_cache_free(policy_cache, new); + return ERR_PTR(-ENOMEM); + } + memcpy(new->v.zonelist, old->v.zonelist, sz); + } + return new; +} + +/* Slow path of a mempolicy comparison */ +int __mpol_equal(struct mempolicy *a, struct mempolicy *b) +{ + if (!a || !b) + return 0; + if (a->policy != b->policy) + return 0; + switch (a->policy) { + case MPOL_DEFAULT: + return 1; + case MPOL_INTERLEAVE: + return bitmap_equal(a->v.nodes, b->v.nodes, MAX_NUMNODES); + case MPOL_PREFERRED: + return a->v.preferred_node == b->v.preferred_node; + case MPOL_BIND: { + int i; + for (i = 0; a->v.zonelist->zones[i]; i++) + if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i]) + return 0; + return b->v.zonelist->zones[i] == NULL; + } + default: + BUG(); + return 0; + } +} + +/* Slow path of a mpol destructor. */ +void __mpol_free(struct mempolicy *p) +{ + if (!atomic_dec_and_test(&p->refcnt)) + return; + if (p->policy == MPOL_BIND) + kfree(p->v.zonelist); + p->policy = MPOL_DEFAULT; + kmem_cache_free(policy_cache, p); +} + +/* + * Hugetlb policy. Same as above, just works with node numbers instead of + * zonelists. + */ + +/* Find first node suitable for an allocation */ +int mpol_first_node(struct vm_area_struct *vma, unsigned long addr) +{ + struct mempolicy *pol = get_vma_policy(vma, addr); + + switch (pol->policy) { + case MPOL_DEFAULT: + return numa_node_id(); + case MPOL_BIND: + return pol->v.zonelist->zones[0]->zone_pgdat->node_id; + case MPOL_INTERLEAVE: + return interleave_nodes(pol); + case MPOL_PREFERRED: + return pol->v.preferred_node >= 0 ? + pol->v.preferred_node : numa_node_id(); + } + BUG(); + return 0; +} + +/* Find secondary valid nodes for an allocation */ +int mpol_node_valid(int nid, struct vm_area_struct *vma, unsigned long addr) +{ + struct mempolicy *pol = get_vma_policy(vma, addr); + + switch (pol->policy) { + case MPOL_PREFERRED: + case MPOL_DEFAULT: + case MPOL_INTERLEAVE: + return 1; + case MPOL_BIND: { + struct zone **z; + for (z = pol->v.zonelist->zones; *z; z++) + if ((*z)->zone_pgdat->node_id == nid) + return 1; + return 0; + } + default: + BUG(); + return 0; + } +} + +/* + * Shared memory backing store policy support. + * + * Remember policies even when nobody has shared memory mapped. + * The policies are kept in Red-Black tree linked from the inode. + * They are protected by the sp->lock spinlock, which should be held + * for any accesses to the tree. + */ + +/* lookup first element intersecting start-end */ +/* Caller holds sp->lock */ +static struct sp_node * +sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end) +{ + struct rb_node *n = sp->root.rb_node; + + while (n) { + struct sp_node *p = rb_entry(n, struct sp_node, nd); + + if (start >= p->end) + n = n->rb_right; + else if (end <= p->start) + n = n->rb_left; + else + break; + } + if (!n) + return NULL; + for (;;) { + struct sp_node *w = NULL; + struct rb_node *prev = rb_prev(n); + if (!prev) + break; + w = rb_entry(prev, struct sp_node, nd); + if (w->end <= start) + break; + n = prev; + } + return rb_entry(n, struct sp_node, nd); +} + +/* Insert a new shared policy into the list. */ +/* Caller holds sp->lock */ +static void sp_insert(struct shared_policy *sp, struct sp_node *new) +{ + struct rb_node **p = &sp->root.rb_node; + struct rb_node *parent = NULL; + struct sp_node *nd; + + while (*p) { + parent = *p; + nd = rb_entry(parent, struct sp_node, nd); + if (new->start < nd->start) + p = &(*p)->rb_left; + else if (new->end > nd->end) + p = &(*p)->rb_right; + else + BUG(); + } + rb_link_node(&new->nd, parent, p); + rb_insert_color(&new->nd, &sp->root); + PDprintk("inserting %lx-%lx: %d\n", new->start, new->end, + new->policy ? new->policy->policy : 0); +} + +/* Find shared policy intersecting idx */ +struct mempolicy * +mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx) +{ + struct mempolicy *pol = NULL; + struct sp_node *sn; + + if (!sp->root.rb_node) + return NULL; + spin_lock(&sp->lock); + sn = sp_lookup(sp, idx, idx+1); + if (sn) { + mpol_get(sn->policy); + pol = sn->policy; + } + spin_unlock(&sp->lock); + return pol; +} + +static void sp_delete(struct shared_policy *sp, struct sp_node *n) +{ + PDprintk("deleting %lx-l%x\n", n->start, n->end); + rb_erase(&n->nd, &sp->root); + mpol_free(n->policy); + kmem_cache_free(sn_cache, n); +} + +struct sp_node * +sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol) +{ + struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL); + + if (!n) + return NULL; + n->start = start; + n->end = end; + mpol_get(pol); + n->policy = pol; + return n; +} + +/* Replace a policy range. */ +static int shared_policy_replace(struct shared_policy *sp, unsigned long start, + unsigned long end, struct sp_node *new) +{ + struct sp_node *n, *new2 = NULL; + +restart: + spin_lock(&sp->lock); + n = sp_lookup(sp, start, end); + /* Take care of old policies in the same range. */ + while (n && n->start < end) { + struct rb_node *next = rb_next(&n->nd); + if (n->start >= start) { + if (n->end <= end) + sp_delete(sp, n); + else + n->start = end; + } else { + /* Old policy spanning whole new range. */ + if (n->end > end) { + if (!new2) { + spin_unlock(&sp->lock); + new2 = sp_alloc(end, n->end, n->policy); + if (!new2) + return -ENOMEM; + goto restart; + } + n->end = start; + sp_insert(sp, new2); + new2 = NULL; + break; + } else + n->end = start; + } + if (!next) + break; + n = rb_entry(next, struct sp_node, nd); + } + if (new) + sp_insert(sp, new); + spin_unlock(&sp->lock); + if (new2) { + mpol_free(new2->policy); + kmem_cache_free(sn_cache, new2); + } + return 0; +} + +int mpol_set_shared_policy(struct shared_policy *info, + struct vm_area_struct *vma, struct mempolicy *npol) +{ + int err; + struct sp_node *new = NULL; + unsigned long sz = vma_pages(vma); + + PDprintk("set_shared_policy %lx sz %lu %d %lx\n", + vma->vm_pgoff, + sz, npol? npol->policy : -1, + npol ? npol->v.nodes[0] : -1); + + if (npol) { + new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); + if (!new) + return -ENOMEM; + } + err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); + if (err && new) + kmem_cache_free(sn_cache, new); + return err; +} + +/* Free a backing policy store on inode delete. */ +void mpol_free_shared_policy(struct shared_policy *p) +{ + struct sp_node *n; + struct rb_node *next; + + if (!p->root.rb_node) + return; + spin_lock(&p->lock); + next = rb_first(&p->root); + while (next) { + n = rb_entry(next, struct sp_node, nd); + next = rb_next(&n->nd); + mpol_free(n->policy); + kmem_cache_free(sn_cache, n); + } + spin_unlock(&p->lock); + p->root = RB_ROOT; +} + +/* assumes fs == KERNEL_DS */ +void __init numa_policy_init(void) +{ + policy_cache = kmem_cache_create("numa_policy", + sizeof(struct mempolicy), + 0, SLAB_PANIC, NULL, NULL); + + sn_cache = kmem_cache_create("shared_policy_node", + sizeof(struct sp_node), + 0, SLAB_PANIC, NULL, NULL); + + /* Set interleaving policy for system init. This way not all + the data structures allocated at system boot end up in node zero. */ + + if (sys_set_mempolicy(MPOL_INTERLEAVE, nodes_addr(node_online_map), + MAX_NUMNODES) < 0) + printk("numa_policy_init: interleaving failed\n"); +} + +/* Reset policy of current process to default. + * Assumes fs == KERNEL_DS */ +void numa_default_policy(void) +{ + sys_set_mempolicy(MPOL_DEFAULT, NULL, 0); +} diff --git a/mm/mempool.c b/mm/mempool.c new file mode 100644 index 000000000000..b014ffeaa413 --- /dev/null +++ b/mm/mempool.c @@ -0,0 +1,290 @@ +/* + * linux/mm/mempool.c + * + * memory buffer pool support. Such pools are mostly used + * for guaranteed, deadlock-free memory allocations during + * extreme VM load. + * + * started by Ingo Molnar, Copyright (C) 2001 + */ + +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/mempool.h> +#include <linux/blkdev.h> +#include <linux/writeback.h> + +static void add_element(mempool_t *pool, void *element) +{ + BUG_ON(pool->curr_nr >= pool->min_nr); + pool->elements[pool->curr_nr++] = element; +} + +static void *remove_element(mempool_t *pool) +{ + BUG_ON(pool->curr_nr <= 0); + return pool->elements[--pool->curr_nr]; +} + +static void free_pool(mempool_t *pool) +{ + while (pool->curr_nr) { + void *element = remove_element(pool); + pool->free(element, pool->pool_data); + } + kfree(pool->elements); + kfree(pool); +} + +/** + * mempool_create - create a memory pool + * @min_nr: the minimum number of elements guaranteed to be + * allocated for this pool. + * @alloc_fn: user-defined element-allocation function. + * @free_fn: user-defined element-freeing function. + * @pool_data: optional private data available to the user-defined functions. + * + * this function creates and allocates a guaranteed size, preallocated + * memory pool. The pool can be used from the mempool_alloc and mempool_free + * functions. This function might sleep. Both the alloc_fn() and the free_fn() + * functions might sleep - as long as the mempool_alloc function is not called + * from IRQ contexts. + */ +mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn, + mempool_free_t *free_fn, void *pool_data) +{ + mempool_t *pool; + + pool = kmalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return NULL; + memset(pool, 0, sizeof(*pool)); + pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL); + if (!pool->elements) { + kfree(pool); + return NULL; + } + spin_lock_init(&pool->lock); + pool->min_nr = min_nr; + pool->pool_data = pool_data; + init_waitqueue_head(&pool->wait); + pool->alloc = alloc_fn; + pool->free = free_fn; + + /* + * First pre-allocate the guaranteed number of buffers. + */ + while (pool->curr_nr < pool->min_nr) { + void *element; + + element = pool->alloc(GFP_KERNEL, pool->pool_data); + if (unlikely(!element)) { + free_pool(pool); + return NULL; + } + add_element(pool, element); + } + return pool; +} +EXPORT_SYMBOL(mempool_create); + +/** + * mempool_resize - resize an existing memory pool + * @pool: pointer to the memory pool which was allocated via + * mempool_create(). + * @new_min_nr: the new minimum number of elements guaranteed to be + * allocated for this pool. + * @gfp_mask: the usual allocation bitmask. + * + * This function shrinks/grows the pool. In the case of growing, + * it cannot be guaranteed that the pool will be grown to the new + * size immediately, but new mempool_free() calls will refill it. + * + * Note, the caller must guarantee that no mempool_destroy is called + * while this function is running. mempool_alloc() & mempool_free() + * might be called (eg. from IRQ contexts) while this function executes. + */ +int mempool_resize(mempool_t *pool, int new_min_nr, unsigned int __nocast gfp_mask) +{ + void *element; + void **new_elements; + unsigned long flags; + + BUG_ON(new_min_nr <= 0); + + spin_lock_irqsave(&pool->lock, flags); + if (new_min_nr <= pool->min_nr) { + while (new_min_nr < pool->curr_nr) { + element = remove_element(pool); + spin_unlock_irqrestore(&pool->lock, flags); + pool->free(element, pool->pool_data); + spin_lock_irqsave(&pool->lock, flags); + } + pool->min_nr = new_min_nr; + goto out_unlock; + } + spin_unlock_irqrestore(&pool->lock, flags); + + /* Grow the pool */ + new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); + if (!new_elements) + return -ENOMEM; + + spin_lock_irqsave(&pool->lock, flags); + if (unlikely(new_min_nr <= pool->min_nr)) { + /* Raced, other resize will do our work */ + spin_unlock_irqrestore(&pool->lock, flags); + kfree(new_elements); + goto out; + } + memcpy(new_elements, pool->elements, + pool->curr_nr * sizeof(*new_elements)); + kfree(pool->elements); + pool->elements = new_elements; + pool->min_nr = new_min_nr; + + while (pool->curr_nr < pool->min_nr) { + spin_unlock_irqrestore(&pool->lock, flags); + element = pool->alloc(gfp_mask, pool->pool_data); + if (!element) + goto out; + spin_lock_irqsave(&pool->lock, flags); + if (pool->curr_nr < pool->min_nr) { + add_element(pool, element); + } else { + spin_unlock_irqrestore(&pool->lock, flags); + pool->free(element, pool->pool_data); /* Raced */ + goto out; + } + } +out_unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + return 0; +} +EXPORT_SYMBOL(mempool_resize); + +/** + * mempool_destroy - deallocate a memory pool + * @pool: pointer to the memory pool which was allocated via + * mempool_create(). + * + * this function only sleeps if the free_fn() function sleeps. The caller + * has to guarantee that all elements have been returned to the pool (ie: + * freed) prior to calling mempool_destroy(). + */ +void mempool_destroy(mempool_t *pool) +{ + if (pool->curr_nr != pool->min_nr) + BUG(); /* There were outstanding elements */ + free_pool(pool); +} +EXPORT_SYMBOL(mempool_destroy); + +/** + * mempool_alloc - allocate an element from a specific memory pool + * @pool: pointer to the memory pool which was allocated via + * mempool_create(). + * @gfp_mask: the usual allocation bitmask. + * + * this function only sleeps if the alloc_fn function sleeps or + * returns NULL. Note that due to preallocation, this function + * *never* fails when called from process contexts. (it might + * fail if called from an IRQ context.) + */ +void * mempool_alloc(mempool_t *pool, unsigned int __nocast gfp_mask) +{ + void *element; + unsigned long flags; + DEFINE_WAIT(wait); + int gfp_nowait = gfp_mask & ~(__GFP_WAIT | __GFP_IO); + + might_sleep_if(gfp_mask & __GFP_WAIT); +repeat_alloc: + element = pool->alloc(gfp_nowait|__GFP_NOWARN, pool->pool_data); + if (likely(element != NULL)) + return element; + + /* + * If the pool is less than 50% full and we can perform effective + * page reclaim then try harder to allocate an element. + */ + mb(); + if ((gfp_mask & __GFP_FS) && (gfp_mask != gfp_nowait) && + (pool->curr_nr <= pool->min_nr/2)) { + element = pool->alloc(gfp_mask, pool->pool_data); + if (likely(element != NULL)) + return element; + } + + /* + * Kick the VM at this point. + */ + wakeup_bdflush(0); + + spin_lock_irqsave(&pool->lock, flags); + if (likely(pool->curr_nr)) { + element = remove_element(pool); + spin_unlock_irqrestore(&pool->lock, flags); + return element; + } + spin_unlock_irqrestore(&pool->lock, flags); + + /* We must not sleep in the GFP_ATOMIC case */ + if (!(gfp_mask & __GFP_WAIT)) + return NULL; + + prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); + mb(); + if (!pool->curr_nr) + io_schedule(); + finish_wait(&pool->wait, &wait); + + goto repeat_alloc; +} +EXPORT_SYMBOL(mempool_alloc); + +/** + * mempool_free - return an element to the pool. + * @element: pool element pointer. + * @pool: pointer to the memory pool which was allocated via + * mempool_create(). + * + * this function only sleeps if the free_fn() function sleeps. + */ +void mempool_free(void *element, mempool_t *pool) +{ + unsigned long flags; + + mb(); + if (pool->curr_nr < pool->min_nr) { + spin_lock_irqsave(&pool->lock, flags); + if (pool->curr_nr < pool->min_nr) { + add_element(pool, element); + spin_unlock_irqrestore(&pool->lock, flags); + wake_up(&pool->wait); + return; + } + spin_unlock_irqrestore(&pool->lock, flags); + } + pool->free(element, pool->pool_data); +} +EXPORT_SYMBOL(mempool_free); + +/* + * A commonly used alloc and free fn. + */ +void *mempool_alloc_slab(unsigned int __nocast gfp_mask, void *pool_data) +{ + kmem_cache_t *mem = (kmem_cache_t *) pool_data; + return kmem_cache_alloc(mem, gfp_mask); +} +EXPORT_SYMBOL(mempool_alloc_slab); + +void mempool_free_slab(void *element, void *pool_data) +{ + kmem_cache_t *mem = (kmem_cache_t *) pool_data; + kmem_cache_free(mem, element); +} +EXPORT_SYMBOL(mempool_free_slab); diff --git a/mm/mincore.c b/mm/mincore.c new file mode 100644 index 000000000000..07833dc5829d --- /dev/null +++ b/mm/mincore.c @@ -0,0 +1,191 @@ +/* + * linux/mm/mincore.c + * + * Copyright (C) 1994-1999 Linus Torvalds + */ + +/* + * The mincore() system call. + */ +#include <linux/slab.h> +#include <linux/pagemap.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/syscalls.h> + +#include <asm/uaccess.h> +#include <asm/pgtable.h> + +/* + * Later we can get more picky about what "in core" means precisely. + * For now, simply check to see if the page is in the page cache, + * and is up to date; i.e. that no page-in operation would be required + * at this time if an application were to map and access this page. + */ +static unsigned char mincore_page(struct vm_area_struct * vma, + unsigned long pgoff) +{ + unsigned char present = 0; + struct address_space * as = vma->vm_file->f_mapping; + struct page * page; + + page = find_get_page(as, pgoff); + if (page) { + present = PageUptodate(page); + page_cache_release(page); + } + + return present; +} + +static long mincore_vma(struct vm_area_struct * vma, + unsigned long start, unsigned long end, unsigned char __user * vec) +{ + long error, i, remaining; + unsigned char * tmp; + + error = -ENOMEM; + if (!vma->vm_file) + return error; + + start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + if (end > vma->vm_end) + end = vma->vm_end; + end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + + error = -EAGAIN; + tmp = (unsigned char *) __get_free_page(GFP_KERNEL); + if (!tmp) + return error; + + /* (end - start) is # of pages, and also # of bytes in "vec */ + remaining = (end - start), + + error = 0; + for (i = 0; remaining > 0; remaining -= PAGE_SIZE, i++) { + int j = 0; + long thispiece = (remaining < PAGE_SIZE) ? + remaining : PAGE_SIZE; + + while (j < thispiece) + tmp[j++] = mincore_page(vma, start++); + + if (copy_to_user(vec + PAGE_SIZE * i, tmp, thispiece)) { + error = -EFAULT; + break; + } + } + + free_page((unsigned long) tmp); + return error; +} + +/* + * The mincore(2) system call. + * + * mincore() returns the memory residency status of the pages in the + * current process's address space specified by [addr, addr + len). + * The status is returned in a vector of bytes. The least significant + * bit of each byte is 1 if the referenced page is in memory, otherwise + * it is zero. + * + * Because the status of a page can change after mincore() checks it + * but before it returns to the application, the returned vector may + * contain stale information. Only locked pages are guaranteed to + * remain in memory. + * + * return values: + * zero - success + * -EFAULT - vec points to an illegal address + * -EINVAL - addr is not a multiple of PAGE_CACHE_SIZE + * -ENOMEM - Addresses in the range [addr, addr + len] are + * invalid for the address space of this process, or + * specify one or more pages which are not currently + * mapped + * -EAGAIN - A kernel resource was temporarily unavailable. + */ +asmlinkage long sys_mincore(unsigned long start, size_t len, + unsigned char __user * vec) +{ + int index = 0; + unsigned long end, limit; + struct vm_area_struct * vma; + size_t max; + int unmapped_error = 0; + long error; + + /* check the arguments */ + if (start & ~PAGE_CACHE_MASK) + goto einval; + + if (start < FIRST_USER_PGD_NR * PGDIR_SIZE) + goto enomem; + + limit = TASK_SIZE; + if (start >= limit) + goto enomem; + + if (!len) + return 0; + + max = limit - start; + len = PAGE_CACHE_ALIGN(len); + if (len > max || !len) + goto enomem; + + end = start + len; + + /* check the output buffer whilst holding the lock */ + error = -EFAULT; + down_read(¤t->mm->mmap_sem); + + if (!access_ok(VERIFY_WRITE, vec, len >> PAGE_SHIFT)) + goto out; + + /* + * If the interval [start,end) covers some unmapped address + * ranges, just ignore them, but return -ENOMEM at the end. + */ + error = 0; + + vma = find_vma(current->mm, start); + while (vma) { + /* Here start < vma->vm_end. */ + if (start < vma->vm_start) { + unmapped_error = -ENOMEM; + start = vma->vm_start; + } + + /* Here vma->vm_start <= start < vma->vm_end. */ + if (end <= vma->vm_end) { + if (start < end) { + error = mincore_vma(vma, start, end, + &vec[index]); + if (error) + goto out; + } + error = unmapped_error; + goto out; + } + + /* Here vma->vm_start <= start < vma->vm_end < end. */ + error = mincore_vma(vma, start, vma->vm_end, &vec[index]); + if (error) + goto out; + index += (vma->vm_end - start) >> PAGE_CACHE_SHIFT; + start = vma->vm_end; + vma = vma->vm_next; + } + + /* we found a hole in the area queried if we arrive here */ + error = -ENOMEM; + +out: + up_read(¤t->mm->mmap_sem); + return error; + +einval: + return -EINVAL; +enomem: + return -ENOMEM; +} diff --git a/mm/mlock.c b/mm/mlock.c new file mode 100644 index 000000000000..4ae3a46ff768 --- /dev/null +++ b/mm/mlock.c @@ -0,0 +1,253 @@ +/* + * linux/mm/mlock.c + * + * (C) Copyright 1995 Linus Torvalds + * (C) Copyright 2002 Christoph Hellwig + */ + +#include <linux/mman.h> +#include <linux/mm.h> +#include <linux/mempolicy.h> +#include <linux/syscalls.h> + + +static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, + unsigned long start, unsigned long end, unsigned int newflags) +{ + struct mm_struct * mm = vma->vm_mm; + pgoff_t pgoff; + int pages; + int ret = 0; + + if (newflags == vma->vm_flags) { + *prev = vma; + goto out; + } + + pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); + *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma, + vma->vm_file, pgoff, vma_policy(vma)); + if (*prev) { + vma = *prev; + goto success; + } + + *prev = vma; + + if (start != vma->vm_start) { + ret = split_vma(mm, vma, start, 1); + if (ret) + goto out; + } + + if (end != vma->vm_end) { + ret = split_vma(mm, vma, end, 0); + if (ret) + goto out; + } + +success: + /* + * vm_flags is protected by the mmap_sem held in write mode. + * It's okay if try_to_unmap_one unmaps a page just after we + * set VM_LOCKED, make_pages_present below will bring it back. + */ + vma->vm_flags = newflags; + + /* + * Keep track of amount of locked VM. + */ + pages = (end - start) >> PAGE_SHIFT; + if (newflags & VM_LOCKED) { + pages = -pages; + if (!(newflags & VM_IO)) + ret = make_pages_present(start, end); + } + + vma->vm_mm->locked_vm -= pages; +out: + if (ret == -ENOMEM) + ret = -EAGAIN; + return ret; +} + +static int do_mlock(unsigned long start, size_t len, int on) +{ + unsigned long nstart, end, tmp; + struct vm_area_struct * vma, * prev; + int error; + + len = PAGE_ALIGN(len); + end = start + len; + if (end < start) + return -EINVAL; + if (end == start) + return 0; + vma = find_vma_prev(current->mm, start, &prev); + if (!vma || vma->vm_start > start) + return -ENOMEM; + + if (start > vma->vm_start) + prev = vma; + + for (nstart = start ; ; ) { + unsigned int newflags; + + /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ + + newflags = vma->vm_flags | VM_LOCKED; + if (!on) + newflags &= ~VM_LOCKED; + + tmp = vma->vm_end; + if (tmp > end) + tmp = end; + error = mlock_fixup(vma, &prev, nstart, tmp, newflags); + if (error) + break; + nstart = tmp; + if (nstart < prev->vm_end) + nstart = prev->vm_end; + if (nstart >= end) + break; + + vma = prev->vm_next; + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + break; + } + } + return error; +} + +asmlinkage long sys_mlock(unsigned long start, size_t len) +{ + unsigned long locked; + unsigned long lock_limit; + int error = -ENOMEM; + + if (!can_do_mlock()) + return -EPERM; + + down_write(¤t->mm->mmap_sem); + len = PAGE_ALIGN(len + (start & ~PAGE_MASK)); + start &= PAGE_MASK; + + locked = len >> PAGE_SHIFT; + locked += current->mm->locked_vm; + + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + lock_limit >>= PAGE_SHIFT; + + /* check against resource limits */ + if ((locked <= lock_limit) || capable(CAP_IPC_LOCK)) + error = do_mlock(start, len, 1); + up_write(¤t->mm->mmap_sem); + return error; +} + +asmlinkage long sys_munlock(unsigned long start, size_t len) +{ + int ret; + + down_write(¤t->mm->mmap_sem); + len = PAGE_ALIGN(len + (start & ~PAGE_MASK)); + start &= PAGE_MASK; + ret = do_mlock(start, len, 0); + up_write(¤t->mm->mmap_sem); + return ret; +} + +static int do_mlockall(int flags) +{ + struct vm_area_struct * vma, * prev = NULL; + unsigned int def_flags = 0; + + if (flags & MCL_FUTURE) + def_flags = VM_LOCKED; + current->mm->def_flags = def_flags; + if (flags == MCL_FUTURE) + goto out; + + for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { + unsigned int newflags; + + newflags = vma->vm_flags | VM_LOCKED; + if (!(flags & MCL_CURRENT)) + newflags &= ~VM_LOCKED; + + /* Ignore errors */ + mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); + } +out: + return 0; +} + +asmlinkage long sys_mlockall(int flags) +{ + unsigned long lock_limit; + int ret = -EINVAL; + + if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE))) + goto out; + + ret = -EPERM; + if (!can_do_mlock()) + goto out; + + down_write(¤t->mm->mmap_sem); + + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + lock_limit >>= PAGE_SHIFT; + + ret = -ENOMEM; + if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) || + capable(CAP_IPC_LOCK)) + ret = do_mlockall(flags); + up_write(¤t->mm->mmap_sem); +out: + return ret; +} + +asmlinkage long sys_munlockall(void) +{ + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_mlockall(0); + up_write(¤t->mm->mmap_sem); + return ret; +} + +/* + * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB + * shm segments) get accounted against the user_struct instead. + */ +static DEFINE_SPINLOCK(shmlock_user_lock); + +int user_shm_lock(size_t size, struct user_struct *user) +{ + unsigned long lock_limit, locked; + int allowed = 0; + + locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + lock_limit >>= PAGE_SHIFT; + spin_lock(&shmlock_user_lock); + if (locked + user->locked_shm > lock_limit && !capable(CAP_IPC_LOCK)) + goto out; + get_uid(user); + user->locked_shm += locked; + allowed = 1; +out: + spin_unlock(&shmlock_user_lock); + return allowed; +} + +void user_shm_unlock(size_t size, struct user_struct *user) +{ + spin_lock(&shmlock_user_lock); + user->locked_shm -= (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + spin_unlock(&shmlock_user_lock); + free_uid(user); +} diff --git a/mm/mmap.c b/mm/mmap.c new file mode 100644 index 000000000000..a95ebda27446 --- /dev/null +++ b/mm/mmap.c @@ -0,0 +1,2082 @@ +/* + * mm/mmap.c + * + * Written by obz. + * + * Address space accounting code <alan@redhat.com> + */ + +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/shm.h> +#include <linux/mman.h> +#include <linux/pagemap.h> +#include <linux/swap.h> +#include <linux/syscalls.h> +#include <linux/init.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/personality.h> +#include <linux/security.h> +#include <linux/hugetlb.h> +#include <linux/profile.h> +#include <linux/module.h> +#include <linux/mount.h> +#include <linux/mempolicy.h> +#include <linux/rmap.h> + +#include <asm/uaccess.h> +#include <asm/cacheflush.h> +#include <asm/tlb.h> + +/* + * WARNING: the debugging will use recursive algorithms so never enable this + * unless you know what you are doing. + */ +#undef DEBUG_MM_RB + +/* description of effects of mapping type and prot in current implementation. + * this is due to the limited x86 page protection hardware. The expected + * behavior is in parens: + * + * map_type prot + * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC + * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes + * w: (no) no w: (no) no w: (yes) yes w: (no) no + * x: (no) no x: (no) yes x: (no) yes x: (yes) yes + * + * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes + * w: (no) no w: (no) no w: (copy) copy w: (no) no + * x: (no) no x: (no) yes x: (no) yes x: (yes) yes + * + */ +pgprot_t protection_map[16] = { + __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111, + __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111 +}; + +int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ +int sysctl_overcommit_ratio = 50; /* default is 50% */ +int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; +atomic_t vm_committed_space = ATOMIC_INIT(0); + +/* + * Check that a process has enough memory to allocate a new virtual + * mapping. 0 means there is enough memory for the allocation to + * succeed and -ENOMEM implies there is not. + * + * We currently support three overcommit policies, which are set via the + * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting + * + * Strict overcommit modes added 2002 Feb 26 by Alan Cox. + * Additional code 2002 Jul 20 by Robert Love. + * + * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise. + * + * Note this is a helper function intended to be used by LSMs which + * wish to use this logic. + */ +int __vm_enough_memory(long pages, int cap_sys_admin) +{ + unsigned long free, allowed; + + vm_acct_memory(pages); + + /* + * Sometimes we want to use more memory than we have + */ + if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS) + return 0; + + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long n; + + free = get_page_cache_size(); + free += nr_swap_pages; + + /* + * Any slabs which are created with the + * SLAB_RECLAIM_ACCOUNT flag claim to have contents + * which are reclaimable, under pressure. The dentry + * cache and most inode caches should fall into this + */ + free += atomic_read(&slab_reclaim_pages); + + /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + free -= free / 32; + + if (free > pages) + return 0; + + /* + * nr_free_pages() is very expensive on large systems, + * only call if we're about to fail. + */ + n = nr_free_pages(); + if (!cap_sys_admin) + n -= n / 32; + free += n; + + if (free > pages) + return 0; + vm_unacct_memory(pages); + return -ENOMEM; + } + + allowed = (totalram_pages - hugetlb_total_pages()) + * sysctl_overcommit_ratio / 100; + /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + allowed -= allowed / 32; + allowed += total_swap_pages; + + /* Don't let a single process grow too big: + leave 3% of the size of this process for other processes */ + allowed -= current->mm->total_vm / 32; + + if (atomic_read(&vm_committed_space) < allowed) + return 0; + + vm_unacct_memory(pages); + + return -ENOMEM; +} + +EXPORT_SYMBOL(sysctl_overcommit_memory); +EXPORT_SYMBOL(sysctl_overcommit_ratio); +EXPORT_SYMBOL(sysctl_max_map_count); +EXPORT_SYMBOL(vm_committed_space); +EXPORT_SYMBOL(__vm_enough_memory); + +/* + * Requires inode->i_mapping->i_mmap_lock + */ +static void __remove_shared_vm_struct(struct vm_area_struct *vma, + struct file *file, struct address_space *mapping) +{ + if (vma->vm_flags & VM_DENYWRITE) + atomic_inc(&file->f_dentry->d_inode->i_writecount); + if (vma->vm_flags & VM_SHARED) + mapping->i_mmap_writable--; + + flush_dcache_mmap_lock(mapping); + if (unlikely(vma->vm_flags & VM_NONLINEAR)) + list_del_init(&vma->shared.vm_set.list); + else + vma_prio_tree_remove(vma, &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); +} + +/* + * Remove one vm structure and free it. + */ +static void remove_vm_struct(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + + might_sleep(); + if (file) { + struct address_space *mapping = file->f_mapping; + spin_lock(&mapping->i_mmap_lock); + __remove_shared_vm_struct(vma, file, mapping); + spin_unlock(&mapping->i_mmap_lock); + } + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); + if (file) + fput(file); + anon_vma_unlink(vma); + mpol_free(vma_policy(vma)); + kmem_cache_free(vm_area_cachep, vma); +} + +/* + * sys_brk() for the most part doesn't need the global kernel + * lock, except when an application is doing something nasty + * like trying to un-brk an area that has already been mapped + * to a regular file. in this case, the unmapping will need + * to invoke file system routines that need the global lock. + */ +asmlinkage unsigned long sys_brk(unsigned long brk) +{ + unsigned long rlim, retval; + unsigned long newbrk, oldbrk; + struct mm_struct *mm = current->mm; + + down_write(&mm->mmap_sem); + + if (brk < mm->end_code) + goto out; + newbrk = PAGE_ALIGN(brk); + oldbrk = PAGE_ALIGN(mm->brk); + if (oldbrk == newbrk) + goto set_brk; + + /* Always allow shrinking brk. */ + if (brk <= mm->brk) { + if (!do_munmap(mm, newbrk, oldbrk-newbrk)) + goto set_brk; + goto out; + } + + /* Check against rlimit.. */ + rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; + if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim) + goto out; + + /* Check against existing mmap mappings. */ + if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) + goto out; + + /* Ok, looks good - let it rip. */ + if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk) + goto out; +set_brk: + mm->brk = brk; +out: + retval = mm->brk; + up_write(&mm->mmap_sem); + return retval; +} + +#ifdef DEBUG_MM_RB +static int browse_rb(struct rb_root *root) +{ + int i = 0, j; + struct rb_node *nd, *pn = NULL; + unsigned long prev = 0, pend = 0; + + for (nd = rb_first(root); nd; nd = rb_next(nd)) { + struct vm_area_struct *vma; + vma = rb_entry(nd, struct vm_area_struct, vm_rb); + if (vma->vm_start < prev) + printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1; + if (vma->vm_start < pend) + printk("vm_start %lx pend %lx\n", vma->vm_start, pend); + if (vma->vm_start > vma->vm_end) + printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start); + i++; + pn = nd; + } + j = 0; + for (nd = pn; nd; nd = rb_prev(nd)) { + j++; + } + if (i != j) + printk("backwards %d, forwards %d\n", j, i), i = 0; + return i; +} + +void validate_mm(struct mm_struct *mm) +{ + int bug = 0; + int i = 0; + struct vm_area_struct *tmp = mm->mmap; + while (tmp) { + tmp = tmp->vm_next; + i++; + } + if (i != mm->map_count) + printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1; + i = browse_rb(&mm->mm_rb); + if (i != mm->map_count) + printk("map_count %d rb %d\n", mm->map_count, i), bug = 1; + if (bug) + BUG(); +} +#else +#define validate_mm(mm) do { } while (0) +#endif + +static struct vm_area_struct * +find_vma_prepare(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **pprev, struct rb_node ***rb_link, + struct rb_node ** rb_parent) +{ + struct vm_area_struct * vma; + struct rb_node ** __rb_link, * __rb_parent, * rb_prev; + + __rb_link = &mm->mm_rb.rb_node; + rb_prev = __rb_parent = NULL; + vma = NULL; + + while (*__rb_link) { + struct vm_area_struct *vma_tmp; + + __rb_parent = *__rb_link; + vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); + + if (vma_tmp->vm_end > addr) { + vma = vma_tmp; + if (vma_tmp->vm_start <= addr) + return vma; + __rb_link = &__rb_parent->rb_left; + } else { + rb_prev = __rb_parent; + __rb_link = &__rb_parent->rb_right; + } + } + + *pprev = NULL; + if (rb_prev) + *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); + *rb_link = __rb_link; + *rb_parent = __rb_parent; + return vma; +} + +static inline void +__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *prev, struct rb_node *rb_parent) +{ + if (prev) { + vma->vm_next = prev->vm_next; + prev->vm_next = vma; + } else { + mm->mmap = vma; + if (rb_parent) + vma->vm_next = rb_entry(rb_parent, + struct vm_area_struct, vm_rb); + else + vma->vm_next = NULL; + } +} + +void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, + struct rb_node **rb_link, struct rb_node *rb_parent) +{ + rb_link_node(&vma->vm_rb, rb_parent, rb_link); + rb_insert_color(&vma->vm_rb, &mm->mm_rb); +} + +static inline void __vma_link_file(struct vm_area_struct *vma) +{ + struct file * file; + + file = vma->vm_file; + if (file) { + struct address_space *mapping = file->f_mapping; + + if (vma->vm_flags & VM_DENYWRITE) + atomic_dec(&file->f_dentry->d_inode->i_writecount); + if (vma->vm_flags & VM_SHARED) + mapping->i_mmap_writable++; + + flush_dcache_mmap_lock(mapping); + if (unlikely(vma->vm_flags & VM_NONLINEAR)) + vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear); + else + vma_prio_tree_insert(vma, &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); + } +} + +static void +__vma_link(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *prev, struct rb_node **rb_link, + struct rb_node *rb_parent) +{ + __vma_link_list(mm, vma, prev, rb_parent); + __vma_link_rb(mm, vma, rb_link, rb_parent); + __anon_vma_link(vma); +} + +static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *prev, struct rb_node **rb_link, + struct rb_node *rb_parent) +{ + struct address_space *mapping = NULL; + + if (vma->vm_file) + mapping = vma->vm_file->f_mapping; + + if (mapping) { + spin_lock(&mapping->i_mmap_lock); + vma->vm_truncate_count = mapping->truncate_count; + } + anon_vma_lock(vma); + + __vma_link(mm, vma, prev, rb_link, rb_parent); + __vma_link_file(vma); + + anon_vma_unlock(vma); + if (mapping) + spin_unlock(&mapping->i_mmap_lock); + + mm->map_count++; + validate_mm(mm); +} + +/* + * Helper for vma_adjust in the split_vma insert case: + * insert vm structure into list and rbtree and anon_vma, + * but it has already been inserted into prio_tree earlier. + */ +static void +__insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma) +{ + struct vm_area_struct * __vma, * prev; + struct rb_node ** rb_link, * rb_parent; + + __vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent); + if (__vma && __vma->vm_start < vma->vm_end) + BUG(); + __vma_link(mm, vma, prev, rb_link, rb_parent); + mm->map_count++; +} + +static inline void +__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *prev) +{ + prev->vm_next = vma->vm_next; + rb_erase(&vma->vm_rb, &mm->mm_rb); + if (mm->mmap_cache == vma) + mm->mmap_cache = prev; +} + +/* + * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that + * is already present in an i_mmap tree without adjusting the tree. + * The following helper function should be used when such adjustments + * are necessary. The "insert" vma (if any) is to be inserted + * before we drop the necessary locks. + */ +void vma_adjust(struct vm_area_struct *vma, unsigned long start, + unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert) +{ + struct mm_struct *mm = vma->vm_mm; + struct vm_area_struct *next = vma->vm_next; + struct vm_area_struct *importer = NULL; + struct address_space *mapping = NULL; + struct prio_tree_root *root = NULL; + struct file *file = vma->vm_file; + struct anon_vma *anon_vma = NULL; + long adjust_next = 0; + int remove_next = 0; + + if (next && !insert) { + if (end >= next->vm_end) { + /* + * vma expands, overlapping all the next, and + * perhaps the one after too (mprotect case 6). + */ +again: remove_next = 1 + (end > next->vm_end); + end = next->vm_end; + anon_vma = next->anon_vma; + importer = vma; + } else if (end > next->vm_start) { + /* + * vma expands, overlapping part of the next: + * mprotect case 5 shifting the boundary up. + */ + adjust_next = (end - next->vm_start) >> PAGE_SHIFT; + anon_vma = next->anon_vma; + importer = vma; + } else if (end < vma->vm_end) { + /* + * vma shrinks, and !insert tells it's not + * split_vma inserting another: so it must be + * mprotect case 4 shifting the boundary down. + */ + adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT); + anon_vma = next->anon_vma; + importer = next; + } + } + + if (file) { + mapping = file->f_mapping; + if (!(vma->vm_flags & VM_NONLINEAR)) + root = &mapping->i_mmap; + spin_lock(&mapping->i_mmap_lock); + if (importer && + vma->vm_truncate_count != next->vm_truncate_count) { + /* + * unmap_mapping_range might be in progress: + * ensure that the expanding vma is rescanned. + */ + importer->vm_truncate_count = 0; + } + if (insert) { + insert->vm_truncate_count = vma->vm_truncate_count; + /* + * Put into prio_tree now, so instantiated pages + * are visible to arm/parisc __flush_dcache_page + * throughout; but we cannot insert into address + * space until vma start or end is updated. + */ + __vma_link_file(insert); + } + } + + /* + * When changing only vma->vm_end, we don't really need + * anon_vma lock: but is that case worth optimizing out? + */ + if (vma->anon_vma) + anon_vma = vma->anon_vma; + if (anon_vma) { + spin_lock(&anon_vma->lock); + /* + * Easily overlooked: when mprotect shifts the boundary, + * make sure the expanding vma has anon_vma set if the + * shrinking vma had, to cover any anon pages imported. + */ + if (importer && !importer->anon_vma) { + importer->anon_vma = anon_vma; + __anon_vma_link(importer); + } + } + + if (root) { + flush_dcache_mmap_lock(mapping); + vma_prio_tree_remove(vma, root); + if (adjust_next) + vma_prio_tree_remove(next, root); + } + + vma->vm_start = start; + vma->vm_end = end; + vma->vm_pgoff = pgoff; + if (adjust_next) { + next->vm_start += adjust_next << PAGE_SHIFT; + next->vm_pgoff += adjust_next; + } + + if (root) { + if (adjust_next) + vma_prio_tree_insert(next, root); + vma_prio_tree_insert(vma, root); + flush_dcache_mmap_unlock(mapping); + } + + if (remove_next) { + /* + * vma_merge has merged next into vma, and needs + * us to remove next before dropping the locks. + */ + __vma_unlink(mm, next, vma); + if (file) + __remove_shared_vm_struct(next, file, mapping); + if (next->anon_vma) + __anon_vma_merge(vma, next); + } else if (insert) { + /* + * split_vma has split insert from vma, and needs + * us to insert it before dropping the locks + * (it may either follow vma or precede it). + */ + __insert_vm_struct(mm, insert); + } + + if (anon_vma) + spin_unlock(&anon_vma->lock); + if (mapping) + spin_unlock(&mapping->i_mmap_lock); + + if (remove_next) { + if (file) + fput(file); + mm->map_count--; + mpol_free(vma_policy(next)); + kmem_cache_free(vm_area_cachep, next); + /* + * In mprotect's case 6 (see comments on vma_merge), + * we must remove another next too. It would clutter + * up the code too much to do both in one go. + */ + if (remove_next == 2) { + next = vma->vm_next; + goto again; + } + } + + validate_mm(mm); +} + +/* + * If the vma has a ->close operation then the driver probably needs to release + * per-vma resources, so we don't attempt to merge those. + */ +#define VM_SPECIAL (VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED) + +static inline int is_mergeable_vma(struct vm_area_struct *vma, + struct file *file, unsigned long vm_flags) +{ + if (vma->vm_flags != vm_flags) + return 0; + if (vma->vm_file != file) + return 0; + if (vma->vm_ops && vma->vm_ops->close) + return 0; + return 1; +} + +static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1, + struct anon_vma *anon_vma2) +{ + return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2); +} + +/* + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) + * in front of (at a lower virtual address and file offset than) the vma. + * + * We cannot merge two vmas if they have differently assigned (non-NULL) + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. + * + * We don't check here for the merged mmap wrapping around the end of pagecache + * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which + * wrap, nor mmaps which cover the final page at index -1UL. + */ +static int +can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, + struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff) +{ + if (is_mergeable_vma(vma, file, vm_flags) && + is_mergeable_anon_vma(anon_vma, vma->anon_vma)) { + if (vma->vm_pgoff == vm_pgoff) + return 1; + } + return 0; +} + +/* + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff) + * beyond (at a higher virtual address and file offset than) the vma. + * + * We cannot merge two vmas if they have differently assigned (non-NULL) + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible. + */ +static int +can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, + struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff) +{ + if (is_mergeable_vma(vma, file, vm_flags) && + is_mergeable_anon_vma(anon_vma, vma->anon_vma)) { + pgoff_t vm_pglen; + vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + if (vma->vm_pgoff + vm_pglen == vm_pgoff) + return 1; + } + return 0; +} + +/* + * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out + * whether that can be merged with its predecessor or its successor. + * Or both (it neatly fills a hole). + * + * In most cases - when called for mmap, brk or mremap - [addr,end) is + * certain not to be mapped by the time vma_merge is called; but when + * called for mprotect, it is certain to be already mapped (either at + * an offset within prev, or at the start of next), and the flags of + * this area are about to be changed to vm_flags - and the no-change + * case has already been eliminated. + * + * The following mprotect cases have to be considered, where AAAA is + * the area passed down from mprotect_fixup, never extending beyond one + * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after: + * + * AAAA AAAA AAAA AAAA + * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX + * cannot merge might become might become might become + * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or + * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or + * mremap move: PPPPNNNNNNNN 8 + * AAAA + * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN + * might become case 1 below case 2 below case 3 below + * + * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX: + * mprotect_fixup updates vm_flags & vm_page_prot on successful return. + */ +struct vm_area_struct *vma_merge(struct mm_struct *mm, + struct vm_area_struct *prev, unsigned long addr, + unsigned long end, unsigned long vm_flags, + struct anon_vma *anon_vma, struct file *file, + pgoff_t pgoff, struct mempolicy *policy) +{ + pgoff_t pglen = (end - addr) >> PAGE_SHIFT; + struct vm_area_struct *area, *next; + + /* + * We later require that vma->vm_flags == vm_flags, + * so this tests vma->vm_flags & VM_SPECIAL, too. + */ + if (vm_flags & VM_SPECIAL) + return NULL; + + if (prev) + next = prev->vm_next; + else + next = mm->mmap; + area = next; + if (next && next->vm_end == end) /* cases 6, 7, 8 */ + next = next->vm_next; + + /* + * Can it merge with the predecessor? + */ + if (prev && prev->vm_end == addr && + mpol_equal(vma_policy(prev), policy) && + can_vma_merge_after(prev, vm_flags, + anon_vma, file, pgoff)) { + /* + * OK, it can. Can we now merge in the successor as well? + */ + if (next && end == next->vm_start && + mpol_equal(policy, vma_policy(next)) && + can_vma_merge_before(next, vm_flags, + anon_vma, file, pgoff+pglen) && + is_mergeable_anon_vma(prev->anon_vma, + next->anon_vma)) { + /* cases 1, 6 */ + vma_adjust(prev, prev->vm_start, + next->vm_end, prev->vm_pgoff, NULL); + } else /* cases 2, 5, 7 */ + vma_adjust(prev, prev->vm_start, + end, prev->vm_pgoff, NULL); + return prev; + } + + /* + * Can this new request be merged in front of next? + */ + if (next && end == next->vm_start && + mpol_equal(policy, vma_policy(next)) && + can_vma_merge_before(next, vm_flags, + anon_vma, file, pgoff+pglen)) { + if (prev && addr < prev->vm_end) /* case 4 */ + vma_adjust(prev, prev->vm_start, + addr, prev->vm_pgoff, NULL); + else /* cases 3, 8 */ + vma_adjust(area, addr, next->vm_end, + next->vm_pgoff - pglen, NULL); + return area; + } + + return NULL; +} + +/* + * find_mergeable_anon_vma is used by anon_vma_prepare, to check + * neighbouring vmas for a suitable anon_vma, before it goes off + * to allocate a new anon_vma. It checks because a repetitive + * sequence of mprotects and faults may otherwise lead to distinct + * anon_vmas being allocated, preventing vma merge in subsequent + * mprotect. + */ +struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) +{ + struct vm_area_struct *near; + unsigned long vm_flags; + + near = vma->vm_next; + if (!near) + goto try_prev; + + /* + * Since only mprotect tries to remerge vmas, match flags + * which might be mprotected into each other later on. + * Neither mlock nor madvise tries to remerge at present, + * so leave their flags as obstructing a merge. + */ + vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC); + vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC); + + if (near->anon_vma && vma->vm_end == near->vm_start && + mpol_equal(vma_policy(vma), vma_policy(near)) && + can_vma_merge_before(near, vm_flags, + NULL, vma->vm_file, vma->vm_pgoff + + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT))) + return near->anon_vma; +try_prev: + /* + * It is potentially slow to have to call find_vma_prev here. + * But it's only on the first write fault on the vma, not + * every time, and we could devise a way to avoid it later + * (e.g. stash info in next's anon_vma_node when assigning + * an anon_vma, or when trying vma_merge). Another time. + */ + if (find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma) + BUG(); + if (!near) + goto none; + + vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC); + vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC); + + if (near->anon_vma && near->vm_end == vma->vm_start && + mpol_equal(vma_policy(near), vma_policy(vma)) && + can_vma_merge_after(near, vm_flags, + NULL, vma->vm_file, vma->vm_pgoff)) + return near->anon_vma; +none: + /* + * There's no absolute need to look only at touching neighbours: + * we could search further afield for "compatible" anon_vmas. + * But it would probably just be a waste of time searching, + * or lead to too many vmas hanging off the same anon_vma. + * We're trying to allow mprotect remerging later on, + * not trying to minimize memory used for anon_vmas. + */ + return NULL; +} + +#ifdef CONFIG_PROC_FS +void __vm_stat_account(struct mm_struct *mm, unsigned long flags, + struct file *file, long pages) +{ + const unsigned long stack_flags + = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN); + +#ifdef CONFIG_HUGETLB + if (flags & VM_HUGETLB) { + if (!(flags & VM_DONTCOPY)) + mm->shared_vm += pages; + return; + } +#endif /* CONFIG_HUGETLB */ + + if (file) { + mm->shared_vm += pages; + if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC) + mm->exec_vm += pages; + } else if (flags & stack_flags) + mm->stack_vm += pages; + if (flags & (VM_RESERVED|VM_IO)) + mm->reserved_vm += pages; +} +#endif /* CONFIG_PROC_FS */ + +/* + * The caller must hold down_write(current->mm->mmap_sem). + */ + +unsigned long do_mmap_pgoff(struct file * file, unsigned long addr, + unsigned long len, unsigned long prot, + unsigned long flags, unsigned long pgoff) +{ + struct mm_struct * mm = current->mm; + struct vm_area_struct * vma, * prev; + struct inode *inode; + unsigned int vm_flags; + int correct_wcount = 0; + int error; + struct rb_node ** rb_link, * rb_parent; + int accountable = 1; + unsigned long charged = 0, reqprot = prot; + + if (file) { + if (is_file_hugepages(file)) + accountable = 0; + + if (!file->f_op || !file->f_op->mmap) + return -ENODEV; + + if ((prot & PROT_EXEC) && + (file->f_vfsmnt->mnt_flags & MNT_NOEXEC)) + return -EPERM; + } + /* + * Does the application expect PROT_READ to imply PROT_EXEC? + * + * (the exception is when the underlying filesystem is noexec + * mounted, in which case we dont add PROT_EXEC.) + */ + if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) + if (!(file && (file->f_vfsmnt->mnt_flags & MNT_NOEXEC))) + prot |= PROT_EXEC; + + if (!len) + return -EINVAL; + + /* Careful about overflows.. */ + len = PAGE_ALIGN(len); + if (!len || len > TASK_SIZE) + return -ENOMEM; + + /* offset overflow? */ + if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + return -EOVERFLOW; + + /* Too many mappings? */ + if (mm->map_count > sysctl_max_map_count) + return -ENOMEM; + + /* Obtain the address to map to. we verify (or select) it and ensure + * that it represents a valid section of the address space. + */ + addr = get_unmapped_area(file, addr, len, pgoff, flags); + if (addr & ~PAGE_MASK) + return addr; + + /* Do simple checking here so the lower-level routines won't have + * to. we assume access permissions have been handled by the open + * of the memory object, so we don't do any here. + */ + vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) | + mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + + if (flags & MAP_LOCKED) { + if (!can_do_mlock()) + return -EPERM; + vm_flags |= VM_LOCKED; + } + /* mlock MCL_FUTURE? */ + if (vm_flags & VM_LOCKED) { + unsigned long locked, lock_limit; + locked = mm->locked_vm << PAGE_SHIFT; + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + locked += len; + if (locked > lock_limit && !capable(CAP_IPC_LOCK)) + return -EAGAIN; + } + + inode = file ? file->f_dentry->d_inode : NULL; + + if (file) { + switch (flags & MAP_TYPE) { + case MAP_SHARED: + if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE)) + return -EACCES; + + /* + * Make sure we don't allow writing to an append-only + * file.. + */ + if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE)) + return -EACCES; + + /* + * Make sure there are no mandatory locks on the file. + */ + if (locks_verify_locked(inode)) + return -EAGAIN; + + vm_flags |= VM_SHARED | VM_MAYSHARE; + if (!(file->f_mode & FMODE_WRITE)) + vm_flags &= ~(VM_MAYWRITE | VM_SHARED); + + /* fall through */ + case MAP_PRIVATE: + if (!(file->f_mode & FMODE_READ)) + return -EACCES; + break; + + default: + return -EINVAL; + } + } else { + switch (flags & MAP_TYPE) { + case MAP_SHARED: + vm_flags |= VM_SHARED | VM_MAYSHARE; + break; + case MAP_PRIVATE: + /* + * Set pgoff according to addr for anon_vma. + */ + pgoff = addr >> PAGE_SHIFT; + break; + default: + return -EINVAL; + } + } + + error = security_file_mmap(file, reqprot, prot, flags); + if (error) + return error; + + /* Clear old maps */ + error = -ENOMEM; +munmap_back: + vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent); + if (vma && vma->vm_start < addr + len) { + if (do_munmap(mm, addr, len)) + return -ENOMEM; + goto munmap_back; + } + + /* Check against address space limit. */ + if ((mm->total_vm << PAGE_SHIFT) + len + > current->signal->rlim[RLIMIT_AS].rlim_cur) + return -ENOMEM; + + if (accountable && (!(flags & MAP_NORESERVE) || + sysctl_overcommit_memory == OVERCOMMIT_NEVER)) { + if (vm_flags & VM_SHARED) { + /* Check memory availability in shmem_file_setup? */ + vm_flags |= VM_ACCOUNT; + } else if (vm_flags & VM_WRITE) { + /* + * Private writable mapping: check memory availability + */ + charged = len >> PAGE_SHIFT; + if (security_vm_enough_memory(charged)) + return -ENOMEM; + vm_flags |= VM_ACCOUNT; + } + } + + /* + * Can we just expand an old private anonymous mapping? + * The VM_SHARED test is necessary because shmem_zero_setup + * will create the file object for a shared anonymous map below. + */ + if (!file && !(vm_flags & VM_SHARED) && + vma_merge(mm, prev, addr, addr + len, vm_flags, + NULL, NULL, pgoff, NULL)) + goto out; + + /* + * Determine the object being mapped and call the appropriate + * specific mapper. the address has already been validated, but + * not unmapped, but the maps are removed from the list. + */ + vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); + if (!vma) { + error = -ENOMEM; + goto unacct_error; + } + memset(vma, 0, sizeof(*vma)); + + vma->vm_mm = mm; + vma->vm_start = addr; + vma->vm_end = addr + len; + vma->vm_flags = vm_flags; + vma->vm_page_prot = protection_map[vm_flags & 0x0f]; + vma->vm_pgoff = pgoff; + + if (file) { + error = -EINVAL; + if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP)) + goto free_vma; + if (vm_flags & VM_DENYWRITE) { + error = deny_write_access(file); + if (error) + goto free_vma; + correct_wcount = 1; + } + vma->vm_file = file; + get_file(file); + error = file->f_op->mmap(file, vma); + if (error) + goto unmap_and_free_vma; + } else if (vm_flags & VM_SHARED) { + error = shmem_zero_setup(vma); + if (error) + goto free_vma; + } + + /* We set VM_ACCOUNT in a shared mapping's vm_flags, to inform + * shmem_zero_setup (perhaps called through /dev/zero's ->mmap) + * that memory reservation must be checked; but that reservation + * belongs to shared memory object, not to vma: so now clear it. + */ + if ((vm_flags & (VM_SHARED|VM_ACCOUNT)) == (VM_SHARED|VM_ACCOUNT)) + vma->vm_flags &= ~VM_ACCOUNT; + + /* Can addr have changed?? + * + * Answer: Yes, several device drivers can do it in their + * f_op->mmap method. -DaveM + */ + addr = vma->vm_start; + pgoff = vma->vm_pgoff; + vm_flags = vma->vm_flags; + + if (!file || !vma_merge(mm, prev, addr, vma->vm_end, + vma->vm_flags, NULL, file, pgoff, vma_policy(vma))) { + file = vma->vm_file; + vma_link(mm, vma, prev, rb_link, rb_parent); + if (correct_wcount) + atomic_inc(&inode->i_writecount); + } else { + if (file) { + if (correct_wcount) + atomic_inc(&inode->i_writecount); + fput(file); + } + mpol_free(vma_policy(vma)); + kmem_cache_free(vm_area_cachep, vma); + } +out: + mm->total_vm += len >> PAGE_SHIFT; + __vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT); + if (vm_flags & VM_LOCKED) { + mm->locked_vm += len >> PAGE_SHIFT; + make_pages_present(addr, addr + len); + } + if (flags & MAP_POPULATE) { + up_write(&mm->mmap_sem); + sys_remap_file_pages(addr, len, 0, + pgoff, flags & MAP_NONBLOCK); + down_write(&mm->mmap_sem); + } + return addr; + +unmap_and_free_vma: + if (correct_wcount) + atomic_inc(&inode->i_writecount); + vma->vm_file = NULL; + fput(file); + + /* Undo any partial mapping done by a device driver. */ + zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start, NULL); +free_vma: + kmem_cache_free(vm_area_cachep, vma); +unacct_error: + if (charged) + vm_unacct_memory(charged); + return error; +} + +EXPORT_SYMBOL(do_mmap_pgoff); + +/* Get an address range which is currently unmapped. + * For shmat() with addr=0. + * + * Ugly calling convention alert: + * Return value with the low bits set means error value, + * ie + * if (ret & ~PAGE_MASK) + * error = ret; + * + * This function "knows" that -ENOMEM has the bits set. + */ +#ifndef HAVE_ARCH_UNMAPPED_AREA +unsigned long +arch_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long start_addr; + + if (len > TASK_SIZE) + return -ENOMEM; + + if (addr) { + addr = PAGE_ALIGN(addr); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + start_addr = addr = mm->free_area_cache; + +full_search: + for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { + /* At this point: (!vma || addr < vma->vm_end). */ + if (TASK_SIZE - len < addr) { + /* + * Start a new search - just in case we missed + * some holes. + */ + if (start_addr != TASK_UNMAPPED_BASE) { + start_addr = addr = TASK_UNMAPPED_BASE; + goto full_search; + } + return -ENOMEM; + } + if (!vma || addr + len <= vma->vm_start) { + /* + * Remember the place where we stopped the search: + */ + mm->free_area_cache = addr + len; + return addr; + } + addr = vma->vm_end; + } +} +#endif + +void arch_unmap_area(struct vm_area_struct *area) +{ + /* + * Is this a new hole at the lowest possible address? + */ + if (area->vm_start >= TASK_UNMAPPED_BASE && + area->vm_start < area->vm_mm->free_area_cache) + area->vm_mm->free_area_cache = area->vm_start; +} + +/* + * This mmap-allocator allocates new areas top-down from below the + * stack's low limit (the base): + */ +#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN +unsigned long +arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, + const unsigned long len, const unsigned long pgoff, + const unsigned long flags) +{ + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + unsigned long addr = addr0; + + /* requested length too big for entire address space */ + if (len > TASK_SIZE) + return -ENOMEM; + + /* requesting a specific address */ + if (addr) { + addr = PAGE_ALIGN(addr); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + + /* either no address requested or can't fit in requested address hole */ + addr = mm->free_area_cache; + + /* make sure it can fit in the remaining address space */ + if (addr >= len) { + vma = find_vma(mm, addr-len); + if (!vma || addr <= vma->vm_start) + /* remember the address as a hint for next time */ + return (mm->free_area_cache = addr-len); + } + + addr = mm->mmap_base-len; + + do { + /* + * Lookup failure means no vma is above this address, + * else if new region fits below vma->vm_start, + * return with success: + */ + vma = find_vma(mm, addr); + if (!vma || addr+len <= vma->vm_start) + /* remember the address as a hint for next time */ + return (mm->free_area_cache = addr); + + /* try just below the current vma->vm_start */ + addr = vma->vm_start-len; + } while (len <= vma->vm_start); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + mm->free_area_cache = TASK_UNMAPPED_BASE; + addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags); + /* + * Restore the topdown base: + */ + mm->free_area_cache = mm->mmap_base; + + return addr; +} +#endif + +void arch_unmap_area_topdown(struct vm_area_struct *area) +{ + /* + * Is this a new hole at the highest possible address? + */ + if (area->vm_end > area->vm_mm->free_area_cache) + area->vm_mm->free_area_cache = area->vm_end; + + /* dont allow allocations above current base */ + if (area->vm_mm->free_area_cache > area->vm_mm->mmap_base) + area->vm_mm->free_area_cache = area->vm_mm->mmap_base; +} + +unsigned long +get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + if (flags & MAP_FIXED) { + unsigned long ret; + + if (addr > TASK_SIZE - len) + return -ENOMEM; + if (addr & ~PAGE_MASK) + return -EINVAL; + if (file && is_file_hugepages(file)) { + /* + * Check if the given range is hugepage aligned, and + * can be made suitable for hugepages. + */ + ret = prepare_hugepage_range(addr, len); + } else { + /* + * Ensure that a normal request is not falling in a + * reserved hugepage range. For some archs like IA-64, + * there is a separate region for hugepages. + */ + ret = is_hugepage_only_range(current->mm, addr, len); + } + if (ret) + return -EINVAL; + return addr; + } + + if (file && file->f_op && file->f_op->get_unmapped_area) + return file->f_op->get_unmapped_area(file, addr, len, + pgoff, flags); + + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +} + +EXPORT_SYMBOL(get_unmapped_area); + +/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ +struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr) +{ + struct vm_area_struct *vma = NULL; + + if (mm) { + /* Check the cache first. */ + /* (Cache hit rate is typically around 35%.) */ + vma = mm->mmap_cache; + if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) { + struct rb_node * rb_node; + + rb_node = mm->mm_rb.rb_node; + vma = NULL; + + while (rb_node) { + struct vm_area_struct * vma_tmp; + + vma_tmp = rb_entry(rb_node, + struct vm_area_struct, vm_rb); + + if (vma_tmp->vm_end > addr) { + vma = vma_tmp; + if (vma_tmp->vm_start <= addr) + break; + rb_node = rb_node->rb_left; + } else + rb_node = rb_node->rb_right; + } + if (vma) + mm->mmap_cache = vma; + } + } + return vma; +} + +EXPORT_SYMBOL(find_vma); + +/* Same as find_vma, but also return a pointer to the previous VMA in *pprev. */ +struct vm_area_struct * +find_vma_prev(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **pprev) +{ + struct vm_area_struct *vma = NULL, *prev = NULL; + struct rb_node * rb_node; + if (!mm) + goto out; + + /* Guard against addr being lower than the first VMA */ + vma = mm->mmap; + + /* Go through the RB tree quickly. */ + rb_node = mm->mm_rb.rb_node; + + while (rb_node) { + struct vm_area_struct *vma_tmp; + vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); + + if (addr < vma_tmp->vm_end) { + rb_node = rb_node->rb_left; + } else { + prev = vma_tmp; + if (!prev->vm_next || (addr < prev->vm_next->vm_end)) + break; + rb_node = rb_node->rb_right; + } + } + +out: + *pprev = prev; + return prev ? prev->vm_next : vma; +} + +/* + * Verify that the stack growth is acceptable and + * update accounting. This is shared with both the + * grow-up and grow-down cases. + */ +static int acct_stack_growth(struct vm_area_struct * vma, unsigned long size, unsigned long grow) +{ + struct mm_struct *mm = vma->vm_mm; + struct rlimit *rlim = current->signal->rlim; + + /* address space limit tests */ + if (mm->total_vm + grow > rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT) + return -ENOMEM; + + /* Stack limit test */ + if (size > rlim[RLIMIT_STACK].rlim_cur) + return -ENOMEM; + + /* mlock limit tests */ + if (vma->vm_flags & VM_LOCKED) { + unsigned long locked; + unsigned long limit; + locked = mm->locked_vm + grow; + limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; + if (locked > limit && !capable(CAP_IPC_LOCK)) + return -ENOMEM; + } + + /* + * Overcommit.. This must be the final test, as it will + * update security statistics. + */ + if (security_vm_enough_memory(grow)) + return -ENOMEM; + + /* Ok, everything looks good - let it rip */ + mm->total_vm += grow; + if (vma->vm_flags & VM_LOCKED) + mm->locked_vm += grow; + __vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow); + return 0; +} + +#ifdef CONFIG_STACK_GROWSUP +/* + * vma is the first one with address > vma->vm_end. Have to extend vma. + */ +int expand_stack(struct vm_area_struct * vma, unsigned long address) +{ + int error; + + if (!(vma->vm_flags & VM_GROWSUP)) + return -EFAULT; + + /* + * We must make sure the anon_vma is allocated + * so that the anon_vma locking is not a noop. + */ + if (unlikely(anon_vma_prepare(vma))) + return -ENOMEM; + anon_vma_lock(vma); + + /* + * vma->vm_start/vm_end cannot change under us because the caller + * is required to hold the mmap_sem in read mode. We need the + * anon_vma lock to serialize against concurrent expand_stacks. + */ + address += 4 + PAGE_SIZE - 1; + address &= PAGE_MASK; + error = 0; + + /* Somebody else might have raced and expanded it already */ + if (address > vma->vm_end) { + unsigned long size, grow; + + size = address - vma->vm_start; + grow = (address - vma->vm_end) >> PAGE_SHIFT; + + error = acct_stack_growth(vma, size, grow); + if (!error) + vma->vm_end = address; + } + anon_vma_unlock(vma); + return error; +} + +struct vm_area_struct * +find_extend_vma(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma, *prev; + + addr &= PAGE_MASK; + vma = find_vma_prev(mm, addr, &prev); + if (vma && (vma->vm_start <= addr)) + return vma; + if (!prev || expand_stack(prev, addr)) + return NULL; + if (prev->vm_flags & VM_LOCKED) { + make_pages_present(addr, prev->vm_end); + } + return prev; +} +#else +/* + * vma is the first one with address < vma->vm_start. Have to extend vma. + */ +int expand_stack(struct vm_area_struct *vma, unsigned long address) +{ + int error; + + /* + * We must make sure the anon_vma is allocated + * so that the anon_vma locking is not a noop. + */ + if (unlikely(anon_vma_prepare(vma))) + return -ENOMEM; + anon_vma_lock(vma); + + /* + * vma->vm_start/vm_end cannot change under us because the caller + * is required to hold the mmap_sem in read mode. We need the + * anon_vma lock to serialize against concurrent expand_stacks. + */ + address &= PAGE_MASK; + error = 0; + + /* Somebody else might have raced and expanded it already */ + if (address < vma->vm_start) { + unsigned long size, grow; + + size = vma->vm_end - address; + grow = (vma->vm_start - address) >> PAGE_SHIFT; + + error = acct_stack_growth(vma, size, grow); + if (!error) { + vma->vm_start = address; + vma->vm_pgoff -= grow; + } + } + anon_vma_unlock(vma); + return error; +} + +struct vm_area_struct * +find_extend_vma(struct mm_struct * mm, unsigned long addr) +{ + struct vm_area_struct * vma; + unsigned long start; + + addr &= PAGE_MASK; + vma = find_vma(mm,addr); + if (!vma) + return NULL; + if (vma->vm_start <= addr) + return vma; + if (!(vma->vm_flags & VM_GROWSDOWN)) + return NULL; + start = vma->vm_start; + if (expand_stack(vma, addr)) + return NULL; + if (vma->vm_flags & VM_LOCKED) { + make_pages_present(addr, start); + } + return vma; +} +#endif + +/* + * Try to free as many page directory entries as we can, + * without having to work very hard at actually scanning + * the page tables themselves. + * + * Right now we try to free page tables if we have a nice + * PGDIR-aligned area that got free'd up. We could be more + * granular if we want to, but this is fast and simple, + * and covers the bad cases. + * + * "prev", if it exists, points to a vma before the one + * we just free'd - but there's no telling how much before. + */ +static void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev, + unsigned long start, unsigned long end) +{ + unsigned long first = start & PGDIR_MASK; + unsigned long last = end + PGDIR_SIZE - 1; + struct mm_struct *mm = tlb->mm; + + if (last > MM_VM_SIZE(mm) || last < end) + last = MM_VM_SIZE(mm); + + if (!prev) { + prev = mm->mmap; + if (!prev) + goto no_mmaps; + if (prev->vm_end > start) { + if (last > prev->vm_start) + last = prev->vm_start; + goto no_mmaps; + } + } + for (;;) { + struct vm_area_struct *next = prev->vm_next; + + if (next) { + if (next->vm_start < start) { + prev = next; + continue; + } + if (last > next->vm_start) + last = next->vm_start; + } + if (prev->vm_end > first) + first = prev->vm_end; + break; + } +no_mmaps: + if (last < first) /* for arches with discontiguous pgd indices */ + return; + if (first < FIRST_USER_PGD_NR * PGDIR_SIZE) + first = FIRST_USER_PGD_NR * PGDIR_SIZE; + /* No point trying to free anything if we're in the same pte page */ + if ((first & PMD_MASK) < (last & PMD_MASK)) { + clear_page_range(tlb, first, last); + flush_tlb_pgtables(mm, first, last); + } +} + +/* Normal function to fix up a mapping + * This function is the default for when an area has no specific + * function. This may be used as part of a more specific routine. + * + * By the time this function is called, the area struct has been + * removed from the process mapping list. + */ +static void unmap_vma(struct mm_struct *mm, struct vm_area_struct *area) +{ + size_t len = area->vm_end - area->vm_start; + + area->vm_mm->total_vm -= len >> PAGE_SHIFT; + if (area->vm_flags & VM_LOCKED) + area->vm_mm->locked_vm -= len >> PAGE_SHIFT; + vm_stat_unaccount(area); + area->vm_mm->unmap_area(area); + remove_vm_struct(area); +} + +/* + * Update the VMA and inode share lists. + * + * Ok - we have the memory areas we should free on the 'free' list, + * so release them, and do the vma updates. + */ +static void unmap_vma_list(struct mm_struct *mm, + struct vm_area_struct *mpnt) +{ + do { + struct vm_area_struct *next = mpnt->vm_next; + unmap_vma(mm, mpnt); + mpnt = next; + } while (mpnt != NULL); + validate_mm(mm); +} + +/* + * Get rid of page table information in the indicated region. + * + * Called with the page table lock held. + */ +static void unmap_region(struct mm_struct *mm, + struct vm_area_struct *vma, + struct vm_area_struct *prev, + unsigned long start, + unsigned long end) +{ + struct mmu_gather *tlb; + unsigned long nr_accounted = 0; + + lru_add_drain(); + tlb = tlb_gather_mmu(mm, 0); + unmap_vmas(&tlb, mm, vma, start, end, &nr_accounted, NULL); + vm_unacct_memory(nr_accounted); + + if (is_hugepage_only_range(mm, start, end - start)) + hugetlb_free_pgtables(tlb, prev, start, end); + else + free_pgtables(tlb, prev, start, end); + tlb_finish_mmu(tlb, start, end); +} + +/* + * Create a list of vma's touched by the unmap, removing them from the mm's + * vma list as we go.. + */ +static void +detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, + struct vm_area_struct *prev, unsigned long end) +{ + struct vm_area_struct **insertion_point; + struct vm_area_struct *tail_vma = NULL; + + insertion_point = (prev ? &prev->vm_next : &mm->mmap); + do { + rb_erase(&vma->vm_rb, &mm->mm_rb); + mm->map_count--; + tail_vma = vma; + vma = vma->vm_next; + } while (vma && vma->vm_start < end); + *insertion_point = vma; + tail_vma->vm_next = NULL; + mm->mmap_cache = NULL; /* Kill the cache. */ +} + +/* + * Split a vma into two pieces at address 'addr', a new vma is allocated + * either for the first part or the the tail. + */ +int split_vma(struct mm_struct * mm, struct vm_area_struct * vma, + unsigned long addr, int new_below) +{ + struct mempolicy *pol; + struct vm_area_struct *new; + + if (is_vm_hugetlb_page(vma) && (addr & ~HPAGE_MASK)) + return -EINVAL; + + if (mm->map_count >= sysctl_max_map_count) + return -ENOMEM; + + new = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); + if (!new) + return -ENOMEM; + + /* most fields are the same, copy all, and then fixup */ + *new = *vma; + + if (new_below) + new->vm_end = addr; + else { + new->vm_start = addr; + new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + } + + pol = mpol_copy(vma_policy(vma)); + if (IS_ERR(pol)) { + kmem_cache_free(vm_area_cachep, new); + return PTR_ERR(pol); + } + vma_set_policy(new, pol); + + if (new->vm_file) + get_file(new->vm_file); + + if (new->vm_ops && new->vm_ops->open) + new->vm_ops->open(new); + + if (new_below) + vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff + + ((addr - new->vm_start) >> PAGE_SHIFT), new); + else + vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); + + return 0; +} + +/* Munmap is split into 2 main parts -- this part which finds + * what needs doing, and the areas themselves, which do the + * work. This now handles partial unmappings. + * Jeremy Fitzhardinge <jeremy@goop.org> + */ +int do_munmap(struct mm_struct *mm, unsigned long start, size_t len) +{ + unsigned long end; + struct vm_area_struct *mpnt, *prev, *last; + + if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start) + return -EINVAL; + + if ((len = PAGE_ALIGN(len)) == 0) + return -EINVAL; + + /* Find the first overlapping VMA */ + mpnt = find_vma_prev(mm, start, &prev); + if (!mpnt) + return 0; + /* we have start < mpnt->vm_end */ + + /* if it doesn't overlap, we have nothing.. */ + end = start + len; + if (mpnt->vm_start >= end) + return 0; + + /* + * If we need to split any vma, do it now to save pain later. + * + * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially + * unmapped vm_area_struct will remain in use: so lower split_vma + * places tmp vma above, and higher split_vma places tmp vma below. + */ + if (start > mpnt->vm_start) { + int error = split_vma(mm, mpnt, start, 0); + if (error) + return error; + prev = mpnt; + } + + /* Does it split the last one? */ + last = find_vma(mm, end); + if (last && end > last->vm_start) { + int error = split_vma(mm, last, end, 1); + if (error) + return error; + } + mpnt = prev? prev->vm_next: mm->mmap; + + /* + * Remove the vma's, and unmap the actual pages + */ + detach_vmas_to_be_unmapped(mm, mpnt, prev, end); + spin_lock(&mm->page_table_lock); + unmap_region(mm, mpnt, prev, start, end); + spin_unlock(&mm->page_table_lock); + + /* Fix up all other VM information */ + unmap_vma_list(mm, mpnt); + + return 0; +} + +EXPORT_SYMBOL(do_munmap); + +asmlinkage long sys_munmap(unsigned long addr, size_t len) +{ + int ret; + struct mm_struct *mm = current->mm; + + profile_munmap(addr); + + down_write(&mm->mmap_sem); + ret = do_munmap(mm, addr, len); + up_write(&mm->mmap_sem); + return ret; +} + +static inline void verify_mm_writelocked(struct mm_struct *mm) +{ +#ifdef CONFIG_DEBUG_KERNEL + if (unlikely(down_read_trylock(&mm->mmap_sem))) { + WARN_ON(1); + up_read(&mm->mmap_sem); + } +#endif +} + +/* + * this is really a simplified "do_mmap". it only handles + * anonymous maps. eventually we may be able to do some + * brk-specific accounting here. + */ +unsigned long do_brk(unsigned long addr, unsigned long len) +{ + struct mm_struct * mm = current->mm; + struct vm_area_struct * vma, * prev; + unsigned long flags; + struct rb_node ** rb_link, * rb_parent; + pgoff_t pgoff = addr >> PAGE_SHIFT; + + len = PAGE_ALIGN(len); + if (!len) + return addr; + + if ((addr + len) > TASK_SIZE || (addr + len) < addr) + return -EINVAL; + + /* + * mlock MCL_FUTURE? + */ + if (mm->def_flags & VM_LOCKED) { + unsigned long locked, lock_limit; + locked = mm->locked_vm << PAGE_SHIFT; + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + locked += len; + if (locked > lock_limit && !capable(CAP_IPC_LOCK)) + return -EAGAIN; + } + + /* + * mm->mmap_sem is required to protect against another thread + * changing the mappings in case we sleep. + */ + verify_mm_writelocked(mm); + + /* + * Clear old maps. this also does some error checking for us + */ + munmap_back: + vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent); + if (vma && vma->vm_start < addr + len) { + if (do_munmap(mm, addr, len)) + return -ENOMEM; + goto munmap_back; + } + + /* Check against address space limits *after* clearing old maps... */ + if ((mm->total_vm << PAGE_SHIFT) + len + > current->signal->rlim[RLIMIT_AS].rlim_cur) + return -ENOMEM; + + if (mm->map_count > sysctl_max_map_count) + return -ENOMEM; + + if (security_vm_enough_memory(len >> PAGE_SHIFT)) + return -ENOMEM; + + flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; + + /* Can we just expand an old private anonymous mapping? */ + if (vma_merge(mm, prev, addr, addr + len, flags, + NULL, NULL, pgoff, NULL)) + goto out; + + /* + * create a vma struct for an anonymous mapping + */ + vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); + if (!vma) { + vm_unacct_memory(len >> PAGE_SHIFT); + return -ENOMEM; + } + memset(vma, 0, sizeof(*vma)); + + vma->vm_mm = mm; + vma->vm_start = addr; + vma->vm_end = addr + len; + vma->vm_pgoff = pgoff; + vma->vm_flags = flags; + vma->vm_page_prot = protection_map[flags & 0x0f]; + vma_link(mm, vma, prev, rb_link, rb_parent); +out: + mm->total_vm += len >> PAGE_SHIFT; + if (flags & VM_LOCKED) { + mm->locked_vm += len >> PAGE_SHIFT; + make_pages_present(addr, addr + len); + } + return addr; +} + +EXPORT_SYMBOL(do_brk); + +/* Release all mmaps. */ +void exit_mmap(struct mm_struct *mm) +{ + struct mmu_gather *tlb; + struct vm_area_struct *vma; + unsigned long nr_accounted = 0; + + lru_add_drain(); + + spin_lock(&mm->page_table_lock); + + tlb = tlb_gather_mmu(mm, 1); + flush_cache_mm(mm); + /* Use ~0UL here to ensure all VMAs in the mm are unmapped */ + mm->map_count -= unmap_vmas(&tlb, mm, mm->mmap, 0, + ~0UL, &nr_accounted, NULL); + vm_unacct_memory(nr_accounted); + BUG_ON(mm->map_count); /* This is just debugging */ + clear_page_range(tlb, FIRST_USER_PGD_NR * PGDIR_SIZE, MM_VM_SIZE(mm)); + + tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm)); + + vma = mm->mmap; + mm->mmap = mm->mmap_cache = NULL; + mm->mm_rb = RB_ROOT; + set_mm_counter(mm, rss, 0); + mm->total_vm = 0; + mm->locked_vm = 0; + + spin_unlock(&mm->page_table_lock); + + /* + * Walk the list again, actually closing and freeing it + * without holding any MM locks. + */ + while (vma) { + struct vm_area_struct *next = vma->vm_next; + remove_vm_struct(vma); + vma = next; + } +} + +/* Insert vm structure into process list sorted by address + * and into the inode's i_mmap tree. If vm_file is non-NULL + * then i_mmap_lock is taken here. + */ +int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma) +{ + struct vm_area_struct * __vma, * prev; + struct rb_node ** rb_link, * rb_parent; + + /* + * The vm_pgoff of a purely anonymous vma should be irrelevant + * until its first write fault, when page's anon_vma and index + * are set. But now set the vm_pgoff it will almost certainly + * end up with (unless mremap moves it elsewhere before that + * first wfault), so /proc/pid/maps tells a consistent story. + * + * By setting it to reflect the virtual start address of the + * vma, merges and splits can happen in a seamless way, just + * using the existing file pgoff checks and manipulations. + * Similarly in do_mmap_pgoff and in do_brk. + */ + if (!vma->vm_file) { + BUG_ON(vma->anon_vma); + vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + } + __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent); + if (__vma && __vma->vm_start < vma->vm_end) + return -ENOMEM; + vma_link(mm, vma, prev, rb_link, rb_parent); + return 0; +} + +/* + * Copy the vma structure to a new location in the same mm, + * prior to moving page table entries, to effect an mremap move. + */ +struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, + unsigned long addr, unsigned long len, pgoff_t pgoff) +{ + struct vm_area_struct *vma = *vmap; + unsigned long vma_start = vma->vm_start; + struct mm_struct *mm = vma->vm_mm; + struct vm_area_struct *new_vma, *prev; + struct rb_node **rb_link, *rb_parent; + struct mempolicy *pol; + + /* + * If anonymous vma has not yet been faulted, update new pgoff + * to match new location, to increase its chance of merging. + */ + if (!vma->vm_file && !vma->anon_vma) + pgoff = addr >> PAGE_SHIFT; + + find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent); + new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, + vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); + if (new_vma) { + /* + * Source vma may have been merged into new_vma + */ + if (vma_start >= new_vma->vm_start && + vma_start < new_vma->vm_end) + *vmap = new_vma; + } else { + new_vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL); + if (new_vma) { + *new_vma = *vma; + pol = mpol_copy(vma_policy(vma)); + if (IS_ERR(pol)) { + kmem_cache_free(vm_area_cachep, new_vma); + return NULL; + } + vma_set_policy(new_vma, pol); + new_vma->vm_start = addr; + new_vma->vm_end = addr + len; + new_vma->vm_pgoff = pgoff; + if (new_vma->vm_file) + get_file(new_vma->vm_file); + if (new_vma->vm_ops && new_vma->vm_ops->open) + new_vma->vm_ops->open(new_vma); + vma_link(mm, new_vma, prev, rb_link, rb_parent); + } + } + return new_vma; +} diff --git a/mm/mprotect.c b/mm/mprotect.c new file mode 100644 index 000000000000..e9fbd013ad9a --- /dev/null +++ b/mm/mprotect.c @@ -0,0 +1,282 @@ +/* + * mm/mprotect.c + * + * (C) Copyright 1994 Linus Torvalds + * (C) Copyright 2002 Christoph Hellwig + * + * Address space accounting code <alan@redhat.com> + * (C) Copyright 2002 Red Hat Inc, All Rights Reserved + */ + +#include <linux/mm.h> +#include <linux/hugetlb.h> +#include <linux/slab.h> +#include <linux/shm.h> +#include <linux/mman.h> +#include <linux/fs.h> +#include <linux/highmem.h> +#include <linux/security.h> +#include <linux/mempolicy.h> +#include <linux/personality.h> +#include <linux/syscalls.h> + +#include <asm/uaccess.h> +#include <asm/pgtable.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> + +static void change_pte_range(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, unsigned long end, pgprot_t newprot) +{ + pte_t *pte; + + pte = pte_offset_map(pmd, addr); + do { + if (pte_present(*pte)) { + pte_t ptent; + + /* Avoid an SMP race with hardware updated dirty/clean + * bits by wiping the pte and then setting the new pte + * into place. + */ + ptent = pte_modify(ptep_get_and_clear(mm, addr, pte), newprot); + set_pte_at(mm, addr, pte, ptent); + lazy_mmu_prot_update(ptent); + } + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); +} + +static inline void change_pmd_range(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, pgprot_t newprot) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + change_pte_range(mm, pmd, addr, next, newprot); + } while (pmd++, addr = next, addr != end); +} + +static inline void change_pud_range(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, pgprot_t newprot) +{ + pud_t *pud; + unsigned long next; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + change_pmd_range(mm, pud, addr, next, newprot); + } while (pud++, addr = next, addr != end); +} + +static void change_protection(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, pgprot_t newprot) +{ + struct mm_struct *mm = vma->vm_mm; + pgd_t *pgd; + unsigned long next; + unsigned long start = addr; + + BUG_ON(addr >= end); + pgd = pgd_offset(mm, addr); + flush_cache_range(vma, addr, end); + spin_lock(&mm->page_table_lock); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + change_pud_range(mm, pgd, addr, next, newprot); + } while (pgd++, addr = next, addr != end); + flush_tlb_range(vma, start, end); + spin_unlock(&mm->page_table_lock); +} + +static int +mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, + unsigned long start, unsigned long end, unsigned long newflags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long oldflags = vma->vm_flags; + long nrpages = (end - start) >> PAGE_SHIFT; + unsigned long charged = 0; + pgprot_t newprot; + pgoff_t pgoff; + int error; + + if (newflags == oldflags) { + *pprev = vma; + return 0; + } + + /* + * If we make a private mapping writable we increase our commit; + * but (without finer accounting) cannot reduce our commit if we + * make it unwritable again. + * + * FIXME? We haven't defined a VM_NORESERVE flag, so mprotecting + * a MAP_NORESERVE private mapping to writable will now reserve. + */ + if (newflags & VM_WRITE) { + if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_SHARED|VM_HUGETLB))) { + charged = nrpages; + if (security_vm_enough_memory(charged)) + return -ENOMEM; + newflags |= VM_ACCOUNT; + } + } + + newprot = protection_map[newflags & 0xf]; + + /* + * First try to merge with previous and/or next vma. + */ + pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); + *pprev = vma_merge(mm, *pprev, start, end, newflags, + vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma)); + if (*pprev) { + vma = *pprev; + goto success; + } + + *pprev = vma; + + if (start != vma->vm_start) { + error = split_vma(mm, vma, start, 1); + if (error) + goto fail; + } + + if (end != vma->vm_end) { + error = split_vma(mm, vma, end, 0); + if (error) + goto fail; + } + +success: + /* + * vm_flags and vm_page_prot are protected by the mmap_sem + * held in write mode. + */ + vma->vm_flags = newflags; + vma->vm_page_prot = newprot; + change_protection(vma, start, end, newprot); + __vm_stat_account(mm, oldflags, vma->vm_file, -nrpages); + __vm_stat_account(mm, newflags, vma->vm_file, nrpages); + return 0; + +fail: + vm_unacct_memory(charged); + return error; +} + +asmlinkage long +sys_mprotect(unsigned long start, size_t len, unsigned long prot) +{ + unsigned long vm_flags, nstart, end, tmp, reqprot; + struct vm_area_struct *vma, *prev; + int error = -EINVAL; + const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); + prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); + if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ + return -EINVAL; + + if (start & ~PAGE_MASK) + return -EINVAL; + if (!len) + return 0; + len = PAGE_ALIGN(len); + end = start + len; + if (end <= start) + return -ENOMEM; + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM)) + return -EINVAL; + + reqprot = prot; + /* + * Does the application expect PROT_READ to imply PROT_EXEC: + */ + if (unlikely((prot & PROT_READ) && + (current->personality & READ_IMPLIES_EXEC))) + prot |= PROT_EXEC; + + vm_flags = calc_vm_prot_bits(prot); + + down_write(¤t->mm->mmap_sem); + + vma = find_vma_prev(current->mm, start, &prev); + error = -ENOMEM; + if (!vma) + goto out; + if (unlikely(grows & PROT_GROWSDOWN)) { + if (vma->vm_start >= end) + goto out; + start = vma->vm_start; + error = -EINVAL; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto out; + } + else { + if (vma->vm_start > start) + goto out; + if (unlikely(grows & PROT_GROWSUP)) { + end = vma->vm_end; + error = -EINVAL; + if (!(vma->vm_flags & VM_GROWSUP)) + goto out; + } + } + if (start > vma->vm_start) + prev = vma; + + for (nstart = start ; ; ) { + unsigned long newflags; + + /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ + + if (is_vm_hugetlb_page(vma)) { + error = -EACCES; + goto out; + } + + newflags = vm_flags | (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC)); + + if ((newflags & ~(newflags >> 4)) & 0xf) { + error = -EACCES; + goto out; + } + + error = security_file_mprotect(vma, reqprot, prot); + if (error) + goto out; + + tmp = vma->vm_end; + if (tmp > end) + tmp = end; + error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); + if (error) + goto out; + nstart = tmp; + + if (nstart < prev->vm_end) + nstart = prev->vm_end; + if (nstart >= end) + goto out; + + vma = prev->vm_next; + if (!vma || vma->vm_start != nstart) { + error = -ENOMEM; + goto out; + } + } +out: + up_write(¤t->mm->mmap_sem); + return error; +} diff --git a/mm/mremap.c b/mm/mremap.c new file mode 100644 index 000000000000..0d1c1b9c7a0a --- /dev/null +++ b/mm/mremap.c @@ -0,0 +1,426 @@ +/* + * mm/mremap.c + * + * (C) Copyright 1996 Linus Torvalds + * + * Address space accounting code <alan@redhat.com> + * (C) Copyright 2002 Red Hat Inc, All Rights Reserved + */ + +#include <linux/mm.h> +#include <linux/hugetlb.h> +#include <linux/slab.h> +#include <linux/shm.h> +#include <linux/mman.h> +#include <linux/swap.h> +#include <linux/fs.h> +#include <linux/highmem.h> +#include <linux/security.h> +#include <linux/syscalls.h> + +#include <asm/uaccess.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> + +static pte_t *get_one_pte_map_nested(struct mm_struct *mm, unsigned long addr) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + if (pgd_none_or_clear_bad(pgd)) + goto end; + + pud = pud_offset(pgd, addr); + if (pud_none_or_clear_bad(pud)) + goto end; + + pmd = pmd_offset(pud, addr); + if (pmd_none_or_clear_bad(pmd)) + goto end; + + pte = pte_offset_map_nested(pmd, addr); + if (pte_none(*pte)) { + pte_unmap_nested(pte); + pte = NULL; + } +end: + return pte; +} + +static pte_t *get_one_pte_map(struct mm_struct *mm, unsigned long addr) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset(mm, addr); + if (pgd_none_or_clear_bad(pgd)) + return NULL; + + pud = pud_offset(pgd, addr); + if (pud_none_or_clear_bad(pud)) + return NULL; + + pmd = pmd_offset(pud, addr); + if (pmd_none_or_clear_bad(pmd)) + return NULL; + + return pte_offset_map(pmd, addr); +} + +static inline pte_t *alloc_one_pte_map(struct mm_struct *mm, unsigned long addr) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + + pud = pud_alloc(mm, pgd, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(mm, pud, addr); + if (pmd) + pte = pte_alloc_map(mm, pmd, addr); + return pte; +} + +static int +move_one_page(struct vm_area_struct *vma, unsigned long old_addr, + struct vm_area_struct *new_vma, unsigned long new_addr) +{ + struct address_space *mapping = NULL; + struct mm_struct *mm = vma->vm_mm; + int error = 0; + pte_t *src, *dst; + + if (vma->vm_file) { + /* + * Subtle point from Rajesh Venkatasubramanian: before + * moving file-based ptes, we must lock vmtruncate out, + * since it might clean the dst vma before the src vma, + * and we propagate stale pages into the dst afterward. + */ + mapping = vma->vm_file->f_mapping; + spin_lock(&mapping->i_mmap_lock); + if (new_vma->vm_truncate_count && + new_vma->vm_truncate_count != vma->vm_truncate_count) + new_vma->vm_truncate_count = 0; + } + spin_lock(&mm->page_table_lock); + + src = get_one_pte_map_nested(mm, old_addr); + if (src) { + /* + * Look to see whether alloc_one_pte_map needs to perform a + * memory allocation. If it does then we need to drop the + * atomic kmap + */ + dst = get_one_pte_map(mm, new_addr); + if (unlikely(!dst)) { + pte_unmap_nested(src); + if (mapping) + spin_unlock(&mapping->i_mmap_lock); + dst = alloc_one_pte_map(mm, new_addr); + if (mapping && !spin_trylock(&mapping->i_mmap_lock)) { + spin_unlock(&mm->page_table_lock); + spin_lock(&mapping->i_mmap_lock); + spin_lock(&mm->page_table_lock); + } + src = get_one_pte_map_nested(mm, old_addr); + } + /* + * Since alloc_one_pte_map can drop and re-acquire + * page_table_lock, we should re-check the src entry... + */ + if (src) { + if (dst) { + pte_t pte; + pte = ptep_clear_flush(vma, old_addr, src); + set_pte_at(mm, new_addr, dst, pte); + } else + error = -ENOMEM; + pte_unmap_nested(src); + } + if (dst) + pte_unmap(dst); + } + spin_unlock(&mm->page_table_lock); + if (mapping) + spin_unlock(&mapping->i_mmap_lock); + return error; +} + +static unsigned long move_page_tables(struct vm_area_struct *vma, + unsigned long old_addr, struct vm_area_struct *new_vma, + unsigned long new_addr, unsigned long len) +{ + unsigned long offset; + + flush_cache_range(vma, old_addr, old_addr + len); + + /* + * This is not the clever way to do this, but we're taking the + * easy way out on the assumption that most remappings will be + * only a few pages.. This also makes error recovery easier. + */ + for (offset = 0; offset < len; offset += PAGE_SIZE) { + if (move_one_page(vma, old_addr + offset, + new_vma, new_addr + offset) < 0) + break; + cond_resched(); + } + return offset; +} + +static unsigned long move_vma(struct vm_area_struct *vma, + unsigned long old_addr, unsigned long old_len, + unsigned long new_len, unsigned long new_addr) +{ + struct mm_struct *mm = vma->vm_mm; + struct vm_area_struct *new_vma; + unsigned long vm_flags = vma->vm_flags; + unsigned long new_pgoff; + unsigned long moved_len; + unsigned long excess = 0; + int split = 0; + + /* + * We'd prefer to avoid failure later on in do_munmap: + * which may split one vma into three before unmapping. + */ + if (mm->map_count >= sysctl_max_map_count - 3) + return -ENOMEM; + + new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); + new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff); + if (!new_vma) + return -ENOMEM; + + moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len); + if (moved_len < old_len) { + /* + * On error, move entries back from new area to old, + * which will succeed since page tables still there, + * and then proceed to unmap new area instead of old. + */ + move_page_tables(new_vma, new_addr, vma, old_addr, moved_len); + vma = new_vma; + old_len = new_len; + old_addr = new_addr; + new_addr = -ENOMEM; + } + + /* Conceal VM_ACCOUNT so old reservation is not undone */ + if (vm_flags & VM_ACCOUNT) { + vma->vm_flags &= ~VM_ACCOUNT; + excess = vma->vm_end - vma->vm_start - old_len; + if (old_addr > vma->vm_start && + old_addr + old_len < vma->vm_end) + split = 1; + } + + if (do_munmap(mm, old_addr, old_len) < 0) { + /* OOM: unable to split vma, just get accounts right */ + vm_unacct_memory(excess >> PAGE_SHIFT); + excess = 0; + } + + /* Restore VM_ACCOUNT if one or two pieces of vma left */ + if (excess) { + vma->vm_flags |= VM_ACCOUNT; + if (split) + vma->vm_next->vm_flags |= VM_ACCOUNT; + } + + mm->total_vm += new_len >> PAGE_SHIFT; + __vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT); + if (vm_flags & VM_LOCKED) { + mm->locked_vm += new_len >> PAGE_SHIFT; + if (new_len > old_len) + make_pages_present(new_addr + old_len, + new_addr + new_len); + } + + return new_addr; +} + +/* + * Expand (or shrink) an existing mapping, potentially moving it at the + * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) + * + * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise + * This option implies MREMAP_MAYMOVE. + */ +unsigned long do_mremap(unsigned long addr, + unsigned long old_len, unsigned long new_len, + unsigned long flags, unsigned long new_addr) +{ + struct vm_area_struct *vma; + unsigned long ret = -EINVAL; + unsigned long charged = 0; + + if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE)) + goto out; + + if (addr & ~PAGE_MASK) + goto out; + + old_len = PAGE_ALIGN(old_len); + new_len = PAGE_ALIGN(new_len); + + /* + * We allow a zero old-len as a special case + * for DOS-emu "duplicate shm area" thing. But + * a zero new-len is nonsensical. + */ + if (!new_len) + goto out; + + /* new_addr is only valid if MREMAP_FIXED is specified */ + if (flags & MREMAP_FIXED) { + if (new_addr & ~PAGE_MASK) + goto out; + if (!(flags & MREMAP_MAYMOVE)) + goto out; + + if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len) + goto out; + + /* Check if the location we're moving into overlaps the + * old location at all, and fail if it does. + */ + if ((new_addr <= addr) && (new_addr+new_len) > addr) + goto out; + + if ((addr <= new_addr) && (addr+old_len) > new_addr) + goto out; + + ret = do_munmap(current->mm, new_addr, new_len); + if (ret) + goto out; + } + + /* + * Always allow a shrinking remap: that just unmaps + * the unnecessary pages.. + * do_munmap does all the needed commit accounting + */ + if (old_len >= new_len) { + ret = do_munmap(current->mm, addr+new_len, old_len - new_len); + if (ret && old_len != new_len) + goto out; + ret = addr; + if (!(flags & MREMAP_FIXED) || (new_addr == addr)) + goto out; + old_len = new_len; + } + + /* + * Ok, we need to grow.. or relocate. + */ + ret = -EFAULT; + vma = find_vma(current->mm, addr); + if (!vma || vma->vm_start > addr) + goto out; + if (is_vm_hugetlb_page(vma)) { + ret = -EINVAL; + goto out; + } + /* We can't remap across vm area boundaries */ + if (old_len > vma->vm_end - addr) + goto out; + if (vma->vm_flags & VM_DONTEXPAND) { + if (new_len > old_len) + goto out; + } + if (vma->vm_flags & VM_LOCKED) { + unsigned long locked, lock_limit; + locked = current->mm->locked_vm << PAGE_SHIFT; + lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; + locked += new_len - old_len; + ret = -EAGAIN; + if (locked > lock_limit && !capable(CAP_IPC_LOCK)) + goto out; + } + ret = -ENOMEM; + if ((current->mm->total_vm << PAGE_SHIFT) + (new_len - old_len) + > current->signal->rlim[RLIMIT_AS].rlim_cur) + goto out; + + if (vma->vm_flags & VM_ACCOUNT) { + charged = (new_len - old_len) >> PAGE_SHIFT; + if (security_vm_enough_memory(charged)) + goto out_nc; + } + + /* old_len exactly to the end of the area.. + * And we're not relocating the area. + */ + if (old_len == vma->vm_end - addr && + !((flags & MREMAP_FIXED) && (addr != new_addr)) && + (old_len != new_len || !(flags & MREMAP_MAYMOVE))) { + unsigned long max_addr = TASK_SIZE; + if (vma->vm_next) + max_addr = vma->vm_next->vm_start; + /* can we just expand the current mapping? */ + if (max_addr - addr >= new_len) { + int pages = (new_len - old_len) >> PAGE_SHIFT; + + vma_adjust(vma, vma->vm_start, + addr + new_len, vma->vm_pgoff, NULL); + + current->mm->total_vm += pages; + __vm_stat_account(vma->vm_mm, vma->vm_flags, + vma->vm_file, pages); + if (vma->vm_flags & VM_LOCKED) { + current->mm->locked_vm += pages; + make_pages_present(addr + old_len, + addr + new_len); + } + ret = addr; + goto out; + } + } + + /* + * We weren't able to just expand or shrink the area, + * we need to create a new one and move it.. + */ + ret = -ENOMEM; + if (flags & MREMAP_MAYMOVE) { + if (!(flags & MREMAP_FIXED)) { + unsigned long map_flags = 0; + if (vma->vm_flags & VM_MAYSHARE) + map_flags |= MAP_SHARED; + + new_addr = get_unmapped_area(vma->vm_file, 0, new_len, + vma->vm_pgoff, map_flags); + ret = new_addr; + if (new_addr & ~PAGE_MASK) + goto out; + } + ret = move_vma(vma, addr, old_len, new_len, new_addr); + } +out: + if (ret & ~PAGE_MASK) + vm_unacct_memory(charged); +out_nc: + return ret; +} + +asmlinkage unsigned long sys_mremap(unsigned long addr, + unsigned long old_len, unsigned long new_len, + unsigned long flags, unsigned long new_addr) +{ + unsigned long ret; + + down_write(¤t->mm->mmap_sem); + ret = do_mremap(addr, old_len, new_len, flags, new_addr); + up_write(¤t->mm->mmap_sem); + return ret; +} diff --git a/mm/msync.c b/mm/msync.c new file mode 100644 index 000000000000..090f426bca7d --- /dev/null +++ b/mm/msync.c @@ -0,0 +1,236 @@ +/* + * linux/mm/msync.c + * + * Copyright (C) 1994-1999 Linus Torvalds + */ + +/* + * The msync() system call. + */ +#include <linux/slab.h> +#include <linux/pagemap.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/hugetlb.h> +#include <linux/syscalls.h> + +#include <asm/pgtable.h> +#include <asm/tlbflush.h> + +/* + * Called with mm->page_table_lock held to protect against other + * threads/the swapper from ripping pte's out from under us. + */ + +static void sync_pte_range(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, unsigned long end) +{ + pte_t *pte; + + pte = pte_offset_map(pmd, addr); + do { + unsigned long pfn; + struct page *page; + + if (!pte_present(*pte)) + continue; + pfn = pte_pfn(*pte); + if (!pfn_valid(pfn)) + continue; + page = pfn_to_page(pfn); + if (PageReserved(page)) + continue; + + if (ptep_clear_flush_dirty(vma, addr, pte) || + page_test_and_clear_dirty(page)) + set_page_dirty(page); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); +} + +static inline void sync_pmd_range(struct vm_area_struct *vma, pud_t *pud, + unsigned long addr, unsigned long end) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + sync_pte_range(vma, pmd, addr, next); + } while (pmd++, addr = next, addr != end); +} + +static inline void sync_pud_range(struct vm_area_struct *vma, pgd_t *pgd, + unsigned long addr, unsigned long end) +{ + pud_t *pud; + unsigned long next; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + sync_pmd_range(vma, pud, addr, next); + } while (pud++, addr = next, addr != end); +} + +static void sync_page_range(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + pgd_t *pgd; + unsigned long next; + + /* For hugepages we can't go walking the page table normally, + * but that's ok, hugetlbfs is memory based, so we don't need + * to do anything more on an msync() */ + if (is_vm_hugetlb_page(vma)) + return; + + BUG_ON(addr >= end); + pgd = pgd_offset(mm, addr); + flush_cache_range(vma, addr, end); + spin_lock(&mm->page_table_lock); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + sync_pud_range(vma, pgd, addr, next); + } while (pgd++, addr = next, addr != end); + spin_unlock(&mm->page_table_lock); +} + +#ifdef CONFIG_PREEMPT +static inline void filemap_sync(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + const size_t chunk = 64 * 1024; /* bytes */ + unsigned long next; + + do { + next = addr + chunk; + if (next > end || next < addr) + next = end; + sync_page_range(vma, addr, next); + cond_resched(); + } while (addr = next, addr != end); +} +#else +static inline void filemap_sync(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + sync_page_range(vma, addr, end); +} +#endif + +/* + * MS_SYNC syncs the entire file - including mappings. + * + * MS_ASYNC does not start I/O (it used to, up to 2.5.67). Instead, it just + * marks the relevant pages dirty. The application may now run fsync() to + * write out the dirty pages and wait on the writeout and check the result. + * Or the application may run fadvise(FADV_DONTNEED) against the fd to start + * async writeout immediately. + * So my _not_ starting I/O in MS_ASYNC we provide complete flexibility to + * applications. + */ +static int msync_interval(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, int flags) +{ + int ret = 0; + struct file *file = vma->vm_file; + + if ((flags & MS_INVALIDATE) && (vma->vm_flags & VM_LOCKED)) + return -EBUSY; + + if (file && (vma->vm_flags & VM_SHARED)) { + filemap_sync(vma, addr, end); + + if (flags & MS_SYNC) { + struct address_space *mapping = file->f_mapping; + int err; + + ret = filemap_fdatawrite(mapping); + if (file->f_op && file->f_op->fsync) { + /* + * We don't take i_sem here because mmap_sem + * is already held. + */ + err = file->f_op->fsync(file,file->f_dentry,1); + if (err && !ret) + ret = err; + } + err = filemap_fdatawait(mapping); + if (!ret) + ret = err; + } + } + return ret; +} + +asmlinkage long sys_msync(unsigned long start, size_t len, int flags) +{ + unsigned long end; + struct vm_area_struct *vma; + int unmapped_error, error = -EINVAL; + + if (flags & MS_SYNC) + current->flags |= PF_SYNCWRITE; + + down_read(¤t->mm->mmap_sem); + if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) + goto out; + if (start & ~PAGE_MASK) + goto out; + if ((flags & MS_ASYNC) && (flags & MS_SYNC)) + goto out; + error = -ENOMEM; + len = (len + ~PAGE_MASK) & PAGE_MASK; + end = start + len; + if (end < start) + goto out; + error = 0; + if (end == start) + goto out; + /* + * If the interval [start,end) covers some unmapped address ranges, + * just ignore them, but return -ENOMEM at the end. + */ + vma = find_vma(current->mm, start); + unmapped_error = 0; + for (;;) { + /* Still start < end. */ + error = -ENOMEM; + if (!vma) + goto out; + /* Here start < vma->vm_end. */ + if (start < vma->vm_start) { + unmapped_error = -ENOMEM; + start = vma->vm_start; + } + /* Here vma->vm_start <= start < vma->vm_end. */ + if (end <= vma->vm_end) { + if (start < end) { + error = msync_interval(vma, start, end, flags); + if (error) + goto out; + } + error = unmapped_error; + goto out; + } + /* Here vma->vm_start <= start < vma->vm_end < end. */ + error = msync_interval(vma, start, vma->vm_end, flags); + if (error) + goto out; + start = vma->vm_end; + vma = vma->vm_next; + } +out: + up_read(¤t->mm->mmap_sem); + current->flags &= ~PF_SYNCWRITE; + return error; +} diff --git a/mm/nommu.c b/mm/nommu.c new file mode 100644 index 000000000000..b293ec1cc4e6 --- /dev/null +++ b/mm/nommu.c @@ -0,0 +1,1180 @@ +/* + * linux/mm/nommu.c + * + * Replacement code for mm functions to support CPU's that don't + * have any form of memory management unit (thus no virtual memory). + * + * See Documentation/nommu-mmap.txt + * + * Copyright (c) 2004-2005 David Howells <dhowells@redhat.com> + * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> + * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> + * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> + */ + +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/swap.h> +#include <linux/file.h> +#include <linux/highmem.h> +#include <linux/pagemap.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/ptrace.h> +#include <linux/blkdev.h> +#include <linux/backing-dev.h> +#include <linux/mount.h> +#include <linux/personality.h> +#include <linux/security.h> +#include <linux/syscalls.h> + +#include <asm/uaccess.h> +#include <asm/tlb.h> +#include <asm/tlbflush.h> + +void *high_memory; +struct page *mem_map; +unsigned long max_mapnr; +unsigned long num_physpages; +unsigned long askedalloc, realalloc; +atomic_t vm_committed_space = ATOMIC_INIT(0); +int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ +int sysctl_overcommit_ratio = 50; /* default is 50% */ +int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; +int heap_stack_gap = 0; + +EXPORT_SYMBOL(mem_map); +EXPORT_SYMBOL(sysctl_max_map_count); +EXPORT_SYMBOL(sysctl_overcommit_memory); +EXPORT_SYMBOL(sysctl_overcommit_ratio); +EXPORT_SYMBOL(vm_committed_space); +EXPORT_SYMBOL(__vm_enough_memory); + +/* list of shareable VMAs */ +struct rb_root nommu_vma_tree = RB_ROOT; +DECLARE_RWSEM(nommu_vma_sem); + +struct vm_operations_struct generic_file_vm_ops = { +}; + +/* + * Handle all mappings that got truncated by a "truncate()" + * system call. + * + * NOTE! We have to be ready to update the memory sharing + * between the file and the memory map for a potential last + * incomplete page. Ugly, but necessary. + */ +int vmtruncate(struct inode *inode, loff_t offset) +{ + struct address_space *mapping = inode->i_mapping; + unsigned long limit; + + if (inode->i_size < offset) + goto do_expand; + i_size_write(inode, offset); + + truncate_inode_pages(mapping, offset); + goto out_truncate; + +do_expand: + limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; + if (limit != RLIM_INFINITY && offset > limit) + goto out_sig; + if (offset > inode->i_sb->s_maxbytes) + goto out; + i_size_write(inode, offset); + +out_truncate: + if (inode->i_op && inode->i_op->truncate) + inode->i_op->truncate(inode); + return 0; +out_sig: + send_sig(SIGXFSZ, current, 0); +out: + return -EFBIG; +} + +EXPORT_SYMBOL(vmtruncate); + +/* + * Return the total memory allocated for this pointer, not + * just what the caller asked for. + * + * Doesn't have to be accurate, i.e. may have races. + */ +unsigned int kobjsize(const void *objp) +{ + struct page *page; + + if (!objp || !((page = virt_to_page(objp)))) + return 0; + + if (PageSlab(page)) + return ksize(objp); + + BUG_ON(page->index < 0); + BUG_ON(page->index >= MAX_ORDER); + + return (PAGE_SIZE << page->index); +} + +/* + * The nommu dodgy version :-) + */ +int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, + unsigned long start, int len, int write, int force, + struct page **pages, struct vm_area_struct **vmas) +{ + int i; + static struct vm_area_struct dummy_vma; + + for (i = 0; i < len; i++) { + if (pages) { + pages[i] = virt_to_page(start); + if (pages[i]) + page_cache_get(pages[i]); + } + if (vmas) + vmas[i] = &dummy_vma; + start += PAGE_SIZE; + } + return(i); +} + +DEFINE_RWLOCK(vmlist_lock); +struct vm_struct *vmlist; + +void vfree(void *addr) +{ + kfree(addr); +} + +void *__vmalloc(unsigned long size, int gfp_mask, pgprot_t prot) +{ + /* + * kmalloc doesn't like __GFP_HIGHMEM for some reason + */ + return kmalloc(size, gfp_mask & ~__GFP_HIGHMEM); +} + +struct page * vmalloc_to_page(void *addr) +{ + return virt_to_page(addr); +} + +unsigned long vmalloc_to_pfn(void *addr) +{ + return page_to_pfn(virt_to_page(addr)); +} + + +long vread(char *buf, char *addr, unsigned long count) +{ + memcpy(buf, addr, count); + return count; +} + +long vwrite(char *buf, char *addr, unsigned long count) +{ + /* Don't allow overflow */ + if ((unsigned long) addr + count < count) + count = -(unsigned long) addr; + + memcpy(addr, buf, count); + return(count); +} + +/* + * vmalloc - allocate virtually continguos memory + * + * @size: allocation size + * + * Allocate enough pages to cover @size from the page level + * allocator and map them into continguos kernel virtual space. + * + * For tight cotrol over page level allocator and protection flags + * use __vmalloc() instead. + */ +void *vmalloc(unsigned long size) +{ + return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); +} + +/* + * vmalloc_32 - allocate virtually continguos memory (32bit addressable) + * + * @size: allocation size + * + * Allocate enough 32bit PA addressable pages to cover @size from the + * page level allocator and map them into continguos kernel virtual space. + */ +void *vmalloc_32(unsigned long size) +{ + return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); +} + +void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) +{ + BUG(); + return NULL; +} + +void vunmap(void *addr) +{ + BUG(); +} + +/* + * sys_brk() for the most part doesn't need the global kernel + * lock, except when an application is doing something nasty + * like trying to un-brk an area that has already been mapped + * to a regular file. in this case, the unmapping will need + * to invoke file system routines that need the global lock. + */ +asmlinkage unsigned long sys_brk(unsigned long brk) +{ + struct mm_struct *mm = current->mm; + + if (brk < mm->start_brk || brk > mm->context.end_brk) + return mm->brk; + + if (mm->brk == brk) + return mm->brk; + + /* + * Always allow shrinking brk + */ + if (brk <= mm->brk) { + mm->brk = brk; + return brk; + } + + /* + * Ok, looks good - let it rip. + */ + return mm->brk = brk; +} + +#ifdef DEBUG +static void show_process_blocks(void) +{ + struct vm_list_struct *vml; + + printk("Process blocks %d:", current->pid); + + for (vml = ¤t->mm->context.vmlist; vml; vml = vml->next) { + printk(" %p: %p", vml, vml->vma); + if (vml->vma) + printk(" (%d @%lx #%d)", + kobjsize((void *) vml->vma->vm_start), + vml->vma->vm_start, + atomic_read(&vml->vma->vm_usage)); + printk(vml->next ? " ->" : ".\n"); + } +} +#endif /* DEBUG */ + +static inline struct vm_area_struct *find_nommu_vma(unsigned long start) +{ + struct vm_area_struct *vma; + struct rb_node *n = nommu_vma_tree.rb_node; + + while (n) { + vma = rb_entry(n, struct vm_area_struct, vm_rb); + + if (start < vma->vm_start) + n = n->rb_left; + else if (start > vma->vm_start) + n = n->rb_right; + else + return vma; + } + + return NULL; +} + +static void add_nommu_vma(struct vm_area_struct *vma) +{ + struct vm_area_struct *pvma; + struct address_space *mapping; + struct rb_node **p = &nommu_vma_tree.rb_node; + struct rb_node *parent = NULL; + + /* add the VMA to the mapping */ + if (vma->vm_file) { + mapping = vma->vm_file->f_mapping; + + flush_dcache_mmap_lock(mapping); + vma_prio_tree_insert(vma, &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); + } + + /* add the VMA to the master list */ + while (*p) { + parent = *p; + pvma = rb_entry(parent, struct vm_area_struct, vm_rb); + + if (vma->vm_start < pvma->vm_start) { + p = &(*p)->rb_left; + } + else if (vma->vm_start > pvma->vm_start) { + p = &(*p)->rb_right; + } + else { + /* mappings are at the same address - this can only + * happen for shared-mem chardevs and shared file + * mappings backed by ramfs/tmpfs */ + BUG_ON(!(pvma->vm_flags & VM_SHARED)); + + if (vma < pvma) + p = &(*p)->rb_left; + else if (vma > pvma) + p = &(*p)->rb_right; + else + BUG(); + } + } + + rb_link_node(&vma->vm_rb, parent, p); + rb_insert_color(&vma->vm_rb, &nommu_vma_tree); +} + +static void delete_nommu_vma(struct vm_area_struct *vma) +{ + struct address_space *mapping; + + /* remove the VMA from the mapping */ + if (vma->vm_file) { + mapping = vma->vm_file->f_mapping; + + flush_dcache_mmap_lock(mapping); + vma_prio_tree_remove(vma, &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); + } + + /* remove from the master list */ + rb_erase(&vma->vm_rb, &nommu_vma_tree); +} + +/* + * determine whether a mapping should be permitted and, if so, what sort of + * mapping we're capable of supporting + */ +static int validate_mmap_request(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long prot, + unsigned long flags, + unsigned long pgoff, + unsigned long *_capabilities) +{ + unsigned long capabilities; + unsigned long reqprot = prot; + int ret; + + /* do the simple checks first */ + if (flags & MAP_FIXED || addr) { + printk(KERN_DEBUG + "%d: Can't do fixed-address/overlay mmap of RAM\n", + current->pid); + return -EINVAL; + } + + if ((flags & MAP_TYPE) != MAP_PRIVATE && + (flags & MAP_TYPE) != MAP_SHARED) + return -EINVAL; + + if (PAGE_ALIGN(len) == 0) + return addr; + + if (len > TASK_SIZE) + return -EINVAL; + + /* offset overflow? */ + if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + return -EINVAL; + + if (file) { + /* validate file mapping requests */ + struct address_space *mapping; + + /* files must support mmap */ + if (!file->f_op || !file->f_op->mmap) + return -ENODEV; + + /* work out if what we've got could possibly be shared + * - we support chardevs that provide their own "memory" + * - we support files/blockdevs that are memory backed + */ + mapping = file->f_mapping; + if (!mapping) + mapping = file->f_dentry->d_inode->i_mapping; + + capabilities = 0; + if (mapping && mapping->backing_dev_info) + capabilities = mapping->backing_dev_info->capabilities; + + if (!capabilities) { + /* no explicit capabilities set, so assume some + * defaults */ + switch (file->f_dentry->d_inode->i_mode & S_IFMT) { + case S_IFREG: + case S_IFBLK: + capabilities = BDI_CAP_MAP_COPY; + break; + + case S_IFCHR: + capabilities = + BDI_CAP_MAP_DIRECT | + BDI_CAP_READ_MAP | + BDI_CAP_WRITE_MAP; + break; + + default: + return -EINVAL; + } + } + + /* eliminate any capabilities that we can't support on this + * device */ + if (!file->f_op->get_unmapped_area) + capabilities &= ~BDI_CAP_MAP_DIRECT; + if (!file->f_op->read) + capabilities &= ~BDI_CAP_MAP_COPY; + + if (flags & MAP_SHARED) { + /* do checks for writing, appending and locking */ + if ((prot & PROT_WRITE) && + !(file->f_mode & FMODE_WRITE)) + return -EACCES; + + if (IS_APPEND(file->f_dentry->d_inode) && + (file->f_mode & FMODE_WRITE)) + return -EACCES; + + if (locks_verify_locked(file->f_dentry->d_inode)) + return -EAGAIN; + + if (!(capabilities & BDI_CAP_MAP_DIRECT)) + return -ENODEV; + + if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) || + ((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) || + ((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP)) + ) { + printk("MAP_SHARED not completely supported on !MMU\n"); + return -EINVAL; + } + + /* we mustn't privatise shared mappings */ + capabilities &= ~BDI_CAP_MAP_COPY; + } + else { + /* we're going to read the file into private memory we + * allocate */ + if (!(capabilities & BDI_CAP_MAP_COPY)) + return -ENODEV; + + /* we don't permit a private writable mapping to be + * shared with the backing device */ + if (prot & PROT_WRITE) + capabilities &= ~BDI_CAP_MAP_DIRECT; + } + + /* handle executable mappings and implied executable + * mappings */ + if (file->f_vfsmnt->mnt_flags & MNT_NOEXEC) { + if (prot & PROT_EXEC) + return -EPERM; + } + else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) { + /* handle implication of PROT_EXEC by PROT_READ */ + if (current->personality & READ_IMPLIES_EXEC) { + if (capabilities & BDI_CAP_EXEC_MAP) + prot |= PROT_EXEC; + } + } + else if ((prot & PROT_READ) && + (prot & PROT_EXEC) && + !(capabilities & BDI_CAP_EXEC_MAP) + ) { + /* backing file is not executable, try to copy */ + capabilities &= ~BDI_CAP_MAP_DIRECT; + } + } + else { + /* anonymous mappings are always memory backed and can be + * privately mapped + */ + capabilities = BDI_CAP_MAP_COPY; + + /* handle PROT_EXEC implication by PROT_READ */ + if ((prot & PROT_READ) && + (current->personality & READ_IMPLIES_EXEC)) + prot |= PROT_EXEC; + } + + /* allow the security API to have its say */ + ret = security_file_mmap(file, reqprot, prot, flags); + if (ret < 0) + return ret; + + /* looks okay */ + *_capabilities = capabilities; + return 0; +} + +/* + * we've determined that we can make the mapping, now translate what we + * now know into VMA flags + */ +static unsigned long determine_vm_flags(struct file *file, + unsigned long prot, + unsigned long flags, + unsigned long capabilities) +{ + unsigned long vm_flags; + + vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags); + vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + /* vm_flags |= mm->def_flags; */ + + if (!(capabilities & BDI_CAP_MAP_DIRECT)) { + /* attempt to share read-only copies of mapped file chunks */ + if (file && !(prot & PROT_WRITE)) + vm_flags |= VM_MAYSHARE; + } + else { + /* overlay a shareable mapping on the backing device or inode + * if possible - used for chardevs, ramfs/tmpfs/shmfs and + * romfs/cramfs */ + if (flags & MAP_SHARED) + vm_flags |= VM_MAYSHARE | VM_SHARED; + else if ((((vm_flags & capabilities) ^ vm_flags) & BDI_CAP_VMFLAGS) == 0) + vm_flags |= VM_MAYSHARE; + } + + /* refuse to let anyone share private mappings with this process if + * it's being traced - otherwise breakpoints set in it may interfere + * with another untraced process + */ + if ((flags & MAP_PRIVATE) && (current->ptrace & PT_PTRACED)) + vm_flags &= ~VM_MAYSHARE; + + return vm_flags; +} + +/* + * set up a shared mapping on a file + */ +static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) +{ + int ret; + + ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); + if (ret != -ENOSYS) + return ret; + + /* getting an ENOSYS error indicates that direct mmap isn't + * possible (as opposed to tried but failed) so we'll fall + * through to making a private copy of the data and mapping + * that if we can */ + return -ENODEV; +} + +/* + * set up a private mapping or an anonymous shared mapping + */ +static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) +{ + void *base; + int ret; + + /* invoke the file's mapping function so that it can keep track of + * shared mappings on devices or memory + * - VM_MAYSHARE will be set if it may attempt to share + */ + if (vma->vm_file) { + ret = vma->vm_file->f_op->mmap(vma->vm_file, vma); + if (ret != -ENOSYS) { + /* shouldn't return success if we're not sharing */ + BUG_ON(ret == 0 && !(vma->vm_flags & VM_MAYSHARE)); + return ret; /* success or a real error */ + } + + /* getting an ENOSYS error indicates that direct mmap isn't + * possible (as opposed to tried but failed) so we'll try to + * make a private copy of the data and map that instead */ + } + + /* allocate some memory to hold the mapping + * - note that this may not return a page-aligned address if the object + * we're allocating is smaller than a page + */ + base = kmalloc(len, GFP_KERNEL); + if (!base) + goto enomem; + + vma->vm_start = (unsigned long) base; + vma->vm_end = vma->vm_start + len; + vma->vm_flags |= VM_MAPPED_COPY; + +#ifdef WARN_ON_SLACK + if (len + WARN_ON_SLACK <= kobjsize(result)) + printk("Allocation of %lu bytes from process %d has %lu bytes of slack\n", + len, current->pid, kobjsize(result) - len); +#endif + + if (vma->vm_file) { + /* read the contents of a file into the copy */ + mm_segment_t old_fs; + loff_t fpos; + + fpos = vma->vm_pgoff; + fpos <<= PAGE_SHIFT; + + old_fs = get_fs(); + set_fs(KERNEL_DS); + ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); + set_fs(old_fs); + + if (ret < 0) + goto error_free; + + /* clear the last little bit */ + if (ret < len) + memset(base + ret, 0, len - ret); + + } else { + /* if it's an anonymous mapping, then just clear it */ + memset(base, 0, len); + } + + return 0; + +error_free: + kfree(base); + vma->vm_start = 0; + return ret; + +enomem: + printk("Allocation of length %lu from process %d failed\n", + len, current->pid); + show_free_areas(); + return -ENOMEM; +} + +/* + * handle mapping creation for uClinux + */ +unsigned long do_mmap_pgoff(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long prot, + unsigned long flags, + unsigned long pgoff) +{ + struct vm_list_struct *vml = NULL; + struct vm_area_struct *vma = NULL; + struct rb_node *rb; + unsigned long capabilities, vm_flags; + void *result; + int ret; + + /* decide whether we should attempt the mapping, and if so what sort of + * mapping */ + ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, + &capabilities); + if (ret < 0) + return ret; + + /* we've determined that we can make the mapping, now translate what we + * now know into VMA flags */ + vm_flags = determine_vm_flags(file, prot, flags, capabilities); + + /* we're going to need to record the mapping if it works */ + vml = kmalloc(sizeof(struct vm_list_struct), GFP_KERNEL); + if (!vml) + goto error_getting_vml; + memset(vml, 0, sizeof(*vml)); + + down_write(&nommu_vma_sem); + + /* if we want to share, we need to check for VMAs created by other + * mmap() calls that overlap with our proposed mapping + * - we can only share with an exact match on most regular files + * - shared mappings on character devices and memory backed files are + * permitted to overlap inexactly as far as we are concerned for in + * these cases, sharing is handled in the driver or filesystem rather + * than here + */ + if (vm_flags & VM_MAYSHARE) { + unsigned long pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + unsigned long vmpglen; + + for (rb = rb_first(&nommu_vma_tree); rb; rb = rb_next(rb)) { + vma = rb_entry(rb, struct vm_area_struct, vm_rb); + + if (!(vma->vm_flags & VM_MAYSHARE)) + continue; + + /* search for overlapping mappings on the same file */ + if (vma->vm_file->f_dentry->d_inode != file->f_dentry->d_inode) + continue; + + if (vma->vm_pgoff >= pgoff + pglen) + continue; + + vmpglen = vma->vm_end - vma->vm_start + PAGE_SIZE - 1; + vmpglen >>= PAGE_SHIFT; + if (pgoff >= vma->vm_pgoff + vmpglen) + continue; + + /* handle inexactly overlapping matches between mappings */ + if (vma->vm_pgoff != pgoff || vmpglen != pglen) { + if (!(capabilities & BDI_CAP_MAP_DIRECT)) + goto sharing_violation; + continue; + } + + /* we've found a VMA we can share */ + atomic_inc(&vma->vm_usage); + + vml->vma = vma; + result = (void *) vma->vm_start; + goto shared; + } + + vma = NULL; + + /* obtain the address at which to make a shared mapping + * - this is the hook for quasi-memory character devices to + * tell us the location of a shared mapping + */ + if (file && file->f_op->get_unmapped_area) { + addr = file->f_op->get_unmapped_area(file, addr, len, + pgoff, flags); + if (IS_ERR((void *) addr)) { + ret = addr; + if (ret != (unsigned long) -ENOSYS) + goto error; + + /* the driver refused to tell us where to site + * the mapping so we'll have to attempt to copy + * it */ + ret = (unsigned long) -ENODEV; + if (!(capabilities & BDI_CAP_MAP_COPY)) + goto error; + + capabilities &= ~BDI_CAP_MAP_DIRECT; + } + } + } + + /* we're going to need a VMA struct as well */ + vma = kmalloc(sizeof(struct vm_area_struct), GFP_KERNEL); + if (!vma) + goto error_getting_vma; + + memset(vma, 0, sizeof(*vma)); + INIT_LIST_HEAD(&vma->anon_vma_node); + atomic_set(&vma->vm_usage, 1); + if (file) + get_file(file); + vma->vm_file = file; + vma->vm_flags = vm_flags; + vma->vm_start = addr; + vma->vm_end = addr + len; + vma->vm_pgoff = pgoff; + + vml->vma = vma; + + /* set up the mapping */ + if (file && vma->vm_flags & VM_SHARED) + ret = do_mmap_shared_file(vma, len); + else + ret = do_mmap_private(vma, len); + if (ret < 0) + goto error; + + /* okay... we have a mapping; now we have to register it */ + result = (void *) vma->vm_start; + + if (vma->vm_flags & VM_MAPPED_COPY) { + realalloc += kobjsize(result); + askedalloc += len; + } + + realalloc += kobjsize(vma); + askedalloc += sizeof(*vma); + + current->mm->total_vm += len >> PAGE_SHIFT; + + add_nommu_vma(vma); + + shared: + realalloc += kobjsize(vml); + askedalloc += sizeof(*vml); + + vml->next = current->mm->context.vmlist; + current->mm->context.vmlist = vml; + + up_write(&nommu_vma_sem); + + if (prot & PROT_EXEC) + flush_icache_range((unsigned long) result, + (unsigned long) result + len); + +#ifdef DEBUG + printk("do_mmap:\n"); + show_process_blocks(); +#endif + + return (unsigned long) result; + + error: + up_write(&nommu_vma_sem); + kfree(vml); + if (vma) { + fput(vma->vm_file); + kfree(vma); + } + return ret; + + sharing_violation: + up_write(&nommu_vma_sem); + printk("Attempt to share mismatched mappings\n"); + kfree(vml); + return -EINVAL; + + error_getting_vma: + up_write(&nommu_vma_sem); + kfree(vml); + printk("Allocation of vml for %lu byte allocation from process %d failed\n", + len, current->pid); + show_free_areas(); + return -ENOMEM; + + error_getting_vml: + printk("Allocation of vml for %lu byte allocation from process %d failed\n", + len, current->pid); + show_free_areas(); + return -ENOMEM; +} + +/* + * handle mapping disposal for uClinux + */ +static void put_vma(struct vm_area_struct *vma) +{ + if (vma) { + down_write(&nommu_vma_sem); + + if (atomic_dec_and_test(&vma->vm_usage)) { + delete_nommu_vma(vma); + + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); + + /* IO memory and memory shared directly out of the pagecache from + * ramfs/tmpfs mustn't be released here */ + if (vma->vm_flags & VM_MAPPED_COPY) { + realalloc -= kobjsize((void *) vma->vm_start); + askedalloc -= vma->vm_end - vma->vm_start; + kfree((void *) vma->vm_start); + } + + realalloc -= kobjsize(vma); + askedalloc -= sizeof(*vma); + + if (vma->vm_file) + fput(vma->vm_file); + kfree(vma); + } + + up_write(&nommu_vma_sem); + } +} + +int do_munmap(struct mm_struct *mm, unsigned long addr, size_t len) +{ + struct vm_list_struct *vml, **parent; + unsigned long end = addr + len; + +#ifdef DEBUG + printk("do_munmap:\n"); +#endif + + for (parent = &mm->context.vmlist; *parent; parent = &(*parent)->next) + if ((*parent)->vma->vm_start == addr && + (*parent)->vma->vm_end == end) + goto found; + + printk("munmap of non-mmaped memory by process %d (%s): %p\n", + current->pid, current->comm, (void *) addr); + return -EINVAL; + + found: + vml = *parent; + + put_vma(vml->vma); + + *parent = vml->next; + realalloc -= kobjsize(vml); + askedalloc -= sizeof(*vml); + kfree(vml); + mm->total_vm -= len >> PAGE_SHIFT; + +#ifdef DEBUG + show_process_blocks(); +#endif + + return 0; +} + +/* Release all mmaps. */ +void exit_mmap(struct mm_struct * mm) +{ + struct vm_list_struct *tmp; + + if (mm) { +#ifdef DEBUG + printk("Exit_mmap:\n"); +#endif + + mm->total_vm = 0; + + while ((tmp = mm->context.vmlist)) { + mm->context.vmlist = tmp->next; + put_vma(tmp->vma); + + realalloc -= kobjsize(tmp); + askedalloc -= sizeof(*tmp); + kfree(tmp); + } + +#ifdef DEBUG + show_process_blocks(); +#endif + } +} + +asmlinkage long sys_munmap(unsigned long addr, size_t len) +{ + int ret; + struct mm_struct *mm = current->mm; + + down_write(&mm->mmap_sem); + ret = do_munmap(mm, addr, len); + up_write(&mm->mmap_sem); + return ret; +} + +unsigned long do_brk(unsigned long addr, unsigned long len) +{ + return -ENOMEM; +} + +/* + * Expand (or shrink) an existing mapping, potentially moving it at the + * same time (controlled by the MREMAP_MAYMOVE flag and available VM space) + * + * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise + * This option implies MREMAP_MAYMOVE. + * + * on uClinux, we only permit changing a mapping's size, and only as long as it stays within the + * hole allocated by the kmalloc() call in do_mmap_pgoff() and the block is not shareable + */ +unsigned long do_mremap(unsigned long addr, + unsigned long old_len, unsigned long new_len, + unsigned long flags, unsigned long new_addr) +{ + struct vm_list_struct *vml = NULL; + + /* insanity checks first */ + if (new_len == 0) + return (unsigned long) -EINVAL; + + if (flags & MREMAP_FIXED && new_addr != addr) + return (unsigned long) -EINVAL; + + for (vml = current->mm->context.vmlist; vml; vml = vml->next) + if (vml->vma->vm_start == addr) + goto found; + + return (unsigned long) -EINVAL; + + found: + if (vml->vma->vm_end != vml->vma->vm_start + old_len) + return (unsigned long) -EFAULT; + + if (vml->vma->vm_flags & VM_MAYSHARE) + return (unsigned long) -EPERM; + + if (new_len > kobjsize((void *) addr)) + return (unsigned long) -ENOMEM; + + /* all checks complete - do it */ + vml->vma->vm_end = vml->vma->vm_start + new_len; + + askedalloc -= old_len; + askedalloc += new_len; + + return vml->vma->vm_start; +} + +/* + * Look up the first VMA which satisfies addr < vm_end, NULL if none + */ +struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +{ + struct vm_list_struct *vml; + + for (vml = mm->context.vmlist; vml; vml = vml->next) + if (addr >= vml->vma->vm_start && addr < vml->vma->vm_end) + return vml->vma; + + return NULL; +} + +EXPORT_SYMBOL(find_vma); + +struct page * follow_page(struct mm_struct *mm, unsigned long addr, int write) +{ + return NULL; +} + +struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) +{ + return NULL; +} + +int remap_pfn_range(struct vm_area_struct *vma, unsigned long from, + unsigned long to, unsigned long size, pgprot_t prot) +{ + return -EPERM; +} + +void swap_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) +{ +} + +unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + return -ENOMEM; +} + +void arch_unmap_area(struct vm_area_struct *area) +{ +} + +void update_mem_hiwater(struct task_struct *tsk) +{ + unsigned long rss = get_mm_counter(tsk->mm, rss); + + if (likely(tsk->mm)) { + if (tsk->mm->hiwater_rss < rss) + tsk->mm->hiwater_rss = rss; + if (tsk->mm->hiwater_vm < tsk->mm->total_vm) + tsk->mm->hiwater_vm = tsk->mm->total_vm; + } +} + +void unmap_mapping_range(struct address_space *mapping, + loff_t const holebegin, loff_t const holelen, + int even_cows) +{ +} + +/* + * Check that a process has enough memory to allocate a new virtual + * mapping. 0 means there is enough memory for the allocation to + * succeed and -ENOMEM implies there is not. + * + * We currently support three overcommit policies, which are set via the + * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting + * + * Strict overcommit modes added 2002 Feb 26 by Alan Cox. + * Additional code 2002 Jul 20 by Robert Love. + * + * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise. + * + * Note this is a helper function intended to be used by LSMs which + * wish to use this logic. + */ +int __vm_enough_memory(long pages, int cap_sys_admin) +{ + unsigned long free, allowed; + + vm_acct_memory(pages); + + /* + * Sometimes we want to use more memory than we have + */ + if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS) + return 0; + + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long n; + + free = get_page_cache_size(); + free += nr_swap_pages; + + /* + * Any slabs which are created with the + * SLAB_RECLAIM_ACCOUNT flag claim to have contents + * which are reclaimable, under pressure. The dentry + * cache and most inode caches should fall into this + */ + free += atomic_read(&slab_reclaim_pages); + + /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + free -= free / 32; + + if (free > pages) + return 0; + + /* + * nr_free_pages() is very expensive on large systems, + * only call if we're about to fail. + */ + n = nr_free_pages(); + if (!cap_sys_admin) + n -= n / 32; + free += n; + + if (free > pages) + return 0; + vm_unacct_memory(pages); + return -ENOMEM; + } + + allowed = totalram_pages * sysctl_overcommit_ratio / 100; + /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + allowed -= allowed / 32; + allowed += total_swap_pages; + + /* Don't let a single process grow too big: + leave 3% of the size of this process for other processes */ + allowed -= current->mm->total_vm / 32; + + if (atomic_read(&vm_committed_space) < allowed) + return 0; + + vm_unacct_memory(pages); + + return -ENOMEM; +} + +int in_gate_area_no_task(unsigned long addr) +{ + return 0; +} diff --git a/mm/oom_kill.c b/mm/oom_kill.c new file mode 100644 index 000000000000..9595a0f6c4b8 --- /dev/null +++ b/mm/oom_kill.c @@ -0,0 +1,292 @@ +/* + * linux/mm/oom_kill.c + * + * Copyright (C) 1998,2000 Rik van Riel + * Thanks go out to Claus Fischer for some serious inspiration and + * for goading me into coding this file... + * + * The routines in this file are used to kill a process when + * we're seriously out of memory. This gets called from kswapd() + * in linux/mm/vmscan.c when we really run out of memory. + * + * Since we won't call these routines often (on a well-configured + * machine) this file will double as a 'coding guide' and a signpost + * for newbie kernel hackers. It features several pointers to major + * kernel subsystems and hints as to where to find out what things do. + */ + +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/swap.h> +#include <linux/timex.h> +#include <linux/jiffies.h> + +/* #define DEBUG */ + +/** + * oom_badness - calculate a numeric value for how bad this task has been + * @p: task struct of which task we should calculate + * @p: current uptime in seconds + * + * The formula used is relatively simple and documented inline in the + * function. The main rationale is that we want to select a good task + * to kill when we run out of memory. + * + * Good in this context means that: + * 1) we lose the minimum amount of work done + * 2) we recover a large amount of memory + * 3) we don't kill anything innocent of eating tons of memory + * 4) we want to kill the minimum amount of processes (one) + * 5) we try to kill the process the user expects us to kill, this + * algorithm has been meticulously tuned to meet the principle + * of least surprise ... (be careful when you change it) + */ + +unsigned long badness(struct task_struct *p, unsigned long uptime) +{ + unsigned long points, cpu_time, run_time, s; + struct list_head *tsk; + + if (!p->mm) + return 0; + + /* + * The memory size of the process is the basis for the badness. + */ + points = p->mm->total_vm; + + /* + * Processes which fork a lot of child processes are likely + * a good choice. We add the vmsize of the childs if they + * have an own mm. This prevents forking servers to flood the + * machine with an endless amount of childs + */ + list_for_each(tsk, &p->children) { + struct task_struct *chld; + chld = list_entry(tsk, struct task_struct, sibling); + if (chld->mm != p->mm && chld->mm) + points += chld->mm->total_vm; + } + + /* + * CPU time is in tens of seconds and run time is in thousands + * of seconds. There is no particular reason for this other than + * that it turned out to work very well in practice. + */ + cpu_time = (cputime_to_jiffies(p->utime) + cputime_to_jiffies(p->stime)) + >> (SHIFT_HZ + 3); + + if (uptime >= p->start_time.tv_sec) + run_time = (uptime - p->start_time.tv_sec) >> 10; + else + run_time = 0; + + s = int_sqrt(cpu_time); + if (s) + points /= s; + s = int_sqrt(int_sqrt(run_time)); + if (s) + points /= s; + + /* + * Niced processes are most likely less important, so double + * their badness points. + */ + if (task_nice(p) > 0) + points *= 2; + + /* + * Superuser processes are usually more important, so we make it + * less likely that we kill those. + */ + if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_ADMIN) || + p->uid == 0 || p->euid == 0) + points /= 4; + + /* + * We don't want to kill a process with direct hardware access. + * Not only could that mess up the hardware, but usually users + * tend to only have this flag set on applications they think + * of as important. + */ + if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) + points /= 4; + + /* + * Adjust the score by oomkilladj. + */ + if (p->oomkilladj) { + if (p->oomkilladj > 0) + points <<= p->oomkilladj; + else + points >>= -(p->oomkilladj); + } + +#ifdef DEBUG + printk(KERN_DEBUG "OOMkill: task %d (%s) got %d points\n", + p->pid, p->comm, points); +#endif + return points; +} + +/* + * Simple selection loop. We chose the process with the highest + * number of 'points'. We expect the caller will lock the tasklist. + * + * (not docbooked, we don't want this one cluttering up the manual) + */ +static struct task_struct * select_bad_process(void) +{ + unsigned long maxpoints = 0; + struct task_struct *g, *p; + struct task_struct *chosen = NULL; + struct timespec uptime; + + do_posix_clock_monotonic_gettime(&uptime); + do_each_thread(g, p) + /* skip the init task with pid == 1 */ + if (p->pid > 1) { + unsigned long points; + + /* + * This is in the process of releasing memory so wait it + * to finish before killing some other task by mistake. + */ + if ((unlikely(test_tsk_thread_flag(p, TIF_MEMDIE)) || (p->flags & PF_EXITING)) && + !(p->flags & PF_DEAD)) + return ERR_PTR(-1UL); + if (p->flags & PF_SWAPOFF) + return p; + + points = badness(p, uptime.tv_sec); + if (points > maxpoints || !chosen) { + chosen = p; + maxpoints = points; + } + } + while_each_thread(g, p); + return chosen; +} + +/** + * We must be careful though to never send SIGKILL a process with + * CAP_SYS_RAW_IO set, send SIGTERM instead (but it's unlikely that + * we select a process with CAP_SYS_RAW_IO set). + */ +static void __oom_kill_task(task_t *p) +{ + if (p->pid == 1) { + WARN_ON(1); + printk(KERN_WARNING "tried to kill init!\n"); + return; + } + + task_lock(p); + if (!p->mm || p->mm == &init_mm) { + WARN_ON(1); + printk(KERN_WARNING "tried to kill an mm-less task!\n"); + task_unlock(p); + return; + } + task_unlock(p); + printk(KERN_ERR "Out of Memory: Killed process %d (%s).\n", p->pid, p->comm); + + /* + * We give our sacrificial lamb high priority and access to + * all the memory it needs. That way it should be able to + * exit() and clear out its resources quickly... + */ + p->time_slice = HZ; + set_tsk_thread_flag(p, TIF_MEMDIE); + + force_sig(SIGKILL, p); +} + +static struct mm_struct *oom_kill_task(task_t *p) +{ + struct mm_struct *mm = get_task_mm(p); + task_t * g, * q; + + if (!mm) + return NULL; + if (mm == &init_mm) { + mmput(mm); + return NULL; + } + + __oom_kill_task(p); + /* + * kill all processes that share the ->mm (i.e. all threads), + * but are in a different thread group + */ + do_each_thread(g, q) + if (q->mm == mm && q->tgid != p->tgid) + __oom_kill_task(q); + while_each_thread(g, q); + + return mm; +} + +static struct mm_struct *oom_kill_process(struct task_struct *p) +{ + struct mm_struct *mm; + struct task_struct *c; + struct list_head *tsk; + + /* Try to kill a child first */ + list_for_each(tsk, &p->children) { + c = list_entry(tsk, struct task_struct, sibling); + if (c->mm == p->mm) + continue; + mm = oom_kill_task(c); + if (mm) + return mm; + } + return oom_kill_task(p); +} + +/** + * oom_kill - kill the "best" process when we run out of memory + * + * If we run out of memory, we have the choice between either + * killing a random task (bad), letting the system crash (worse) + * OR try to be smart about which process to kill. Note that we + * don't have to be perfect here, we just have to be good. + */ +void out_of_memory(unsigned int __nocast gfp_mask) +{ + struct mm_struct *mm = NULL; + task_t * p; + + read_lock(&tasklist_lock); +retry: + p = select_bad_process(); + + if (PTR_ERR(p) == -1UL) + goto out; + + /* Found nothing?!?! Either we hang forever, or we panic. */ + if (!p) { + read_unlock(&tasklist_lock); + show_free_areas(); + panic("Out of memory and no killable processes...\n"); + } + + printk("oom-killer: gfp_mask=0x%x\n", gfp_mask); + show_free_areas(); + mm = oom_kill_process(p); + if (!mm) + goto retry; + + out: + read_unlock(&tasklist_lock); + if (mm) + mmput(mm); + + /* + * Give "p" a good chance of killing itself before we + * retry to allocate memory. + */ + __set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(1); +} diff --git a/mm/page-writeback.c b/mm/page-writeback.c new file mode 100644 index 000000000000..6ddd6a29c73b --- /dev/null +++ b/mm/page-writeback.c @@ -0,0 +1,819 @@ +/* + * mm/page-writeback.c. + * + * Copyright (C) 2002, Linus Torvalds. + * + * Contains functions related to writing back dirty pages at the + * address_space level. + * + * 10Apr2002 akpm@zip.com.au + * Initial version + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/slab.h> +#include <linux/pagemap.h> +#include <linux/writeback.h> +#include <linux/init.h> +#include <linux/backing-dev.h> +#include <linux/blkdev.h> +#include <linux/mpage.h> +#include <linux/percpu.h> +#include <linux/notifier.h> +#include <linux/smp.h> +#include <linux/sysctl.h> +#include <linux/cpu.h> +#include <linux/syscalls.h> + +/* + * The maximum number of pages to writeout in a single bdflush/kupdate + * operation. We do this so we don't hold I_LOCK against an inode for + * enormous amounts of time, which would block a userspace task which has + * been forced to throttle against that inode. Also, the code reevaluates + * the dirty each time it has written this many pages. + */ +#define MAX_WRITEBACK_PAGES 1024 + +/* + * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited + * will look to see if it needs to force writeback or throttling. + */ +static long ratelimit_pages = 32; + +static long total_pages; /* The total number of pages in the machine. */ +static int dirty_exceeded; /* Dirty mem may be over limit */ + +/* + * When balance_dirty_pages decides that the caller needs to perform some + * non-background writeback, this is how many pages it will attempt to write. + * It should be somewhat larger than RATELIMIT_PAGES to ensure that reasonably + * large amounts of I/O are submitted. + */ +static inline long sync_writeback_pages(void) +{ + return ratelimit_pages + ratelimit_pages / 2; +} + +/* The following parameters are exported via /proc/sys/vm */ + +/* + * Start background writeback (via pdflush) at this percentage + */ +int dirty_background_ratio = 10; + +/* + * The generator of dirty data starts writeback at this percentage + */ +int vm_dirty_ratio = 40; + +/* + * The interval between `kupdate'-style writebacks, in centiseconds + * (hundredths of a second) + */ +int dirty_writeback_centisecs = 5 * 100; + +/* + * The longest number of centiseconds for which data is allowed to remain dirty + */ +int dirty_expire_centisecs = 30 * 100; + +/* + * Flag that makes the machine dump writes/reads and block dirtyings. + */ +int block_dump; + +/* + * Flag that puts the machine in "laptop mode". + */ +int laptop_mode; + +EXPORT_SYMBOL(laptop_mode); + +/* End of sysctl-exported parameters */ + + +static void background_writeout(unsigned long _min_pages); + +struct writeback_state +{ + unsigned long nr_dirty; + unsigned long nr_unstable; + unsigned long nr_mapped; + unsigned long nr_writeback; +}; + +static void get_writeback_state(struct writeback_state *wbs) +{ + wbs->nr_dirty = read_page_state(nr_dirty); + wbs->nr_unstable = read_page_state(nr_unstable); + wbs->nr_mapped = read_page_state(nr_mapped); + wbs->nr_writeback = read_page_state(nr_writeback); +} + +/* + * Work out the current dirty-memory clamping and background writeout + * thresholds. + * + * The main aim here is to lower them aggressively if there is a lot of mapped + * memory around. To avoid stressing page reclaim with lots of unreclaimable + * pages. It is better to clamp down on writers than to start swapping, and + * performing lots of scanning. + * + * We only allow 1/2 of the currently-unmapped memory to be dirtied. + * + * We don't permit the clamping level to fall below 5% - that is getting rather + * excessive. + * + * We make sure that the background writeout level is below the adjusted + * clamping level. + */ +static void +get_dirty_limits(struct writeback_state *wbs, long *pbackground, long *pdirty, + struct address_space *mapping) +{ + int background_ratio; /* Percentages */ + int dirty_ratio; + int unmapped_ratio; + long background; + long dirty; + unsigned long available_memory = total_pages; + struct task_struct *tsk; + + get_writeback_state(wbs); + +#ifdef CONFIG_HIGHMEM + /* + * If this mapping can only allocate from low memory, + * we exclude high memory from our count. + */ + if (mapping && !(mapping_gfp_mask(mapping) & __GFP_HIGHMEM)) + available_memory -= totalhigh_pages; +#endif + + + unmapped_ratio = 100 - (wbs->nr_mapped * 100) / total_pages; + + dirty_ratio = vm_dirty_ratio; + if (dirty_ratio > unmapped_ratio / 2) + dirty_ratio = unmapped_ratio / 2; + + if (dirty_ratio < 5) + dirty_ratio = 5; + + background_ratio = dirty_background_ratio; + if (background_ratio >= dirty_ratio) + background_ratio = dirty_ratio / 2; + + background = (background_ratio * available_memory) / 100; + dirty = (dirty_ratio * available_memory) / 100; + tsk = current; + if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { + background += background / 4; + dirty += dirty / 4; + } + *pbackground = background; + *pdirty = dirty; +} + +/* + * balance_dirty_pages() must be called by processes which are generating dirty + * data. It looks at the number of dirty pages in the machine and will force + * the caller to perform writeback if the system is over `vm_dirty_ratio'. + * If we're over `background_thresh' then pdflush is woken to perform some + * writeout. + */ +static void balance_dirty_pages(struct address_space *mapping) +{ + struct writeback_state wbs; + long nr_reclaimable; + long background_thresh; + long dirty_thresh; + unsigned long pages_written = 0; + unsigned long write_chunk = sync_writeback_pages(); + + struct backing_dev_info *bdi = mapping->backing_dev_info; + + for (;;) { + struct writeback_control wbc = { + .bdi = bdi, + .sync_mode = WB_SYNC_NONE, + .older_than_this = NULL, + .nr_to_write = write_chunk, + }; + + get_dirty_limits(&wbs, &background_thresh, + &dirty_thresh, mapping); + nr_reclaimable = wbs.nr_dirty + wbs.nr_unstable; + if (nr_reclaimable + wbs.nr_writeback <= dirty_thresh) + break; + + dirty_exceeded = 1; + + /* Note: nr_reclaimable denotes nr_dirty + nr_unstable. + * Unstable writes are a feature of certain networked + * filesystems (i.e. NFS) in which data may have been + * written to the server's write cache, but has not yet + * been flushed to permanent storage. + */ + if (nr_reclaimable) { + writeback_inodes(&wbc); + get_dirty_limits(&wbs, &background_thresh, + &dirty_thresh, mapping); + nr_reclaimable = wbs.nr_dirty + wbs.nr_unstable; + if (nr_reclaimable + wbs.nr_writeback <= dirty_thresh) + break; + pages_written += write_chunk - wbc.nr_to_write; + if (pages_written >= write_chunk) + break; /* We've done our duty */ + } + blk_congestion_wait(WRITE, HZ/10); + } + + if (nr_reclaimable + wbs.nr_writeback <= dirty_thresh) + dirty_exceeded = 0; + + if (writeback_in_progress(bdi)) + return; /* pdflush is already working this queue */ + + /* + * In laptop mode, we wait until hitting the higher threshold before + * starting background writeout, and then write out all the way down + * to the lower threshold. So slow writers cause minimal disk activity. + * + * In normal mode, we start background writeout at the lower + * background_thresh, to keep the amount of dirty memory low. + */ + if ((laptop_mode && pages_written) || + (!laptop_mode && (nr_reclaimable > background_thresh))) + pdflush_operation(background_writeout, 0); +} + +/** + * balance_dirty_pages_ratelimited - balance dirty memory state + * @mapping - address_space which was dirtied + * + * Processes which are dirtying memory should call in here once for each page + * which was newly dirtied. The function will periodically check the system's + * dirty state and will initiate writeback if needed. + * + * On really big machines, get_writeback_state is expensive, so try to avoid + * calling it too often (ratelimiting). But once we're over the dirty memory + * limit we decrease the ratelimiting by a lot, to prevent individual processes + * from overshooting the limit by (ratelimit_pages) each. + */ +void balance_dirty_pages_ratelimited(struct address_space *mapping) +{ + static DEFINE_PER_CPU(int, ratelimits) = 0; + long ratelimit; + + ratelimit = ratelimit_pages; + if (dirty_exceeded) + ratelimit = 8; + + /* + * Check the rate limiting. Also, we do not want to throttle real-time + * tasks in balance_dirty_pages(). Period. + */ + if (get_cpu_var(ratelimits)++ >= ratelimit) { + __get_cpu_var(ratelimits) = 0; + put_cpu_var(ratelimits); + balance_dirty_pages(mapping); + return; + } + put_cpu_var(ratelimits); +} +EXPORT_SYMBOL(balance_dirty_pages_ratelimited); + +void throttle_vm_writeout(void) +{ + struct writeback_state wbs; + long background_thresh; + long dirty_thresh; + + for ( ; ; ) { + get_dirty_limits(&wbs, &background_thresh, &dirty_thresh, NULL); + + /* + * Boost the allowable dirty threshold a bit for page + * allocators so they don't get DoS'ed by heavy writers + */ + dirty_thresh += dirty_thresh / 10; /* wheeee... */ + + if (wbs.nr_unstable + wbs.nr_writeback <= dirty_thresh) + break; + blk_congestion_wait(WRITE, HZ/10); + } +} + + +/* + * writeback at least _min_pages, and keep writing until the amount of dirty + * memory is less than the background threshold, or until we're all clean. + */ +static void background_writeout(unsigned long _min_pages) +{ + long min_pages = _min_pages; + struct writeback_control wbc = { + .bdi = NULL, + .sync_mode = WB_SYNC_NONE, + .older_than_this = NULL, + .nr_to_write = 0, + .nonblocking = 1, + }; + + for ( ; ; ) { + struct writeback_state wbs; + long background_thresh; + long dirty_thresh; + + get_dirty_limits(&wbs, &background_thresh, &dirty_thresh, NULL); + if (wbs.nr_dirty + wbs.nr_unstable < background_thresh + && min_pages <= 0) + break; + wbc.encountered_congestion = 0; + wbc.nr_to_write = MAX_WRITEBACK_PAGES; + wbc.pages_skipped = 0; + writeback_inodes(&wbc); + min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; + if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) { + /* Wrote less than expected */ + blk_congestion_wait(WRITE, HZ/10); + if (!wbc.encountered_congestion) + break; + } + } +} + +/* + * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back + * the whole world. Returns 0 if a pdflush thread was dispatched. Returns + * -1 if all pdflush threads were busy. + */ +int wakeup_bdflush(long nr_pages) +{ + if (nr_pages == 0) { + struct writeback_state wbs; + + get_writeback_state(&wbs); + nr_pages = wbs.nr_dirty + wbs.nr_unstable; + } + return pdflush_operation(background_writeout, nr_pages); +} + +static void wb_timer_fn(unsigned long unused); +static void laptop_timer_fn(unsigned long unused); + +static struct timer_list wb_timer = + TIMER_INITIALIZER(wb_timer_fn, 0, 0); +static struct timer_list laptop_mode_wb_timer = + TIMER_INITIALIZER(laptop_timer_fn, 0, 0); + +/* + * Periodic writeback of "old" data. + * + * Define "old": the first time one of an inode's pages is dirtied, we mark the + * dirtying-time in the inode's address_space. So this periodic writeback code + * just walks the superblock inode list, writing back any inodes which are + * older than a specific point in time. + * + * Try to run once per dirty_writeback_centisecs. But if a writeback event + * takes longer than a dirty_writeback_centisecs interval, then leave a + * one-second gap. + * + * older_than_this takes precedence over nr_to_write. So we'll only write back + * all dirty pages if they are all attached to "old" mappings. + */ +static void wb_kupdate(unsigned long arg) +{ + unsigned long oldest_jif; + unsigned long start_jif; + unsigned long next_jif; + long nr_to_write; + struct writeback_state wbs; + struct writeback_control wbc = { + .bdi = NULL, + .sync_mode = WB_SYNC_NONE, + .older_than_this = &oldest_jif, + .nr_to_write = 0, + .nonblocking = 1, + .for_kupdate = 1, + }; + + sync_supers(); + + get_writeback_state(&wbs); + oldest_jif = jiffies - (dirty_expire_centisecs * HZ) / 100; + start_jif = jiffies; + next_jif = start_jif + (dirty_writeback_centisecs * HZ) / 100; + nr_to_write = wbs.nr_dirty + wbs.nr_unstable + + (inodes_stat.nr_inodes - inodes_stat.nr_unused); + while (nr_to_write > 0) { + wbc.encountered_congestion = 0; + wbc.nr_to_write = MAX_WRITEBACK_PAGES; + writeback_inodes(&wbc); + if (wbc.nr_to_write > 0) { + if (wbc.encountered_congestion) + blk_congestion_wait(WRITE, HZ/10); + else + break; /* All the old data is written */ + } + nr_to_write -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; + } + if (time_before(next_jif, jiffies + HZ)) + next_jif = jiffies + HZ; + if (dirty_writeback_centisecs) + mod_timer(&wb_timer, next_jif); +} + +/* + * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs + */ +int dirty_writeback_centisecs_handler(ctl_table *table, int write, + struct file *file, void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec(table, write, file, buffer, length, ppos); + if (dirty_writeback_centisecs) { + mod_timer(&wb_timer, + jiffies + (dirty_writeback_centisecs * HZ) / 100); + } else { + del_timer(&wb_timer); + } + return 0; +} + +static void wb_timer_fn(unsigned long unused) +{ + if (pdflush_operation(wb_kupdate, 0) < 0) + mod_timer(&wb_timer, jiffies + HZ); /* delay 1 second */ +} + +static void laptop_flush(unsigned long unused) +{ + sys_sync(); +} + +static void laptop_timer_fn(unsigned long unused) +{ + pdflush_operation(laptop_flush, 0); +} + +/* + * We've spun up the disk and we're in laptop mode: schedule writeback + * of all dirty data a few seconds from now. If the flush is already scheduled + * then push it back - the user is still using the disk. + */ +void laptop_io_completion(void) +{ + mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode * HZ); +} + +/* + * We're in laptop mode and we've just synced. The sync's writes will have + * caused another writeback to be scheduled by laptop_io_completion. + * Nothing needs to be written back anymore, so we unschedule the writeback. + */ +void laptop_sync_completion(void) +{ + del_timer(&laptop_mode_wb_timer); +} + +/* + * If ratelimit_pages is too high then we can get into dirty-data overload + * if a large number of processes all perform writes at the same time. + * If it is too low then SMP machines will call the (expensive) + * get_writeback_state too often. + * + * Here we set ratelimit_pages to a level which ensures that when all CPUs are + * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory + * thresholds before writeback cuts in. + * + * But the limit should not be set too high. Because it also controls the + * amount of memory which the balance_dirty_pages() caller has to write back. + * If this is too large then the caller will block on the IO queue all the + * time. So limit it to four megabytes - the balance_dirty_pages() caller + * will write six megabyte chunks, max. + */ + +static void set_ratelimit(void) +{ + ratelimit_pages = total_pages / (num_online_cpus() * 32); + if (ratelimit_pages < 16) + ratelimit_pages = 16; + if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024) + ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE; +} + +static int +ratelimit_handler(struct notifier_block *self, unsigned long u, void *v) +{ + set_ratelimit(); + return 0; +} + +static struct notifier_block ratelimit_nb = { + .notifier_call = ratelimit_handler, + .next = NULL, +}; + +/* + * If the machine has a large highmem:lowmem ratio then scale back the default + * dirty memory thresholds: allowing too much dirty highmem pins an excessive + * number of buffer_heads. + */ +void __init page_writeback_init(void) +{ + long buffer_pages = nr_free_buffer_pages(); + long correction; + + total_pages = nr_free_pagecache_pages(); + + correction = (100 * 4 * buffer_pages) / total_pages; + + if (correction < 100) { + dirty_background_ratio *= correction; + dirty_background_ratio /= 100; + vm_dirty_ratio *= correction; + vm_dirty_ratio /= 100; + + if (dirty_background_ratio <= 0) + dirty_background_ratio = 1; + if (vm_dirty_ratio <= 0) + vm_dirty_ratio = 1; + } + mod_timer(&wb_timer, jiffies + (dirty_writeback_centisecs * HZ) / 100); + set_ratelimit(); + register_cpu_notifier(&ratelimit_nb); +} + +int do_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + if (wbc->nr_to_write <= 0) + return 0; + if (mapping->a_ops->writepages) + return mapping->a_ops->writepages(mapping, wbc); + return generic_writepages(mapping, wbc); +} + +/** + * write_one_page - write out a single page and optionally wait on I/O + * + * @page - the page to write + * @wait - if true, wait on writeout + * + * The page must be locked by the caller and will be unlocked upon return. + * + * write_one_page() returns a negative error code if I/O failed. + */ +int write_one_page(struct page *page, int wait) +{ + struct address_space *mapping = page->mapping; + int ret = 0; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 1, + }; + + BUG_ON(!PageLocked(page)); + + if (wait) + wait_on_page_writeback(page); + + if (clear_page_dirty_for_io(page)) { + page_cache_get(page); + ret = mapping->a_ops->writepage(page, &wbc); + if (ret == 0 && wait) { + wait_on_page_writeback(page); + if (PageError(page)) + ret = -EIO; + } + page_cache_release(page); + } else { + unlock_page(page); + } + return ret; +} +EXPORT_SYMBOL(write_one_page); + +/* + * For address_spaces which do not use buffers. Just tag the page as dirty in + * its radix tree. + * + * This is also used when a single buffer is being dirtied: we want to set the + * page dirty in that case, but not all the buffers. This is a "bottom-up" + * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. + * + * Most callers have locked the page, which pins the address_space in memory. + * But zap_pte_range() does not lock the page, however in that case the + * mapping is pinned by the vma's ->vm_file reference. + * + * We take care to handle the case where the page was truncated from the + * mapping by re-checking page_mapping() insode tree_lock. + */ +int __set_page_dirty_nobuffers(struct page *page) +{ + int ret = 0; + + if (!TestSetPageDirty(page)) { + struct address_space *mapping = page_mapping(page); + struct address_space *mapping2; + + if (mapping) { + write_lock_irq(&mapping->tree_lock); + mapping2 = page_mapping(page); + if (mapping2) { /* Race with truncate? */ + BUG_ON(mapping2 != mapping); + if (mapping_cap_account_dirty(mapping)) + inc_page_state(nr_dirty); + radix_tree_tag_set(&mapping->page_tree, + page_index(page), PAGECACHE_TAG_DIRTY); + } + write_unlock_irq(&mapping->tree_lock); + if (mapping->host) { + /* !PageAnon && !swapper_space */ + __mark_inode_dirty(mapping->host, + I_DIRTY_PAGES); + } + } + } + return ret; +} +EXPORT_SYMBOL(__set_page_dirty_nobuffers); + +/* + * When a writepage implementation decides that it doesn't want to write this + * page for some reason, it should redirty the locked page via + * redirty_page_for_writepage() and it should then unlock the page and return 0 + */ +int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) +{ + wbc->pages_skipped++; + return __set_page_dirty_nobuffers(page); +} +EXPORT_SYMBOL(redirty_page_for_writepage); + +/* + * If the mapping doesn't provide a set_page_dirty a_op, then + * just fall through and assume that it wants buffer_heads. + */ +int fastcall set_page_dirty(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + + if (likely(mapping)) { + int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; + if (spd) + return (*spd)(page); + return __set_page_dirty_buffers(page); + } + if (!PageDirty(page)) + SetPageDirty(page); + return 0; +} +EXPORT_SYMBOL(set_page_dirty); + +/* + * set_page_dirty() is racy if the caller has no reference against + * page->mapping->host, and if the page is unlocked. This is because another + * CPU could truncate the page off the mapping and then free the mapping. + * + * Usually, the page _is_ locked, or the caller is a user-space process which + * holds a reference on the inode by having an open file. + * + * In other cases, the page should be locked before running set_page_dirty(). + */ +int set_page_dirty_lock(struct page *page) +{ + int ret; + + lock_page(page); + ret = set_page_dirty(page); + unlock_page(page); + return ret; +} +EXPORT_SYMBOL(set_page_dirty_lock); + +/* + * Clear a page's dirty flag, while caring for dirty memory accounting. + * Returns true if the page was previously dirty. + */ +int test_clear_page_dirty(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + unsigned long flags; + + if (mapping) { + write_lock_irqsave(&mapping->tree_lock, flags); + if (TestClearPageDirty(page)) { + radix_tree_tag_clear(&mapping->page_tree, + page_index(page), + PAGECACHE_TAG_DIRTY); + write_unlock_irqrestore(&mapping->tree_lock, flags); + if (mapping_cap_account_dirty(mapping)) + dec_page_state(nr_dirty); + return 1; + } + write_unlock_irqrestore(&mapping->tree_lock, flags); + return 0; + } + return TestClearPageDirty(page); +} +EXPORT_SYMBOL(test_clear_page_dirty); + +/* + * Clear a page's dirty flag, while caring for dirty memory accounting. + * Returns true if the page was previously dirty. + * + * This is for preparing to put the page under writeout. We leave the page + * tagged as dirty in the radix tree so that a concurrent write-for-sync + * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage + * implementation will run either set_page_writeback() or set_page_dirty(), + * at which stage we bring the page's dirty flag and radix-tree dirty tag + * back into sync. + * + * This incoherency between the page's dirty flag and radix-tree tag is + * unfortunate, but it only exists while the page is locked. + */ +int clear_page_dirty_for_io(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + + if (mapping) { + if (TestClearPageDirty(page)) { + if (mapping_cap_account_dirty(mapping)) + dec_page_state(nr_dirty); + return 1; + } + return 0; + } + return TestClearPageDirty(page); +} +EXPORT_SYMBOL(clear_page_dirty_for_io); + +int test_clear_page_writeback(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + int ret; + + if (mapping) { + unsigned long flags; + + write_lock_irqsave(&mapping->tree_lock, flags); + ret = TestClearPageWriteback(page); + if (ret) + radix_tree_tag_clear(&mapping->page_tree, + page_index(page), + PAGECACHE_TAG_WRITEBACK); + write_unlock_irqrestore(&mapping->tree_lock, flags); + } else { + ret = TestClearPageWriteback(page); + } + return ret; +} + +int test_set_page_writeback(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + int ret; + + if (mapping) { + unsigned long flags; + + write_lock_irqsave(&mapping->tree_lock, flags); + ret = TestSetPageWriteback(page); + if (!ret) + radix_tree_tag_set(&mapping->page_tree, + page_index(page), + PAGECACHE_TAG_WRITEBACK); + if (!PageDirty(page)) + radix_tree_tag_clear(&mapping->page_tree, + page_index(page), + PAGECACHE_TAG_DIRTY); + write_unlock_irqrestore(&mapping->tree_lock, flags); + } else { + ret = TestSetPageWriteback(page); + } + return ret; + +} +EXPORT_SYMBOL(test_set_page_writeback); + +/* + * Return true if any of the pages in the mapping are marged with the + * passed tag. + */ +int mapping_tagged(struct address_space *mapping, int tag) +{ + unsigned long flags; + int ret; + + read_lock_irqsave(&mapping->tree_lock, flags); + ret = radix_tree_tagged(&mapping->page_tree, tag); + read_unlock_irqrestore(&mapping->tree_lock, flags); + return ret; +} +EXPORT_SYMBOL(mapping_tagged); diff --git a/mm/page_alloc.c b/mm/page_alloc.c new file mode 100644 index 000000000000..c73dbbc1cd8f --- /dev/null +++ b/mm/page_alloc.c @@ -0,0 +1,2220 @@ +/* + * linux/mm/page_alloc.c + * + * Manages the free list, the system allocates free pages here. + * Note that kmalloc() lives in slab.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * Swap reorganised 29.12.95, Stephen Tweedie + * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 + * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 + * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 + * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 + * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 + * (lots of bits borrowed from Ingo Molnar & Andrew Morton) + */ + +#include <linux/config.h> +#include <linux/stddef.h> +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/interrupt.h> +#include <linux/pagemap.h> +#include <linux/bootmem.h> +#include <linux/compiler.h> +#include <linux/module.h> +#include <linux/suspend.h> +#include <linux/pagevec.h> +#include <linux/blkdev.h> +#include <linux/slab.h> +#include <linux/notifier.h> +#include <linux/topology.h> +#include <linux/sysctl.h> +#include <linux/cpu.h> +#include <linux/cpuset.h> +#include <linux/nodemask.h> +#include <linux/vmalloc.h> + +#include <asm/tlbflush.h> +#include "internal.h" + +/* + * MCD - HACK: Find somewhere to initialize this EARLY, or make this + * initializer cleaner + */ +nodemask_t node_online_map = { { [0] = 1UL } }; +nodemask_t node_possible_map = NODE_MASK_ALL; +struct pglist_data *pgdat_list; +unsigned long totalram_pages; +unsigned long totalhigh_pages; +long nr_swap_pages; + +/* + * results with 256, 32 in the lowmem_reserve sysctl: + * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) + * 1G machine -> (16M dma, 784M normal, 224M high) + * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA + * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL + * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA + */ +int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { 256, 32 }; + +EXPORT_SYMBOL(totalram_pages); +EXPORT_SYMBOL(nr_swap_pages); + +/* + * Used by page_zone() to look up the address of the struct zone whose + * id is encoded in the upper bits of page->flags + */ +struct zone *zone_table[1 << (ZONES_SHIFT + NODES_SHIFT)]; +EXPORT_SYMBOL(zone_table); + +static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" }; +int min_free_kbytes = 1024; + +unsigned long __initdata nr_kernel_pages; +unsigned long __initdata nr_all_pages; + +/* + * Temporary debugging check for pages not lying within a given zone. + */ +static int bad_range(struct zone *zone, struct page *page) +{ + if (page_to_pfn(page) >= zone->zone_start_pfn + zone->spanned_pages) + return 1; + if (page_to_pfn(page) < zone->zone_start_pfn) + return 1; +#ifdef CONFIG_HOLES_IN_ZONE + if (!pfn_valid(page_to_pfn(page))) + return 1; +#endif + if (zone != page_zone(page)) + return 1; + return 0; +} + +static void bad_page(const char *function, struct page *page) +{ + printk(KERN_EMERG "Bad page state at %s (in process '%s', page %p)\n", + function, current->comm, page); + printk(KERN_EMERG "flags:0x%0*lx mapping:%p mapcount:%d count:%d\n", + (int)(2*sizeof(page_flags_t)), (unsigned long)page->flags, + page->mapping, page_mapcount(page), page_count(page)); + printk(KERN_EMERG "Backtrace:\n"); + dump_stack(); + printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n"); + page->flags &= ~(1 << PG_private | + 1 << PG_locked | + 1 << PG_lru | + 1 << PG_active | + 1 << PG_dirty | + 1 << PG_swapcache | + 1 << PG_writeback); + set_page_count(page, 0); + reset_page_mapcount(page); + page->mapping = NULL; + tainted |= TAINT_BAD_PAGE; +} + +#ifndef CONFIG_HUGETLB_PAGE +#define prep_compound_page(page, order) do { } while (0) +#define destroy_compound_page(page, order) do { } while (0) +#else +/* + * Higher-order pages are called "compound pages". They are structured thusly: + * + * The first PAGE_SIZE page is called the "head page". + * + * The remaining PAGE_SIZE pages are called "tail pages". + * + * All pages have PG_compound set. All pages have their ->private pointing at + * the head page (even the head page has this). + * + * The first tail page's ->mapping, if non-zero, holds the address of the + * compound page's put_page() function. + * + * The order of the allocation is stored in the first tail page's ->index + * This is only for debug at present. This usage means that zero-order pages + * may not be compound. + */ +static void prep_compound_page(struct page *page, unsigned long order) +{ + int i; + int nr_pages = 1 << order; + + page[1].mapping = NULL; + page[1].index = order; + for (i = 0; i < nr_pages; i++) { + struct page *p = page + i; + + SetPageCompound(p); + p->private = (unsigned long)page; + } +} + +static void destroy_compound_page(struct page *page, unsigned long order) +{ + int i; + int nr_pages = 1 << order; + + if (!PageCompound(page)) + return; + + if (page[1].index != order) + bad_page(__FUNCTION__, page); + + for (i = 0; i < nr_pages; i++) { + struct page *p = page + i; + + if (!PageCompound(p)) + bad_page(__FUNCTION__, page); + if (p->private != (unsigned long)page) + bad_page(__FUNCTION__, page); + ClearPageCompound(p); + } +} +#endif /* CONFIG_HUGETLB_PAGE */ + +/* + * function for dealing with page's order in buddy system. + * zone->lock is already acquired when we use these. + * So, we don't need atomic page->flags operations here. + */ +static inline unsigned long page_order(struct page *page) { + return page->private; +} + +static inline void set_page_order(struct page *page, int order) { + page->private = order; + __SetPagePrivate(page); +} + +static inline void rmv_page_order(struct page *page) +{ + __ClearPagePrivate(page); + page->private = 0; +} + +/* + * Locate the struct page for both the matching buddy in our + * pair (buddy1) and the combined O(n+1) page they form (page). + * + * 1) Any buddy B1 will have an order O twin B2 which satisfies + * the following equation: + * B2 = B1 ^ (1 << O) + * For example, if the starting buddy (buddy2) is #8 its order + * 1 buddy is #10: + * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 + * + * 2) Any buddy B will have an order O+1 parent P which + * satisfies the following equation: + * P = B & ~(1 << O) + * + * Assumption: *_mem_map is contigious at least up to MAX_ORDER + */ +static inline struct page * +__page_find_buddy(struct page *page, unsigned long page_idx, unsigned int order) +{ + unsigned long buddy_idx = page_idx ^ (1 << order); + + return page + (buddy_idx - page_idx); +} + +static inline unsigned long +__find_combined_index(unsigned long page_idx, unsigned int order) +{ + return (page_idx & ~(1 << order)); +} + +/* + * This function checks whether a page is free && is the buddy + * we can do coalesce a page and its buddy if + * (a) the buddy is free && + * (b) the buddy is on the buddy system && + * (c) a page and its buddy have the same order. + * for recording page's order, we use page->private and PG_private. + * + */ +static inline int page_is_buddy(struct page *page, int order) +{ + if (PagePrivate(page) && + (page_order(page) == order) && + !PageReserved(page) && + page_count(page) == 0) + return 1; + return 0; +} + +/* + * Freeing function for a buddy system allocator. + * + * The concept of a buddy system is to maintain direct-mapped table + * (containing bit values) for memory blocks of various "orders". + * The bottom level table contains the map for the smallest allocatable + * units of memory (here, pages), and each level above it describes + * pairs of units from the levels below, hence, "buddies". + * At a high level, all that happens here is marking the table entry + * at the bottom level available, and propagating the changes upward + * as necessary, plus some accounting needed to play nicely with other + * parts of the VM system. + * At each level, we keep a list of pages, which are heads of continuous + * free pages of length of (1 << order) and marked with PG_Private.Page's + * order is recorded in page->private field. + * So when we are allocating or freeing one, we can derive the state of the + * other. That is, if we allocate a small block, and both were + * free, the remainder of the region must be split into blocks. + * If a block is freed, and its buddy is also free, then this + * triggers coalescing into a block of larger size. + * + * -- wli + */ + +static inline void __free_pages_bulk (struct page *page, + struct zone *zone, unsigned int order) +{ + unsigned long page_idx; + int order_size = 1 << order; + + if (unlikely(order)) + destroy_compound_page(page, order); + + page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); + + BUG_ON(page_idx & (order_size - 1)); + BUG_ON(bad_range(zone, page)); + + zone->free_pages += order_size; + while (order < MAX_ORDER-1) { + unsigned long combined_idx; + struct free_area *area; + struct page *buddy; + + combined_idx = __find_combined_index(page_idx, order); + buddy = __page_find_buddy(page, page_idx, order); + + if (bad_range(zone, buddy)) + break; + if (!page_is_buddy(buddy, order)) + break; /* Move the buddy up one level. */ + list_del(&buddy->lru); + area = zone->free_area + order; + area->nr_free--; + rmv_page_order(buddy); + page = page + (combined_idx - page_idx); + page_idx = combined_idx; + order++; + } + set_page_order(page, order); + list_add(&page->lru, &zone->free_area[order].free_list); + zone->free_area[order].nr_free++; +} + +static inline void free_pages_check(const char *function, struct page *page) +{ + if ( page_mapcount(page) || + page->mapping != NULL || + page_count(page) != 0 || + (page->flags & ( + 1 << PG_lru | + 1 << PG_private | + 1 << PG_locked | + 1 << PG_active | + 1 << PG_reclaim | + 1 << PG_slab | + 1 << PG_swapcache | + 1 << PG_writeback ))) + bad_page(function, page); + if (PageDirty(page)) + ClearPageDirty(page); +} + +/* + * Frees a list of pages. + * Assumes all pages on list are in same zone, and of same order. + * count is the number of pages to free, or 0 for all on the list. + * + * If the zone was previously in an "all pages pinned" state then look to + * see if this freeing clears that state. + * + * And clear the zone's pages_scanned counter, to hold off the "all pages are + * pinned" detection logic. + */ +static int +free_pages_bulk(struct zone *zone, int count, + struct list_head *list, unsigned int order) +{ + unsigned long flags; + struct page *page = NULL; + int ret = 0; + + spin_lock_irqsave(&zone->lock, flags); + zone->all_unreclaimable = 0; + zone->pages_scanned = 0; + while (!list_empty(list) && count--) { + page = list_entry(list->prev, struct page, lru); + /* have to delete it as __free_pages_bulk list manipulates */ + list_del(&page->lru); + __free_pages_bulk(page, zone, order); + ret++; + } + spin_unlock_irqrestore(&zone->lock, flags); + return ret; +} + +void __free_pages_ok(struct page *page, unsigned int order) +{ + LIST_HEAD(list); + int i; + + arch_free_page(page, order); + + mod_page_state(pgfree, 1 << order); + +#ifndef CONFIG_MMU + if (order > 0) + for (i = 1 ; i < (1 << order) ; ++i) + __put_page(page + i); +#endif + + for (i = 0 ; i < (1 << order) ; ++i) + free_pages_check(__FUNCTION__, page + i); + list_add(&page->lru, &list); + kernel_map_pages(page, 1<<order, 0); + free_pages_bulk(page_zone(page), 1, &list, order); +} + + +/* + * The order of subdivision here is critical for the IO subsystem. + * Please do not alter this order without good reasons and regression + * testing. Specifically, as large blocks of memory are subdivided, + * the order in which smaller blocks are delivered depends on the order + * they're subdivided in this function. This is the primary factor + * influencing the order in which pages are delivered to the IO + * subsystem according to empirical testing, and this is also justified + * by considering the behavior of a buddy system containing a single + * large block of memory acted on by a series of small allocations. + * This behavior is a critical factor in sglist merging's success. + * + * -- wli + */ +static inline struct page * +expand(struct zone *zone, struct page *page, + int low, int high, struct free_area *area) +{ + unsigned long size = 1 << high; + + while (high > low) { + area--; + high--; + size >>= 1; + BUG_ON(bad_range(zone, &page[size])); + list_add(&page[size].lru, &area->free_list); + area->nr_free++; + set_page_order(&page[size], high); + } + return page; +} + +void set_page_refs(struct page *page, int order) +{ +#ifdef CONFIG_MMU + set_page_count(page, 1); +#else + int i; + + /* + * We need to reference all the pages for this order, otherwise if + * anyone accesses one of the pages with (get/put) it will be freed. + * - eg: access_process_vm() + */ + for (i = 0; i < (1 << order); i++) + set_page_count(page + i, 1); +#endif /* CONFIG_MMU */ +} + +/* + * This page is about to be returned from the page allocator + */ +static void prep_new_page(struct page *page, int order) +{ + if (page->mapping || page_mapcount(page) || + (page->flags & ( + 1 << PG_private | + 1 << PG_locked | + 1 << PG_lru | + 1 << PG_active | + 1 << PG_dirty | + 1 << PG_reclaim | + 1 << PG_swapcache | + 1 << PG_writeback ))) + bad_page(__FUNCTION__, page); + + page->flags &= ~(1 << PG_uptodate | 1 << PG_error | + 1 << PG_referenced | 1 << PG_arch_1 | + 1 << PG_checked | 1 << PG_mappedtodisk); + page->private = 0; + set_page_refs(page, order); + kernel_map_pages(page, 1 << order, 1); +} + +/* + * Do the hard work of removing an element from the buddy allocator. + * Call me with the zone->lock already held. + */ +static struct page *__rmqueue(struct zone *zone, unsigned int order) +{ + struct free_area * area; + unsigned int current_order; + struct page *page; + + for (current_order = order; current_order < MAX_ORDER; ++current_order) { + area = zone->free_area + current_order; + if (list_empty(&area->free_list)) + continue; + + page = list_entry(area->free_list.next, struct page, lru); + list_del(&page->lru); + rmv_page_order(page); + area->nr_free--; + zone->free_pages -= 1UL << order; + return expand(zone, page, order, current_order, area); + } + + return NULL; +} + +/* + * Obtain a specified number of elements from the buddy allocator, all under + * a single hold of the lock, for efficiency. Add them to the supplied list. + * Returns the number of new pages which were placed at *list. + */ +static int rmqueue_bulk(struct zone *zone, unsigned int order, + unsigned long count, struct list_head *list) +{ + unsigned long flags; + int i; + int allocated = 0; + struct page *page; + + spin_lock_irqsave(&zone->lock, flags); + for (i = 0; i < count; ++i) { + page = __rmqueue(zone, order); + if (page == NULL) + break; + allocated++; + list_add_tail(&page->lru, list); + } + spin_unlock_irqrestore(&zone->lock, flags); + return allocated; +} + +#if defined(CONFIG_PM) || defined(CONFIG_HOTPLUG_CPU) +static void __drain_pages(unsigned int cpu) +{ + struct zone *zone; + int i; + + for_each_zone(zone) { + struct per_cpu_pageset *pset; + + pset = &zone->pageset[cpu]; + for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) { + struct per_cpu_pages *pcp; + + pcp = &pset->pcp[i]; + pcp->count -= free_pages_bulk(zone, pcp->count, + &pcp->list, 0); + } + } +} +#endif /* CONFIG_PM || CONFIG_HOTPLUG_CPU */ + +#ifdef CONFIG_PM + +void mark_free_pages(struct zone *zone) +{ + unsigned long zone_pfn, flags; + int order; + struct list_head *curr; + + if (!zone->spanned_pages) + return; + + spin_lock_irqsave(&zone->lock, flags); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) + ClearPageNosaveFree(pfn_to_page(zone_pfn + zone->zone_start_pfn)); + + for (order = MAX_ORDER - 1; order >= 0; --order) + list_for_each(curr, &zone->free_area[order].free_list) { + unsigned long start_pfn, i; + + start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); + + for (i=0; i < (1<<order); i++) + SetPageNosaveFree(pfn_to_page(start_pfn+i)); + } + spin_unlock_irqrestore(&zone->lock, flags); +} + +/* + * Spill all of this CPU's per-cpu pages back into the buddy allocator. + */ +void drain_local_pages(void) +{ + unsigned long flags; + + local_irq_save(flags); + __drain_pages(smp_processor_id()); + local_irq_restore(flags); +} +#endif /* CONFIG_PM */ + +static void zone_statistics(struct zonelist *zonelist, struct zone *z) +{ +#ifdef CONFIG_NUMA + unsigned long flags; + int cpu; + pg_data_t *pg = z->zone_pgdat; + pg_data_t *orig = zonelist->zones[0]->zone_pgdat; + struct per_cpu_pageset *p; + + local_irq_save(flags); + cpu = smp_processor_id(); + p = &z->pageset[cpu]; + if (pg == orig) { + z->pageset[cpu].numa_hit++; + } else { + p->numa_miss++; + zonelist->zones[0]->pageset[cpu].numa_foreign++; + } + if (pg == NODE_DATA(numa_node_id())) + p->local_node++; + else + p->other_node++; + local_irq_restore(flags); +#endif +} + +/* + * Free a 0-order page + */ +static void FASTCALL(free_hot_cold_page(struct page *page, int cold)); +static void fastcall free_hot_cold_page(struct page *page, int cold) +{ + struct zone *zone = page_zone(page); + struct per_cpu_pages *pcp; + unsigned long flags; + + arch_free_page(page, 0); + + kernel_map_pages(page, 1, 0); + inc_page_state(pgfree); + if (PageAnon(page)) + page->mapping = NULL; + free_pages_check(__FUNCTION__, page); + pcp = &zone->pageset[get_cpu()].pcp[cold]; + local_irq_save(flags); + if (pcp->count >= pcp->high) + pcp->count -= free_pages_bulk(zone, pcp->batch, &pcp->list, 0); + list_add(&page->lru, &pcp->list); + pcp->count++; + local_irq_restore(flags); + put_cpu(); +} + +void fastcall free_hot_page(struct page *page) +{ + free_hot_cold_page(page, 0); +} + +void fastcall free_cold_page(struct page *page) +{ + free_hot_cold_page(page, 1); +} + +static inline void prep_zero_page(struct page *page, int order, unsigned int __nocast gfp_flags) +{ + int i; + + BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM); + for(i = 0; i < (1 << order); i++) + clear_highpage(page + i); +} + +/* + * Really, prep_compound_page() should be called from __rmqueue_bulk(). But + * we cheat by calling it from here, in the order > 0 path. Saves a branch + * or two. + */ +static struct page * +buffered_rmqueue(struct zone *zone, int order, unsigned int __nocast gfp_flags) +{ + unsigned long flags; + struct page *page = NULL; + int cold = !!(gfp_flags & __GFP_COLD); + + if (order == 0) { + struct per_cpu_pages *pcp; + + pcp = &zone->pageset[get_cpu()].pcp[cold]; + local_irq_save(flags); + if (pcp->count <= pcp->low) + pcp->count += rmqueue_bulk(zone, 0, + pcp->batch, &pcp->list); + if (pcp->count) { + page = list_entry(pcp->list.next, struct page, lru); + list_del(&page->lru); + pcp->count--; + } + local_irq_restore(flags); + put_cpu(); + } + + if (page == NULL) { + spin_lock_irqsave(&zone->lock, flags); + page = __rmqueue(zone, order); + spin_unlock_irqrestore(&zone->lock, flags); + } + + if (page != NULL) { + BUG_ON(bad_range(zone, page)); + mod_page_state_zone(zone, pgalloc, 1 << order); + prep_new_page(page, order); + + if (gfp_flags & __GFP_ZERO) + prep_zero_page(page, order, gfp_flags); + + if (order && (gfp_flags & __GFP_COMP)) + prep_compound_page(page, order); + } + return page; +} + +/* + * Return 1 if free pages are above 'mark'. This takes into account the order + * of the allocation. + */ +int zone_watermark_ok(struct zone *z, int order, unsigned long mark, + int classzone_idx, int can_try_harder, int gfp_high) +{ + /* free_pages my go negative - that's OK */ + long min = mark, free_pages = z->free_pages - (1 << order) + 1; + int o; + + if (gfp_high) + min -= min / 2; + if (can_try_harder) + min -= min / 4; + + if (free_pages <= min + z->lowmem_reserve[classzone_idx]) + return 0; + for (o = 0; o < order; o++) { + /* At the next order, this order's pages become unavailable */ + free_pages -= z->free_area[o].nr_free << o; + + /* Require fewer higher order pages to be free */ + min >>= 1; + + if (free_pages <= min) + return 0; + } + return 1; +} + +/* + * This is the 'heart' of the zoned buddy allocator. + */ +struct page * fastcall +__alloc_pages(unsigned int __nocast gfp_mask, unsigned int order, + struct zonelist *zonelist) +{ + const int wait = gfp_mask & __GFP_WAIT; + struct zone **zones, *z; + struct page *page; + struct reclaim_state reclaim_state; + struct task_struct *p = current; + int i; + int classzone_idx; + int do_retry; + int can_try_harder; + int did_some_progress; + + might_sleep_if(wait); + + /* + * The caller may dip into page reserves a bit more if the caller + * cannot run direct reclaim, or is the caller has realtime scheduling + * policy + */ + can_try_harder = (unlikely(rt_task(p)) && !in_interrupt()) || !wait; + + zones = zonelist->zones; /* the list of zones suitable for gfp_mask */ + + if (unlikely(zones[0] == NULL)) { + /* Should this ever happen?? */ + return NULL; + } + + classzone_idx = zone_idx(zones[0]); + + restart: + /* Go through the zonelist once, looking for a zone with enough free */ + for (i = 0; (z = zones[i]) != NULL; i++) { + + if (!zone_watermark_ok(z, order, z->pages_low, + classzone_idx, 0, 0)) + continue; + + if (!cpuset_zone_allowed(z)) + continue; + + page = buffered_rmqueue(z, order, gfp_mask); + if (page) + goto got_pg; + } + + for (i = 0; (z = zones[i]) != NULL; i++) + wakeup_kswapd(z, order); + + /* + * Go through the zonelist again. Let __GFP_HIGH and allocations + * coming from realtime tasks to go deeper into reserves + * + * This is the last chance, in general, before the goto nopage. + * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc. + */ + for (i = 0; (z = zones[i]) != NULL; i++) { + if (!zone_watermark_ok(z, order, z->pages_min, + classzone_idx, can_try_harder, + gfp_mask & __GFP_HIGH)) + continue; + + if (wait && !cpuset_zone_allowed(z)) + continue; + + page = buffered_rmqueue(z, order, gfp_mask); + if (page) + goto got_pg; + } + + /* This allocation should allow future memory freeing. */ + if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE))) && !in_interrupt()) { + /* go through the zonelist yet again, ignoring mins */ + for (i = 0; (z = zones[i]) != NULL; i++) { + if (!cpuset_zone_allowed(z)) + continue; + page = buffered_rmqueue(z, order, gfp_mask); + if (page) + goto got_pg; + } + goto nopage; + } + + /* Atomic allocations - we can't balance anything */ + if (!wait) + goto nopage; + +rebalance: + cond_resched(); + + /* We now go into synchronous reclaim */ + p->flags |= PF_MEMALLOC; + reclaim_state.reclaimed_slab = 0; + p->reclaim_state = &reclaim_state; + + did_some_progress = try_to_free_pages(zones, gfp_mask, order); + + p->reclaim_state = NULL; + p->flags &= ~PF_MEMALLOC; + + cond_resched(); + + if (likely(did_some_progress)) { + /* + * Go through the zonelist yet one more time, keep + * very high watermark here, this is only to catch + * a parallel oom killing, we must fail if we're still + * under heavy pressure. + */ + for (i = 0; (z = zones[i]) != NULL; i++) { + if (!zone_watermark_ok(z, order, z->pages_min, + classzone_idx, can_try_harder, + gfp_mask & __GFP_HIGH)) + continue; + + if (!cpuset_zone_allowed(z)) + continue; + + page = buffered_rmqueue(z, order, gfp_mask); + if (page) + goto got_pg; + } + } else if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { + /* + * Go through the zonelist yet one more time, keep + * very high watermark here, this is only to catch + * a parallel oom killing, we must fail if we're still + * under heavy pressure. + */ + for (i = 0; (z = zones[i]) != NULL; i++) { + if (!zone_watermark_ok(z, order, z->pages_high, + classzone_idx, 0, 0)) + continue; + + if (!cpuset_zone_allowed(z)) + continue; + + page = buffered_rmqueue(z, order, gfp_mask); + if (page) + goto got_pg; + } + + out_of_memory(gfp_mask); + goto restart; + } + + /* + * Don't let big-order allocations loop unless the caller explicitly + * requests that. Wait for some write requests to complete then retry. + * + * In this implementation, __GFP_REPEAT means __GFP_NOFAIL for order + * <= 3, but that may not be true in other implementations. + */ + do_retry = 0; + if (!(gfp_mask & __GFP_NORETRY)) { + if ((order <= 3) || (gfp_mask & __GFP_REPEAT)) + do_retry = 1; + if (gfp_mask & __GFP_NOFAIL) + do_retry = 1; + } + if (do_retry) { + blk_congestion_wait(WRITE, HZ/50); + goto rebalance; + } + +nopage: + if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) { + printk(KERN_WARNING "%s: page allocation failure." + " order:%d, mode:0x%x\n", + p->comm, order, gfp_mask); + dump_stack(); + } + return NULL; +got_pg: + zone_statistics(zonelist, z); + return page; +} + +EXPORT_SYMBOL(__alloc_pages); + +/* + * Common helper functions. + */ +fastcall unsigned long __get_free_pages(unsigned int __nocast gfp_mask, unsigned int order) +{ + struct page * page; + page = alloc_pages(gfp_mask, order); + if (!page) + return 0; + return (unsigned long) page_address(page); +} + +EXPORT_SYMBOL(__get_free_pages); + +fastcall unsigned long get_zeroed_page(unsigned int __nocast gfp_mask) +{ + struct page * page; + + /* + * get_zeroed_page() returns a 32-bit address, which cannot represent + * a highmem page + */ + BUG_ON(gfp_mask & __GFP_HIGHMEM); + + page = alloc_pages(gfp_mask | __GFP_ZERO, 0); + if (page) + return (unsigned long) page_address(page); + return 0; +} + +EXPORT_SYMBOL(get_zeroed_page); + +void __pagevec_free(struct pagevec *pvec) +{ + int i = pagevec_count(pvec); + + while (--i >= 0) + free_hot_cold_page(pvec->pages[i], pvec->cold); +} + +fastcall void __free_pages(struct page *page, unsigned int order) +{ + if (!PageReserved(page) && put_page_testzero(page)) { + if (order == 0) + free_hot_page(page); + else + __free_pages_ok(page, order); + } +} + +EXPORT_SYMBOL(__free_pages); + +fastcall void free_pages(unsigned long addr, unsigned int order) +{ + if (addr != 0) { + BUG_ON(!virt_addr_valid((void *)addr)); + __free_pages(virt_to_page((void *)addr), order); + } +} + +EXPORT_SYMBOL(free_pages); + +/* + * Total amount of free (allocatable) RAM: + */ +unsigned int nr_free_pages(void) +{ + unsigned int sum = 0; + struct zone *zone; + + for_each_zone(zone) + sum += zone->free_pages; + + return sum; +} + +EXPORT_SYMBOL(nr_free_pages); + +#ifdef CONFIG_NUMA +unsigned int nr_free_pages_pgdat(pg_data_t *pgdat) +{ + unsigned int i, sum = 0; + + for (i = 0; i < MAX_NR_ZONES; i++) + sum += pgdat->node_zones[i].free_pages; + + return sum; +} +#endif + +static unsigned int nr_free_zone_pages(int offset) +{ + pg_data_t *pgdat; + unsigned int sum = 0; + + for_each_pgdat(pgdat) { + struct zonelist *zonelist = pgdat->node_zonelists + offset; + struct zone **zonep = zonelist->zones; + struct zone *zone; + + for (zone = *zonep++; zone; zone = *zonep++) { + unsigned long size = zone->present_pages; + unsigned long high = zone->pages_high; + if (size > high) + sum += size - high; + } + } + + return sum; +} + +/* + * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL + */ +unsigned int nr_free_buffer_pages(void) +{ + return nr_free_zone_pages(GFP_USER & GFP_ZONEMASK); +} + +/* + * Amount of free RAM allocatable within all zones + */ +unsigned int nr_free_pagecache_pages(void) +{ + return nr_free_zone_pages(GFP_HIGHUSER & GFP_ZONEMASK); +} + +#ifdef CONFIG_HIGHMEM +unsigned int nr_free_highpages (void) +{ + pg_data_t *pgdat; + unsigned int pages = 0; + + for_each_pgdat(pgdat) + pages += pgdat->node_zones[ZONE_HIGHMEM].free_pages; + + return pages; +} +#endif + +#ifdef CONFIG_NUMA +static void show_node(struct zone *zone) +{ + printk("Node %d ", zone->zone_pgdat->node_id); +} +#else +#define show_node(zone) do { } while (0) +#endif + +/* + * Accumulate the page_state information across all CPUs. + * The result is unavoidably approximate - it can change + * during and after execution of this function. + */ +static DEFINE_PER_CPU(struct page_state, page_states) = {0}; + +atomic_t nr_pagecache = ATOMIC_INIT(0); +EXPORT_SYMBOL(nr_pagecache); +#ifdef CONFIG_SMP +DEFINE_PER_CPU(long, nr_pagecache_local) = 0; +#endif + +void __get_page_state(struct page_state *ret, int nr) +{ + int cpu = 0; + + memset(ret, 0, sizeof(*ret)); + + cpu = first_cpu(cpu_online_map); + while (cpu < NR_CPUS) { + unsigned long *in, *out, off; + + in = (unsigned long *)&per_cpu(page_states, cpu); + + cpu = next_cpu(cpu, cpu_online_map); + + if (cpu < NR_CPUS) + prefetch(&per_cpu(page_states, cpu)); + + out = (unsigned long *)ret; + for (off = 0; off < nr; off++) + *out++ += *in++; + } +} + +void get_page_state(struct page_state *ret) +{ + int nr; + + nr = offsetof(struct page_state, GET_PAGE_STATE_LAST); + nr /= sizeof(unsigned long); + + __get_page_state(ret, nr + 1); +} + +void get_full_page_state(struct page_state *ret) +{ + __get_page_state(ret, sizeof(*ret) / sizeof(unsigned long)); +} + +unsigned long __read_page_state(unsigned offset) +{ + unsigned long ret = 0; + int cpu; + + for_each_online_cpu(cpu) { + unsigned long in; + + in = (unsigned long)&per_cpu(page_states, cpu) + offset; + ret += *((unsigned long *)in); + } + return ret; +} + +void __mod_page_state(unsigned offset, unsigned long delta) +{ + unsigned long flags; + void* ptr; + + local_irq_save(flags); + ptr = &__get_cpu_var(page_states); + *(unsigned long*)(ptr + offset) += delta; + local_irq_restore(flags); +} + +EXPORT_SYMBOL(__mod_page_state); + +void __get_zone_counts(unsigned long *active, unsigned long *inactive, + unsigned long *free, struct pglist_data *pgdat) +{ + struct zone *zones = pgdat->node_zones; + int i; + + *active = 0; + *inactive = 0; + *free = 0; + for (i = 0; i < MAX_NR_ZONES; i++) { + *active += zones[i].nr_active; + *inactive += zones[i].nr_inactive; + *free += zones[i].free_pages; + } +} + +void get_zone_counts(unsigned long *active, + unsigned long *inactive, unsigned long *free) +{ + struct pglist_data *pgdat; + + *active = 0; + *inactive = 0; + *free = 0; + for_each_pgdat(pgdat) { + unsigned long l, m, n; + __get_zone_counts(&l, &m, &n, pgdat); + *active += l; + *inactive += m; + *free += n; + } +} + +void si_meminfo(struct sysinfo *val) +{ + val->totalram = totalram_pages; + val->sharedram = 0; + val->freeram = nr_free_pages(); + val->bufferram = nr_blockdev_pages(); +#ifdef CONFIG_HIGHMEM + val->totalhigh = totalhigh_pages; + val->freehigh = nr_free_highpages(); +#else + val->totalhigh = 0; + val->freehigh = 0; +#endif + val->mem_unit = PAGE_SIZE; +} + +EXPORT_SYMBOL(si_meminfo); + +#ifdef CONFIG_NUMA +void si_meminfo_node(struct sysinfo *val, int nid) +{ + pg_data_t *pgdat = NODE_DATA(nid); + + val->totalram = pgdat->node_present_pages; + val->freeram = nr_free_pages_pgdat(pgdat); + val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages; + val->freehigh = pgdat->node_zones[ZONE_HIGHMEM].free_pages; + val->mem_unit = PAGE_SIZE; +} +#endif + +#define K(x) ((x) << (PAGE_SHIFT-10)) + +/* + * Show free area list (used inside shift_scroll-lock stuff) + * We also calculate the percentage fragmentation. We do this by counting the + * memory on each free list with the exception of the first item on the list. + */ +void show_free_areas(void) +{ + struct page_state ps; + int cpu, temperature; + unsigned long active; + unsigned long inactive; + unsigned long free; + struct zone *zone; + + for_each_zone(zone) { + show_node(zone); + printk("%s per-cpu:", zone->name); + + if (!zone->present_pages) { + printk(" empty\n"); + continue; + } else + printk("\n"); + + for (cpu = 0; cpu < NR_CPUS; ++cpu) { + struct per_cpu_pageset *pageset; + + if (!cpu_possible(cpu)) + continue; + + pageset = zone->pageset + cpu; + + for (temperature = 0; temperature < 2; temperature++) + printk("cpu %d %s: low %d, high %d, batch %d\n", + cpu, + temperature ? "cold" : "hot", + pageset->pcp[temperature].low, + pageset->pcp[temperature].high, + pageset->pcp[temperature].batch); + } + } + + get_page_state(&ps); + get_zone_counts(&active, &inactive, &free); + + printk("\nFree pages: %11ukB (%ukB HighMem)\n", + K(nr_free_pages()), + K(nr_free_highpages())); + + printk("Active:%lu inactive:%lu dirty:%lu writeback:%lu " + "unstable:%lu free:%u slab:%lu mapped:%lu pagetables:%lu\n", + active, + inactive, + ps.nr_dirty, + ps.nr_writeback, + ps.nr_unstable, + nr_free_pages(), + ps.nr_slab, + ps.nr_mapped, + ps.nr_page_table_pages); + + for_each_zone(zone) { + int i; + + show_node(zone); + printk("%s" + " free:%lukB" + " min:%lukB" + " low:%lukB" + " high:%lukB" + " active:%lukB" + " inactive:%lukB" + " present:%lukB" + " pages_scanned:%lu" + " all_unreclaimable? %s" + "\n", + zone->name, + K(zone->free_pages), + K(zone->pages_min), + K(zone->pages_low), + K(zone->pages_high), + K(zone->nr_active), + K(zone->nr_inactive), + K(zone->present_pages), + zone->pages_scanned, + (zone->all_unreclaimable ? "yes" : "no") + ); + printk("lowmem_reserve[]:"); + for (i = 0; i < MAX_NR_ZONES; i++) + printk(" %lu", zone->lowmem_reserve[i]); + printk("\n"); + } + + for_each_zone(zone) { + unsigned long nr, flags, order, total = 0; + + show_node(zone); + printk("%s: ", zone->name); + if (!zone->present_pages) { + printk("empty\n"); + continue; + } + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + nr = zone->free_area[order].nr_free; + total += nr << order; + printk("%lu*%lukB ", nr, K(1UL) << order); + } + spin_unlock_irqrestore(&zone->lock, flags); + printk("= %lukB\n", K(total)); + } + + show_swap_cache_info(); +} + +/* + * Builds allocation fallback zone lists. + */ +static int __init build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, int j, int k) +{ + switch (k) { + struct zone *zone; + default: + BUG(); + case ZONE_HIGHMEM: + zone = pgdat->node_zones + ZONE_HIGHMEM; + if (zone->present_pages) { +#ifndef CONFIG_HIGHMEM + BUG(); +#endif + zonelist->zones[j++] = zone; + } + case ZONE_NORMAL: + zone = pgdat->node_zones + ZONE_NORMAL; + if (zone->present_pages) + zonelist->zones[j++] = zone; + case ZONE_DMA: + zone = pgdat->node_zones + ZONE_DMA; + if (zone->present_pages) + zonelist->zones[j++] = zone; + } + + return j; +} + +#ifdef CONFIG_NUMA +#define MAX_NODE_LOAD (num_online_nodes()) +static int __initdata node_load[MAX_NUMNODES]; +/** + * find_next_best_node - find the next node that should appear in a given + * node's fallback list + * @node: node whose fallback list we're appending + * @used_node_mask: nodemask_t of already used nodes + * + * We use a number of factors to determine which is the next node that should + * appear on a given node's fallback list. The node should not have appeared + * already in @node's fallback list, and it should be the next closest node + * according to the distance array (which contains arbitrary distance values + * from each node to each node in the system), and should also prefer nodes + * with no CPUs, since presumably they'll have very little allocation pressure + * on them otherwise. + * It returns -1 if no node is found. + */ +static int __init find_next_best_node(int node, nodemask_t *used_node_mask) +{ + int i, n, val; + int min_val = INT_MAX; + int best_node = -1; + + for_each_online_node(i) { + cpumask_t tmp; + + /* Start from local node */ + n = (node+i) % num_online_nodes(); + + /* Don't want a node to appear more than once */ + if (node_isset(n, *used_node_mask)) + continue; + + /* Use the local node if we haven't already */ + if (!node_isset(node, *used_node_mask)) { + best_node = node; + break; + } + + /* Use the distance array to find the distance */ + val = node_distance(node, n); + + /* Give preference to headless and unused nodes */ + tmp = node_to_cpumask(n); + if (!cpus_empty(tmp)) + val += PENALTY_FOR_NODE_WITH_CPUS; + + /* Slight preference for less loaded node */ + val *= (MAX_NODE_LOAD*MAX_NUMNODES); + val += node_load[n]; + + if (val < min_val) { + min_val = val; + best_node = n; + } + } + + if (best_node >= 0) + node_set(best_node, *used_node_mask); + + return best_node; +} + +static void __init build_zonelists(pg_data_t *pgdat) +{ + int i, j, k, node, local_node; + int prev_node, load; + struct zonelist *zonelist; + nodemask_t used_mask; + + /* initialize zonelists */ + for (i = 0; i < GFP_ZONETYPES; i++) { + zonelist = pgdat->node_zonelists + i; + zonelist->zones[0] = NULL; + } + + /* NUMA-aware ordering of nodes */ + local_node = pgdat->node_id; + load = num_online_nodes(); + prev_node = local_node; + nodes_clear(used_mask); + while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { + /* + * We don't want to pressure a particular node. + * So adding penalty to the first node in same + * distance group to make it round-robin. + */ + if (node_distance(local_node, node) != + node_distance(local_node, prev_node)) + node_load[node] += load; + prev_node = node; + load--; + for (i = 0; i < GFP_ZONETYPES; i++) { + zonelist = pgdat->node_zonelists + i; + for (j = 0; zonelist->zones[j] != NULL; j++); + + k = ZONE_NORMAL; + if (i & __GFP_HIGHMEM) + k = ZONE_HIGHMEM; + if (i & __GFP_DMA) + k = ZONE_DMA; + + j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); + zonelist->zones[j] = NULL; + } + } +} + +#else /* CONFIG_NUMA */ + +static void __init build_zonelists(pg_data_t *pgdat) +{ + int i, j, k, node, local_node; + + local_node = pgdat->node_id; + for (i = 0; i < GFP_ZONETYPES; i++) { + struct zonelist *zonelist; + + zonelist = pgdat->node_zonelists + i; + + j = 0; + k = ZONE_NORMAL; + if (i & __GFP_HIGHMEM) + k = ZONE_HIGHMEM; + if (i & __GFP_DMA) + k = ZONE_DMA; + + j = build_zonelists_node(pgdat, zonelist, j, k); + /* + * Now we build the zonelist so that it contains the zones + * of all the other nodes. + * We don't want to pressure a particular node, so when + * building the zones for node N, we make sure that the + * zones coming right after the local ones are those from + * node N+1 (modulo N) + */ + for (node = local_node + 1; node < MAX_NUMNODES; node++) { + if (!node_online(node)) + continue; + j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); + } + for (node = 0; node < local_node; node++) { + if (!node_online(node)) + continue; + j = build_zonelists_node(NODE_DATA(node), zonelist, j, k); + } + + zonelist->zones[j] = NULL; + } +} + +#endif /* CONFIG_NUMA */ + +void __init build_all_zonelists(void) +{ + int i; + + for_each_online_node(i) + build_zonelists(NODE_DATA(i)); + printk("Built %i zonelists\n", num_online_nodes()); + cpuset_init_current_mems_allowed(); +} + +/* + * Helper functions to size the waitqueue hash table. + * Essentially these want to choose hash table sizes sufficiently + * large so that collisions trying to wait on pages are rare. + * But in fact, the number of active page waitqueues on typical + * systems is ridiculously low, less than 200. So this is even + * conservative, even though it seems large. + * + * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to + * waitqueues, i.e. the size of the waitq table given the number of pages. + */ +#define PAGES_PER_WAITQUEUE 256 + +static inline unsigned long wait_table_size(unsigned long pages) +{ + unsigned long size = 1; + + pages /= PAGES_PER_WAITQUEUE; + + while (size < pages) + size <<= 1; + + /* + * Once we have dozens or even hundreds of threads sleeping + * on IO we've got bigger problems than wait queue collision. + * Limit the size of the wait table to a reasonable size. + */ + size = min(size, 4096UL); + + return max(size, 4UL); +} + +/* + * This is an integer logarithm so that shifts can be used later + * to extract the more random high bits from the multiplicative + * hash function before the remainder is taken. + */ +static inline unsigned long wait_table_bits(unsigned long size) +{ + return ffz(~size); +} + +#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) + +static void __init calculate_zone_totalpages(struct pglist_data *pgdat, + unsigned long *zones_size, unsigned long *zholes_size) +{ + unsigned long realtotalpages, totalpages = 0; + int i; + + for (i = 0; i < MAX_NR_ZONES; i++) + totalpages += zones_size[i]; + pgdat->node_spanned_pages = totalpages; + + realtotalpages = totalpages; + if (zholes_size) + for (i = 0; i < MAX_NR_ZONES; i++) + realtotalpages -= zholes_size[i]; + pgdat->node_present_pages = realtotalpages; + printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); +} + + +/* + * Initially all pages are reserved - free ones are freed + * up by free_all_bootmem() once the early boot process is + * done. Non-atomic initialization, single-pass. + */ +void __init memmap_init_zone(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn) +{ + struct page *start = pfn_to_page(start_pfn); + struct page *page; + + for (page = start; page < (start + size); page++) { + set_page_zone(page, NODEZONE(nid, zone)); + set_page_count(page, 0); + reset_page_mapcount(page); + SetPageReserved(page); + INIT_LIST_HEAD(&page->lru); +#ifdef WANT_PAGE_VIRTUAL + /* The shift won't overflow because ZONE_NORMAL is below 4G. */ + if (!is_highmem_idx(zone)) + set_page_address(page, __va(start_pfn << PAGE_SHIFT)); +#endif + start_pfn++; + } +} + +void zone_init_free_lists(struct pglist_data *pgdat, struct zone *zone, + unsigned long size) +{ + int order; + for (order = 0; order < MAX_ORDER ; order++) { + INIT_LIST_HEAD(&zone->free_area[order].free_list); + zone->free_area[order].nr_free = 0; + } +} + +#ifndef __HAVE_ARCH_MEMMAP_INIT +#define memmap_init(size, nid, zone, start_pfn) \ + memmap_init_zone((size), (nid), (zone), (start_pfn)) +#endif + +/* + * Set up the zone data structures: + * - mark all pages reserved + * - mark all memory queues empty + * - clear the memory bitmaps + */ +static void __init free_area_init_core(struct pglist_data *pgdat, + unsigned long *zones_size, unsigned long *zholes_size) +{ + unsigned long i, j; + const unsigned long zone_required_alignment = 1UL << (MAX_ORDER-1); + int cpu, nid = pgdat->node_id; + unsigned long zone_start_pfn = pgdat->node_start_pfn; + + pgdat->nr_zones = 0; + init_waitqueue_head(&pgdat->kswapd_wait); + pgdat->kswapd_max_order = 0; + + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = pgdat->node_zones + j; + unsigned long size, realsize; + unsigned long batch; + + zone_table[NODEZONE(nid, j)] = zone; + realsize = size = zones_size[j]; + if (zholes_size) + realsize -= zholes_size[j]; + + if (j == ZONE_DMA || j == ZONE_NORMAL) + nr_kernel_pages += realsize; + nr_all_pages += realsize; + + zone->spanned_pages = size; + zone->present_pages = realsize; + zone->name = zone_names[j]; + spin_lock_init(&zone->lock); + spin_lock_init(&zone->lru_lock); + zone->zone_pgdat = pgdat; + zone->free_pages = 0; + + zone->temp_priority = zone->prev_priority = DEF_PRIORITY; + + /* + * The per-cpu-pages pools are set to around 1000th of the + * size of the zone. But no more than 1/4 of a meg - there's + * no point in going beyond the size of L2 cache. + * + * OK, so we don't know how big the cache is. So guess. + */ + batch = zone->present_pages / 1024; + if (batch * PAGE_SIZE > 256 * 1024) + batch = (256 * 1024) / PAGE_SIZE; + batch /= 4; /* We effectively *= 4 below */ + if (batch < 1) + batch = 1; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + struct per_cpu_pages *pcp; + + pcp = &zone->pageset[cpu].pcp[0]; /* hot */ + pcp->count = 0; + pcp->low = 2 * batch; + pcp->high = 6 * batch; + pcp->batch = 1 * batch; + INIT_LIST_HEAD(&pcp->list); + + pcp = &zone->pageset[cpu].pcp[1]; /* cold */ + pcp->count = 0; + pcp->low = 0; + pcp->high = 2 * batch; + pcp->batch = 1 * batch; + INIT_LIST_HEAD(&pcp->list); + } + printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", + zone_names[j], realsize, batch); + INIT_LIST_HEAD(&zone->active_list); + INIT_LIST_HEAD(&zone->inactive_list); + zone->nr_scan_active = 0; + zone->nr_scan_inactive = 0; + zone->nr_active = 0; + zone->nr_inactive = 0; + if (!size) + continue; + + /* + * The per-page waitqueue mechanism uses hashed waitqueues + * per zone. + */ + zone->wait_table_size = wait_table_size(size); + zone->wait_table_bits = + wait_table_bits(zone->wait_table_size); + zone->wait_table = (wait_queue_head_t *) + alloc_bootmem_node(pgdat, zone->wait_table_size + * sizeof(wait_queue_head_t)); + + for(i = 0; i < zone->wait_table_size; ++i) + init_waitqueue_head(zone->wait_table + i); + + pgdat->nr_zones = j+1; + + zone->zone_mem_map = pfn_to_page(zone_start_pfn); + zone->zone_start_pfn = zone_start_pfn; + + if ((zone_start_pfn) & (zone_required_alignment-1)) + printk(KERN_CRIT "BUG: wrong zone alignment, it will crash\n"); + + memmap_init(size, nid, j, zone_start_pfn); + + zone_start_pfn += size; + + zone_init_free_lists(pgdat, zone, zone->spanned_pages); + } +} + +static void __init alloc_node_mem_map(struct pglist_data *pgdat) +{ + unsigned long size; + + /* Skip empty nodes */ + if (!pgdat->node_spanned_pages) + return; + + /* ia64 gets its own node_mem_map, before this, without bootmem */ + if (!pgdat->node_mem_map) { + size = (pgdat->node_spanned_pages + 1) * sizeof(struct page); + pgdat->node_mem_map = alloc_bootmem_node(pgdat, size); + } +#ifndef CONFIG_DISCONTIGMEM + /* + * With no DISCONTIG, the global mem_map is just set as node 0's + */ + if (pgdat == NODE_DATA(0)) + mem_map = NODE_DATA(0)->node_mem_map; +#endif +} + +void __init free_area_init_node(int nid, struct pglist_data *pgdat, + unsigned long *zones_size, unsigned long node_start_pfn, + unsigned long *zholes_size) +{ + pgdat->node_id = nid; + pgdat->node_start_pfn = node_start_pfn; + calculate_zone_totalpages(pgdat, zones_size, zholes_size); + + alloc_node_mem_map(pgdat); + + free_area_init_core(pgdat, zones_size, zholes_size); +} + +#ifndef CONFIG_DISCONTIGMEM +static bootmem_data_t contig_bootmem_data; +struct pglist_data contig_page_data = { .bdata = &contig_bootmem_data }; + +EXPORT_SYMBOL(contig_page_data); + +void __init free_area_init(unsigned long *zones_size) +{ + free_area_init_node(0, &contig_page_data, zones_size, + __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); +} +#endif + +#ifdef CONFIG_PROC_FS + +#include <linux/seq_file.h> + +static void *frag_start(struct seq_file *m, loff_t *pos) +{ + pg_data_t *pgdat; + loff_t node = *pos; + + for (pgdat = pgdat_list; pgdat && node; pgdat = pgdat->pgdat_next) + --node; + + return pgdat; +} + +static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) +{ + pg_data_t *pgdat = (pg_data_t *)arg; + + (*pos)++; + return pgdat->pgdat_next; +} + +static void frag_stop(struct seq_file *m, void *arg) +{ +} + +/* + * This walks the free areas for each zone. + */ +static int frag_show(struct seq_file *m, void *arg) +{ + pg_data_t *pgdat = (pg_data_t *)arg; + struct zone *zone; + struct zone *node_zones = pgdat->node_zones; + unsigned long flags; + int order; + + for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { + if (!zone->present_pages) + continue; + + spin_lock_irqsave(&zone->lock, flags); + seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); + for (order = 0; order < MAX_ORDER; ++order) + seq_printf(m, "%6lu ", zone->free_area[order].nr_free); + spin_unlock_irqrestore(&zone->lock, flags); + seq_putc(m, '\n'); + } + return 0; +} + +struct seq_operations fragmentation_op = { + .start = frag_start, + .next = frag_next, + .stop = frag_stop, + .show = frag_show, +}; + +static char *vmstat_text[] = { + "nr_dirty", + "nr_writeback", + "nr_unstable", + "nr_page_table_pages", + "nr_mapped", + "nr_slab", + + "pgpgin", + "pgpgout", + "pswpin", + "pswpout", + "pgalloc_high", + + "pgalloc_normal", + "pgalloc_dma", + "pgfree", + "pgactivate", + "pgdeactivate", + + "pgfault", + "pgmajfault", + "pgrefill_high", + "pgrefill_normal", + "pgrefill_dma", + + "pgsteal_high", + "pgsteal_normal", + "pgsteal_dma", + "pgscan_kswapd_high", + "pgscan_kswapd_normal", + + "pgscan_kswapd_dma", + "pgscan_direct_high", + "pgscan_direct_normal", + "pgscan_direct_dma", + "pginodesteal", + + "slabs_scanned", + "kswapd_steal", + "kswapd_inodesteal", + "pageoutrun", + "allocstall", + + "pgrotated", +}; + +static void *vmstat_start(struct seq_file *m, loff_t *pos) +{ + struct page_state *ps; + + if (*pos >= ARRAY_SIZE(vmstat_text)) + return NULL; + + ps = kmalloc(sizeof(*ps), GFP_KERNEL); + m->private = ps; + if (!ps) + return ERR_PTR(-ENOMEM); + get_full_page_state(ps); + ps->pgpgin /= 2; /* sectors -> kbytes */ + ps->pgpgout /= 2; + return (unsigned long *)ps + *pos; +} + +static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) +{ + (*pos)++; + if (*pos >= ARRAY_SIZE(vmstat_text)) + return NULL; + return (unsigned long *)m->private + *pos; +} + +static int vmstat_show(struct seq_file *m, void *arg) +{ + unsigned long *l = arg; + unsigned long off = l - (unsigned long *)m->private; + + seq_printf(m, "%s %lu\n", vmstat_text[off], *l); + return 0; +} + +static void vmstat_stop(struct seq_file *m, void *arg) +{ + kfree(m->private); + m->private = NULL; +} + +struct seq_operations vmstat_op = { + .start = vmstat_start, + .next = vmstat_next, + .stop = vmstat_stop, + .show = vmstat_show, +}; + +#endif /* CONFIG_PROC_FS */ + +#ifdef CONFIG_HOTPLUG_CPU +static int page_alloc_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + int cpu = (unsigned long)hcpu; + long *count; + unsigned long *src, *dest; + + if (action == CPU_DEAD) { + int i; + + /* Drain local pagecache count. */ + count = &per_cpu(nr_pagecache_local, cpu); + atomic_add(*count, &nr_pagecache); + *count = 0; + local_irq_disable(); + __drain_pages(cpu); + + /* Add dead cpu's page_states to our own. */ + dest = (unsigned long *)&__get_cpu_var(page_states); + src = (unsigned long *)&per_cpu(page_states, cpu); + + for (i = 0; i < sizeof(struct page_state)/sizeof(unsigned long); + i++) { + dest[i] += src[i]; + src[i] = 0; + } + + local_irq_enable(); + } + return NOTIFY_OK; +} +#endif /* CONFIG_HOTPLUG_CPU */ + +void __init page_alloc_init(void) +{ + hotcpu_notifier(page_alloc_cpu_notify, 0); +} + +/* + * setup_per_zone_lowmem_reserve - called whenever + * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone + * has a correct pages reserved value, so an adequate number of + * pages are left in the zone after a successful __alloc_pages(). + */ +static void setup_per_zone_lowmem_reserve(void) +{ + struct pglist_data *pgdat; + int j, idx; + + for_each_pgdat(pgdat) { + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = pgdat->node_zones + j; + unsigned long present_pages = zone->present_pages; + + zone->lowmem_reserve[j] = 0; + + for (idx = j-1; idx >= 0; idx--) { + struct zone *lower_zone; + + if (sysctl_lowmem_reserve_ratio[idx] < 1) + sysctl_lowmem_reserve_ratio[idx] = 1; + + lower_zone = pgdat->node_zones + idx; + lower_zone->lowmem_reserve[j] = present_pages / + sysctl_lowmem_reserve_ratio[idx]; + present_pages += lower_zone->present_pages; + } + } + } +} + +/* + * setup_per_zone_pages_min - called when min_free_kbytes changes. Ensures + * that the pages_{min,low,high} values for each zone are set correctly + * with respect to min_free_kbytes. + */ +static void setup_per_zone_pages_min(void) +{ + unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); + unsigned long lowmem_pages = 0; + struct zone *zone; + unsigned long flags; + + /* Calculate total number of !ZONE_HIGHMEM pages */ + for_each_zone(zone) { + if (!is_highmem(zone)) + lowmem_pages += zone->present_pages; + } + + for_each_zone(zone) { + spin_lock_irqsave(&zone->lru_lock, flags); + if (is_highmem(zone)) { + /* + * Often, highmem doesn't need to reserve any pages. + * But the pages_min/low/high values are also used for + * batching up page reclaim activity so we need a + * decent value here. + */ + int min_pages; + + min_pages = zone->present_pages / 1024; + if (min_pages < SWAP_CLUSTER_MAX) + min_pages = SWAP_CLUSTER_MAX; + if (min_pages > 128) + min_pages = 128; + zone->pages_min = min_pages; + } else { + /* if it's a lowmem zone, reserve a number of pages + * proportionate to the zone's size. + */ + zone->pages_min = (pages_min * zone->present_pages) / + lowmem_pages; + } + + /* + * When interpreting these watermarks, just keep in mind that: + * zone->pages_min == (zone->pages_min * 4) / 4; + */ + zone->pages_low = (zone->pages_min * 5) / 4; + zone->pages_high = (zone->pages_min * 6) / 4; + spin_unlock_irqrestore(&zone->lru_lock, flags); + } +} + +/* + * Initialise min_free_kbytes. + * + * For small machines we want it small (128k min). For large machines + * we want it large (64MB max). But it is not linear, because network + * bandwidth does not increase linearly with machine size. We use + * + * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: + * min_free_kbytes = sqrt(lowmem_kbytes * 16) + * + * which yields + * + * 16MB: 512k + * 32MB: 724k + * 64MB: 1024k + * 128MB: 1448k + * 256MB: 2048k + * 512MB: 2896k + * 1024MB: 4096k + * 2048MB: 5792k + * 4096MB: 8192k + * 8192MB: 11584k + * 16384MB: 16384k + */ +static int __init init_per_zone_pages_min(void) +{ + unsigned long lowmem_kbytes; + + lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); + + min_free_kbytes = int_sqrt(lowmem_kbytes * 16); + if (min_free_kbytes < 128) + min_free_kbytes = 128; + if (min_free_kbytes > 65536) + min_free_kbytes = 65536; + setup_per_zone_pages_min(); + setup_per_zone_lowmem_reserve(); + return 0; +} +module_init(init_per_zone_pages_min) + +/* + * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so + * that we can call two helper functions whenever min_free_kbytes + * changes. + */ +int min_free_kbytes_sysctl_handler(ctl_table *table, int write, + struct file *file, void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec(table, write, file, buffer, length, ppos); + setup_per_zone_pages_min(); + return 0; +} + +/* + * lowmem_reserve_ratio_sysctl_handler - just a wrapper around + * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() + * whenever sysctl_lowmem_reserve_ratio changes. + * + * The reserve ratio obviously has absolutely no relation with the + * pages_min watermarks. The lowmem reserve ratio can only make sense + * if in function of the boot time zone sizes. + */ +int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write, + struct file *file, void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec_minmax(table, write, file, buffer, length, ppos); + setup_per_zone_lowmem_reserve(); + return 0; +} + +__initdata int hashdist = HASHDIST_DEFAULT; + +#ifdef CONFIG_NUMA +static int __init set_hashdist(char *str) +{ + if (!str) + return 0; + hashdist = simple_strtoul(str, &str, 0); + return 1; +} +__setup("hashdist=", set_hashdist); +#endif + +/* + * allocate a large system hash table from bootmem + * - it is assumed that the hash table must contain an exact power-of-2 + * quantity of entries + * - limit is the number of hash buckets, not the total allocation size + */ +void *__init alloc_large_system_hash(const char *tablename, + unsigned long bucketsize, + unsigned long numentries, + int scale, + int flags, + unsigned int *_hash_shift, + unsigned int *_hash_mask, + unsigned long limit) +{ + unsigned long long max = limit; + unsigned long log2qty, size; + void *table = NULL; + + /* allow the kernel cmdline to have a say */ + if (!numentries) { + /* round applicable memory size up to nearest megabyte */ + numentries = (flags & HASH_HIGHMEM) ? nr_all_pages : nr_kernel_pages; + numentries += (1UL << (20 - PAGE_SHIFT)) - 1; + numentries >>= 20 - PAGE_SHIFT; + numentries <<= 20 - PAGE_SHIFT; + + /* limit to 1 bucket per 2^scale bytes of low memory */ + if (scale > PAGE_SHIFT) + numentries >>= (scale - PAGE_SHIFT); + else + numentries <<= (PAGE_SHIFT - scale); + } + /* rounded up to nearest power of 2 in size */ + numentries = 1UL << (long_log2(numentries) + 1); + + /* limit allocation size to 1/16 total memory by default */ + if (max == 0) { + max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; + do_div(max, bucketsize); + } + + if (numentries > max) + numentries = max; + + log2qty = long_log2(numentries); + + do { + size = bucketsize << log2qty; + if (flags & HASH_EARLY) + table = alloc_bootmem(size); + else if (hashdist) + table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); + else { + unsigned long order; + for (order = 0; ((1UL << order) << PAGE_SHIFT) < size; order++) + ; + table = (void*) __get_free_pages(GFP_ATOMIC, order); + } + } while (!table && size > PAGE_SIZE && --log2qty); + + if (!table) + panic("Failed to allocate %s hash table\n", tablename); + + printk("%s hash table entries: %d (order: %d, %lu bytes)\n", + tablename, + (1U << log2qty), + long_log2(size) - PAGE_SHIFT, + size); + + if (_hash_shift) + *_hash_shift = log2qty; + if (_hash_mask) + *_hash_mask = (1 << log2qty) - 1; + + return table; +} diff --git a/mm/page_io.c b/mm/page_io.c new file mode 100644 index 000000000000..667c76df1ec2 --- /dev/null +++ b/mm/page_io.c @@ -0,0 +1,160 @@ +/* + * linux/mm/page_io.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * + * Swap reorganised 29.12.95, + * Asynchronous swapping added 30.12.95. Stephen Tweedie + * Removed race in async swapping. 14.4.1996. Bruno Haible + * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie + * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman + */ + +#include <linux/mm.h> +#include <linux/kernel_stat.h> +#include <linux/pagemap.h> +#include <linux/swap.h> +#include <linux/bio.h> +#include <linux/swapops.h> +#include <linux/writeback.h> +#include <asm/pgtable.h> + +static struct bio *get_swap_bio(unsigned int __nocast gfp_flags, pgoff_t index, + struct page *page, bio_end_io_t end_io) +{ + struct bio *bio; + + bio = bio_alloc(gfp_flags, 1); + if (bio) { + struct swap_info_struct *sis; + swp_entry_t entry = { .val = index, }; + + sis = get_swap_info_struct(swp_type(entry)); + bio->bi_sector = map_swap_page(sis, swp_offset(entry)) * + (PAGE_SIZE >> 9); + bio->bi_bdev = sis->bdev; + bio->bi_io_vec[0].bv_page = page; + bio->bi_io_vec[0].bv_len = PAGE_SIZE; + bio->bi_io_vec[0].bv_offset = 0; + bio->bi_vcnt = 1; + bio->bi_idx = 0; + bio->bi_size = PAGE_SIZE; + bio->bi_end_io = end_io; + } + return bio; +} + +static int end_swap_bio_write(struct bio *bio, unsigned int bytes_done, int err) +{ + const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); + struct page *page = bio->bi_io_vec[0].bv_page; + + if (bio->bi_size) + return 1; + + if (!uptodate) + SetPageError(page); + end_page_writeback(page); + bio_put(bio); + return 0; +} + +static int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err) +{ + const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); + struct page *page = bio->bi_io_vec[0].bv_page; + + if (bio->bi_size) + return 1; + + if (!uptodate) { + SetPageError(page); + ClearPageUptodate(page); + } else { + SetPageUptodate(page); + } + unlock_page(page); + bio_put(bio); + return 0; +} + +/* + * We may have stale swap cache pages in memory: notice + * them here and get rid of the unnecessary final write. + */ +int swap_writepage(struct page *page, struct writeback_control *wbc) +{ + struct bio *bio; + int ret = 0, rw = WRITE; + + if (remove_exclusive_swap_page(page)) { + unlock_page(page); + goto out; + } + bio = get_swap_bio(GFP_NOIO, page->private, page, end_swap_bio_write); + if (bio == NULL) { + set_page_dirty(page); + unlock_page(page); + ret = -ENOMEM; + goto out; + } + if (wbc->sync_mode == WB_SYNC_ALL) + rw |= (1 << BIO_RW_SYNC); + inc_page_state(pswpout); + set_page_writeback(page); + unlock_page(page); + submit_bio(rw, bio); +out: + return ret; +} + +int swap_readpage(struct file *file, struct page *page) +{ + struct bio *bio; + int ret = 0; + + BUG_ON(!PageLocked(page)); + ClearPageUptodate(page); + bio = get_swap_bio(GFP_KERNEL, page->private, page, end_swap_bio_read); + if (bio == NULL) { + unlock_page(page); + ret = -ENOMEM; + goto out; + } + inc_page_state(pswpin); + submit_bio(READ, bio); +out: + return ret; +} + +#if defined(CONFIG_SOFTWARE_SUSPEND) || defined(CONFIG_PM_DISK) +/* + * A scruffy utility function to read or write an arbitrary swap page + * and wait on the I/O. The caller must have a ref on the page. + * + * We use end_swap_bio_read() even for writes, because it happens to do what + * we want. + */ +int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page) +{ + struct bio *bio; + int ret = 0; + + lock_page(page); + + bio = get_swap_bio(GFP_KERNEL, entry.val, page, end_swap_bio_read); + if (bio == NULL) { + unlock_page(page); + ret = -ENOMEM; + goto out; + } + + submit_bio(rw | (1 << BIO_RW_SYNC), bio); + wait_on_page_locked(page); + + if (!PageUptodate(page) || PageError(page)) + ret = -EIO; +out: + return ret; +} +#endif diff --git a/mm/pdflush.c b/mm/pdflush.c new file mode 100644 index 000000000000..38ce279cc8cd --- /dev/null +++ b/mm/pdflush.c @@ -0,0 +1,228 @@ +/* + * mm/pdflush.c - worker threads for writing back filesystem data + * + * Copyright (C) 2002, Linus Torvalds. + * + * 09Apr2002 akpm@zip.com.au + * Initial version + * 29Feb2004 kaos@sgi.com + * Move worker thread creation to kthread to avoid chewing + * up stack space with nested calls to kernel_thread. + */ + +#include <linux/sched.h> +#include <linux/list.h> +#include <linux/signal.h> +#include <linux/spinlock.h> +#include <linux/gfp.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/fs.h> // Needed by writeback.h +#include <linux/writeback.h> // Prototypes pdflush_operation() +#include <linux/kthread.h> + + +/* + * Minimum and maximum number of pdflush instances + */ +#define MIN_PDFLUSH_THREADS 2 +#define MAX_PDFLUSH_THREADS 8 + +static void start_one_pdflush_thread(void); + + +/* + * The pdflush threads are worker threads for writing back dirty data. + * Ideally, we'd like one thread per active disk spindle. But the disk + * topology is very hard to divine at this level. Instead, we take + * care in various places to prevent more than one pdflush thread from + * performing writeback against a single filesystem. pdflush threads + * have the PF_FLUSHER flag set in current->flags to aid in this. + */ + +/* + * All the pdflush threads. Protected by pdflush_lock + */ +static LIST_HEAD(pdflush_list); +static DEFINE_SPINLOCK(pdflush_lock); + +/* + * The count of currently-running pdflush threads. Protected + * by pdflush_lock. + * + * Readable by sysctl, but not writable. Published to userspace at + * /proc/sys/vm/nr_pdflush_threads. + */ +int nr_pdflush_threads = 0; + +/* + * The time at which the pdflush thread pool last went empty + */ +static unsigned long last_empty_jifs; + +/* + * The pdflush thread. + * + * Thread pool management algorithm: + * + * - The minimum and maximum number of pdflush instances are bound + * by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS. + * + * - If there have been no idle pdflush instances for 1 second, create + * a new one. + * + * - If the least-recently-went-to-sleep pdflush thread has been asleep + * for more than one second, terminate a thread. + */ + +/* + * A structure for passing work to a pdflush thread. Also for passing + * state information between pdflush threads. Protected by pdflush_lock. + */ +struct pdflush_work { + struct task_struct *who; /* The thread */ + void (*fn)(unsigned long); /* A callback function */ + unsigned long arg0; /* An argument to the callback */ + struct list_head list; /* On pdflush_list, when idle */ + unsigned long when_i_went_to_sleep; +}; + +static int __pdflush(struct pdflush_work *my_work) +{ + current->flags |= PF_FLUSHER; + my_work->fn = NULL; + my_work->who = current; + INIT_LIST_HEAD(&my_work->list); + + spin_lock_irq(&pdflush_lock); + nr_pdflush_threads++; + for ( ; ; ) { + struct pdflush_work *pdf; + + set_current_state(TASK_INTERRUPTIBLE); + list_move(&my_work->list, &pdflush_list); + my_work->when_i_went_to_sleep = jiffies; + spin_unlock_irq(&pdflush_lock); + + schedule(); + if (try_to_freeze(PF_FREEZE)) { + spin_lock_irq(&pdflush_lock); + continue; + } + + spin_lock_irq(&pdflush_lock); + if (!list_empty(&my_work->list)) { + printk("pdflush: bogus wakeup!\n"); + my_work->fn = NULL; + continue; + } + if (my_work->fn == NULL) { + printk("pdflush: NULL work function\n"); + continue; + } + spin_unlock_irq(&pdflush_lock); + + (*my_work->fn)(my_work->arg0); + + /* + * Thread creation: For how long have there been zero + * available threads? + */ + if (jiffies - last_empty_jifs > 1 * HZ) { + /* unlocked list_empty() test is OK here */ + if (list_empty(&pdflush_list)) { + /* unlocked test is OK here */ + if (nr_pdflush_threads < MAX_PDFLUSH_THREADS) + start_one_pdflush_thread(); + } + } + + spin_lock_irq(&pdflush_lock); + my_work->fn = NULL; + + /* + * Thread destruction: For how long has the sleepiest + * thread slept? + */ + if (list_empty(&pdflush_list)) + continue; + if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS) + continue; + pdf = list_entry(pdflush_list.prev, struct pdflush_work, list); + if (jiffies - pdf->when_i_went_to_sleep > 1 * HZ) { + /* Limit exit rate */ + pdf->when_i_went_to_sleep = jiffies; + break; /* exeunt */ + } + } + nr_pdflush_threads--; + spin_unlock_irq(&pdflush_lock); + return 0; +} + +/* + * Of course, my_work wants to be just a local in __pdflush(). It is + * separated out in this manner to hopefully prevent the compiler from + * performing unfortunate optimisations against the auto variables. Because + * these are visible to other tasks and CPUs. (No problem has actually + * been observed. This is just paranoia). + */ +static int pdflush(void *dummy) +{ + struct pdflush_work my_work; + + /* + * pdflush can spend a lot of time doing encryption via dm-crypt. We + * don't want to do that at keventd's priority. + */ + set_user_nice(current, 0); + return __pdflush(&my_work); +} + +/* + * Attempt to wake up a pdflush thread, and get it to do some work for you. + * Returns zero if it indeed managed to find a worker thread, and passed your + * payload to it. + */ +int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0) +{ + unsigned long flags; + int ret = 0; + + if (fn == NULL) + BUG(); /* Hard to diagnose if it's deferred */ + + spin_lock_irqsave(&pdflush_lock, flags); + if (list_empty(&pdflush_list)) { + spin_unlock_irqrestore(&pdflush_lock, flags); + ret = -1; + } else { + struct pdflush_work *pdf; + + pdf = list_entry(pdflush_list.next, struct pdflush_work, list); + list_del_init(&pdf->list); + if (list_empty(&pdflush_list)) + last_empty_jifs = jiffies; + pdf->fn = fn; + pdf->arg0 = arg0; + wake_up_process(pdf->who); + spin_unlock_irqrestore(&pdflush_lock, flags); + } + return ret; +} + +static void start_one_pdflush_thread(void) +{ + kthread_run(pdflush, NULL, "pdflush"); +} + +static int __init pdflush_init(void) +{ + int i; + + for (i = 0; i < MIN_PDFLUSH_THREADS; i++) + start_one_pdflush_thread(); + return 0; +} + +module_init(pdflush_init); diff --git a/mm/prio_tree.c b/mm/prio_tree.c new file mode 100644 index 000000000000..b4e76c25f953 --- /dev/null +++ b/mm/prio_tree.c @@ -0,0 +1,207 @@ +/* + * mm/prio_tree.c - priority search tree for mapping->i_mmap + * + * Copyright (C) 2004, Rajesh Venkatasubramanian <vrajesh@umich.edu> + * + * This file is released under the GPL v2. + * + * Based on the radix priority search tree proposed by Edward M. McCreight + * SIAM Journal of Computing, vol. 14, no.2, pages 257-276, May 1985 + * + * 02Feb2004 Initial version + */ + +#include <linux/mm.h> +#include <linux/prio_tree.h> + +/* + * See lib/prio_tree.c for details on the general radix priority search tree + * code. + */ + +/* + * The following #defines are mirrored from lib/prio_tree.c. They're only used + * for debugging, and should be removed (along with the debugging code using + * them) when switching also VMAs to the regular prio_tree code. + */ + +#define RADIX_INDEX(vma) ((vma)->vm_pgoff) +#define VMA_SIZE(vma) (((vma)->vm_end - (vma)->vm_start) >> PAGE_SHIFT) +/* avoid overflow */ +#define HEAP_INDEX(vma) ((vma)->vm_pgoff + (VMA_SIZE(vma) - 1)) + +/* + * Radix priority search tree for address_space->i_mmap + * + * For each vma that map a unique set of file pages i.e., unique [radix_index, + * heap_index] value, we have a corresponing priority search tree node. If + * multiple vmas have identical [radix_index, heap_index] value, then one of + * them is used as a tree node and others are stored in a vm_set list. The tree + * node points to the first vma (head) of the list using vm_set.head. + * + * prio_tree_root + * | + * A vm_set.head + * / \ / + * L R -> H-I-J-K-M-N-O-P-Q-S + * ^ ^ <-- vm_set.list --> + * tree nodes + * + * We need some way to identify whether a vma is a tree node, head of a vm_set + * list, or just a member of a vm_set list. We cannot use vm_flags to store + * such information. The reason is, in the above figure, it is possible that + * vm_flags' of R and H are covered by the different mmap_sems. When R is + * removed under R->mmap_sem, H replaces R as a tree node. Since we do not hold + * H->mmap_sem, we cannot use H->vm_flags for marking that H is a tree node now. + * That's why some trick involving shared.vm_set.parent is used for identifying + * tree nodes and list head nodes. + * + * vma radix priority search tree node rules: + * + * vma->shared.vm_set.parent != NULL ==> a tree node + * vma->shared.vm_set.head != NULL ==> list of others mapping same range + * vma->shared.vm_set.head == NULL ==> no others map the same range + * + * vma->shared.vm_set.parent == NULL + * vma->shared.vm_set.head != NULL ==> list head of vmas mapping same range + * vma->shared.vm_set.head == NULL ==> a list node + */ + +/* + * Add a new vma known to map the same set of pages as the old vma: + * useful for fork's dup_mmap as well as vma_prio_tree_insert below. + * Note that it just happens to work correctly on i_mmap_nonlinear too. + */ +void vma_prio_tree_add(struct vm_area_struct *vma, struct vm_area_struct *old) +{ + /* Leave these BUG_ONs till prio_tree patch stabilizes */ + BUG_ON(RADIX_INDEX(vma) != RADIX_INDEX(old)); + BUG_ON(HEAP_INDEX(vma) != HEAP_INDEX(old)); + + vma->shared.vm_set.head = NULL; + vma->shared.vm_set.parent = NULL; + + if (!old->shared.vm_set.parent) + list_add(&vma->shared.vm_set.list, + &old->shared.vm_set.list); + else if (old->shared.vm_set.head) + list_add_tail(&vma->shared.vm_set.list, + &old->shared.vm_set.head->shared.vm_set.list); + else { + INIT_LIST_HEAD(&vma->shared.vm_set.list); + vma->shared.vm_set.head = old; + old->shared.vm_set.head = vma; + } +} + +void vma_prio_tree_insert(struct vm_area_struct *vma, + struct prio_tree_root *root) +{ + struct prio_tree_node *ptr; + struct vm_area_struct *old; + + vma->shared.vm_set.head = NULL; + + ptr = raw_prio_tree_insert(root, &vma->shared.prio_tree_node); + if (ptr != (struct prio_tree_node *) &vma->shared.prio_tree_node) { + old = prio_tree_entry(ptr, struct vm_area_struct, + shared.prio_tree_node); + vma_prio_tree_add(vma, old); + } +} + +void vma_prio_tree_remove(struct vm_area_struct *vma, + struct prio_tree_root *root) +{ + struct vm_area_struct *node, *head, *new_head; + + if (!vma->shared.vm_set.head) { + if (!vma->shared.vm_set.parent) + list_del_init(&vma->shared.vm_set.list); + else + raw_prio_tree_remove(root, &vma->shared.prio_tree_node); + } else { + /* Leave this BUG_ON till prio_tree patch stabilizes */ + BUG_ON(vma->shared.vm_set.head->shared.vm_set.head != vma); + if (vma->shared.vm_set.parent) { + head = vma->shared.vm_set.head; + if (!list_empty(&head->shared.vm_set.list)) { + new_head = list_entry( + head->shared.vm_set.list.next, + struct vm_area_struct, + shared.vm_set.list); + list_del_init(&head->shared.vm_set.list); + } else + new_head = NULL; + + raw_prio_tree_replace(root, &vma->shared.prio_tree_node, + &head->shared.prio_tree_node); + head->shared.vm_set.head = new_head; + if (new_head) + new_head->shared.vm_set.head = head; + + } else { + node = vma->shared.vm_set.head; + if (!list_empty(&vma->shared.vm_set.list)) { + new_head = list_entry( + vma->shared.vm_set.list.next, + struct vm_area_struct, + shared.vm_set.list); + list_del_init(&vma->shared.vm_set.list); + node->shared.vm_set.head = new_head; + new_head->shared.vm_set.head = node; + } else + node->shared.vm_set.head = NULL; + } + } +} + +/* + * Helper function to enumerate vmas that map a given file page or a set of + * contiguous file pages. The function returns vmas that at least map a single + * page in the given range of contiguous file pages. + */ +struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma, + struct prio_tree_iter *iter) +{ + struct prio_tree_node *ptr; + struct vm_area_struct *next; + + if (!vma) { + /* + * First call is with NULL vma + */ + ptr = prio_tree_next(iter); + if (ptr) { + next = prio_tree_entry(ptr, struct vm_area_struct, + shared.prio_tree_node); + prefetch(next->shared.vm_set.head); + return next; + } else + return NULL; + } + + if (vma->shared.vm_set.parent) { + if (vma->shared.vm_set.head) { + next = vma->shared.vm_set.head; + prefetch(next->shared.vm_set.list.next); + return next; + } + } else { + next = list_entry(vma->shared.vm_set.list.next, + struct vm_area_struct, shared.vm_set.list); + if (!next->shared.vm_set.head) { + prefetch(next->shared.vm_set.list.next); + return next; + } + } + + ptr = prio_tree_next(iter); + if (ptr) { + next = prio_tree_entry(ptr, struct vm_area_struct, + shared.prio_tree_node); + prefetch(next->shared.vm_set.head); + return next; + } else + return NULL; +} diff --git a/mm/readahead.c b/mm/readahead.c new file mode 100644 index 000000000000..b840e7c6ea74 --- /dev/null +++ b/mm/readahead.c @@ -0,0 +1,557 @@ +/* + * mm/readahead.c - address_space-level file readahead. + * + * Copyright (C) 2002, Linus Torvalds + * + * 09Apr2002 akpm@zip.com.au + * Initial version. + */ + +#include <linux/kernel.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/blkdev.h> +#include <linux/backing-dev.h> +#include <linux/pagevec.h> + +void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) +{ +} +EXPORT_SYMBOL(default_unplug_io_fn); + +struct backing_dev_info default_backing_dev_info = { + .ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE, + .state = 0, + .capabilities = BDI_CAP_MAP_COPY, + .unplug_io_fn = default_unplug_io_fn, +}; +EXPORT_SYMBOL_GPL(default_backing_dev_info); + +/* + * Initialise a struct file's readahead state. Assumes that the caller has + * memset *ra to zero. + */ +void +file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) +{ + ra->ra_pages = mapping->backing_dev_info->ra_pages; + ra->prev_page = -1; +} + +/* + * Return max readahead size for this inode in number-of-pages. + */ +static inline unsigned long get_max_readahead(struct file_ra_state *ra) +{ + return ra->ra_pages; +} + +static inline unsigned long get_min_readahead(struct file_ra_state *ra) +{ + return (VM_MIN_READAHEAD * 1024) / PAGE_CACHE_SIZE; +} + +static inline void ra_off(struct file_ra_state *ra) +{ + ra->start = 0; + ra->flags = 0; + ra->size = 0; + ra->ahead_start = 0; + ra->ahead_size = 0; + return; +} + +/* + * Set the initial window size, round to next power of 2 and square + * for small size, x 4 for medium, and x 2 for large + * for 128k (32 page) max ra + * 1-8 page = 32k initial, > 8 page = 128k initial + */ +static unsigned long get_init_ra_size(unsigned long size, unsigned long max) +{ + unsigned long newsize = roundup_pow_of_two(size); + + if (newsize <= max / 64) + newsize = newsize * newsize; + else if (newsize <= max / 4) + newsize = max / 4; + else + newsize = max; + return newsize; +} + +/* + * Set the new window size, this is called only when I/O is to be submitted, + * not for each call to readahead. If a cache miss occured, reduce next I/O + * size, else increase depending on how close to max we are. + */ +static inline unsigned long get_next_ra_size(struct file_ra_state *ra) +{ + unsigned long max = get_max_readahead(ra); + unsigned long min = get_min_readahead(ra); + unsigned long cur = ra->size; + unsigned long newsize; + + if (ra->flags & RA_FLAG_MISS) { + ra->flags &= ~RA_FLAG_MISS; + newsize = max((cur - 2), min); + } else if (cur < max / 16) { + newsize = 4 * cur; + } else { + newsize = 2 * cur; + } + return min(newsize, max); +} + +#define list_to_page(head) (list_entry((head)->prev, struct page, lru)) + +/** + * read_cache_pages - populate an address space with some pages, and + * start reads against them. + * @mapping: the address_space + * @pages: The address of a list_head which contains the target pages. These + * pages have their ->index populated and are otherwise uninitialised. + * @filler: callback routine for filling a single page. + * @data: private data for the callback routine. + * + * Hides the details of the LRU cache etc from the filesystems. + */ +int read_cache_pages(struct address_space *mapping, struct list_head *pages, + int (*filler)(void *, struct page *), void *data) +{ + struct page *page; + struct pagevec lru_pvec; + int ret = 0; + + pagevec_init(&lru_pvec, 0); + + while (!list_empty(pages)) { + page = list_to_page(pages); + list_del(&page->lru); + if (add_to_page_cache(page, mapping, page->index, GFP_KERNEL)) { + page_cache_release(page); + continue; + } + ret = filler(data, page); + if (!pagevec_add(&lru_pvec, page)) + __pagevec_lru_add(&lru_pvec); + if (ret) { + while (!list_empty(pages)) { + struct page *victim; + + victim = list_to_page(pages); + list_del(&victim->lru); + page_cache_release(victim); + } + break; + } + } + pagevec_lru_add(&lru_pvec); + return ret; +} + +EXPORT_SYMBOL(read_cache_pages); + +static int read_pages(struct address_space *mapping, struct file *filp, + struct list_head *pages, unsigned nr_pages) +{ + unsigned page_idx; + struct pagevec lru_pvec; + int ret = 0; + + if (mapping->a_ops->readpages) { + ret = mapping->a_ops->readpages(filp, mapping, pages, nr_pages); + goto out; + } + + pagevec_init(&lru_pvec, 0); + for (page_idx = 0; page_idx < nr_pages; page_idx++) { + struct page *page = list_to_page(pages); + list_del(&page->lru); + if (!add_to_page_cache(page, mapping, + page->index, GFP_KERNEL)) { + mapping->a_ops->readpage(filp, page); + if (!pagevec_add(&lru_pvec, page)) + __pagevec_lru_add(&lru_pvec); + } else { + page_cache_release(page); + } + } + pagevec_lru_add(&lru_pvec); +out: + return ret; +} + +/* + * Readahead design. + * + * The fields in struct file_ra_state represent the most-recently-executed + * readahead attempt: + * + * start: Page index at which we started the readahead + * size: Number of pages in that read + * Together, these form the "current window". + * Together, start and size represent the `readahead window'. + * prev_page: The page which the readahead algorithm most-recently inspected. + * It is mainly used to detect sequential file reading. + * If page_cache_readahead sees that it is again being called for + * a page which it just looked at, it can return immediately without + * making any state changes. + * ahead_start, + * ahead_size: Together, these form the "ahead window". + * ra_pages: The externally controlled max readahead for this fd. + * + * When readahead is in the off state (size == 0), readahead is disabled. + * In this state, prev_page is used to detect the resumption of sequential I/O. + * + * The readahead code manages two windows - the "current" and the "ahead" + * windows. The intent is that while the application is walking the pages + * in the current window, I/O is underway on the ahead window. When the + * current window is fully traversed, it is replaced by the ahead window + * and the ahead window is invalidated. When this copying happens, the + * new current window's pages are probably still locked. So + * we submit a new batch of I/O immediately, creating a new ahead window. + * + * So: + * + * ----|----------------|----------------|----- + * ^start ^start+size + * ^ahead_start ^ahead_start+ahead_size + * + * ^ When this page is read, we submit I/O for the + * ahead window. + * + * A `readahead hit' occurs when a read request is made against a page which is + * the next sequential page. Ahead window calculations are done only when it + * is time to submit a new IO. The code ramps up the size agressively at first, + * but slow down as it approaches max_readhead. + * + * Any seek/ramdom IO will result in readahead being turned off. It will resume + * at the first sequential access. + * + * There is a special-case: if the first page which the application tries to + * read happens to be the first page of the file, it is assumed that a linear + * read is about to happen and the window is immediately set to the initial size + * based on I/O request size and the max_readahead. + * + * This function is to be called for every read request, rather than when + * it is time to perform readahead. It is called only once for the entire I/O + * regardless of size unless readahead is unable to start enough I/O to satisfy + * the request (I/O request > max_readahead). + */ + +/* + * do_page_cache_readahead actually reads a chunk of disk. It allocates all + * the pages first, then submits them all for I/O. This avoids the very bad + * behaviour which would occur if page allocations are causing VM writeback. + * We really don't want to intermingle reads and writes like that. + * + * Returns the number of pages requested, or the maximum amount of I/O allowed. + * + * do_page_cache_readahead() returns -1 if it encountered request queue + * congestion. + */ +static int +__do_page_cache_readahead(struct address_space *mapping, struct file *filp, + unsigned long offset, unsigned long nr_to_read) +{ + struct inode *inode = mapping->host; + struct page *page; + unsigned long end_index; /* The last page we want to read */ + LIST_HEAD(page_pool); + int page_idx; + int ret = 0; + loff_t isize = i_size_read(inode); + + if (isize == 0) + goto out; + + end_index = ((isize - 1) >> PAGE_CACHE_SHIFT); + + /* + * Preallocate as many pages as we will need. + */ + read_lock_irq(&mapping->tree_lock); + for (page_idx = 0; page_idx < nr_to_read; page_idx++) { + unsigned long page_offset = offset + page_idx; + + if (page_offset > end_index) + break; + + page = radix_tree_lookup(&mapping->page_tree, page_offset); + if (page) + continue; + + read_unlock_irq(&mapping->tree_lock); + page = page_cache_alloc_cold(mapping); + read_lock_irq(&mapping->tree_lock); + if (!page) + break; + page->index = page_offset; + list_add(&page->lru, &page_pool); + ret++; + } + read_unlock_irq(&mapping->tree_lock); + + /* + * Now start the IO. We ignore I/O errors - if the page is not + * uptodate then the caller will launch readpage again, and + * will then handle the error. + */ + if (ret) + read_pages(mapping, filp, &page_pool, ret); + BUG_ON(!list_empty(&page_pool)); +out: + return ret; +} + +/* + * Chunk the readahead into 2 megabyte units, so that we don't pin too much + * memory at once. + */ +int force_page_cache_readahead(struct address_space *mapping, struct file *filp, + unsigned long offset, unsigned long nr_to_read) +{ + int ret = 0; + + if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages)) + return -EINVAL; + + while (nr_to_read) { + int err; + + unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_CACHE_SIZE; + + if (this_chunk > nr_to_read) + this_chunk = nr_to_read; + err = __do_page_cache_readahead(mapping, filp, + offset, this_chunk); + if (err < 0) { + ret = err; + break; + } + ret += err; + offset += this_chunk; + nr_to_read -= this_chunk; + } + return ret; +} + +/* + * Check how effective readahead is being. If the amount of started IO is + * less than expected then the file is partly or fully in pagecache and + * readahead isn't helping. + * + */ +static inline int check_ra_success(struct file_ra_state *ra, + unsigned long nr_to_read, unsigned long actual) +{ + if (actual == 0) { + ra->cache_hit += nr_to_read; + if (ra->cache_hit >= VM_MAX_CACHE_HIT) { + ra_off(ra); + ra->flags |= RA_FLAG_INCACHE; + return 0; + } + } else { + ra->cache_hit=0; + } + return 1; +} + +/* + * This version skips the IO if the queue is read-congested, and will tell the + * block layer to abandon the readahead if request allocation would block. + * + * force_page_cache_readahead() will ignore queue congestion and will block on + * request queues. + */ +int do_page_cache_readahead(struct address_space *mapping, struct file *filp, + unsigned long offset, unsigned long nr_to_read) +{ + if (bdi_read_congested(mapping->backing_dev_info)) + return -1; + + return __do_page_cache_readahead(mapping, filp, offset, nr_to_read); +} + +/* + * Read 'nr_to_read' pages starting at page 'offset'. If the flag 'block' + * is set wait till the read completes. Otherwise attempt to read without + * blocking. + * Returns 1 meaning 'success' if read is succesfull without switching off + * readhaead mode. Otherwise return failure. + */ +static int +blockable_page_cache_readahead(struct address_space *mapping, struct file *filp, + unsigned long offset, unsigned long nr_to_read, + struct file_ra_state *ra, int block) +{ + int actual; + + if (!block && bdi_read_congested(mapping->backing_dev_info)) + return 0; + + actual = __do_page_cache_readahead(mapping, filp, offset, nr_to_read); + + return check_ra_success(ra, nr_to_read, actual); +} + +static int make_ahead_window(struct address_space *mapping, struct file *filp, + struct file_ra_state *ra, int force) +{ + int block, ret; + + ra->ahead_size = get_next_ra_size(ra); + ra->ahead_start = ra->start + ra->size; + + block = force || (ra->prev_page >= ra->ahead_start); + ret = blockable_page_cache_readahead(mapping, filp, + ra->ahead_start, ra->ahead_size, ra, block); + + if (!ret && !force) { + /* A read failure in blocking mode, implies pages are + * all cached. So we can safely assume we have taken + * care of all the pages requested in this call. + * A read failure in non-blocking mode, implies we are + * reading more pages than requested in this call. So + * we safely assume we have taken care of all the pages + * requested in this call. + * + * Just reset the ahead window in case we failed due to + * congestion. The ahead window will any way be closed + * in case we failed due to excessive page cache hits. + */ + ra->ahead_start = 0; + ra->ahead_size = 0; + } + + return ret; +} + +/* + * page_cache_readahead is the main function. If performs the adaptive + * readahead window size management and submits the readahead I/O. + */ +unsigned long +page_cache_readahead(struct address_space *mapping, struct file_ra_state *ra, + struct file *filp, unsigned long offset, + unsigned long req_size) +{ + unsigned long max, newsize; + int sequential; + + /* + * We avoid doing extra work and bogusly perturbing the readahead + * window expansion logic. + */ + if (offset == ra->prev_page && --req_size) + ++offset; + + /* Note that prev_page == -1 if it is a first read */ + sequential = (offset == ra->prev_page + 1); + ra->prev_page = offset; + + max = get_max_readahead(ra); + newsize = min(req_size, max); + + /* No readahead or sub-page sized read or file already in cache */ + if (newsize == 0 || (ra->flags & RA_FLAG_INCACHE)) + goto out; + + ra->prev_page += newsize - 1; + + /* + * Special case - first read at start of file. We'll assume it's + * a whole-file read and grow the window fast. Or detect first + * sequential access + */ + if (sequential && ra->size == 0) { + ra->size = get_init_ra_size(newsize, max); + ra->start = offset; + if (!blockable_page_cache_readahead(mapping, filp, offset, + ra->size, ra, 1)) + goto out; + + /* + * If the request size is larger than our max readahead, we + * at least want to be sure that we get 2 IOs in flight and + * we know that we will definitly need the new I/O. + * once we do this, subsequent calls should be able to overlap + * IOs,* thus preventing stalls. so issue the ahead window + * immediately. + */ + if (req_size >= max) + make_ahead_window(mapping, filp, ra, 1); + + goto out; + } + + /* + * Now handle the random case: + * partial page reads and first access were handled above, + * so this must be the next page otherwise it is random + */ + if (!sequential) { + ra_off(ra); + blockable_page_cache_readahead(mapping, filp, offset, + newsize, ra, 1); + goto out; + } + + /* + * If we get here we are doing sequential IO and this was not the first + * occurence (ie we have an existing window) + */ + + if (ra->ahead_start == 0) { /* no ahead window yet */ + if (!make_ahead_window(mapping, filp, ra, 0)) + goto out; + } + /* + * Already have an ahead window, check if we crossed into it. + * If so, shift windows and issue a new ahead window. + * Only return the #pages that are in the current window, so that + * we get called back on the first page of the ahead window which + * will allow us to submit more IO. + */ + if (ra->prev_page >= ra->ahead_start) { + ra->start = ra->ahead_start; + ra->size = ra->ahead_size; + make_ahead_window(mapping, filp, ra, 0); + } + +out: + return ra->prev_page + 1; +} + +/* + * handle_ra_miss() is called when it is known that a page which should have + * been present in the pagecache (we just did some readahead there) was in fact + * not found. This will happen if it was evicted by the VM (readahead + * thrashing) + * + * Turn on the cache miss flag in the RA struct, this will cause the RA code + * to reduce the RA size on the next read. + */ +void handle_ra_miss(struct address_space *mapping, + struct file_ra_state *ra, pgoff_t offset) +{ + ra->flags |= RA_FLAG_MISS; + ra->flags &= ~RA_FLAG_INCACHE; +} + +/* + * Given a desired number of PAGE_CACHE_SIZE readahead pages, return a + * sensible upper limit. + */ +unsigned long max_sane_readahead(unsigned long nr) +{ + unsigned long active; + unsigned long inactive; + unsigned long free; + + __get_zone_counts(&active, &inactive, &free, NODE_DATA(numa_node_id())); + return min(nr, (inactive + free) / 2); +} diff --git a/mm/rmap.c b/mm/rmap.c new file mode 100644 index 000000000000..884d6d1928bc --- /dev/null +++ b/mm/rmap.c @@ -0,0 +1,862 @@ +/* + * mm/rmap.c - physical to virtual reverse mappings + * + * Copyright 2001, Rik van Riel <riel@conectiva.com.br> + * Released under the General Public License (GPL). + * + * Simple, low overhead reverse mapping scheme. + * Please try to keep this thing as modular as possible. + * + * Provides methods for unmapping each kind of mapped page: + * the anon methods track anonymous pages, and + * the file methods track pages belonging to an inode. + * + * Original design by Rik van Riel <riel@conectiva.com.br> 2001 + * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 + * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 + * Contributions by Hugh Dickins <hugh@veritas.com> 2003, 2004 + */ + +/* + * Lock ordering in mm: + * + * inode->i_sem (while writing or truncating, not reading or faulting) + * inode->i_alloc_sem + * + * When a page fault occurs in writing from user to file, down_read + * of mmap_sem nests within i_sem; in sys_msync, i_sem nests within + * down_read of mmap_sem; i_sem and down_write of mmap_sem are never + * taken together; in truncation, i_sem is taken outermost. + * + * mm->mmap_sem + * page->flags PG_locked (lock_page) + * mapping->i_mmap_lock + * anon_vma->lock + * mm->page_table_lock + * zone->lru_lock (in mark_page_accessed) + * swap_list_lock (in swap_free etc's swap_info_get) + * mmlist_lock (in mmput, drain_mmlist and others) + * swap_device_lock (in swap_duplicate, swap_info_get) + * mapping->private_lock (in __set_page_dirty_buffers) + * inode_lock (in set_page_dirty's __mark_inode_dirty) + * sb_lock (within inode_lock in fs/fs-writeback.c) + * mapping->tree_lock (widely used, in set_page_dirty, + * in arch-dependent flush_dcache_mmap_lock, + * within inode_lock in __sync_single_inode) + */ + +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/swap.h> +#include <linux/swapops.h> +#include <linux/slab.h> +#include <linux/init.h> +#include <linux/rmap.h> +#include <linux/rcupdate.h> + +#include <asm/tlbflush.h> + +//#define RMAP_DEBUG /* can be enabled only for debugging */ + +kmem_cache_t *anon_vma_cachep; + +static inline void validate_anon_vma(struct vm_area_struct *find_vma) +{ +#ifdef RMAP_DEBUG + struct anon_vma *anon_vma = find_vma->anon_vma; + struct vm_area_struct *vma; + unsigned int mapcount = 0; + int found = 0; + + list_for_each_entry(vma, &anon_vma->head, anon_vma_node) { + mapcount++; + BUG_ON(mapcount > 100000); + if (vma == find_vma) + found = 1; + } + BUG_ON(!found); +#endif +} + +/* This must be called under the mmap_sem. */ +int anon_vma_prepare(struct vm_area_struct *vma) +{ + struct anon_vma *anon_vma = vma->anon_vma; + + might_sleep(); + if (unlikely(!anon_vma)) { + struct mm_struct *mm = vma->vm_mm; + struct anon_vma *allocated, *locked; + + anon_vma = find_mergeable_anon_vma(vma); + if (anon_vma) { + allocated = NULL; + locked = anon_vma; + spin_lock(&locked->lock); + } else { + anon_vma = anon_vma_alloc(); + if (unlikely(!anon_vma)) + return -ENOMEM; + allocated = anon_vma; + locked = NULL; + } + + /* page_table_lock to protect against threads */ + spin_lock(&mm->page_table_lock); + if (likely(!vma->anon_vma)) { + vma->anon_vma = anon_vma; + list_add(&vma->anon_vma_node, &anon_vma->head); + allocated = NULL; + } + spin_unlock(&mm->page_table_lock); + + if (locked) + spin_unlock(&locked->lock); + if (unlikely(allocated)) + anon_vma_free(allocated); + } + return 0; +} + +void __anon_vma_merge(struct vm_area_struct *vma, struct vm_area_struct *next) +{ + BUG_ON(vma->anon_vma != next->anon_vma); + list_del(&next->anon_vma_node); +} + +void __anon_vma_link(struct vm_area_struct *vma) +{ + struct anon_vma *anon_vma = vma->anon_vma; + + if (anon_vma) { + list_add(&vma->anon_vma_node, &anon_vma->head); + validate_anon_vma(vma); + } +} + +void anon_vma_link(struct vm_area_struct *vma) +{ + struct anon_vma *anon_vma = vma->anon_vma; + + if (anon_vma) { + spin_lock(&anon_vma->lock); + list_add(&vma->anon_vma_node, &anon_vma->head); + validate_anon_vma(vma); + spin_unlock(&anon_vma->lock); + } +} + +void anon_vma_unlink(struct vm_area_struct *vma) +{ + struct anon_vma *anon_vma = vma->anon_vma; + int empty; + + if (!anon_vma) + return; + + spin_lock(&anon_vma->lock); + validate_anon_vma(vma); + list_del(&vma->anon_vma_node); + + /* We must garbage collect the anon_vma if it's empty */ + empty = list_empty(&anon_vma->head); + spin_unlock(&anon_vma->lock); + + if (empty) + anon_vma_free(anon_vma); +} + +static void anon_vma_ctor(void *data, kmem_cache_t *cachep, unsigned long flags) +{ + if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == + SLAB_CTOR_CONSTRUCTOR) { + struct anon_vma *anon_vma = data; + + spin_lock_init(&anon_vma->lock); + INIT_LIST_HEAD(&anon_vma->head); + } +} + +void __init anon_vma_init(void) +{ + anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), + 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor, NULL); +} + +/* + * Getting a lock on a stable anon_vma from a page off the LRU is + * tricky: page_lock_anon_vma rely on RCU to guard against the races. + */ +static struct anon_vma *page_lock_anon_vma(struct page *page) +{ + struct anon_vma *anon_vma = NULL; + unsigned long anon_mapping; + + rcu_read_lock(); + anon_mapping = (unsigned long) page->mapping; + if (!(anon_mapping & PAGE_MAPPING_ANON)) + goto out; + if (!page_mapped(page)) + goto out; + + anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); + spin_lock(&anon_vma->lock); +out: + rcu_read_unlock(); + return anon_vma; +} + +/* + * At what user virtual address is page expected in vma? + */ +static inline unsigned long +vma_address(struct page *page, struct vm_area_struct *vma) +{ + pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); + unsigned long address; + + address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (unlikely(address < vma->vm_start || address >= vma->vm_end)) { + /* page should be within any vma from prio_tree_next */ + BUG_ON(!PageAnon(page)); + return -EFAULT; + } + return address; +} + +/* + * At what user virtual address is page expected in vma? checking that the + * page matches the vma: currently only used by unuse_process, on anon pages. + */ +unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) +{ + if (PageAnon(page)) { + if ((void *)vma->anon_vma != + (void *)page->mapping - PAGE_MAPPING_ANON) + return -EFAULT; + } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) { + if (vma->vm_file->f_mapping != page->mapping) + return -EFAULT; + } else + return -EFAULT; + return vma_address(page, vma); +} + +/* + * Subfunctions of page_referenced: page_referenced_one called + * repeatedly from either page_referenced_anon or page_referenced_file. + */ +static int page_referenced_one(struct page *page, + struct vm_area_struct *vma, unsigned int *mapcount, int ignore_token) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + int referenced = 0; + + if (!get_mm_counter(mm, rss)) + goto out; + address = vma_address(page, vma); + if (address == -EFAULT) + goto out; + + spin_lock(&mm->page_table_lock); + + pgd = pgd_offset(mm, address); + if (!pgd_present(*pgd)) + goto out_unlock; + + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); + if (!pmd_present(*pmd)) + goto out_unlock; + + pte = pte_offset_map(pmd, address); + if (!pte_present(*pte)) + goto out_unmap; + + if (page_to_pfn(page) != pte_pfn(*pte)) + goto out_unmap; + + if (ptep_clear_flush_young(vma, address, pte)) + referenced++; + + if (mm != current->mm && !ignore_token && has_swap_token(mm)) + referenced++; + + (*mapcount)--; + +out_unmap: + pte_unmap(pte); +out_unlock: + spin_unlock(&mm->page_table_lock); +out: + return referenced; +} + +static int page_referenced_anon(struct page *page, int ignore_token) +{ + unsigned int mapcount; + struct anon_vma *anon_vma; + struct vm_area_struct *vma; + int referenced = 0; + + anon_vma = page_lock_anon_vma(page); + if (!anon_vma) + return referenced; + + mapcount = page_mapcount(page); + list_for_each_entry(vma, &anon_vma->head, anon_vma_node) { + referenced += page_referenced_one(page, vma, &mapcount, + ignore_token); + if (!mapcount) + break; + } + spin_unlock(&anon_vma->lock); + return referenced; +} + +/** + * page_referenced_file - referenced check for object-based rmap + * @page: the page we're checking references on. + * + * For an object-based mapped page, find all the places it is mapped and + * check/clear the referenced flag. This is done by following the page->mapping + * pointer, then walking the chain of vmas it holds. It returns the number + * of references it found. + * + * This function is only called from page_referenced for object-based pages. + */ +static int page_referenced_file(struct page *page, int ignore_token) +{ + unsigned int mapcount; + struct address_space *mapping = page->mapping; + pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); + struct vm_area_struct *vma; + struct prio_tree_iter iter; + int referenced = 0; + + /* + * The caller's checks on page->mapping and !PageAnon have made + * sure that this is a file page: the check for page->mapping + * excludes the case just before it gets set on an anon page. + */ + BUG_ON(PageAnon(page)); + + /* + * The page lock not only makes sure that page->mapping cannot + * suddenly be NULLified by truncation, it makes sure that the + * structure at mapping cannot be freed and reused yet, + * so we can safely take mapping->i_mmap_lock. + */ + BUG_ON(!PageLocked(page)); + + spin_lock(&mapping->i_mmap_lock); + + /* + * i_mmap_lock does not stabilize mapcount at all, but mapcount + * is more likely to be accurate if we note it after spinning. + */ + mapcount = page_mapcount(page); + + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { + if ((vma->vm_flags & (VM_LOCKED|VM_MAYSHARE)) + == (VM_LOCKED|VM_MAYSHARE)) { + referenced++; + break; + } + referenced += page_referenced_one(page, vma, &mapcount, + ignore_token); + if (!mapcount) + break; + } + + spin_unlock(&mapping->i_mmap_lock); + return referenced; +} + +/** + * page_referenced - test if the page was referenced + * @page: the page to test + * @is_locked: caller holds lock on the page + * + * Quick test_and_clear_referenced for all mappings to a page, + * returns the number of ptes which referenced the page. + */ +int page_referenced(struct page *page, int is_locked, int ignore_token) +{ + int referenced = 0; + + if (!swap_token_default_timeout) + ignore_token = 1; + + if (page_test_and_clear_young(page)) + referenced++; + + if (TestClearPageReferenced(page)) + referenced++; + + if (page_mapped(page) && page->mapping) { + if (PageAnon(page)) + referenced += page_referenced_anon(page, ignore_token); + else if (is_locked) + referenced += page_referenced_file(page, ignore_token); + else if (TestSetPageLocked(page)) + referenced++; + else { + if (page->mapping) + referenced += page_referenced_file(page, + ignore_token); + unlock_page(page); + } + } + return referenced; +} + +/** + * page_add_anon_rmap - add pte mapping to an anonymous page + * @page: the page to add the mapping to + * @vma: the vm area in which the mapping is added + * @address: the user virtual address mapped + * + * The caller needs to hold the mm->page_table_lock. + */ +void page_add_anon_rmap(struct page *page, + struct vm_area_struct *vma, unsigned long address) +{ + struct anon_vma *anon_vma = vma->anon_vma; + pgoff_t index; + + BUG_ON(PageReserved(page)); + BUG_ON(!anon_vma); + + inc_mm_counter(vma->vm_mm, anon_rss); + + anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; + index = (address - vma->vm_start) >> PAGE_SHIFT; + index += vma->vm_pgoff; + index >>= PAGE_CACHE_SHIFT - PAGE_SHIFT; + + if (atomic_inc_and_test(&page->_mapcount)) { + page->index = index; + page->mapping = (struct address_space *) anon_vma; + inc_page_state(nr_mapped); + } + /* else checking page index and mapping is racy */ +} + +/** + * page_add_file_rmap - add pte mapping to a file page + * @page: the page to add the mapping to + * + * The caller needs to hold the mm->page_table_lock. + */ +void page_add_file_rmap(struct page *page) +{ + BUG_ON(PageAnon(page)); + if (!pfn_valid(page_to_pfn(page)) || PageReserved(page)) + return; + + if (atomic_inc_and_test(&page->_mapcount)) + inc_page_state(nr_mapped); +} + +/** + * page_remove_rmap - take down pte mapping from a page + * @page: page to remove mapping from + * + * Caller needs to hold the mm->page_table_lock. + */ +void page_remove_rmap(struct page *page) +{ + BUG_ON(PageReserved(page)); + + if (atomic_add_negative(-1, &page->_mapcount)) { + BUG_ON(page_mapcount(page) < 0); + /* + * It would be tidy to reset the PageAnon mapping here, + * but that might overwrite a racing page_add_anon_rmap + * which increments mapcount after us but sets mapping + * before us: so leave the reset to free_hot_cold_page, + * and remember that it's only reliable while mapped. + * Leaving it set also helps swapoff to reinstate ptes + * faster for those pages still in swapcache. + */ + if (page_test_and_clear_dirty(page)) + set_page_dirty(page); + dec_page_state(nr_mapped); + } +} + +/* + * Subfunctions of try_to_unmap: try_to_unmap_one called + * repeatedly from either try_to_unmap_anon or try_to_unmap_file. + */ +static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t pteval; + int ret = SWAP_AGAIN; + + if (!get_mm_counter(mm, rss)) + goto out; + address = vma_address(page, vma); + if (address == -EFAULT) + goto out; + + /* + * We need the page_table_lock to protect us from page faults, + * munmap, fork, etc... + */ + spin_lock(&mm->page_table_lock); + + pgd = pgd_offset(mm, address); + if (!pgd_present(*pgd)) + goto out_unlock; + + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); + if (!pmd_present(*pmd)) + goto out_unlock; + + pte = pte_offset_map(pmd, address); + if (!pte_present(*pte)) + goto out_unmap; + + if (page_to_pfn(page) != pte_pfn(*pte)) + goto out_unmap; + + /* + * If the page is mlock()d, we cannot swap it out. + * If it's recently referenced (perhaps page_referenced + * skipped over this mm) then we should reactivate it. + */ + if ((vma->vm_flags & (VM_LOCKED|VM_RESERVED)) || + ptep_clear_flush_young(vma, address, pte)) { + ret = SWAP_FAIL; + goto out_unmap; + } + + /* + * Don't pull an anonymous page out from under get_user_pages. + * GUP carefully breaks COW and raises page count (while holding + * page_table_lock, as we have here) to make sure that the page + * cannot be freed. If we unmap that page here, a user write + * access to the virtual address will bring back the page, but + * its raised count will (ironically) be taken to mean it's not + * an exclusive swap page, do_wp_page will replace it by a copy + * page, and the user never get to see the data GUP was holding + * the original page for. + * + * This test is also useful for when swapoff (unuse_process) has + * to drop page lock: its reference to the page stops existing + * ptes from being unmapped, so swapoff can make progress. + */ + if (PageSwapCache(page) && + page_count(page) != page_mapcount(page) + 2) { + ret = SWAP_FAIL; + goto out_unmap; + } + + /* Nuke the page table entry. */ + flush_cache_page(vma, address, page_to_pfn(page)); + pteval = ptep_clear_flush(vma, address, pte); + + /* Move the dirty bit to the physical page now the pte is gone. */ + if (pte_dirty(pteval)) + set_page_dirty(page); + + if (PageAnon(page)) { + swp_entry_t entry = { .val = page->private }; + /* + * Store the swap location in the pte. + * See handle_pte_fault() ... + */ + BUG_ON(!PageSwapCache(page)); + swap_duplicate(entry); + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); + BUG_ON(pte_file(*pte)); + dec_mm_counter(mm, anon_rss); + } + + inc_mm_counter(mm, rss); + page_remove_rmap(page); + page_cache_release(page); + +out_unmap: + pte_unmap(pte); +out_unlock: + spin_unlock(&mm->page_table_lock); +out: + return ret; +} + +/* + * objrmap doesn't work for nonlinear VMAs because the assumption that + * offset-into-file correlates with offset-into-virtual-addresses does not hold. + * Consequently, given a particular page and its ->index, we cannot locate the + * ptes which are mapping that page without an exhaustive linear search. + * + * So what this code does is a mini "virtual scan" of each nonlinear VMA which + * maps the file to which the target page belongs. The ->vm_private_data field + * holds the current cursor into that scan. Successive searches will circulate + * around the vma's virtual address space. + * + * So as more replacement pressure is applied to the pages in a nonlinear VMA, + * more scanning pressure is placed against them as well. Eventually pages + * will become fully unmapped and are eligible for eviction. + * + * For very sparsely populated VMAs this is a little inefficient - chances are + * there there won't be many ptes located within the scan cluster. In this case + * maybe we could scan further - to the end of the pte page, perhaps. + */ +#define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE) +#define CLUSTER_MASK (~(CLUSTER_SIZE - 1)) + +static void try_to_unmap_cluster(unsigned long cursor, + unsigned int *mapcount, struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t pteval; + struct page *page; + unsigned long address; + unsigned long end; + unsigned long pfn; + + /* + * We need the page_table_lock to protect us from page faults, + * munmap, fork, etc... + */ + spin_lock(&mm->page_table_lock); + + address = (vma->vm_start + cursor) & CLUSTER_MASK; + end = address + CLUSTER_SIZE; + if (address < vma->vm_start) + address = vma->vm_start; + if (end > vma->vm_end) + end = vma->vm_end; + + pgd = pgd_offset(mm, address); + if (!pgd_present(*pgd)) + goto out_unlock; + + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); + if (!pmd_present(*pmd)) + goto out_unlock; + + for (pte = pte_offset_map(pmd, address); + address < end; pte++, address += PAGE_SIZE) { + + if (!pte_present(*pte)) + continue; + + pfn = pte_pfn(*pte); + if (!pfn_valid(pfn)) + continue; + + page = pfn_to_page(pfn); + BUG_ON(PageAnon(page)); + if (PageReserved(page)) + continue; + + if (ptep_clear_flush_young(vma, address, pte)) + continue; + + /* Nuke the page table entry. */ + flush_cache_page(vma, address, pfn); + pteval = ptep_clear_flush(vma, address, pte); + + /* If nonlinear, store the file page offset in the pte. */ + if (page->index != linear_page_index(vma, address)) + set_pte_at(mm, address, pte, pgoff_to_pte(page->index)); + + /* Move the dirty bit to the physical page now the pte is gone. */ + if (pte_dirty(pteval)) + set_page_dirty(page); + + page_remove_rmap(page); + page_cache_release(page); + dec_mm_counter(mm, rss); + (*mapcount)--; + } + + pte_unmap(pte); + +out_unlock: + spin_unlock(&mm->page_table_lock); +} + +static int try_to_unmap_anon(struct page *page) +{ + struct anon_vma *anon_vma; + struct vm_area_struct *vma; + int ret = SWAP_AGAIN; + + anon_vma = page_lock_anon_vma(page); + if (!anon_vma) + return ret; + + list_for_each_entry(vma, &anon_vma->head, anon_vma_node) { + ret = try_to_unmap_one(page, vma); + if (ret == SWAP_FAIL || !page_mapped(page)) + break; + } + spin_unlock(&anon_vma->lock); + return ret; +} + +/** + * try_to_unmap_file - unmap file page using the object-based rmap method + * @page: the page to unmap + * + * Find all the mappings of a page using the mapping pointer and the vma chains + * contained in the address_space struct it points to. + * + * This function is only called from try_to_unmap for object-based pages. + */ +static int try_to_unmap_file(struct page *page) +{ + struct address_space *mapping = page->mapping; + pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); + struct vm_area_struct *vma; + struct prio_tree_iter iter; + int ret = SWAP_AGAIN; + unsigned long cursor; + unsigned long max_nl_cursor = 0; + unsigned long max_nl_size = 0; + unsigned int mapcount; + + spin_lock(&mapping->i_mmap_lock); + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) { + ret = try_to_unmap_one(page, vma); + if (ret == SWAP_FAIL || !page_mapped(page)) + goto out; + } + + if (list_empty(&mapping->i_mmap_nonlinear)) + goto out; + + list_for_each_entry(vma, &mapping->i_mmap_nonlinear, + shared.vm_set.list) { + if (vma->vm_flags & (VM_LOCKED|VM_RESERVED)) + continue; + cursor = (unsigned long) vma->vm_private_data; + if (cursor > max_nl_cursor) + max_nl_cursor = cursor; + cursor = vma->vm_end - vma->vm_start; + if (cursor > max_nl_size) + max_nl_size = cursor; + } + + if (max_nl_size == 0) { /* any nonlinears locked or reserved */ + ret = SWAP_FAIL; + goto out; + } + + /* + * We don't try to search for this page in the nonlinear vmas, + * and page_referenced wouldn't have found it anyway. Instead + * just walk the nonlinear vmas trying to age and unmap some. + * The mapcount of the page we came in with is irrelevant, + * but even so use it as a guide to how hard we should try? + */ + mapcount = page_mapcount(page); + if (!mapcount) + goto out; + cond_resched_lock(&mapping->i_mmap_lock); + + max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK; + if (max_nl_cursor == 0) + max_nl_cursor = CLUSTER_SIZE; + + do { + list_for_each_entry(vma, &mapping->i_mmap_nonlinear, + shared.vm_set.list) { + if (vma->vm_flags & (VM_LOCKED|VM_RESERVED)) + continue; + cursor = (unsigned long) vma->vm_private_data; + while (get_mm_counter(vma->vm_mm, rss) && + cursor < max_nl_cursor && + cursor < vma->vm_end - vma->vm_start) { + try_to_unmap_cluster(cursor, &mapcount, vma); + cursor += CLUSTER_SIZE; + vma->vm_private_data = (void *) cursor; + if ((int)mapcount <= 0) + goto out; + } + vma->vm_private_data = (void *) max_nl_cursor; + } + cond_resched_lock(&mapping->i_mmap_lock); + max_nl_cursor += CLUSTER_SIZE; + } while (max_nl_cursor <= max_nl_size); + + /* + * Don't loop forever (perhaps all the remaining pages are + * in locked vmas). Reset cursor on all unreserved nonlinear + * vmas, now forgetting on which ones it had fallen behind. + */ + list_for_each_entry(vma, &mapping->i_mmap_nonlinear, + shared.vm_set.list) { + if (!(vma->vm_flags & VM_RESERVED)) + vma->vm_private_data = NULL; + } +out: + spin_unlock(&mapping->i_mmap_lock); + return ret; +} + +/** + * try_to_unmap - try to remove all page table mappings to a page + * @page: the page to get unmapped + * + * Tries to remove all the page table entries which are mapping this + * page, used in the pageout path. Caller must hold the page lock. + * Return values are: + * + * SWAP_SUCCESS - we succeeded in removing all mappings + * SWAP_AGAIN - we missed a mapping, try again later + * SWAP_FAIL - the page is unswappable + */ +int try_to_unmap(struct page *page) +{ + int ret; + + BUG_ON(PageReserved(page)); + BUG_ON(!PageLocked(page)); + + if (PageAnon(page)) + ret = try_to_unmap_anon(page); + else + ret = try_to_unmap_file(page); + + if (!page_mapped(page)) + ret = SWAP_SUCCESS; + return ret; +} diff --git a/mm/shmem.c b/mm/shmem.c new file mode 100644 index 000000000000..61574b81d979 --- /dev/null +++ b/mm/shmem.c @@ -0,0 +1,2326 @@ +/* + * Resizable virtual memory filesystem for Linux. + * + * Copyright (C) 2000 Linus Torvalds. + * 2000 Transmeta Corp. + * 2000-2001 Christoph Rohland + * 2000-2001 SAP AG + * 2002 Red Hat Inc. + * Copyright (C) 2002-2004 Hugh Dickins. + * Copyright (C) 2002-2004 VERITAS Software Corporation. + * Copyright (C) 2004 Andi Kleen, SuSE Labs + * + * Extended attribute support for tmpfs: + * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> + * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> + * + * This file is released under the GPL. + */ + +/* + * This virtual memory filesystem is heavily based on the ramfs. It + * extends ramfs by the ability to use swap and honor resource limits + * which makes it a completely usable filesystem. + */ + +#include <linux/config.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/devfs_fs_kernel.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/file.h> +#include <linux/swap.h> +#include <linux/pagemap.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/backing-dev.h> +#include <linux/shmem_fs.h> +#include <linux/mount.h> +#include <linux/writeback.h> +#include <linux/vfs.h> +#include <linux/blkdev.h> +#include <linux/security.h> +#include <linux/swapops.h> +#include <linux/mempolicy.h> +#include <linux/namei.h> +#include <linux/xattr.h> +#include <asm/uaccess.h> +#include <asm/div64.h> +#include <asm/pgtable.h> + +/* This magic number is used in glibc for posix shared memory */ +#define TMPFS_MAGIC 0x01021994 + +#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long)) +#define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE) +#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512) + +#define SHMEM_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1)) +#define SHMEM_MAX_BYTES ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT) + +#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT) + +/* info->flags needs VM_flags to handle pagein/truncate races efficiently */ +#define SHMEM_PAGEIN VM_READ +#define SHMEM_TRUNCATE VM_WRITE + +/* Definition to limit shmem_truncate's steps between cond_rescheds */ +#define LATENCY_LIMIT 64 + +/* Pretend that each entry is of this size in directory's i_size */ +#define BOGO_DIRENT_SIZE 20 + +/* Keep swapped page count in private field of indirect struct page */ +#define nr_swapped private + +/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */ +enum sgp_type { + SGP_QUICK, /* don't try more than file page cache lookup */ + SGP_READ, /* don't exceed i_size, don't allocate page */ + SGP_CACHE, /* don't exceed i_size, may allocate page */ + SGP_WRITE, /* may exceed i_size, may allocate page */ +}; + +static int shmem_getpage(struct inode *inode, unsigned long idx, + struct page **pagep, enum sgp_type sgp, int *type); + +static inline struct page *shmem_dir_alloc(unsigned int gfp_mask) +{ + /* + * The above definition of ENTRIES_PER_PAGE, and the use of + * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE: + * might be reconsidered if it ever diverges from PAGE_SIZE. + */ + return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT); +} + +static inline void shmem_dir_free(struct page *page) +{ + __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT); +} + +static struct page **shmem_dir_map(struct page *page) +{ + return (struct page **)kmap_atomic(page, KM_USER0); +} + +static inline void shmem_dir_unmap(struct page **dir) +{ + kunmap_atomic(dir, KM_USER0); +} + +static swp_entry_t *shmem_swp_map(struct page *page) +{ + return (swp_entry_t *)kmap_atomic(page, KM_USER1); +} + +static inline void shmem_swp_balance_unmap(void) +{ + /* + * When passing a pointer to an i_direct entry, to code which + * also handles indirect entries and so will shmem_swp_unmap, + * we must arrange for the preempt count to remain in balance. + * What kmap_atomic of a lowmem page does depends on config + * and architecture, so pretend to kmap_atomic some lowmem page. + */ + (void) kmap_atomic(ZERO_PAGE(0), KM_USER1); +} + +static inline void shmem_swp_unmap(swp_entry_t *entry) +{ + kunmap_atomic(entry, KM_USER1); +} + +static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* + * shmem_file_setup pre-accounts the whole fixed size of a VM object, + * for shared memory and for shared anonymous (/dev/zero) mappings + * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), + * consistent with the pre-accounting of private mappings ... + */ +static inline int shmem_acct_size(unsigned long flags, loff_t size) +{ + return (flags & VM_ACCOUNT)? + security_vm_enough_memory(VM_ACCT(size)): 0; +} + +static inline void shmem_unacct_size(unsigned long flags, loff_t size) +{ + if (flags & VM_ACCOUNT) + vm_unacct_memory(VM_ACCT(size)); +} + +/* + * ... whereas tmpfs objects are accounted incrementally as + * pages are allocated, in order to allow huge sparse files. + * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, + * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. + */ +static inline int shmem_acct_block(unsigned long flags) +{ + return (flags & VM_ACCOUNT)? + 0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE)); +} + +static inline void shmem_unacct_blocks(unsigned long flags, long pages) +{ + if (!(flags & VM_ACCOUNT)) + vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE)); +} + +static struct super_operations shmem_ops; +static struct address_space_operations shmem_aops; +static struct file_operations shmem_file_operations; +static struct inode_operations shmem_inode_operations; +static struct inode_operations shmem_dir_inode_operations; +static struct inode_operations shmem_special_inode_operations; +static struct vm_operations_struct shmem_vm_ops; + +static struct backing_dev_info shmem_backing_dev_info = { + .ra_pages = 0, /* No readahead */ + .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK, + .unplug_io_fn = default_unplug_io_fn, +}; + +static LIST_HEAD(shmem_swaplist); +static DEFINE_SPINLOCK(shmem_swaplist_lock); + +static void shmem_free_blocks(struct inode *inode, long pages) +{ + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + sbinfo->free_blocks += pages; + inode->i_blocks -= pages*BLOCKS_PER_PAGE; + spin_unlock(&sbinfo->stat_lock); + } +} + +/* + * shmem_recalc_inode - recalculate the size of an inode + * + * @inode: inode to recalc + * + * We have to calculate the free blocks since the mm can drop + * undirtied hole pages behind our back. + * + * But normally info->alloced == inode->i_mapping->nrpages + info->swapped + * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) + * + * It has to be called with the spinlock held. + */ +static void shmem_recalc_inode(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + long freed; + + freed = info->alloced - info->swapped - inode->i_mapping->nrpages; + if (freed > 0) { + info->alloced -= freed; + shmem_unacct_blocks(info->flags, freed); + shmem_free_blocks(inode, freed); + } +} + +/* + * shmem_swp_entry - find the swap vector position in the info structure + * + * @info: info structure for the inode + * @index: index of the page to find + * @page: optional page to add to the structure. Has to be preset to + * all zeros + * + * If there is no space allocated yet it will return NULL when + * page is NULL, else it will use the page for the needed block, + * setting it to NULL on return to indicate that it has been used. + * + * The swap vector is organized the following way: + * + * There are SHMEM_NR_DIRECT entries directly stored in the + * shmem_inode_info structure. So small files do not need an addional + * allocation. + * + * For pages with index > SHMEM_NR_DIRECT there is the pointer + * i_indirect which points to a page which holds in the first half + * doubly indirect blocks, in the second half triple indirect blocks: + * + * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the + * following layout (for SHMEM_NR_DIRECT == 16): + * + * i_indirect -> dir --> 16-19 + * | +-> 20-23 + * | + * +-->dir2 --> 24-27 + * | +-> 28-31 + * | +-> 32-35 + * | +-> 36-39 + * | + * +-->dir3 --> 40-43 + * +-> 44-47 + * +-> 48-51 + * +-> 52-55 + */ +static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page) +{ + unsigned long offset; + struct page **dir; + struct page *subdir; + + if (index < SHMEM_NR_DIRECT) { + shmem_swp_balance_unmap(); + return info->i_direct+index; + } + if (!info->i_indirect) { + if (page) { + info->i_indirect = *page; + *page = NULL; + } + return NULL; /* need another page */ + } + + index -= SHMEM_NR_DIRECT; + offset = index % ENTRIES_PER_PAGE; + index /= ENTRIES_PER_PAGE; + dir = shmem_dir_map(info->i_indirect); + + if (index >= ENTRIES_PER_PAGE/2) { + index -= ENTRIES_PER_PAGE/2; + dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE; + index %= ENTRIES_PER_PAGE; + subdir = *dir; + if (!subdir) { + if (page) { + *dir = *page; + *page = NULL; + } + shmem_dir_unmap(dir); + return NULL; /* need another page */ + } + shmem_dir_unmap(dir); + dir = shmem_dir_map(subdir); + } + + dir += index; + subdir = *dir; + if (!subdir) { + if (!page || !(subdir = *page)) { + shmem_dir_unmap(dir); + return NULL; /* need a page */ + } + *dir = subdir; + *page = NULL; + } + shmem_dir_unmap(dir); + return shmem_swp_map(subdir) + offset; +} + +static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value) +{ + long incdec = value? 1: -1; + + entry->val = value; + info->swapped += incdec; + if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) + kmap_atomic_to_page(entry)->nr_swapped += incdec; +} + +/* + * shmem_swp_alloc - get the position of the swap entry for the page. + * If it does not exist allocate the entry. + * + * @info: info structure for the inode + * @index: index of the page to find + * @sgp: check and recheck i_size? skip allocation? + */ +static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp) +{ + struct inode *inode = &info->vfs_inode; + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct page *page = NULL; + swp_entry_t *entry; + + if (sgp != SGP_WRITE && + ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + return ERR_PTR(-EINVAL); + + while (!(entry = shmem_swp_entry(info, index, &page))) { + if (sgp == SGP_READ) + return shmem_swp_map(ZERO_PAGE(0)); + /* + * Test free_blocks against 1 not 0, since we have 1 data + * page (and perhaps indirect index pages) yet to allocate: + * a waste to allocate index if we cannot allocate data. + */ + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + if (sbinfo->free_blocks <= 1) { + spin_unlock(&sbinfo->stat_lock); + return ERR_PTR(-ENOSPC); + } + sbinfo->free_blocks--; + inode->i_blocks += BLOCKS_PER_PAGE; + spin_unlock(&sbinfo->stat_lock); + } + + spin_unlock(&info->lock); + page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO); + if (page) { + page->nr_swapped = 0; + } + spin_lock(&info->lock); + + if (!page) { + shmem_free_blocks(inode, 1); + return ERR_PTR(-ENOMEM); + } + if (sgp != SGP_WRITE && + ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { + entry = ERR_PTR(-EINVAL); + break; + } + if (info->next_index <= index) + info->next_index = index + 1; + } + if (page) { + /* another task gave its page, or truncated the file */ + shmem_free_blocks(inode, 1); + shmem_dir_free(page); + } + if (info->next_index <= index && !IS_ERR(entry)) + info->next_index = index + 1; + return entry; +} + +/* + * shmem_free_swp - free some swap entries in a directory + * + * @dir: pointer to the directory + * @edir: pointer after last entry of the directory + */ +static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir) +{ + swp_entry_t *ptr; + int freed = 0; + + for (ptr = dir; ptr < edir; ptr++) { + if (ptr->val) { + free_swap_and_cache(*ptr); + *ptr = (swp_entry_t){0}; + freed++; + } + } + return freed; +} + +static int shmem_map_and_free_swp(struct page *subdir, + int offset, int limit, struct page ***dir) +{ + swp_entry_t *ptr; + int freed = 0; + + ptr = shmem_swp_map(subdir); + for (; offset < limit; offset += LATENCY_LIMIT) { + int size = limit - offset; + if (size > LATENCY_LIMIT) + size = LATENCY_LIMIT; + freed += shmem_free_swp(ptr+offset, ptr+offset+size); + if (need_resched()) { + shmem_swp_unmap(ptr); + if (*dir) { + shmem_dir_unmap(*dir); + *dir = NULL; + } + cond_resched(); + ptr = shmem_swp_map(subdir); + } + } + shmem_swp_unmap(ptr); + return freed; +} + +static void shmem_free_pages(struct list_head *next) +{ + struct page *page; + int freed = 0; + + do { + page = container_of(next, struct page, lru); + next = next->next; + shmem_dir_free(page); + freed++; + if (freed >= LATENCY_LIMIT) { + cond_resched(); + freed = 0; + } + } while (next); +} + +static void shmem_truncate(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + unsigned long idx; + unsigned long size; + unsigned long limit; + unsigned long stage; + unsigned long diroff; + struct page **dir; + struct page *topdir; + struct page *middir; + struct page *subdir; + swp_entry_t *ptr; + LIST_HEAD(pages_to_free); + long nr_pages_to_free = 0; + long nr_swaps_freed = 0; + int offset; + int freed; + + inode->i_ctime = inode->i_mtime = CURRENT_TIME; + idx = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; + if (idx >= info->next_index) + return; + + spin_lock(&info->lock); + info->flags |= SHMEM_TRUNCATE; + limit = info->next_index; + info->next_index = idx; + topdir = info->i_indirect; + if (topdir && idx <= SHMEM_NR_DIRECT) { + info->i_indirect = NULL; + nr_pages_to_free++; + list_add(&topdir->lru, &pages_to_free); + } + spin_unlock(&info->lock); + + if (info->swapped && idx < SHMEM_NR_DIRECT) { + ptr = info->i_direct; + size = limit; + if (size > SHMEM_NR_DIRECT) + size = SHMEM_NR_DIRECT; + nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size); + } + if (!topdir) + goto done2; + + BUG_ON(limit <= SHMEM_NR_DIRECT); + limit -= SHMEM_NR_DIRECT; + idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0; + offset = idx % ENTRIES_PER_PAGE; + idx -= offset; + + dir = shmem_dir_map(topdir); + stage = ENTRIES_PER_PAGEPAGE/2; + if (idx < ENTRIES_PER_PAGEPAGE/2) { + middir = topdir; + diroff = idx/ENTRIES_PER_PAGE; + } else { + dir += ENTRIES_PER_PAGE/2; + dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE; + while (stage <= idx) + stage += ENTRIES_PER_PAGEPAGE; + middir = *dir; + if (*dir) { + diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) % + ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE; + if (!diroff && !offset) { + *dir = NULL; + nr_pages_to_free++; + list_add(&middir->lru, &pages_to_free); + } + shmem_dir_unmap(dir); + dir = shmem_dir_map(middir); + } else { + diroff = 0; + offset = 0; + idx = stage; + } + } + + for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) { + if (unlikely(idx == stage)) { + shmem_dir_unmap(dir); + dir = shmem_dir_map(topdir) + + ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; + while (!*dir) { + dir++; + idx += ENTRIES_PER_PAGEPAGE; + if (idx >= limit) + goto done1; + } + stage = idx + ENTRIES_PER_PAGEPAGE; + middir = *dir; + *dir = NULL; + nr_pages_to_free++; + list_add(&middir->lru, &pages_to_free); + shmem_dir_unmap(dir); + cond_resched(); + dir = shmem_dir_map(middir); + diroff = 0; + } + subdir = dir[diroff]; + if (subdir && subdir->nr_swapped) { + size = limit - idx; + if (size > ENTRIES_PER_PAGE) + size = ENTRIES_PER_PAGE; + freed = shmem_map_and_free_swp(subdir, + offset, size, &dir); + if (!dir) + dir = shmem_dir_map(middir); + nr_swaps_freed += freed; + if (offset) + spin_lock(&info->lock); + subdir->nr_swapped -= freed; + if (offset) + spin_unlock(&info->lock); + BUG_ON(subdir->nr_swapped > offset); + } + if (offset) + offset = 0; + else if (subdir) { + dir[diroff] = NULL; + nr_pages_to_free++; + list_add(&subdir->lru, &pages_to_free); + } + } +done1: + shmem_dir_unmap(dir); +done2: + if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) { + /* + * Call truncate_inode_pages again: racing shmem_unuse_inode + * may have swizzled a page in from swap since vmtruncate or + * generic_delete_inode did it, before we lowered next_index. + * Also, though shmem_getpage checks i_size before adding to + * cache, no recheck after: so fix the narrow window there too. + */ + truncate_inode_pages(inode->i_mapping, inode->i_size); + } + + spin_lock(&info->lock); + info->flags &= ~SHMEM_TRUNCATE; + info->swapped -= nr_swaps_freed; + if (nr_pages_to_free) + shmem_free_blocks(inode, nr_pages_to_free); + shmem_recalc_inode(inode); + spin_unlock(&info->lock); + + /* + * Empty swap vector directory pages to be freed? + */ + if (!list_empty(&pages_to_free)) { + pages_to_free.prev->next = NULL; + shmem_free_pages(pages_to_free.next); + } +} + +static int shmem_notify_change(struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = dentry->d_inode; + struct page *page = NULL; + int error; + + if (attr->ia_valid & ATTR_SIZE) { + if (attr->ia_size < inode->i_size) { + /* + * If truncating down to a partial page, then + * if that page is already allocated, hold it + * in memory until the truncation is over, so + * truncate_partial_page cannnot miss it were + * it assigned to swap. + */ + if (attr->ia_size & (PAGE_CACHE_SIZE-1)) { + (void) shmem_getpage(inode, + attr->ia_size>>PAGE_CACHE_SHIFT, + &page, SGP_READ, NULL); + } + /* + * Reset SHMEM_PAGEIN flag so that shmem_truncate can + * detect if any pages might have been added to cache + * after truncate_inode_pages. But we needn't bother + * if it's being fully truncated to zero-length: the + * nrpages check is efficient enough in that case. + */ + if (attr->ia_size) { + struct shmem_inode_info *info = SHMEM_I(inode); + spin_lock(&info->lock); + info->flags &= ~SHMEM_PAGEIN; + spin_unlock(&info->lock); + } + } + } + + error = inode_change_ok(inode, attr); + if (!error) + error = inode_setattr(inode, attr); + if (page) + page_cache_release(page); + return error; +} + +static void shmem_delete_inode(struct inode *inode) +{ + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct shmem_inode_info *info = SHMEM_I(inode); + + if (inode->i_op->truncate == shmem_truncate) { + shmem_unacct_size(info->flags, inode->i_size); + inode->i_size = 0; + shmem_truncate(inode); + if (!list_empty(&info->swaplist)) { + spin_lock(&shmem_swaplist_lock); + list_del_init(&info->swaplist); + spin_unlock(&shmem_swaplist_lock); + } + } + if (sbinfo) { + BUG_ON(inode->i_blocks); + spin_lock(&sbinfo->stat_lock); + sbinfo->free_inodes++; + spin_unlock(&sbinfo->stat_lock); + } + clear_inode(inode); +} + +static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir) +{ + swp_entry_t *ptr; + + for (ptr = dir; ptr < edir; ptr++) { + if (ptr->val == entry.val) + return ptr - dir; + } + return -1; +} + +static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page) +{ + struct inode *inode; + unsigned long idx; + unsigned long size; + unsigned long limit; + unsigned long stage; + struct page **dir; + struct page *subdir; + swp_entry_t *ptr; + int offset; + + idx = 0; + ptr = info->i_direct; + spin_lock(&info->lock); + limit = info->next_index; + size = limit; + if (size > SHMEM_NR_DIRECT) + size = SHMEM_NR_DIRECT; + offset = shmem_find_swp(entry, ptr, ptr+size); + if (offset >= 0) { + shmem_swp_balance_unmap(); + goto found; + } + if (!info->i_indirect) + goto lost2; + + dir = shmem_dir_map(info->i_indirect); + stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2; + + for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) { + if (unlikely(idx == stage)) { + shmem_dir_unmap(dir-1); + dir = shmem_dir_map(info->i_indirect) + + ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; + while (!*dir) { + dir++; + idx += ENTRIES_PER_PAGEPAGE; + if (idx >= limit) + goto lost1; + } + stage = idx + ENTRIES_PER_PAGEPAGE; + subdir = *dir; + shmem_dir_unmap(dir); + dir = shmem_dir_map(subdir); + } + subdir = *dir; + if (subdir && subdir->nr_swapped) { + ptr = shmem_swp_map(subdir); + size = limit - idx; + if (size > ENTRIES_PER_PAGE) + size = ENTRIES_PER_PAGE; + offset = shmem_find_swp(entry, ptr, ptr+size); + if (offset >= 0) { + shmem_dir_unmap(dir); + goto found; + } + shmem_swp_unmap(ptr); + } + } +lost1: + shmem_dir_unmap(dir-1); +lost2: + spin_unlock(&info->lock); + return 0; +found: + idx += offset; + inode = &info->vfs_inode; + if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) { + info->flags |= SHMEM_PAGEIN; + shmem_swp_set(info, ptr + offset, 0); + } + shmem_swp_unmap(ptr); + spin_unlock(&info->lock); + /* + * Decrement swap count even when the entry is left behind: + * try_to_unuse will skip over mms, then reincrement count. + */ + swap_free(entry); + return 1; +} + +/* + * shmem_unuse() search for an eventually swapped out shmem page. + */ +int shmem_unuse(swp_entry_t entry, struct page *page) +{ + struct list_head *p, *next; + struct shmem_inode_info *info; + int found = 0; + + spin_lock(&shmem_swaplist_lock); + list_for_each_safe(p, next, &shmem_swaplist) { + info = list_entry(p, struct shmem_inode_info, swaplist); + if (!info->swapped) + list_del_init(&info->swaplist); + else if (shmem_unuse_inode(info, entry, page)) { + /* move head to start search for next from here */ + list_move_tail(&shmem_swaplist, &info->swaplist); + found = 1; + break; + } + } + spin_unlock(&shmem_swaplist_lock); + return found; +} + +/* + * Move the page from the page cache to the swap cache. + */ +static int shmem_writepage(struct page *page, struct writeback_control *wbc) +{ + struct shmem_inode_info *info; + swp_entry_t *entry, swap; + struct address_space *mapping; + unsigned long index; + struct inode *inode; + + BUG_ON(!PageLocked(page)); + BUG_ON(page_mapped(page)); + + mapping = page->mapping; + index = page->index; + inode = mapping->host; + info = SHMEM_I(inode); + if (info->flags & VM_LOCKED) + goto redirty; + swap = get_swap_page(); + if (!swap.val) + goto redirty; + + spin_lock(&info->lock); + shmem_recalc_inode(inode); + if (index >= info->next_index) { + BUG_ON(!(info->flags & SHMEM_TRUNCATE)); + goto unlock; + } + entry = shmem_swp_entry(info, index, NULL); + BUG_ON(!entry); + BUG_ON(entry->val); + + if (move_to_swap_cache(page, swap) == 0) { + shmem_swp_set(info, entry, swap.val); + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + if (list_empty(&info->swaplist)) { + spin_lock(&shmem_swaplist_lock); + /* move instead of add in case we're racing */ + list_move_tail(&info->swaplist, &shmem_swaplist); + spin_unlock(&shmem_swaplist_lock); + } + unlock_page(page); + return 0; + } + + shmem_swp_unmap(entry); +unlock: + spin_unlock(&info->lock); + swap_free(swap); +redirty: + set_page_dirty(page); + return WRITEPAGE_ACTIVATE; /* Return with the page locked */ +} + +#ifdef CONFIG_NUMA +static struct page *shmem_swapin_async(struct shared_policy *p, + swp_entry_t entry, unsigned long idx) +{ + struct page *page; + struct vm_area_struct pvma; + + /* Create a pseudo vma that just contains the policy */ + memset(&pvma, 0, sizeof(struct vm_area_struct)); + pvma.vm_end = PAGE_SIZE; + pvma.vm_pgoff = idx; + pvma.vm_policy = mpol_shared_policy_lookup(p, idx); + page = read_swap_cache_async(entry, &pvma, 0); + mpol_free(pvma.vm_policy); + return page; +} + +struct page *shmem_swapin(struct shmem_inode_info *info, swp_entry_t entry, + unsigned long idx) +{ + struct shared_policy *p = &info->policy; + int i, num; + struct page *page; + unsigned long offset; + + num = valid_swaphandles(entry, &offset); + for (i = 0; i < num; offset++, i++) { + page = shmem_swapin_async(p, + swp_entry(swp_type(entry), offset), idx); + if (!page) + break; + page_cache_release(page); + } + lru_add_drain(); /* Push any new pages onto the LRU now */ + return shmem_swapin_async(p, entry, idx); +} + +static struct page * +shmem_alloc_page(unsigned long gfp, struct shmem_inode_info *info, + unsigned long idx) +{ + struct vm_area_struct pvma; + struct page *page; + + memset(&pvma, 0, sizeof(struct vm_area_struct)); + pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); + pvma.vm_pgoff = idx; + pvma.vm_end = PAGE_SIZE; + page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0); + mpol_free(pvma.vm_policy); + return page; +} +#else +static inline struct page * +shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx) +{ + swapin_readahead(entry, 0, NULL); + return read_swap_cache_async(entry, NULL, 0); +} + +static inline struct page * +shmem_alloc_page(unsigned int __nocast gfp,struct shmem_inode_info *info, + unsigned long idx) +{ + return alloc_page(gfp | __GFP_ZERO); +} +#endif + +/* + * shmem_getpage - either get the page from swap or allocate a new one + * + * If we allocate a new one we do not mark it dirty. That's up to the + * vm. If we swap it in we mark it dirty since we also free the swap + * entry since a page cannot live in both the swap and page cache + */ +static int shmem_getpage(struct inode *inode, unsigned long idx, + struct page **pagep, enum sgp_type sgp, int *type) +{ + struct address_space *mapping = inode->i_mapping; + struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo; + struct page *filepage = *pagep; + struct page *swappage; + swp_entry_t *entry; + swp_entry_t swap; + int error; + + if (idx >= SHMEM_MAX_INDEX) + return -EFBIG; + /* + * Normally, filepage is NULL on entry, and either found + * uptodate immediately, or allocated and zeroed, or read + * in under swappage, which is then assigned to filepage. + * But shmem_prepare_write passes in a locked filepage, + * which may be found not uptodate by other callers too, + * and may need to be copied from the swappage read in. + */ +repeat: + if (!filepage) + filepage = find_lock_page(mapping, idx); + if (filepage && PageUptodate(filepage)) + goto done; + error = 0; + if (sgp == SGP_QUICK) + goto failed; + + spin_lock(&info->lock); + shmem_recalc_inode(inode); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) { + spin_unlock(&info->lock); + error = PTR_ERR(entry); + goto failed; + } + swap = *entry; + + if (swap.val) { + /* Look it up and read it in.. */ + swappage = lookup_swap_cache(swap); + if (!swappage) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + /* here we actually do the io */ + if (type && *type == VM_FAULT_MINOR) { + inc_page_state(pgmajfault); + *type = VM_FAULT_MAJOR; + } + swappage = shmem_swapin(info, swap, idx); + if (!swappage) { + spin_lock(&info->lock); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) + error = PTR_ERR(entry); + else { + if (entry->val == swap.val) + error = -ENOMEM; + shmem_swp_unmap(entry); + } + spin_unlock(&info->lock); + if (error) + goto failed; + goto repeat; + } + wait_on_page_locked(swappage); + page_cache_release(swappage); + goto repeat; + } + + /* We have to do this with page locked to prevent races */ + if (TestSetPageLocked(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + wait_on_page_locked(swappage); + page_cache_release(swappage); + goto repeat; + } + if (PageWriteback(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + wait_on_page_writeback(swappage); + unlock_page(swappage); + page_cache_release(swappage); + goto repeat; + } + if (!PageUptodate(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + unlock_page(swappage); + page_cache_release(swappage); + error = -EIO; + goto failed; + } + + if (filepage) { + shmem_swp_set(info, entry, 0); + shmem_swp_unmap(entry); + delete_from_swap_cache(swappage); + spin_unlock(&info->lock); + copy_highpage(filepage, swappage); + unlock_page(swappage); + page_cache_release(swappage); + flush_dcache_page(filepage); + SetPageUptodate(filepage); + set_page_dirty(filepage); + swap_free(swap); + } else if (!(error = move_from_swap_cache( + swappage, idx, mapping))) { + info->flags |= SHMEM_PAGEIN; + shmem_swp_set(info, entry, 0); + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + filepage = swappage; + swap_free(swap); + } else { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + unlock_page(swappage); + page_cache_release(swappage); + if (error == -ENOMEM) { + /* let kswapd refresh zone for GFP_ATOMICs */ + blk_congestion_wait(WRITE, HZ/50); + } + goto repeat; + } + } else if (sgp == SGP_READ && !filepage) { + shmem_swp_unmap(entry); + filepage = find_get_page(mapping, idx); + if (filepage && + (!PageUptodate(filepage) || TestSetPageLocked(filepage))) { + spin_unlock(&info->lock); + wait_on_page_locked(filepage); + page_cache_release(filepage); + filepage = NULL; + goto repeat; + } + spin_unlock(&info->lock); + } else { + shmem_swp_unmap(entry); + sbinfo = SHMEM_SB(inode->i_sb); + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + if (sbinfo->free_blocks == 0 || + shmem_acct_block(info->flags)) { + spin_unlock(&sbinfo->stat_lock); + spin_unlock(&info->lock); + error = -ENOSPC; + goto failed; + } + sbinfo->free_blocks--; + inode->i_blocks += BLOCKS_PER_PAGE; + spin_unlock(&sbinfo->stat_lock); + } else if (shmem_acct_block(info->flags)) { + spin_unlock(&info->lock); + error = -ENOSPC; + goto failed; + } + + if (!filepage) { + spin_unlock(&info->lock); + filepage = shmem_alloc_page(mapping_gfp_mask(mapping), + info, + idx); + if (!filepage) { + shmem_unacct_blocks(info->flags, 1); + shmem_free_blocks(inode, 1); + error = -ENOMEM; + goto failed; + } + + spin_lock(&info->lock); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) + error = PTR_ERR(entry); + else { + swap = *entry; + shmem_swp_unmap(entry); + } + if (error || swap.val || 0 != add_to_page_cache_lru( + filepage, mapping, idx, GFP_ATOMIC)) { + spin_unlock(&info->lock); + page_cache_release(filepage); + shmem_unacct_blocks(info->flags, 1); + shmem_free_blocks(inode, 1); + filepage = NULL; + if (error) + goto failed; + goto repeat; + } + info->flags |= SHMEM_PAGEIN; + } + + info->alloced++; + spin_unlock(&info->lock); + flush_dcache_page(filepage); + SetPageUptodate(filepage); + } +done: + if (*pagep != filepage) { + unlock_page(filepage); + *pagep = filepage; + } + return 0; + +failed: + if (*pagep != filepage) { + unlock_page(filepage); + page_cache_release(filepage); + } + return error; +} + +struct page *shmem_nopage(struct vm_area_struct *vma, unsigned long address, int *type) +{ + struct inode *inode = vma->vm_file->f_dentry->d_inode; + struct page *page = NULL; + unsigned long idx; + int error; + + idx = (address - vma->vm_start) >> PAGE_SHIFT; + idx += vma->vm_pgoff; + idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT; + if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + return NOPAGE_SIGBUS; + + error = shmem_getpage(inode, idx, &page, SGP_CACHE, type); + if (error) + return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS; + + mark_page_accessed(page); + return page; +} + +static int shmem_populate(struct vm_area_struct *vma, + unsigned long addr, unsigned long len, + pgprot_t prot, unsigned long pgoff, int nonblock) +{ + struct inode *inode = vma->vm_file->f_dentry->d_inode; + struct mm_struct *mm = vma->vm_mm; + enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE; + unsigned long size; + + size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size) + return -EINVAL; + + while ((long) len > 0) { + struct page *page = NULL; + int err; + /* + * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE + */ + err = shmem_getpage(inode, pgoff, &page, sgp, NULL); + if (err) + return err; + if (page) { + mark_page_accessed(page); + err = install_page(mm, vma, addr, page, prot); + if (err) { + page_cache_release(page); + return err; + } + } else if (nonblock) { + err = install_file_pte(mm, vma, addr, pgoff, prot); + if (err) + return err; + } + + len -= PAGE_SIZE; + addr += PAGE_SIZE; + pgoff++; + } + return 0; +} + +#ifdef CONFIG_NUMA +int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new) +{ + struct inode *i = vma->vm_file->f_dentry->d_inode; + return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new); +} + +struct mempolicy * +shmem_get_policy(struct vm_area_struct *vma, unsigned long addr) +{ + struct inode *i = vma->vm_file->f_dentry->d_inode; + unsigned long idx; + + idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx); +} +#endif + +int shmem_lock(struct file *file, int lock, struct user_struct *user) +{ + struct inode *inode = file->f_dentry->d_inode; + struct shmem_inode_info *info = SHMEM_I(inode); + int retval = -ENOMEM; + + spin_lock(&info->lock); + if (lock && !(info->flags & VM_LOCKED)) { + if (!user_shm_lock(inode->i_size, user)) + goto out_nomem; + info->flags |= VM_LOCKED; + } + if (!lock && (info->flags & VM_LOCKED) && user) { + user_shm_unlock(inode->i_size, user); + info->flags &= ~VM_LOCKED; + } + retval = 0; +out_nomem: + spin_unlock(&info->lock); + return retval; +} + +static int shmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + file_accessed(file); + vma->vm_ops = &shmem_vm_ops; + return 0; +} + +static struct inode * +shmem_get_inode(struct super_block *sb, int mode, dev_t dev) +{ + struct inode *inode; + struct shmem_inode_info *info; + struct shmem_sb_info *sbinfo = SHMEM_SB(sb); + + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + if (!sbinfo->free_inodes) { + spin_unlock(&sbinfo->stat_lock); + return NULL; + } + sbinfo->free_inodes--; + spin_unlock(&sbinfo->stat_lock); + } + + inode = new_inode(sb); + if (inode) { + inode->i_mode = mode; + inode->i_uid = current->fsuid; + inode->i_gid = current->fsgid; + inode->i_blksize = PAGE_CACHE_SIZE; + inode->i_blocks = 0; + inode->i_mapping->a_ops = &shmem_aops; + inode->i_mapping->backing_dev_info = &shmem_backing_dev_info; + inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; + info = SHMEM_I(inode); + memset(info, 0, (char *)inode - (char *)info); + spin_lock_init(&info->lock); + INIT_LIST_HEAD(&info->swaplist); + + switch (mode & S_IFMT) { + default: + inode->i_op = &shmem_special_inode_operations; + init_special_inode(inode, mode, dev); + break; + case S_IFREG: + inode->i_op = &shmem_inode_operations; + inode->i_fop = &shmem_file_operations; + mpol_shared_policy_init(&info->policy); + break; + case S_IFDIR: + inode->i_nlink++; + /* Some things misbehave if size == 0 on a directory */ + inode->i_size = 2 * BOGO_DIRENT_SIZE; + inode->i_op = &shmem_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + break; + case S_IFLNK: + /* + * Must not load anything in the rbtree, + * mpol_free_shared_policy will not be called. + */ + mpol_shared_policy_init(&info->policy); + break; + } + } else if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + sbinfo->free_inodes++; + spin_unlock(&sbinfo->stat_lock); + } + return inode; +} + +#ifdef CONFIG_TMPFS + +static int shmem_set_size(struct shmem_sb_info *sbinfo, + unsigned long max_blocks, unsigned long max_inodes) +{ + int error; + unsigned long blocks, inodes; + + spin_lock(&sbinfo->stat_lock); + blocks = sbinfo->max_blocks - sbinfo->free_blocks; + inodes = sbinfo->max_inodes - sbinfo->free_inodes; + error = -EINVAL; + if (max_blocks < blocks) + goto out; + if (max_inodes < inodes) + goto out; + error = 0; + sbinfo->max_blocks = max_blocks; + sbinfo->free_blocks = max_blocks - blocks; + sbinfo->max_inodes = max_inodes; + sbinfo->free_inodes = max_inodes - inodes; +out: + spin_unlock(&sbinfo->stat_lock); + return error; +} + +static struct inode_operations shmem_symlink_inode_operations; +static struct inode_operations shmem_symlink_inline_operations; + +/* + * Normally tmpfs makes no use of shmem_prepare_write, but it + * lets a tmpfs file be used read-write below the loop driver. + */ +static int +shmem_prepare_write(struct file *file, struct page *page, unsigned offset, unsigned to) +{ + struct inode *inode = page->mapping->host; + return shmem_getpage(inode, page->index, &page, SGP_WRITE, NULL); +} + +static ssize_t +shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) +{ + struct inode *inode = file->f_dentry->d_inode; + loff_t pos; + unsigned long written; + ssize_t err; + + if ((ssize_t) count < 0) + return -EINVAL; + + if (!access_ok(VERIFY_READ, buf, count)) + return -EFAULT; + + down(&inode->i_sem); + + pos = *ppos; + written = 0; + + err = generic_write_checks(file, &pos, &count, 0); + if (err || !count) + goto out; + + err = remove_suid(file->f_dentry); + if (err) + goto out; + + inode->i_ctime = inode->i_mtime = CURRENT_TIME; + + do { + struct page *page = NULL; + unsigned long bytes, index, offset; + char *kaddr; + int left; + + offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ + index = pos >> PAGE_CACHE_SHIFT; + bytes = PAGE_CACHE_SIZE - offset; + if (bytes > count) + bytes = count; + + /* + * We don't hold page lock across copy from user - + * what would it guard against? - so no deadlock here. + * But it still may be a good idea to prefault below. + */ + + err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL); + if (err) + break; + + left = bytes; + if (PageHighMem(page)) { + volatile unsigned char dummy; + __get_user(dummy, buf); + __get_user(dummy, buf + bytes - 1); + + kaddr = kmap_atomic(page, KM_USER0); + left = __copy_from_user_inatomic(kaddr + offset, + buf, bytes); + kunmap_atomic(kaddr, KM_USER0); + } + if (left) { + kaddr = kmap(page); + left = __copy_from_user(kaddr + offset, buf, bytes); + kunmap(page); + } + + written += bytes; + count -= bytes; + pos += bytes; + buf += bytes; + if (pos > inode->i_size) + i_size_write(inode, pos); + + flush_dcache_page(page); + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + + if (left) { + pos -= left; + written -= left; + err = -EFAULT; + break; + } + + /* + * Our dirty pages are not counted in nr_dirty, + * and we do not attempt to balance dirty pages. + */ + + cond_resched(); + } while (count); + + *ppos = pos; + if (written) + err = written; +out: + up(&inode->i_sem); + return err; +} + +static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor) +{ + struct inode *inode = filp->f_dentry->d_inode; + struct address_space *mapping = inode->i_mapping; + unsigned long index, offset; + + index = *ppos >> PAGE_CACHE_SHIFT; + offset = *ppos & ~PAGE_CACHE_MASK; + + for (;;) { + struct page *page = NULL; + unsigned long end_index, nr, ret; + loff_t i_size = i_size_read(inode); + + end_index = i_size >> PAGE_CACHE_SHIFT; + if (index > end_index) + break; + if (index == end_index) { + nr = i_size & ~PAGE_CACHE_MASK; + if (nr <= offset) + break; + } + + desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL); + if (desc->error) { + if (desc->error == -EINVAL) + desc->error = 0; + break; + } + + /* + * We must evaluate after, since reads (unlike writes) + * are called without i_sem protection against truncate + */ + nr = PAGE_CACHE_SIZE; + i_size = i_size_read(inode); + end_index = i_size >> PAGE_CACHE_SHIFT; + if (index == end_index) { + nr = i_size & ~PAGE_CACHE_MASK; + if (nr <= offset) { + if (page) + page_cache_release(page); + break; + } + } + nr -= offset; + + if (page) { + /* + * If users can be writing to this page using arbitrary + * virtual addresses, take care about potential aliasing + * before reading the page on the kernel side. + */ + if (mapping_writably_mapped(mapping)) + flush_dcache_page(page); + /* + * Mark the page accessed if we read the beginning. + */ + if (!offset) + mark_page_accessed(page); + } else + page = ZERO_PAGE(0); + + /* + * Ok, we have the page, and it's up-to-date, so + * now we can copy it to user space... + * + * The actor routine returns how many bytes were actually used.. + * NOTE! This may not be the same as how much of a user buffer + * we filled up (we may be padding etc), so we can only update + * "pos" here (the actor routine has to update the user buffer + * pointers and the remaining count). + */ + ret = actor(desc, page, offset, nr); + offset += ret; + index += offset >> PAGE_CACHE_SHIFT; + offset &= ~PAGE_CACHE_MASK; + + page_cache_release(page); + if (ret != nr || !desc->count) + break; + + cond_resched(); + } + + *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset; + file_accessed(filp); +} + +static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos) +{ + read_descriptor_t desc; + + if ((ssize_t) count < 0) + return -EINVAL; + if (!access_ok(VERIFY_WRITE, buf, count)) + return -EFAULT; + if (!count) + return 0; + + desc.written = 0; + desc.count = count; + desc.arg.buf = buf; + desc.error = 0; + + do_shmem_file_read(filp, ppos, &desc, file_read_actor); + if (desc.written) + return desc.written; + return desc.error; +} + +static ssize_t shmem_file_sendfile(struct file *in_file, loff_t *ppos, + size_t count, read_actor_t actor, void *target) +{ + read_descriptor_t desc; + + if (!count) + return 0; + + desc.written = 0; + desc.count = count; + desc.arg.data = target; + desc.error = 0; + + do_shmem_file_read(in_file, ppos, &desc, actor); + if (desc.written) + return desc.written; + return desc.error; +} + +static int shmem_statfs(struct super_block *sb, struct kstatfs *buf) +{ + struct shmem_sb_info *sbinfo = SHMEM_SB(sb); + + buf->f_type = TMPFS_MAGIC; + buf->f_bsize = PAGE_CACHE_SIZE; + buf->f_namelen = NAME_MAX; + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + buf->f_blocks = sbinfo->max_blocks; + buf->f_bavail = buf->f_bfree = sbinfo->free_blocks; + buf->f_files = sbinfo->max_inodes; + buf->f_ffree = sbinfo->free_inodes; + spin_unlock(&sbinfo->stat_lock); + } + /* else leave those fields 0 like simple_statfs */ + return 0; +} + +/* + * File creation. Allocate an inode, and we're done.. + */ +static int +shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev) +{ + struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev); + int error = -ENOSPC; + + if (inode) { + if (dir->i_mode & S_ISGID) { + inode->i_gid = dir->i_gid; + if (S_ISDIR(mode)) + inode->i_mode |= S_ISGID; + } + dir->i_size += BOGO_DIRENT_SIZE; + dir->i_ctime = dir->i_mtime = CURRENT_TIME; + d_instantiate(dentry, inode); + dget(dentry); /* Extra count - pin the dentry in core */ + error = 0; + } + return error; +} + +static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode) +{ + int error; + + if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) + return error; + dir->i_nlink++; + return 0; +} + +static int shmem_create(struct inode *dir, struct dentry *dentry, int mode, + struct nameidata *nd) +{ + return shmem_mknod(dir, dentry, mode | S_IFREG, 0); +} + +/* + * Link a file.. + */ +static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = old_dentry->d_inode; + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + + /* + * No ordinary (disk based) filesystem counts links as inodes; + * but each new link needs a new dentry, pinning lowmem, and + * tmpfs dentries cannot be pruned until they are unlinked. + */ + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + if (!sbinfo->free_inodes) { + spin_unlock(&sbinfo->stat_lock); + return -ENOSPC; + } + sbinfo->free_inodes--; + spin_unlock(&sbinfo->stat_lock); + } + + dir->i_size += BOGO_DIRENT_SIZE; + inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; + inode->i_nlink++; + atomic_inc(&inode->i_count); /* New dentry reference */ + dget(dentry); /* Extra pinning count for the created dentry */ + d_instantiate(dentry, inode); + return 0; +} + +static int shmem_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + + if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) { + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + sbinfo->free_inodes++; + spin_unlock(&sbinfo->stat_lock); + } + } + + dir->i_size -= BOGO_DIRENT_SIZE; + inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME; + inode->i_nlink--; + dput(dentry); /* Undo the count from "create" - this does all the work */ + return 0; +} + +static int shmem_rmdir(struct inode *dir, struct dentry *dentry) +{ + if (!simple_empty(dentry)) + return -ENOTEMPTY; + + dir->i_nlink--; + return shmem_unlink(dir, dentry); +} + +/* + * The VFS layer already does all the dentry stuff for rename, + * we just have to decrement the usage count for the target if + * it exists so that the VFS layer correctly free's it when it + * gets overwritten. + */ +static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) +{ + struct inode *inode = old_dentry->d_inode; + int they_are_dirs = S_ISDIR(inode->i_mode); + + if (!simple_empty(new_dentry)) + return -ENOTEMPTY; + + if (new_dentry->d_inode) { + (void) shmem_unlink(new_dir, new_dentry); + if (they_are_dirs) + old_dir->i_nlink--; + } else if (they_are_dirs) { + old_dir->i_nlink--; + new_dir->i_nlink++; + } + + old_dir->i_size -= BOGO_DIRENT_SIZE; + new_dir->i_size += BOGO_DIRENT_SIZE; + old_dir->i_ctime = old_dir->i_mtime = + new_dir->i_ctime = new_dir->i_mtime = + inode->i_ctime = CURRENT_TIME; + return 0; +} + +static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +{ + int error; + int len; + struct inode *inode; + struct page *page = NULL; + char *kaddr; + struct shmem_inode_info *info; + + len = strlen(symname) + 1; + if (len > PAGE_CACHE_SIZE) + return -ENAMETOOLONG; + + inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0); + if (!inode) + return -ENOSPC; + + info = SHMEM_I(inode); + inode->i_size = len-1; + if (len <= (char *)inode - (char *)info) { + /* do it inline */ + memcpy(info, symname, len); + inode->i_op = &shmem_symlink_inline_operations; + } else { + error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); + if (error) { + iput(inode); + return error; + } + inode->i_op = &shmem_symlink_inode_operations; + kaddr = kmap_atomic(page, KM_USER0); + memcpy(kaddr, symname, len); + kunmap_atomic(kaddr, KM_USER0); + set_page_dirty(page); + page_cache_release(page); + } + if (dir->i_mode & S_ISGID) + inode->i_gid = dir->i_gid; + dir->i_size += BOGO_DIRENT_SIZE; + dir->i_ctime = dir->i_mtime = CURRENT_TIME; + d_instantiate(dentry, inode); + dget(dentry); + return 0; +} + +static int shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd) +{ + nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode)); + return 0; +} + +static int shmem_follow_link(struct dentry *dentry, struct nameidata *nd) +{ + struct page *page = NULL; + int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); + nd_set_link(nd, res ? ERR_PTR(res) : kmap(page)); + return 0; +} + +static void shmem_put_link(struct dentry *dentry, struct nameidata *nd) +{ + if (!IS_ERR(nd_get_link(nd))) { + struct page *page; + + page = find_get_page(dentry->d_inode->i_mapping, 0); + if (!page) + BUG(); + kunmap(page); + mark_page_accessed(page); + page_cache_release(page); + page_cache_release(page); + } +} + +static struct inode_operations shmem_symlink_inline_operations = { + .readlink = generic_readlink, + .follow_link = shmem_follow_link_inline, +#ifdef CONFIG_TMPFS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = generic_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +static struct inode_operations shmem_symlink_inode_operations = { + .truncate = shmem_truncate, + .readlink = generic_readlink, + .follow_link = shmem_follow_link, + .put_link = shmem_put_link, +#ifdef CONFIG_TMPFS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = generic_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +static int shmem_parse_options(char *options, int *mode, uid_t *uid, gid_t *gid, unsigned long *blocks, unsigned long *inodes) +{ + char *this_char, *value, *rest; + + while ((this_char = strsep(&options, ",")) != NULL) { + if (!*this_char) + continue; + if ((value = strchr(this_char,'=')) != NULL) { + *value++ = 0; + } else { + printk(KERN_ERR + "tmpfs: No value for mount option '%s'\n", + this_char); + return 1; + } + + if (!strcmp(this_char,"size")) { + unsigned long long size; + size = memparse(value,&rest); + if (*rest == '%') { + size <<= PAGE_SHIFT; + size *= totalram_pages; + do_div(size, 100); + rest++; + } + if (*rest) + goto bad_val; + *blocks = size >> PAGE_CACHE_SHIFT; + } else if (!strcmp(this_char,"nr_blocks")) { + *blocks = memparse(value,&rest); + if (*rest) + goto bad_val; + } else if (!strcmp(this_char,"nr_inodes")) { + *inodes = memparse(value,&rest); + if (*rest) + goto bad_val; + } else if (!strcmp(this_char,"mode")) { + if (!mode) + continue; + *mode = simple_strtoul(value,&rest,8); + if (*rest) + goto bad_val; + } else if (!strcmp(this_char,"uid")) { + if (!uid) + continue; + *uid = simple_strtoul(value,&rest,0); + if (*rest) + goto bad_val; + } else if (!strcmp(this_char,"gid")) { + if (!gid) + continue; + *gid = simple_strtoul(value,&rest,0); + if (*rest) + goto bad_val; + } else { + printk(KERN_ERR "tmpfs: Bad mount option %s\n", + this_char); + return 1; + } + } + return 0; + +bad_val: + printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n", + value, this_char); + return 1; + +} + +static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) +{ + struct shmem_sb_info *sbinfo = SHMEM_SB(sb); + unsigned long max_blocks = 0; + unsigned long max_inodes = 0; + + if (sbinfo) { + max_blocks = sbinfo->max_blocks; + max_inodes = sbinfo->max_inodes; + } + if (shmem_parse_options(data, NULL, NULL, NULL, &max_blocks, &max_inodes)) + return -EINVAL; + /* Keep it simple: disallow limited <-> unlimited remount */ + if ((max_blocks || max_inodes) == !sbinfo) + return -EINVAL; + /* But allow the pointless unlimited -> unlimited remount */ + if (!sbinfo) + return 0; + return shmem_set_size(sbinfo, max_blocks, max_inodes); +} +#endif + +static void shmem_put_super(struct super_block *sb) +{ + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; +} + +#ifdef CONFIG_TMPFS_XATTR +static struct xattr_handler *shmem_xattr_handlers[]; +#else +#define shmem_xattr_handlers NULL +#endif + +static int shmem_fill_super(struct super_block *sb, + void *data, int silent) +{ + struct inode *inode; + struct dentry *root; + int mode = S_IRWXUGO | S_ISVTX; + uid_t uid = current->fsuid; + gid_t gid = current->fsgid; + int err = -ENOMEM; + +#ifdef CONFIG_TMPFS + unsigned long blocks = 0; + unsigned long inodes = 0; + + /* + * Per default we only allow half of the physical ram per + * tmpfs instance, limiting inodes to one per page of lowmem; + * but the internal instance is left unlimited. + */ + if (!(sb->s_flags & MS_NOUSER)) { + blocks = totalram_pages / 2; + inodes = totalram_pages - totalhigh_pages; + if (inodes > blocks) + inodes = blocks; + + if (shmem_parse_options(data, &mode, + &uid, &gid, &blocks, &inodes)) + return -EINVAL; + } + + if (blocks || inodes) { + struct shmem_sb_info *sbinfo; + sbinfo = kmalloc(sizeof(struct shmem_sb_info), GFP_KERNEL); + if (!sbinfo) + return -ENOMEM; + sb->s_fs_info = sbinfo; + spin_lock_init(&sbinfo->stat_lock); + sbinfo->max_blocks = blocks; + sbinfo->free_blocks = blocks; + sbinfo->max_inodes = inodes; + sbinfo->free_inodes = inodes; + } + sb->s_xattr = shmem_xattr_handlers; +#else + sb->s_flags |= MS_NOUSER; +#endif + + sb->s_maxbytes = SHMEM_MAX_BYTES; + sb->s_blocksize = PAGE_CACHE_SIZE; + sb->s_blocksize_bits = PAGE_CACHE_SHIFT; + sb->s_magic = TMPFS_MAGIC; + sb->s_op = &shmem_ops; + inode = shmem_get_inode(sb, S_IFDIR | mode, 0); + if (!inode) + goto failed; + inode->i_uid = uid; + inode->i_gid = gid; + root = d_alloc_root(inode); + if (!root) + goto failed_iput; + sb->s_root = root; + return 0; + +failed_iput: + iput(inode); +failed: + shmem_put_super(sb); + return err; +} + +static kmem_cache_t *shmem_inode_cachep; + +static struct inode *shmem_alloc_inode(struct super_block *sb) +{ + struct shmem_inode_info *p; + p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, SLAB_KERNEL); + if (!p) + return NULL; + return &p->vfs_inode; +} + +static void shmem_destroy_inode(struct inode *inode) +{ + if ((inode->i_mode & S_IFMT) == S_IFREG) { + /* only struct inode is valid if it's an inline symlink */ + mpol_free_shared_policy(&SHMEM_I(inode)->policy); + } + kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); +} + +static void init_once(void *foo, kmem_cache_t *cachep, unsigned long flags) +{ + struct shmem_inode_info *p = (struct shmem_inode_info *) foo; + + if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == + SLAB_CTOR_CONSTRUCTOR) { + inode_init_once(&p->vfs_inode); + } +} + +static int init_inodecache(void) +{ + shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", + sizeof(struct shmem_inode_info), + 0, 0, init_once, NULL); + if (shmem_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + if (kmem_cache_destroy(shmem_inode_cachep)) + printk(KERN_INFO "shmem_inode_cache: not all structures were freed\n"); +} + +static struct address_space_operations shmem_aops = { + .writepage = shmem_writepage, + .set_page_dirty = __set_page_dirty_nobuffers, +#ifdef CONFIG_TMPFS + .prepare_write = shmem_prepare_write, + .commit_write = simple_commit_write, +#endif +}; + +static struct file_operations shmem_file_operations = { + .mmap = shmem_mmap, +#ifdef CONFIG_TMPFS + .llseek = generic_file_llseek, + .read = shmem_file_read, + .write = shmem_file_write, + .fsync = simple_sync_file, + .sendfile = shmem_file_sendfile, +#endif +}; + +static struct inode_operations shmem_inode_operations = { + .truncate = shmem_truncate, + .setattr = shmem_notify_change, +#ifdef CONFIG_TMPFS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = generic_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +static struct inode_operations shmem_dir_inode_operations = { +#ifdef CONFIG_TMPFS + .create = shmem_create, + .lookup = simple_lookup, + .link = shmem_link, + .unlink = shmem_unlink, + .symlink = shmem_symlink, + .mkdir = shmem_mkdir, + .rmdir = shmem_rmdir, + .mknod = shmem_mknod, + .rename = shmem_rename, +#ifdef CONFIG_TMPFS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = generic_listxattr, + .removexattr = generic_removexattr, +#endif +#endif +}; + +static struct inode_operations shmem_special_inode_operations = { +#ifdef CONFIG_TMPFS_XATTR + .setxattr = generic_setxattr, + .getxattr = generic_getxattr, + .listxattr = generic_listxattr, + .removexattr = generic_removexattr, +#endif +}; + +static struct super_operations shmem_ops = { + .alloc_inode = shmem_alloc_inode, + .destroy_inode = shmem_destroy_inode, +#ifdef CONFIG_TMPFS + .statfs = shmem_statfs, + .remount_fs = shmem_remount_fs, +#endif + .delete_inode = shmem_delete_inode, + .drop_inode = generic_delete_inode, + .put_super = shmem_put_super, +}; + +static struct vm_operations_struct shmem_vm_ops = { + .nopage = shmem_nopage, + .populate = shmem_populate, +#ifdef CONFIG_NUMA + .set_policy = shmem_set_policy, + .get_policy = shmem_get_policy, +#endif +}; + + +#ifdef CONFIG_TMPFS_SECURITY + +static size_t shmem_xattr_security_list(struct inode *inode, char *list, size_t list_len, + const char *name, size_t name_len) +{ + return security_inode_listsecurity(inode, list, list_len); +} + +static int shmem_xattr_security_get(struct inode *inode, const char *name, void *buffer, size_t size) +{ + if (strcmp(name, "") == 0) + return -EINVAL; + return security_inode_getsecurity(inode, name, buffer, size); +} + +static int shmem_xattr_security_set(struct inode *inode, const char *name, const void *value, size_t size, int flags) +{ + if (strcmp(name, "") == 0) + return -EINVAL; + return security_inode_setsecurity(inode, name, value, size, flags); +} + +static struct xattr_handler shmem_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .list = shmem_xattr_security_list, + .get = shmem_xattr_security_get, + .set = shmem_xattr_security_set, +}; + +#endif /* CONFIG_TMPFS_SECURITY */ + +#ifdef CONFIG_TMPFS_XATTR + +static struct xattr_handler *shmem_xattr_handlers[] = { +#ifdef CONFIG_TMPFS_SECURITY + &shmem_xattr_security_handler, +#endif + NULL +}; + +#endif /* CONFIG_TMPFS_XATTR */ + +static struct super_block *shmem_get_sb(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) +{ + return get_sb_nodev(fs_type, flags, data, shmem_fill_super); +} + +static struct file_system_type tmpfs_fs_type = { + .owner = THIS_MODULE, + .name = "tmpfs", + .get_sb = shmem_get_sb, + .kill_sb = kill_litter_super, +}; +static struct vfsmount *shm_mnt; + +static int __init init_tmpfs(void) +{ + int error; + + error = init_inodecache(); + if (error) + goto out3; + + error = register_filesystem(&tmpfs_fs_type); + if (error) { + printk(KERN_ERR "Could not register tmpfs\n"); + goto out2; + } +#ifdef CONFIG_TMPFS + devfs_mk_dir("shm"); +#endif + shm_mnt = do_kern_mount(tmpfs_fs_type.name, MS_NOUSER, + tmpfs_fs_type.name, NULL); + if (IS_ERR(shm_mnt)) { + error = PTR_ERR(shm_mnt); + printk(KERN_ERR "Could not kern_mount tmpfs\n"); + goto out1; + } + return 0; + +out1: + unregister_filesystem(&tmpfs_fs_type); +out2: + destroy_inodecache(); +out3: + shm_mnt = ERR_PTR(error); + return error; +} +module_init(init_tmpfs) + +/* + * shmem_file_setup - get an unlinked file living in tmpfs + * + * @name: name for dentry (to be seen in /proc/<pid>/maps + * @size: size to be set for the file + * + */ +struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags) +{ + int error; + struct file *file; + struct inode *inode; + struct dentry *dentry, *root; + struct qstr this; + + if (IS_ERR(shm_mnt)) + return (void *)shm_mnt; + + if (size < 0 || size > SHMEM_MAX_BYTES) + return ERR_PTR(-EINVAL); + + if (shmem_acct_size(flags, size)) + return ERR_PTR(-ENOMEM); + + error = -ENOMEM; + this.name = name; + this.len = strlen(name); + this.hash = 0; /* will go */ + root = shm_mnt->mnt_root; + dentry = d_alloc(root, &this); + if (!dentry) + goto put_memory; + + error = -ENFILE; + file = get_empty_filp(); + if (!file) + goto put_dentry; + + error = -ENOSPC; + inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0); + if (!inode) + goto close_file; + + SHMEM_I(inode)->flags = flags & VM_ACCOUNT; + d_instantiate(dentry, inode); + inode->i_size = size; + inode->i_nlink = 0; /* It is unlinked */ + file->f_vfsmnt = mntget(shm_mnt); + file->f_dentry = dentry; + file->f_mapping = inode->i_mapping; + file->f_op = &shmem_file_operations; + file->f_mode = FMODE_WRITE | FMODE_READ; + return file; + +close_file: + put_filp(file); +put_dentry: + dput(dentry); +put_memory: + shmem_unacct_size(flags, size); + return ERR_PTR(error); +} + +/* + * shmem_zero_setup - setup a shared anonymous mapping + * + * @vma: the vma to be mmapped is prepared by do_mmap_pgoff + */ +int shmem_zero_setup(struct vm_area_struct *vma) +{ + struct file *file; + loff_t size = vma->vm_end - vma->vm_start; + + file = shmem_file_setup("dev/zero", size, vma->vm_flags); + if (IS_ERR(file)) + return PTR_ERR(file); + + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = file; + vma->vm_ops = &shmem_vm_ops; + return 0; +} diff --git a/mm/slab.c b/mm/slab.c new file mode 100644 index 000000000000..ec660d85ddd7 --- /dev/null +++ b/mm/slab.c @@ -0,0 +1,3060 @@ +/* + * linux/mm/slab.c + * Written by Mark Hemment, 1996/97. + * (markhe@nextd.demon.co.uk) + * + * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli + * + * Major cleanup, different bufctl logic, per-cpu arrays + * (c) 2000 Manfred Spraul + * + * Cleanup, make the head arrays unconditional, preparation for NUMA + * (c) 2002 Manfred Spraul + * + * An implementation of the Slab Allocator as described in outline in; + * UNIX Internals: The New Frontiers by Uresh Vahalia + * Pub: Prentice Hall ISBN 0-13-101908-2 + * or with a little more detail in; + * The Slab Allocator: An Object-Caching Kernel Memory Allocator + * Jeff Bonwick (Sun Microsystems). + * Presented at: USENIX Summer 1994 Technical Conference + * + * The memory is organized in caches, one cache for each object type. + * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct) + * Each cache consists out of many slabs (they are small (usually one + * page long) and always contiguous), and each slab contains multiple + * initialized objects. + * + * This means, that your constructor is used only for newly allocated + * slabs and you must pass objects with the same intializations to + * kmem_cache_free. + * + * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM, + * normal). If you need a special memory type, then must create a new + * cache for that memory type. + * + * In order to reduce fragmentation, the slabs are sorted in 3 groups: + * full slabs with 0 free objects + * partial slabs + * empty slabs with no allocated objects + * + * If partial slabs exist, then new allocations come from these slabs, + * otherwise from empty slabs or new slabs are allocated. + * + * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache + * during kmem_cache_destroy(). The caller must prevent concurrent allocs. + * + * Each cache has a short per-cpu head array, most allocs + * and frees go into that array, and if that array overflows, then 1/2 + * of the entries in the array are given back into the global cache. + * The head array is strictly LIFO and should improve the cache hit rates. + * On SMP, it additionally reduces the spinlock operations. + * + * The c_cpuarray may not be read with enabled local interrupts - + * it's changed with a smp_call_function(). + * + * SMP synchronization: + * constructors and destructors are called without any locking. + * Several members in kmem_cache_t and struct slab never change, they + * are accessed without any locking. + * The per-cpu arrays are never accessed from the wrong cpu, no locking, + * and local interrupts are disabled so slab code is preempt-safe. + * The non-constant members are protected with a per-cache irq spinlock. + * + * Many thanks to Mark Hemment, who wrote another per-cpu slab patch + * in 2000 - many ideas in the current implementation are derived from + * his patch. + * + * Further notes from the original documentation: + * + * 11 April '97. Started multi-threading - markhe + * The global cache-chain is protected by the semaphore 'cache_chain_sem'. + * The sem is only needed when accessing/extending the cache-chain, which + * can never happen inside an interrupt (kmem_cache_create(), + * kmem_cache_shrink() and kmem_cache_reap()). + * + * At present, each engine can be growing a cache. This should be blocked. + * + */ + +#include <linux/config.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/cache.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/compiler.h> +#include <linux/seq_file.h> +#include <linux/notifier.h> +#include <linux/kallsyms.h> +#include <linux/cpu.h> +#include <linux/sysctl.h> +#include <linux/module.h> +#include <linux/rcupdate.h> + +#include <asm/uaccess.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/page.h> + +/* + * DEBUG - 1 for kmem_cache_create() to honour; SLAB_DEBUG_INITIAL, + * SLAB_RED_ZONE & SLAB_POISON. + * 0 for faster, smaller code (especially in the critical paths). + * + * STATS - 1 to collect stats for /proc/slabinfo. + * 0 for faster, smaller code (especially in the critical paths). + * + * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible) + */ + +#ifdef CONFIG_DEBUG_SLAB +#define DEBUG 1 +#define STATS 1 +#define FORCED_DEBUG 1 +#else +#define DEBUG 0 +#define STATS 0 +#define FORCED_DEBUG 0 +#endif + + +/* Shouldn't this be in a header file somewhere? */ +#define BYTES_PER_WORD sizeof(void *) + +#ifndef cache_line_size +#define cache_line_size() L1_CACHE_BYTES +#endif + +#ifndef ARCH_KMALLOC_MINALIGN +/* + * Enforce a minimum alignment for the kmalloc caches. + * Usually, the kmalloc caches are cache_line_size() aligned, except when + * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned. + * Some archs want to perform DMA into kmalloc caches and need a guaranteed + * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that. + * Note that this flag disables some debug features. + */ +#define ARCH_KMALLOC_MINALIGN 0 +#endif + +#ifndef ARCH_SLAB_MINALIGN +/* + * Enforce a minimum alignment for all caches. + * Intended for archs that get misalignment faults even for BYTES_PER_WORD + * aligned buffers. Includes ARCH_KMALLOC_MINALIGN. + * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables + * some debug features. + */ +#define ARCH_SLAB_MINALIGN 0 +#endif + +#ifndef ARCH_KMALLOC_FLAGS +#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN +#endif + +/* Legal flag mask for kmem_cache_create(). */ +#if DEBUG +# define CREATE_MASK (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \ + SLAB_POISON | SLAB_HWCACHE_ALIGN | \ + SLAB_NO_REAP | SLAB_CACHE_DMA | \ + SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \ + SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ + SLAB_DESTROY_BY_RCU) +#else +# define CREATE_MASK (SLAB_HWCACHE_ALIGN | SLAB_NO_REAP | \ + SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \ + SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ + SLAB_DESTROY_BY_RCU) +#endif + +/* + * kmem_bufctl_t: + * + * Bufctl's are used for linking objs within a slab + * linked offsets. + * + * This implementation relies on "struct page" for locating the cache & + * slab an object belongs to. + * This allows the bufctl structure to be small (one int), but limits + * the number of objects a slab (not a cache) can contain when off-slab + * bufctls are used. The limit is the size of the largest general cache + * that does not use off-slab slabs. + * For 32bit archs with 4 kB pages, is this 56. + * This is not serious, as it is only for large objects, when it is unwise + * to have too many per slab. + * Note: This limit can be raised by introducing a general cache whose size + * is less than 512 (PAGE_SIZE<<3), but greater than 256. + */ + +#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0) +#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1) +#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-2) + +/* Max number of objs-per-slab for caches which use off-slab slabs. + * Needed to avoid a possible looping condition in cache_grow(). + */ +static unsigned long offslab_limit; + +/* + * struct slab + * + * Manages the objs in a slab. Placed either at the beginning of mem allocated + * for a slab, or allocated from an general cache. + * Slabs are chained into three list: fully used, partial, fully free slabs. + */ +struct slab { + struct list_head list; + unsigned long colouroff; + void *s_mem; /* including colour offset */ + unsigned int inuse; /* num of objs active in slab */ + kmem_bufctl_t free; +}; + +/* + * struct slab_rcu + * + * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to + * arrange for kmem_freepages to be called via RCU. This is useful if + * we need to approach a kernel structure obliquely, from its address + * obtained without the usual locking. We can lock the structure to + * stabilize it and check it's still at the given address, only if we + * can be sure that the memory has not been meanwhile reused for some + * other kind of object (which our subsystem's lock might corrupt). + * + * rcu_read_lock before reading the address, then rcu_read_unlock after + * taking the spinlock within the structure expected at that address. + * + * We assume struct slab_rcu can overlay struct slab when destroying. + */ +struct slab_rcu { + struct rcu_head head; + kmem_cache_t *cachep; + void *addr; +}; + +/* + * struct array_cache + * + * Per cpu structures + * Purpose: + * - LIFO ordering, to hand out cache-warm objects from _alloc + * - reduce the number of linked list operations + * - reduce spinlock operations + * + * The limit is stored in the per-cpu structure to reduce the data cache + * footprint. + * + */ +struct array_cache { + unsigned int avail; + unsigned int limit; + unsigned int batchcount; + unsigned int touched; +}; + +/* bootstrap: The caches do not work without cpuarrays anymore, + * but the cpuarrays are allocated from the generic caches... + */ +#define BOOT_CPUCACHE_ENTRIES 1 +struct arraycache_init { + struct array_cache cache; + void * entries[BOOT_CPUCACHE_ENTRIES]; +}; + +/* + * The slab lists of all objects. + * Hopefully reduce the internal fragmentation + * NUMA: The spinlock could be moved from the kmem_cache_t + * into this structure, too. Figure out what causes + * fewer cross-node spinlock operations. + */ +struct kmem_list3 { + struct list_head slabs_partial; /* partial list first, better asm code */ + struct list_head slabs_full; + struct list_head slabs_free; + unsigned long free_objects; + int free_touched; + unsigned long next_reap; + struct array_cache *shared; +}; + +#define LIST3_INIT(parent) \ + { \ + .slabs_full = LIST_HEAD_INIT(parent.slabs_full), \ + .slabs_partial = LIST_HEAD_INIT(parent.slabs_partial), \ + .slabs_free = LIST_HEAD_INIT(parent.slabs_free) \ + } +#define list3_data(cachep) \ + (&(cachep)->lists) + +/* NUMA: per-node */ +#define list3_data_ptr(cachep, ptr) \ + list3_data(cachep) + +/* + * kmem_cache_t + * + * manages a cache. + */ + +struct kmem_cache_s { +/* 1) per-cpu data, touched during every alloc/free */ + struct array_cache *array[NR_CPUS]; + unsigned int batchcount; + unsigned int limit; +/* 2) touched by every alloc & free from the backend */ + struct kmem_list3 lists; + /* NUMA: kmem_3list_t *nodelists[MAX_NUMNODES] */ + unsigned int objsize; + unsigned int flags; /* constant flags */ + unsigned int num; /* # of objs per slab */ + unsigned int free_limit; /* upper limit of objects in the lists */ + spinlock_t spinlock; + +/* 3) cache_grow/shrink */ + /* order of pgs per slab (2^n) */ + unsigned int gfporder; + + /* force GFP flags, e.g. GFP_DMA */ + unsigned int gfpflags; + + size_t colour; /* cache colouring range */ + unsigned int colour_off; /* colour offset */ + unsigned int colour_next; /* cache colouring */ + kmem_cache_t *slabp_cache; + unsigned int slab_size; + unsigned int dflags; /* dynamic flags */ + + /* constructor func */ + void (*ctor)(void *, kmem_cache_t *, unsigned long); + + /* de-constructor func */ + void (*dtor)(void *, kmem_cache_t *, unsigned long); + +/* 4) cache creation/removal */ + const char *name; + struct list_head next; + +/* 5) statistics */ +#if STATS + unsigned long num_active; + unsigned long num_allocations; + unsigned long high_mark; + unsigned long grown; + unsigned long reaped; + unsigned long errors; + unsigned long max_freeable; + unsigned long node_allocs; + atomic_t allochit; + atomic_t allocmiss; + atomic_t freehit; + atomic_t freemiss; +#endif +#if DEBUG + int dbghead; + int reallen; +#endif +}; + +#define CFLGS_OFF_SLAB (0x80000000UL) +#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB) + +#define BATCHREFILL_LIMIT 16 +/* Optimization question: fewer reaps means less + * probability for unnessary cpucache drain/refill cycles. + * + * OTHO the cpuarrays can contain lots of objects, + * which could lock up otherwise freeable slabs. + */ +#define REAPTIMEOUT_CPUC (2*HZ) +#define REAPTIMEOUT_LIST3 (4*HZ) + +#if STATS +#define STATS_INC_ACTIVE(x) ((x)->num_active++) +#define STATS_DEC_ACTIVE(x) ((x)->num_active--) +#define STATS_INC_ALLOCED(x) ((x)->num_allocations++) +#define STATS_INC_GROWN(x) ((x)->grown++) +#define STATS_INC_REAPED(x) ((x)->reaped++) +#define STATS_SET_HIGH(x) do { if ((x)->num_active > (x)->high_mark) \ + (x)->high_mark = (x)->num_active; \ + } while (0) +#define STATS_INC_ERR(x) ((x)->errors++) +#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++) +#define STATS_SET_FREEABLE(x, i) \ + do { if ((x)->max_freeable < i) \ + (x)->max_freeable = i; \ + } while (0) + +#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit) +#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss) +#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit) +#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss) +#else +#define STATS_INC_ACTIVE(x) do { } while (0) +#define STATS_DEC_ACTIVE(x) do { } while (0) +#define STATS_INC_ALLOCED(x) do { } while (0) +#define STATS_INC_GROWN(x) do { } while (0) +#define STATS_INC_REAPED(x) do { } while (0) +#define STATS_SET_HIGH(x) do { } while (0) +#define STATS_INC_ERR(x) do { } while (0) +#define STATS_INC_NODEALLOCS(x) do { } while (0) +#define STATS_SET_FREEABLE(x, i) \ + do { } while (0) + +#define STATS_INC_ALLOCHIT(x) do { } while (0) +#define STATS_INC_ALLOCMISS(x) do { } while (0) +#define STATS_INC_FREEHIT(x) do { } while (0) +#define STATS_INC_FREEMISS(x) do { } while (0) +#endif + +#if DEBUG +/* Magic nums for obj red zoning. + * Placed in the first word before and the first word after an obj. + */ +#define RED_INACTIVE 0x5A2CF071UL /* when obj is inactive */ +#define RED_ACTIVE 0x170FC2A5UL /* when obj is active */ + +/* ...and for poisoning */ +#define POISON_INUSE 0x5a /* for use-uninitialised poisoning */ +#define POISON_FREE 0x6b /* for use-after-free poisoning */ +#define POISON_END 0xa5 /* end-byte of poisoning */ + +/* memory layout of objects: + * 0 : objp + * 0 .. cachep->dbghead - BYTES_PER_WORD - 1: padding. This ensures that + * the end of an object is aligned with the end of the real + * allocation. Catches writes behind the end of the allocation. + * cachep->dbghead - BYTES_PER_WORD .. cachep->dbghead - 1: + * redzone word. + * cachep->dbghead: The real object. + * cachep->objsize - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] + * cachep->objsize - 1* BYTES_PER_WORD: last caller address [BYTES_PER_WORD long] + */ +static int obj_dbghead(kmem_cache_t *cachep) +{ + return cachep->dbghead; +} + +static int obj_reallen(kmem_cache_t *cachep) +{ + return cachep->reallen; +} + +static unsigned long *dbg_redzone1(kmem_cache_t *cachep, void *objp) +{ + BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); + return (unsigned long*) (objp+obj_dbghead(cachep)-BYTES_PER_WORD); +} + +static unsigned long *dbg_redzone2(kmem_cache_t *cachep, void *objp) +{ + BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); + if (cachep->flags & SLAB_STORE_USER) + return (unsigned long*) (objp+cachep->objsize-2*BYTES_PER_WORD); + return (unsigned long*) (objp+cachep->objsize-BYTES_PER_WORD); +} + +static void **dbg_userword(kmem_cache_t *cachep, void *objp) +{ + BUG_ON(!(cachep->flags & SLAB_STORE_USER)); + return (void**)(objp+cachep->objsize-BYTES_PER_WORD); +} + +#else + +#define obj_dbghead(x) 0 +#define obj_reallen(cachep) (cachep->objsize) +#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long *)NULL;}) +#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long *)NULL;}) +#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) + +#endif + +/* + * Maximum size of an obj (in 2^order pages) + * and absolute limit for the gfp order. + */ +#if defined(CONFIG_LARGE_ALLOCS) +#define MAX_OBJ_ORDER 13 /* up to 32Mb */ +#define MAX_GFP_ORDER 13 /* up to 32Mb */ +#elif defined(CONFIG_MMU) +#define MAX_OBJ_ORDER 5 /* 32 pages */ +#define MAX_GFP_ORDER 5 /* 32 pages */ +#else +#define MAX_OBJ_ORDER 8 /* up to 1Mb */ +#define MAX_GFP_ORDER 8 /* up to 1Mb */ +#endif + +/* + * Do not go above this order unless 0 objects fit into the slab. + */ +#define BREAK_GFP_ORDER_HI 1 +#define BREAK_GFP_ORDER_LO 0 +static int slab_break_gfp_order = BREAK_GFP_ORDER_LO; + +/* Macros for storing/retrieving the cachep and or slab from the + * global 'mem_map'. These are used to find the slab an obj belongs to. + * With kfree(), these are used to find the cache which an obj belongs to. + */ +#define SET_PAGE_CACHE(pg,x) ((pg)->lru.next = (struct list_head *)(x)) +#define GET_PAGE_CACHE(pg) ((kmem_cache_t *)(pg)->lru.next) +#define SET_PAGE_SLAB(pg,x) ((pg)->lru.prev = (struct list_head *)(x)) +#define GET_PAGE_SLAB(pg) ((struct slab *)(pg)->lru.prev) + +/* These are the default caches for kmalloc. Custom caches can have other sizes. */ +struct cache_sizes malloc_sizes[] = { +#define CACHE(x) { .cs_size = (x) }, +#include <linux/kmalloc_sizes.h> + CACHE(ULONG_MAX) +#undef CACHE +}; +EXPORT_SYMBOL(malloc_sizes); + +/* Must match cache_sizes above. Out of line to keep cache footprint low. */ +struct cache_names { + char *name; + char *name_dma; +}; + +static struct cache_names __initdata cache_names[] = { +#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, +#include <linux/kmalloc_sizes.h> + { NULL, } +#undef CACHE +}; + +static struct arraycache_init initarray_cache __initdata = + { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; +static struct arraycache_init initarray_generic = + { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; + +/* internal cache of cache description objs */ +static kmem_cache_t cache_cache = { + .lists = LIST3_INIT(cache_cache.lists), + .batchcount = 1, + .limit = BOOT_CPUCACHE_ENTRIES, + .objsize = sizeof(kmem_cache_t), + .flags = SLAB_NO_REAP, + .spinlock = SPIN_LOCK_UNLOCKED, + .name = "kmem_cache", +#if DEBUG + .reallen = sizeof(kmem_cache_t), +#endif +}; + +/* Guard access to the cache-chain. */ +static struct semaphore cache_chain_sem; +static struct list_head cache_chain; + +/* + * vm_enough_memory() looks at this to determine how many + * slab-allocated pages are possibly freeable under pressure + * + * SLAB_RECLAIM_ACCOUNT turns this on per-slab + */ +atomic_t slab_reclaim_pages; +EXPORT_SYMBOL(slab_reclaim_pages); + +/* + * chicken and egg problem: delay the per-cpu array allocation + * until the general caches are up. + */ +static enum { + NONE, + PARTIAL, + FULL +} g_cpucache_up; + +static DEFINE_PER_CPU(struct work_struct, reap_work); + +static void free_block(kmem_cache_t* cachep, void** objpp, int len); +static void enable_cpucache (kmem_cache_t *cachep); +static void cache_reap (void *unused); + +static inline void **ac_entry(struct array_cache *ac) +{ + return (void**)(ac+1); +} + +static inline struct array_cache *ac_data(kmem_cache_t *cachep) +{ + return cachep->array[smp_processor_id()]; +} + +static inline kmem_cache_t *kmem_find_general_cachep(size_t size, int gfpflags) +{ + struct cache_sizes *csizep = malloc_sizes; + +#if DEBUG + /* This happens if someone tries to call + * kmem_cache_create(), or __kmalloc(), before + * the generic caches are initialized. + */ + BUG_ON(csizep->cs_cachep == NULL); +#endif + while (size > csizep->cs_size) + csizep++; + + /* + * Really subtile: The last entry with cs->cs_size==ULONG_MAX + * has cs_{dma,}cachep==NULL. Thus no special case + * for large kmalloc calls required. + */ + if (unlikely(gfpflags & GFP_DMA)) + return csizep->cs_dmacachep; + return csizep->cs_cachep; +} + +/* Cal the num objs, wastage, and bytes left over for a given slab size. */ +static void cache_estimate(unsigned long gfporder, size_t size, size_t align, + int flags, size_t *left_over, unsigned int *num) +{ + int i; + size_t wastage = PAGE_SIZE<<gfporder; + size_t extra = 0; + size_t base = 0; + + if (!(flags & CFLGS_OFF_SLAB)) { + base = sizeof(struct slab); + extra = sizeof(kmem_bufctl_t); + } + i = 0; + while (i*size + ALIGN(base+i*extra, align) <= wastage) + i++; + if (i > 0) + i--; + + if (i > SLAB_LIMIT) + i = SLAB_LIMIT; + + *num = i; + wastage -= i*size; + wastage -= ALIGN(base+i*extra, align); + *left_over = wastage; +} + +#define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg) + +static void __slab_error(const char *function, kmem_cache_t *cachep, char *msg) +{ + printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", + function, cachep->name, msg); + dump_stack(); +} + +/* + * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz + * via the workqueue/eventd. + * Add the CPU number into the expiration time to minimize the possibility of + * the CPUs getting into lockstep and contending for the global cache chain + * lock. + */ +static void __devinit start_cpu_timer(int cpu) +{ + struct work_struct *reap_work = &per_cpu(reap_work, cpu); + + /* + * When this gets called from do_initcalls via cpucache_init(), + * init_workqueues() has already run, so keventd will be setup + * at that time. + */ + if (keventd_up() && reap_work->func == NULL) { + INIT_WORK(reap_work, cache_reap, NULL); + schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu); + } +} + +static struct array_cache *alloc_arraycache(int cpu, int entries, + int batchcount) +{ + int memsize = sizeof(void*)*entries+sizeof(struct array_cache); + struct array_cache *nc = NULL; + + if (cpu != -1) { + kmem_cache_t *cachep; + cachep = kmem_find_general_cachep(memsize, GFP_KERNEL); + if (cachep) + nc = kmem_cache_alloc_node(cachep, cpu_to_node(cpu)); + } + if (!nc) + nc = kmalloc(memsize, GFP_KERNEL); + if (nc) { + nc->avail = 0; + nc->limit = entries; + nc->batchcount = batchcount; + nc->touched = 0; + } + return nc; +} + +static int __devinit cpuup_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + long cpu = (long)hcpu; + kmem_cache_t* cachep; + + switch (action) { + case CPU_UP_PREPARE: + down(&cache_chain_sem); + list_for_each_entry(cachep, &cache_chain, next) { + struct array_cache *nc; + + nc = alloc_arraycache(cpu, cachep->limit, cachep->batchcount); + if (!nc) + goto bad; + + spin_lock_irq(&cachep->spinlock); + cachep->array[cpu] = nc; + cachep->free_limit = (1+num_online_cpus())*cachep->batchcount + + cachep->num; + spin_unlock_irq(&cachep->spinlock); + + } + up(&cache_chain_sem); + break; + case CPU_ONLINE: + start_cpu_timer(cpu); + break; +#ifdef CONFIG_HOTPLUG_CPU + case CPU_DEAD: + /* fall thru */ + case CPU_UP_CANCELED: + down(&cache_chain_sem); + + list_for_each_entry(cachep, &cache_chain, next) { + struct array_cache *nc; + + spin_lock_irq(&cachep->spinlock); + /* cpu is dead; no one can alloc from it. */ + nc = cachep->array[cpu]; + cachep->array[cpu] = NULL; + cachep->free_limit -= cachep->batchcount; + free_block(cachep, ac_entry(nc), nc->avail); + spin_unlock_irq(&cachep->spinlock); + kfree(nc); + } + up(&cache_chain_sem); + break; +#endif + } + return NOTIFY_OK; +bad: + up(&cache_chain_sem); + return NOTIFY_BAD; +} + +static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 }; + +/* Initialisation. + * Called after the gfp() functions have been enabled, and before smp_init(). + */ +void __init kmem_cache_init(void) +{ + size_t left_over; + struct cache_sizes *sizes; + struct cache_names *names; + + /* + * Fragmentation resistance on low memory - only use bigger + * page orders on machines with more than 32MB of memory. + */ + if (num_physpages > (32 << 20) >> PAGE_SHIFT) + slab_break_gfp_order = BREAK_GFP_ORDER_HI; + + + /* Bootstrap is tricky, because several objects are allocated + * from caches that do not exist yet: + * 1) initialize the cache_cache cache: it contains the kmem_cache_t + * structures of all caches, except cache_cache itself: cache_cache + * is statically allocated. + * Initially an __init data area is used for the head array, it's + * replaced with a kmalloc allocated array at the end of the bootstrap. + * 2) Create the first kmalloc cache. + * The kmem_cache_t for the new cache is allocated normally. An __init + * data area is used for the head array. + * 3) Create the remaining kmalloc caches, with minimally sized head arrays. + * 4) Replace the __init data head arrays for cache_cache and the first + * kmalloc cache with kmalloc allocated arrays. + * 5) Resize the head arrays of the kmalloc caches to their final sizes. + */ + + /* 1) create the cache_cache */ + init_MUTEX(&cache_chain_sem); + INIT_LIST_HEAD(&cache_chain); + list_add(&cache_cache.next, &cache_chain); + cache_cache.colour_off = cache_line_size(); + cache_cache.array[smp_processor_id()] = &initarray_cache.cache; + + cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size()); + + cache_estimate(0, cache_cache.objsize, cache_line_size(), 0, + &left_over, &cache_cache.num); + if (!cache_cache.num) + BUG(); + + cache_cache.colour = left_over/cache_cache.colour_off; + cache_cache.colour_next = 0; + cache_cache.slab_size = ALIGN(cache_cache.num*sizeof(kmem_bufctl_t) + + sizeof(struct slab), cache_line_size()); + + /* 2+3) create the kmalloc caches */ + sizes = malloc_sizes; + names = cache_names; + + while (sizes->cs_size != ULONG_MAX) { + /* For performance, all the general caches are L1 aligned. + * This should be particularly beneficial on SMP boxes, as it + * eliminates "false sharing". + * Note for systems short on memory removing the alignment will + * allow tighter packing of the smaller caches. */ + sizes->cs_cachep = kmem_cache_create(names->name, + sizes->cs_size, ARCH_KMALLOC_MINALIGN, + (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); + + /* Inc off-slab bufctl limit until the ceiling is hit. */ + if (!(OFF_SLAB(sizes->cs_cachep))) { + offslab_limit = sizes->cs_size-sizeof(struct slab); + offslab_limit /= sizeof(kmem_bufctl_t); + } + + sizes->cs_dmacachep = kmem_cache_create(names->name_dma, + sizes->cs_size, ARCH_KMALLOC_MINALIGN, + (ARCH_KMALLOC_FLAGS | SLAB_CACHE_DMA | SLAB_PANIC), + NULL, NULL); + + sizes++; + names++; + } + /* 4) Replace the bootstrap head arrays */ + { + void * ptr; + + ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); + local_irq_disable(); + BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache); + memcpy(ptr, ac_data(&cache_cache), sizeof(struct arraycache_init)); + cache_cache.array[smp_processor_id()] = ptr; + local_irq_enable(); + + ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); + local_irq_disable(); + BUG_ON(ac_data(malloc_sizes[0].cs_cachep) != &initarray_generic.cache); + memcpy(ptr, ac_data(malloc_sizes[0].cs_cachep), + sizeof(struct arraycache_init)); + malloc_sizes[0].cs_cachep->array[smp_processor_id()] = ptr; + local_irq_enable(); + } + + /* 5) resize the head arrays to their final sizes */ + { + kmem_cache_t *cachep; + down(&cache_chain_sem); + list_for_each_entry(cachep, &cache_chain, next) + enable_cpucache(cachep); + up(&cache_chain_sem); + } + + /* Done! */ + g_cpucache_up = FULL; + + /* Register a cpu startup notifier callback + * that initializes ac_data for all new cpus + */ + register_cpu_notifier(&cpucache_notifier); + + + /* The reap timers are started later, with a module init call: + * That part of the kernel is not yet operational. + */ +} + +static int __init cpucache_init(void) +{ + int cpu; + + /* + * Register the timers that return unneeded + * pages to gfp. + */ + for (cpu = 0; cpu < NR_CPUS; cpu++) { + if (cpu_online(cpu)) + start_cpu_timer(cpu); + } + + return 0; +} + +__initcall(cpucache_init); + +/* + * Interface to system's page allocator. No need to hold the cache-lock. + * + * If we requested dmaable memory, we will get it. Even if we + * did not request dmaable memory, we might get it, but that + * would be relatively rare and ignorable. + */ +static void *kmem_getpages(kmem_cache_t *cachep, unsigned int __nocast flags, int nodeid) +{ + struct page *page; + void *addr; + int i; + + flags |= cachep->gfpflags; + if (likely(nodeid == -1)) { + page = alloc_pages(flags, cachep->gfporder); + } else { + page = alloc_pages_node(nodeid, flags, cachep->gfporder); + } + if (!page) + return NULL; + addr = page_address(page); + + i = (1 << cachep->gfporder); + if (cachep->flags & SLAB_RECLAIM_ACCOUNT) + atomic_add(i, &slab_reclaim_pages); + add_page_state(nr_slab, i); + while (i--) { + SetPageSlab(page); + page++; + } + return addr; +} + +/* + * Interface to system's page release. + */ +static void kmem_freepages(kmem_cache_t *cachep, void *addr) +{ + unsigned long i = (1<<cachep->gfporder); + struct page *page = virt_to_page(addr); + const unsigned long nr_freed = i; + + while (i--) { + if (!TestClearPageSlab(page)) + BUG(); + page++; + } + sub_page_state(nr_slab, nr_freed); + if (current->reclaim_state) + current->reclaim_state->reclaimed_slab += nr_freed; + free_pages((unsigned long)addr, cachep->gfporder); + if (cachep->flags & SLAB_RECLAIM_ACCOUNT) + atomic_sub(1<<cachep->gfporder, &slab_reclaim_pages); +} + +static void kmem_rcu_free(struct rcu_head *head) +{ + struct slab_rcu *slab_rcu = (struct slab_rcu *) head; + kmem_cache_t *cachep = slab_rcu->cachep; + + kmem_freepages(cachep, slab_rcu->addr); + if (OFF_SLAB(cachep)) + kmem_cache_free(cachep->slabp_cache, slab_rcu); +} + +#if DEBUG + +#ifdef CONFIG_DEBUG_PAGEALLOC +static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, + unsigned long caller) +{ + int size = obj_reallen(cachep); + + addr = (unsigned long *)&((char*)addr)[obj_dbghead(cachep)]; + + if (size < 5*sizeof(unsigned long)) + return; + + *addr++=0x12345678; + *addr++=caller; + *addr++=smp_processor_id(); + size -= 3*sizeof(unsigned long); + { + unsigned long *sptr = &caller; + unsigned long svalue; + + while (!kstack_end(sptr)) { + svalue = *sptr++; + if (kernel_text_address(svalue)) { + *addr++=svalue; + size -= sizeof(unsigned long); + if (size <= sizeof(unsigned long)) + break; + } + } + + } + *addr++=0x87654321; +} +#endif + +static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val) +{ + int size = obj_reallen(cachep); + addr = &((char*)addr)[obj_dbghead(cachep)]; + + memset(addr, val, size); + *(unsigned char *)(addr+size-1) = POISON_END; +} + +static void dump_line(char *data, int offset, int limit) +{ + int i; + printk(KERN_ERR "%03x:", offset); + for (i=0;i<limit;i++) { + printk(" %02x", (unsigned char)data[offset+i]); + } + printk("\n"); +} +#endif + +#if DEBUG + +static void print_objinfo(kmem_cache_t *cachep, void *objp, int lines) +{ + int i, size; + char *realobj; + + if (cachep->flags & SLAB_RED_ZONE) { + printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n", + *dbg_redzone1(cachep, objp), + *dbg_redzone2(cachep, objp)); + } + + if (cachep->flags & SLAB_STORE_USER) { + printk(KERN_ERR "Last user: [<%p>]", + *dbg_userword(cachep, objp)); + print_symbol("(%s)", + (unsigned long)*dbg_userword(cachep, objp)); + printk("\n"); + } + realobj = (char*)objp+obj_dbghead(cachep); + size = obj_reallen(cachep); + for (i=0; i<size && lines;i+=16, lines--) { + int limit; + limit = 16; + if (i+limit > size) + limit = size-i; + dump_line(realobj, i, limit); + } +} + +static void check_poison_obj(kmem_cache_t *cachep, void *objp) +{ + char *realobj; + int size, i; + int lines = 0; + + realobj = (char*)objp+obj_dbghead(cachep); + size = obj_reallen(cachep); + + for (i=0;i<size;i++) { + char exp = POISON_FREE; + if (i == size-1) + exp = POISON_END; + if (realobj[i] != exp) { + int limit; + /* Mismatch ! */ + /* Print header */ + if (lines == 0) { + printk(KERN_ERR "Slab corruption: start=%p, len=%d\n", + realobj, size); + print_objinfo(cachep, objp, 0); + } + /* Hexdump the affected line */ + i = (i/16)*16; + limit = 16; + if (i+limit > size) + limit = size-i; + dump_line(realobj, i, limit); + i += 16; + lines++; + /* Limit to 5 lines */ + if (lines > 5) + break; + } + } + if (lines != 0) { + /* Print some data about the neighboring objects, if they + * exist: + */ + struct slab *slabp = GET_PAGE_SLAB(virt_to_page(objp)); + int objnr; + + objnr = (objp-slabp->s_mem)/cachep->objsize; + if (objnr) { + objp = slabp->s_mem+(objnr-1)*cachep->objsize; + realobj = (char*)objp+obj_dbghead(cachep); + printk(KERN_ERR "Prev obj: start=%p, len=%d\n", + realobj, size); + print_objinfo(cachep, objp, 2); + } + if (objnr+1 < cachep->num) { + objp = slabp->s_mem+(objnr+1)*cachep->objsize; + realobj = (char*)objp+obj_dbghead(cachep); + printk(KERN_ERR "Next obj: start=%p, len=%d\n", + realobj, size); + print_objinfo(cachep, objp, 2); + } + } +} +#endif + +/* Destroy all the objs in a slab, and release the mem back to the system. + * Before calling the slab must have been unlinked from the cache. + * The cache-lock is not held/needed. + */ +static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) +{ + void *addr = slabp->s_mem - slabp->colouroff; + +#if DEBUG + int i; + for (i = 0; i < cachep->num; i++) { + void *objp = slabp->s_mem + cachep->objsize * i; + + if (cachep->flags & SLAB_POISON) { +#ifdef CONFIG_DEBUG_PAGEALLOC + if ((cachep->objsize%PAGE_SIZE)==0 && OFF_SLAB(cachep)) + kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE,1); + else + check_poison_obj(cachep, objp); +#else + check_poison_obj(cachep, objp); +#endif + } + if (cachep->flags & SLAB_RED_ZONE) { + if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) + slab_error(cachep, "start of a freed object " + "was overwritten"); + if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) + slab_error(cachep, "end of a freed object " + "was overwritten"); + } + if (cachep->dtor && !(cachep->flags & SLAB_POISON)) + (cachep->dtor)(objp+obj_dbghead(cachep), cachep, 0); + } +#else + if (cachep->dtor) { + int i; + for (i = 0; i < cachep->num; i++) { + void* objp = slabp->s_mem+cachep->objsize*i; + (cachep->dtor)(objp, cachep, 0); + } + } +#endif + + if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { + struct slab_rcu *slab_rcu; + + slab_rcu = (struct slab_rcu *) slabp; + slab_rcu->cachep = cachep; + slab_rcu->addr = addr; + call_rcu(&slab_rcu->head, kmem_rcu_free); + } else { + kmem_freepages(cachep, addr); + if (OFF_SLAB(cachep)) + kmem_cache_free(cachep->slabp_cache, slabp); + } +} + +/** + * kmem_cache_create - Create a cache. + * @name: A string which is used in /proc/slabinfo to identify this cache. + * @size: The size of objects to be created in this cache. + * @align: The required alignment for the objects. + * @flags: SLAB flags + * @ctor: A constructor for the objects. + * @dtor: A destructor for the objects. + * + * Returns a ptr to the cache on success, NULL on failure. + * Cannot be called within a int, but can be interrupted. + * The @ctor is run when new pages are allocated by the cache + * and the @dtor is run before the pages are handed back. + * + * @name must be valid until the cache is destroyed. This implies that + * the module calling this has to destroy the cache before getting + * unloaded. + * + * The flags are + * + * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) + * to catch references to uninitialised memory. + * + * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check + * for buffer overruns. + * + * %SLAB_NO_REAP - Don't automatically reap this cache when we're under + * memory pressure. + * + * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware + * cacheline. This can be beneficial if you're counting cycles as closely + * as davem. + */ +kmem_cache_t * +kmem_cache_create (const char *name, size_t size, size_t align, + unsigned long flags, void (*ctor)(void*, kmem_cache_t *, unsigned long), + void (*dtor)(void*, kmem_cache_t *, unsigned long)) +{ + size_t left_over, slab_size, ralign; + kmem_cache_t *cachep = NULL; + + /* + * Sanity checks... these are all serious usage bugs. + */ + if ((!name) || + in_interrupt() || + (size < BYTES_PER_WORD) || + (size > (1<<MAX_OBJ_ORDER)*PAGE_SIZE) || + (dtor && !ctor)) { + printk(KERN_ERR "%s: Early error in slab %s\n", + __FUNCTION__, name); + BUG(); + } + +#if DEBUG + WARN_ON(strchr(name, ' ')); /* It confuses parsers */ + if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { + /* No constructor, but inital state check requested */ + printk(KERN_ERR "%s: No con, but init state check " + "requested - %s\n", __FUNCTION__, name); + flags &= ~SLAB_DEBUG_INITIAL; + } + +#if FORCED_DEBUG + /* + * Enable redzoning and last user accounting, except for caches with + * large objects, if the increased size would increase the object size + * above the next power of two: caches with object sizes just above a + * power of two have a significant amount of internal fragmentation. + */ + if ((size < 4096 || fls(size-1) == fls(size-1+3*BYTES_PER_WORD))) + flags |= SLAB_RED_ZONE|SLAB_STORE_USER; + if (!(flags & SLAB_DESTROY_BY_RCU)) + flags |= SLAB_POISON; +#endif + if (flags & SLAB_DESTROY_BY_RCU) + BUG_ON(flags & SLAB_POISON); +#endif + if (flags & SLAB_DESTROY_BY_RCU) + BUG_ON(dtor); + + /* + * Always checks flags, a caller might be expecting debug + * support which isn't available. + */ + if (flags & ~CREATE_MASK) + BUG(); + + /* Check that size is in terms of words. This is needed to avoid + * unaligned accesses for some archs when redzoning is used, and makes + * sure any on-slab bufctl's are also correctly aligned. + */ + if (size & (BYTES_PER_WORD-1)) { + size += (BYTES_PER_WORD-1); + size &= ~(BYTES_PER_WORD-1); + } + + /* calculate out the final buffer alignment: */ + /* 1) arch recommendation: can be overridden for debug */ + if (flags & SLAB_HWCACHE_ALIGN) { + /* Default alignment: as specified by the arch code. + * Except if an object is really small, then squeeze multiple + * objects into one cacheline. + */ + ralign = cache_line_size(); + while (size <= ralign/2) + ralign /= 2; + } else { + ralign = BYTES_PER_WORD; + } + /* 2) arch mandated alignment: disables debug if necessary */ + if (ralign < ARCH_SLAB_MINALIGN) { + ralign = ARCH_SLAB_MINALIGN; + if (ralign > BYTES_PER_WORD) + flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); + } + /* 3) caller mandated alignment: disables debug if necessary */ + if (ralign < align) { + ralign = align; + if (ralign > BYTES_PER_WORD) + flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); + } + /* 4) Store it. Note that the debug code below can reduce + * the alignment to BYTES_PER_WORD. + */ + align = ralign; + + /* Get cache's description obj. */ + cachep = (kmem_cache_t *) kmem_cache_alloc(&cache_cache, SLAB_KERNEL); + if (!cachep) + goto opps; + memset(cachep, 0, sizeof(kmem_cache_t)); + +#if DEBUG + cachep->reallen = size; + + if (flags & SLAB_RED_ZONE) { + /* redzoning only works with word aligned caches */ + align = BYTES_PER_WORD; + + /* add space for red zone words */ + cachep->dbghead += BYTES_PER_WORD; + size += 2*BYTES_PER_WORD; + } + if (flags & SLAB_STORE_USER) { + /* user store requires word alignment and + * one word storage behind the end of the real + * object. + */ + align = BYTES_PER_WORD; + size += BYTES_PER_WORD; + } +#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) + if (size > 128 && cachep->reallen > cache_line_size() && size < PAGE_SIZE) { + cachep->dbghead += PAGE_SIZE - size; + size = PAGE_SIZE; + } +#endif +#endif + + /* Determine if the slab management is 'on' or 'off' slab. */ + if (size >= (PAGE_SIZE>>3)) + /* + * Size is large, assume best to place the slab management obj + * off-slab (should allow better packing of objs). + */ + flags |= CFLGS_OFF_SLAB; + + size = ALIGN(size, align); + + if ((flags & SLAB_RECLAIM_ACCOUNT) && size <= PAGE_SIZE) { + /* + * A VFS-reclaimable slab tends to have most allocations + * as GFP_NOFS and we really don't want to have to be allocating + * higher-order pages when we are unable to shrink dcache. + */ + cachep->gfporder = 0; + cache_estimate(cachep->gfporder, size, align, flags, + &left_over, &cachep->num); + } else { + /* + * Calculate size (in pages) of slabs, and the num of objs per + * slab. This could be made much more intelligent. For now, + * try to avoid using high page-orders for slabs. When the + * gfp() funcs are more friendly towards high-order requests, + * this should be changed. + */ + do { + unsigned int break_flag = 0; +cal_wastage: + cache_estimate(cachep->gfporder, size, align, flags, + &left_over, &cachep->num); + if (break_flag) + break; + if (cachep->gfporder >= MAX_GFP_ORDER) + break; + if (!cachep->num) + goto next; + if (flags & CFLGS_OFF_SLAB && + cachep->num > offslab_limit) { + /* This num of objs will cause problems. */ + cachep->gfporder--; + break_flag++; + goto cal_wastage; + } + + /* + * Large num of objs is good, but v. large slabs are + * currently bad for the gfp()s. + */ + if (cachep->gfporder >= slab_break_gfp_order) + break; + + if ((left_over*8) <= (PAGE_SIZE<<cachep->gfporder)) + break; /* Acceptable internal fragmentation. */ +next: + cachep->gfporder++; + } while (1); + } + + if (!cachep->num) { + printk("kmem_cache_create: couldn't create cache %s.\n", name); + kmem_cache_free(&cache_cache, cachep); + cachep = NULL; + goto opps; + } + slab_size = ALIGN(cachep->num*sizeof(kmem_bufctl_t) + + sizeof(struct slab), align); + + /* + * If the slab has been placed off-slab, and we have enough space then + * move it on-slab. This is at the expense of any extra colouring. + */ + if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) { + flags &= ~CFLGS_OFF_SLAB; + left_over -= slab_size; + } + + if (flags & CFLGS_OFF_SLAB) { + /* really off slab. No need for manual alignment */ + slab_size = cachep->num*sizeof(kmem_bufctl_t)+sizeof(struct slab); + } + + cachep->colour_off = cache_line_size(); + /* Offset must be a multiple of the alignment. */ + if (cachep->colour_off < align) + cachep->colour_off = align; + cachep->colour = left_over/cachep->colour_off; + cachep->slab_size = slab_size; + cachep->flags = flags; + cachep->gfpflags = 0; + if (flags & SLAB_CACHE_DMA) + cachep->gfpflags |= GFP_DMA; + spin_lock_init(&cachep->spinlock); + cachep->objsize = size; + /* NUMA */ + INIT_LIST_HEAD(&cachep->lists.slabs_full); + INIT_LIST_HEAD(&cachep->lists.slabs_partial); + INIT_LIST_HEAD(&cachep->lists.slabs_free); + + if (flags & CFLGS_OFF_SLAB) + cachep->slabp_cache = kmem_find_general_cachep(slab_size,0); + cachep->ctor = ctor; + cachep->dtor = dtor; + cachep->name = name; + + /* Don't let CPUs to come and go */ + lock_cpu_hotplug(); + + if (g_cpucache_up == FULL) { + enable_cpucache(cachep); + } else { + if (g_cpucache_up == NONE) { + /* Note: the first kmem_cache_create must create + * the cache that's used by kmalloc(24), otherwise + * the creation of further caches will BUG(). + */ + cachep->array[smp_processor_id()] = &initarray_generic.cache; + g_cpucache_up = PARTIAL; + } else { + cachep->array[smp_processor_id()] = kmalloc(sizeof(struct arraycache_init),GFP_KERNEL); + } + BUG_ON(!ac_data(cachep)); + ac_data(cachep)->avail = 0; + ac_data(cachep)->limit = BOOT_CPUCACHE_ENTRIES; + ac_data(cachep)->batchcount = 1; + ac_data(cachep)->touched = 0; + cachep->batchcount = 1; + cachep->limit = BOOT_CPUCACHE_ENTRIES; + cachep->free_limit = (1+num_online_cpus())*cachep->batchcount + + cachep->num; + } + + cachep->lists.next_reap = jiffies + REAPTIMEOUT_LIST3 + + ((unsigned long)cachep)%REAPTIMEOUT_LIST3; + + /* Need the semaphore to access the chain. */ + down(&cache_chain_sem); + { + struct list_head *p; + mm_segment_t old_fs; + + old_fs = get_fs(); + set_fs(KERNEL_DS); + list_for_each(p, &cache_chain) { + kmem_cache_t *pc = list_entry(p, kmem_cache_t, next); + char tmp; + /* This happens when the module gets unloaded and doesn't + destroy its slab cache and noone else reuses the vmalloc + area of the module. Print a warning. */ + if (__get_user(tmp,pc->name)) { + printk("SLAB: cache with size %d has lost its name\n", + pc->objsize); + continue; + } + if (!strcmp(pc->name,name)) { + printk("kmem_cache_create: duplicate cache %s\n",name); + up(&cache_chain_sem); + unlock_cpu_hotplug(); + BUG(); + } + } + set_fs(old_fs); + } + + /* cache setup completed, link it into the list */ + list_add(&cachep->next, &cache_chain); + up(&cache_chain_sem); + unlock_cpu_hotplug(); +opps: + if (!cachep && (flags & SLAB_PANIC)) + panic("kmem_cache_create(): failed to create slab `%s'\n", + name); + return cachep; +} +EXPORT_SYMBOL(kmem_cache_create); + +#if DEBUG +static void check_irq_off(void) +{ + BUG_ON(!irqs_disabled()); +} + +static void check_irq_on(void) +{ + BUG_ON(irqs_disabled()); +} + +static void check_spinlock_acquired(kmem_cache_t *cachep) +{ +#ifdef CONFIG_SMP + check_irq_off(); + BUG_ON(spin_trylock(&cachep->spinlock)); +#endif +} +#else +#define check_irq_off() do { } while(0) +#define check_irq_on() do { } while(0) +#define check_spinlock_acquired(x) do { } while(0) +#endif + +/* + * Waits for all CPUs to execute func(). + */ +static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg) +{ + check_irq_on(); + preempt_disable(); + + local_irq_disable(); + func(arg); + local_irq_enable(); + + if (smp_call_function(func, arg, 1, 1)) + BUG(); + + preempt_enable(); +} + +static void drain_array_locked(kmem_cache_t* cachep, + struct array_cache *ac, int force); + +static void do_drain(void *arg) +{ + kmem_cache_t *cachep = (kmem_cache_t*)arg; + struct array_cache *ac; + + check_irq_off(); + ac = ac_data(cachep); + spin_lock(&cachep->spinlock); + free_block(cachep, &ac_entry(ac)[0], ac->avail); + spin_unlock(&cachep->spinlock); + ac->avail = 0; +} + +static void drain_cpu_caches(kmem_cache_t *cachep) +{ + smp_call_function_all_cpus(do_drain, cachep); + check_irq_on(); + spin_lock_irq(&cachep->spinlock); + if (cachep->lists.shared) + drain_array_locked(cachep, cachep->lists.shared, 1); + spin_unlock_irq(&cachep->spinlock); +} + + +/* NUMA shrink all list3s */ +static int __cache_shrink(kmem_cache_t *cachep) +{ + struct slab *slabp; + int ret; + + drain_cpu_caches(cachep); + + check_irq_on(); + spin_lock_irq(&cachep->spinlock); + + for(;;) { + struct list_head *p; + + p = cachep->lists.slabs_free.prev; + if (p == &cachep->lists.slabs_free) + break; + + slabp = list_entry(cachep->lists.slabs_free.prev, struct slab, list); +#if DEBUG + if (slabp->inuse) + BUG(); +#endif + list_del(&slabp->list); + + cachep->lists.free_objects -= cachep->num; + spin_unlock_irq(&cachep->spinlock); + slab_destroy(cachep, slabp); + spin_lock_irq(&cachep->spinlock); + } + ret = !list_empty(&cachep->lists.slabs_full) || + !list_empty(&cachep->lists.slabs_partial); + spin_unlock_irq(&cachep->spinlock); + return ret; +} + +/** + * kmem_cache_shrink - Shrink a cache. + * @cachep: The cache to shrink. + * + * Releases as many slabs as possible for a cache. + * To help debugging, a zero exit status indicates all slabs were released. + */ +int kmem_cache_shrink(kmem_cache_t *cachep) +{ + if (!cachep || in_interrupt()) + BUG(); + + return __cache_shrink(cachep); +} +EXPORT_SYMBOL(kmem_cache_shrink); + +/** + * kmem_cache_destroy - delete a cache + * @cachep: the cache to destroy + * + * Remove a kmem_cache_t object from the slab cache. + * Returns 0 on success. + * + * It is expected this function will be called by a module when it is + * unloaded. This will remove the cache completely, and avoid a duplicate + * cache being allocated each time a module is loaded and unloaded, if the + * module doesn't have persistent in-kernel storage across loads and unloads. + * + * The cache must be empty before calling this function. + * + * The caller must guarantee that noone will allocate memory from the cache + * during the kmem_cache_destroy(). + */ +int kmem_cache_destroy(kmem_cache_t * cachep) +{ + int i; + + if (!cachep || in_interrupt()) + BUG(); + + /* Don't let CPUs to come and go */ + lock_cpu_hotplug(); + + /* Find the cache in the chain of caches. */ + down(&cache_chain_sem); + /* + * the chain is never empty, cache_cache is never destroyed + */ + list_del(&cachep->next); + up(&cache_chain_sem); + + if (__cache_shrink(cachep)) { + slab_error(cachep, "Can't free all objects"); + down(&cache_chain_sem); + list_add(&cachep->next,&cache_chain); + up(&cache_chain_sem); + unlock_cpu_hotplug(); + return 1; + } + + if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) + synchronize_kernel(); + + /* no cpu_online check required here since we clear the percpu + * array on cpu offline and set this to NULL. + */ + for (i = 0; i < NR_CPUS; i++) + kfree(cachep->array[i]); + + /* NUMA: free the list3 structures */ + kfree(cachep->lists.shared); + cachep->lists.shared = NULL; + kmem_cache_free(&cache_cache, cachep); + + unlock_cpu_hotplug(); + + return 0; +} +EXPORT_SYMBOL(kmem_cache_destroy); + +/* Get the memory for a slab management obj. */ +static struct slab* alloc_slabmgmt(kmem_cache_t *cachep, + void *objp, int colour_off, unsigned int __nocast local_flags) +{ + struct slab *slabp; + + if (OFF_SLAB(cachep)) { + /* Slab management obj is off-slab. */ + slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags); + if (!slabp) + return NULL; + } else { + slabp = objp+colour_off; + colour_off += cachep->slab_size; + } + slabp->inuse = 0; + slabp->colouroff = colour_off; + slabp->s_mem = objp+colour_off; + + return slabp; +} + +static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) +{ + return (kmem_bufctl_t *)(slabp+1); +} + +static void cache_init_objs(kmem_cache_t *cachep, + struct slab *slabp, unsigned long ctor_flags) +{ + int i; + + for (i = 0; i < cachep->num; i++) { + void* objp = slabp->s_mem+cachep->objsize*i; +#if DEBUG + /* need to poison the objs? */ + if (cachep->flags & SLAB_POISON) + poison_obj(cachep, objp, POISON_FREE); + if (cachep->flags & SLAB_STORE_USER) + *dbg_userword(cachep, objp) = NULL; + + if (cachep->flags & SLAB_RED_ZONE) { + *dbg_redzone1(cachep, objp) = RED_INACTIVE; + *dbg_redzone2(cachep, objp) = RED_INACTIVE; + } + /* + * Constructors are not allowed to allocate memory from + * the same cache which they are a constructor for. + * Otherwise, deadlock. They must also be threaded. + */ + if (cachep->ctor && !(cachep->flags & SLAB_POISON)) + cachep->ctor(objp+obj_dbghead(cachep), cachep, ctor_flags); + + if (cachep->flags & SLAB_RED_ZONE) { + if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) + slab_error(cachep, "constructor overwrote the" + " end of an object"); + if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) + slab_error(cachep, "constructor overwrote the" + " start of an object"); + } + if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) + kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); +#else + if (cachep->ctor) + cachep->ctor(objp, cachep, ctor_flags); +#endif + slab_bufctl(slabp)[i] = i+1; + } + slab_bufctl(slabp)[i-1] = BUFCTL_END; + slabp->free = 0; +} + +static void kmem_flagcheck(kmem_cache_t *cachep, unsigned int flags) +{ + if (flags & SLAB_DMA) { + if (!(cachep->gfpflags & GFP_DMA)) + BUG(); + } else { + if (cachep->gfpflags & GFP_DMA) + BUG(); + } +} + +static void set_slab_attr(kmem_cache_t *cachep, struct slab *slabp, void *objp) +{ + int i; + struct page *page; + + /* Nasty!!!!!! I hope this is OK. */ + i = 1 << cachep->gfporder; + page = virt_to_page(objp); + do { + SET_PAGE_CACHE(page, cachep); + SET_PAGE_SLAB(page, slabp); + page++; + } while (--i); +} + +/* + * Grow (by 1) the number of slabs within a cache. This is called by + * kmem_cache_alloc() when there are no active objs left in a cache. + */ +static int cache_grow(kmem_cache_t *cachep, unsigned int __nocast flags, int nodeid) +{ + struct slab *slabp; + void *objp; + size_t offset; + unsigned int local_flags; + unsigned long ctor_flags; + + /* Be lazy and only check for valid flags here, + * keeping it out of the critical path in kmem_cache_alloc(). + */ + if (flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)) + BUG(); + if (flags & SLAB_NO_GROW) + return 0; + + ctor_flags = SLAB_CTOR_CONSTRUCTOR; + local_flags = (flags & SLAB_LEVEL_MASK); + if (!(local_flags & __GFP_WAIT)) + /* + * Not allowed to sleep. Need to tell a constructor about + * this - it might need to know... + */ + ctor_flags |= SLAB_CTOR_ATOMIC; + + /* About to mess with non-constant members - lock. */ + check_irq_off(); + spin_lock(&cachep->spinlock); + + /* Get colour for the slab, and cal the next value. */ + offset = cachep->colour_next; + cachep->colour_next++; + if (cachep->colour_next >= cachep->colour) + cachep->colour_next = 0; + offset *= cachep->colour_off; + + spin_unlock(&cachep->spinlock); + + if (local_flags & __GFP_WAIT) + local_irq_enable(); + + /* + * The test for missing atomic flag is performed here, rather than + * the more obvious place, simply to reduce the critical path length + * in kmem_cache_alloc(). If a caller is seriously mis-behaving they + * will eventually be caught here (where it matters). + */ + kmem_flagcheck(cachep, flags); + + + /* Get mem for the objs. */ + if (!(objp = kmem_getpages(cachep, flags, nodeid))) + goto failed; + + /* Get slab management. */ + if (!(slabp = alloc_slabmgmt(cachep, objp, offset, local_flags))) + goto opps1; + + set_slab_attr(cachep, slabp, objp); + + cache_init_objs(cachep, slabp, ctor_flags); + + if (local_flags & __GFP_WAIT) + local_irq_disable(); + check_irq_off(); + spin_lock(&cachep->spinlock); + + /* Make slab active. */ + list_add_tail(&slabp->list, &(list3_data(cachep)->slabs_free)); + STATS_INC_GROWN(cachep); + list3_data(cachep)->free_objects += cachep->num; + spin_unlock(&cachep->spinlock); + return 1; +opps1: + kmem_freepages(cachep, objp); +failed: + if (local_flags & __GFP_WAIT) + local_irq_disable(); + return 0; +} + +#if DEBUG + +/* + * Perform extra freeing checks: + * - detect bad pointers. + * - POISON/RED_ZONE checking + * - destructor calls, for caches with POISON+dtor + */ +static void kfree_debugcheck(const void *objp) +{ + struct page *page; + + if (!virt_addr_valid(objp)) { + printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", + (unsigned long)objp); + BUG(); + } + page = virt_to_page(objp); + if (!PageSlab(page)) { + printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", (unsigned long)objp); + BUG(); + } +} + +static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, + void *caller) +{ + struct page *page; + unsigned int objnr; + struct slab *slabp; + + objp -= obj_dbghead(cachep); + kfree_debugcheck(objp); + page = virt_to_page(objp); + + if (GET_PAGE_CACHE(page) != cachep) { + printk(KERN_ERR "mismatch in kmem_cache_free: expected cache %p, got %p\n", + GET_PAGE_CACHE(page),cachep); + printk(KERN_ERR "%p is %s.\n", cachep, cachep->name); + printk(KERN_ERR "%p is %s.\n", GET_PAGE_CACHE(page), GET_PAGE_CACHE(page)->name); + WARN_ON(1); + } + slabp = GET_PAGE_SLAB(page); + + if (cachep->flags & SLAB_RED_ZONE) { + if (*dbg_redzone1(cachep, objp) != RED_ACTIVE || *dbg_redzone2(cachep, objp) != RED_ACTIVE) { + slab_error(cachep, "double free, or memory outside" + " object was overwritten"); + printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", + objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); + } + *dbg_redzone1(cachep, objp) = RED_INACTIVE; + *dbg_redzone2(cachep, objp) = RED_INACTIVE; + } + if (cachep->flags & SLAB_STORE_USER) + *dbg_userword(cachep, objp) = caller; + + objnr = (objp-slabp->s_mem)/cachep->objsize; + + BUG_ON(objnr >= cachep->num); + BUG_ON(objp != slabp->s_mem + objnr*cachep->objsize); + + if (cachep->flags & SLAB_DEBUG_INITIAL) { + /* Need to call the slab's constructor so the + * caller can perform a verify of its state (debugging). + * Called without the cache-lock held. + */ + cachep->ctor(objp+obj_dbghead(cachep), + cachep, SLAB_CTOR_CONSTRUCTOR|SLAB_CTOR_VERIFY); + } + if (cachep->flags & SLAB_POISON && cachep->dtor) { + /* we want to cache poison the object, + * call the destruction callback + */ + cachep->dtor(objp+obj_dbghead(cachep), cachep, 0); + } + if (cachep->flags & SLAB_POISON) { +#ifdef CONFIG_DEBUG_PAGEALLOC + if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) { + store_stackinfo(cachep, objp, (unsigned long)caller); + kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); + } else { + poison_obj(cachep, objp, POISON_FREE); + } +#else + poison_obj(cachep, objp, POISON_FREE); +#endif + } + return objp; +} + +static void check_slabp(kmem_cache_t *cachep, struct slab *slabp) +{ + kmem_bufctl_t i; + int entries = 0; + + check_spinlock_acquired(cachep); + /* Check slab's freelist to see if this obj is there. */ + for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { + entries++; + if (entries > cachep->num || i >= cachep->num) + goto bad; + } + if (entries != cachep->num - slabp->inuse) { +bad: + printk(KERN_ERR "slab: Internal list corruption detected in cache '%s'(%d), slabp %p(%d). Hexdump:\n", + cachep->name, cachep->num, slabp, slabp->inuse); + for (i=0;i<sizeof(slabp)+cachep->num*sizeof(kmem_bufctl_t);i++) { + if ((i%16)==0) + printk("\n%03x:", i); + printk(" %02x", ((unsigned char*)slabp)[i]); + } + printk("\n"); + BUG(); + } +} +#else +#define kfree_debugcheck(x) do { } while(0) +#define cache_free_debugcheck(x,objp,z) (objp) +#define check_slabp(x,y) do { } while(0) +#endif + +static void *cache_alloc_refill(kmem_cache_t *cachep, unsigned int __nocast flags) +{ + int batchcount; + struct kmem_list3 *l3; + struct array_cache *ac; + + check_irq_off(); + ac = ac_data(cachep); +retry: + batchcount = ac->batchcount; + if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { + /* if there was little recent activity on this + * cache, then perform only a partial refill. + * Otherwise we could generate refill bouncing. + */ + batchcount = BATCHREFILL_LIMIT; + } + l3 = list3_data(cachep); + + BUG_ON(ac->avail > 0); + spin_lock(&cachep->spinlock); + if (l3->shared) { + struct array_cache *shared_array = l3->shared; + if (shared_array->avail) { + if (batchcount > shared_array->avail) + batchcount = shared_array->avail; + shared_array->avail -= batchcount; + ac->avail = batchcount; + memcpy(ac_entry(ac), &ac_entry(shared_array)[shared_array->avail], + sizeof(void*)*batchcount); + shared_array->touched = 1; + goto alloc_done; + } + } + while (batchcount > 0) { + struct list_head *entry; + struct slab *slabp; + /* Get slab alloc is to come from. */ + entry = l3->slabs_partial.next; + if (entry == &l3->slabs_partial) { + l3->free_touched = 1; + entry = l3->slabs_free.next; + if (entry == &l3->slabs_free) + goto must_grow; + } + + slabp = list_entry(entry, struct slab, list); + check_slabp(cachep, slabp); + check_spinlock_acquired(cachep); + while (slabp->inuse < cachep->num && batchcount--) { + kmem_bufctl_t next; + STATS_INC_ALLOCED(cachep); + STATS_INC_ACTIVE(cachep); + STATS_SET_HIGH(cachep); + + /* get obj pointer */ + ac_entry(ac)[ac->avail++] = slabp->s_mem + slabp->free*cachep->objsize; + + slabp->inuse++; + next = slab_bufctl(slabp)[slabp->free]; +#if DEBUG + slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; +#endif + slabp->free = next; + } + check_slabp(cachep, slabp); + + /* move slabp to correct slabp list: */ + list_del(&slabp->list); + if (slabp->free == BUFCTL_END) + list_add(&slabp->list, &l3->slabs_full); + else + list_add(&slabp->list, &l3->slabs_partial); + } + +must_grow: + l3->free_objects -= ac->avail; +alloc_done: + spin_unlock(&cachep->spinlock); + + if (unlikely(!ac->avail)) { + int x; + x = cache_grow(cachep, flags, -1); + + // cache_grow can reenable interrupts, then ac could change. + ac = ac_data(cachep); + if (!x && ac->avail == 0) // no objects in sight? abort + return NULL; + + if (!ac->avail) // objects refilled by interrupt? + goto retry; + } + ac->touched = 1; + return ac_entry(ac)[--ac->avail]; +} + +static inline void +cache_alloc_debugcheck_before(kmem_cache_t *cachep, unsigned int __nocast flags) +{ + might_sleep_if(flags & __GFP_WAIT); +#if DEBUG + kmem_flagcheck(cachep, flags); +#endif +} + +#if DEBUG +static void * +cache_alloc_debugcheck_after(kmem_cache_t *cachep, + unsigned long flags, void *objp, void *caller) +{ + if (!objp) + return objp; + if (cachep->flags & SLAB_POISON) { +#ifdef CONFIG_DEBUG_PAGEALLOC + if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) + kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 1); + else + check_poison_obj(cachep, objp); +#else + check_poison_obj(cachep, objp); +#endif + poison_obj(cachep, objp, POISON_INUSE); + } + if (cachep->flags & SLAB_STORE_USER) + *dbg_userword(cachep, objp) = caller; + + if (cachep->flags & SLAB_RED_ZONE) { + if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || *dbg_redzone2(cachep, objp) != RED_INACTIVE) { + slab_error(cachep, "double free, or memory outside" + " object was overwritten"); + printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", + objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); + } + *dbg_redzone1(cachep, objp) = RED_ACTIVE; + *dbg_redzone2(cachep, objp) = RED_ACTIVE; + } + objp += obj_dbghead(cachep); + if (cachep->ctor && cachep->flags & SLAB_POISON) { + unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR; + + if (!(flags & __GFP_WAIT)) + ctor_flags |= SLAB_CTOR_ATOMIC; + + cachep->ctor(objp, cachep, ctor_flags); + } + return objp; +} +#else +#define cache_alloc_debugcheck_after(a,b,objp,d) (objp) +#endif + + +static inline void *__cache_alloc(kmem_cache_t *cachep, unsigned int __nocast flags) +{ + unsigned long save_flags; + void* objp; + struct array_cache *ac; + + cache_alloc_debugcheck_before(cachep, flags); + + local_irq_save(save_flags); + ac = ac_data(cachep); + if (likely(ac->avail)) { + STATS_INC_ALLOCHIT(cachep); + ac->touched = 1; + objp = ac_entry(ac)[--ac->avail]; + } else { + STATS_INC_ALLOCMISS(cachep); + objp = cache_alloc_refill(cachep, flags); + } + local_irq_restore(save_flags); + objp = cache_alloc_debugcheck_after(cachep, flags, objp, __builtin_return_address(0)); + return objp; +} + +/* + * NUMA: different approach needed if the spinlock is moved into + * the l3 structure + */ + +static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects) +{ + int i; + + check_spinlock_acquired(cachep); + + /* NUMA: move add into loop */ + cachep->lists.free_objects += nr_objects; + + for (i = 0; i < nr_objects; i++) { + void *objp = objpp[i]; + struct slab *slabp; + unsigned int objnr; + + slabp = GET_PAGE_SLAB(virt_to_page(objp)); + list_del(&slabp->list); + objnr = (objp - slabp->s_mem) / cachep->objsize; + check_slabp(cachep, slabp); +#if DEBUG + if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) { + printk(KERN_ERR "slab: double free detected in cache '%s', objp %p.\n", + cachep->name, objp); + BUG(); + } +#endif + slab_bufctl(slabp)[objnr] = slabp->free; + slabp->free = objnr; + STATS_DEC_ACTIVE(cachep); + slabp->inuse--; + check_slabp(cachep, slabp); + + /* fixup slab chains */ + if (slabp->inuse == 0) { + if (cachep->lists.free_objects > cachep->free_limit) { + cachep->lists.free_objects -= cachep->num; + slab_destroy(cachep, slabp); + } else { + list_add(&slabp->list, + &list3_data_ptr(cachep, objp)->slabs_free); + } + } else { + /* Unconditionally move a slab to the end of the + * partial list on free - maximum time for the + * other objects to be freed, too. + */ + list_add_tail(&slabp->list, + &list3_data_ptr(cachep, objp)->slabs_partial); + } + } +} + +static void cache_flusharray(kmem_cache_t *cachep, struct array_cache *ac) +{ + int batchcount; + + batchcount = ac->batchcount; +#if DEBUG + BUG_ON(!batchcount || batchcount > ac->avail); +#endif + check_irq_off(); + spin_lock(&cachep->spinlock); + if (cachep->lists.shared) { + struct array_cache *shared_array = cachep->lists.shared; + int max = shared_array->limit-shared_array->avail; + if (max) { + if (batchcount > max) + batchcount = max; + memcpy(&ac_entry(shared_array)[shared_array->avail], + &ac_entry(ac)[0], + sizeof(void*)*batchcount); + shared_array->avail += batchcount; + goto free_done; + } + } + + free_block(cachep, &ac_entry(ac)[0], batchcount); +free_done: +#if STATS + { + int i = 0; + struct list_head *p; + + p = list3_data(cachep)->slabs_free.next; + while (p != &(list3_data(cachep)->slabs_free)) { + struct slab *slabp; + + slabp = list_entry(p, struct slab, list); + BUG_ON(slabp->inuse); + + i++; + p = p->next; + } + STATS_SET_FREEABLE(cachep, i); + } +#endif + spin_unlock(&cachep->spinlock); + ac->avail -= batchcount; + memmove(&ac_entry(ac)[0], &ac_entry(ac)[batchcount], + sizeof(void*)*ac->avail); +} + +/* + * __cache_free + * Release an obj back to its cache. If the obj has a constructed + * state, it must be in this state _before_ it is released. + * + * Called with disabled ints. + */ +static inline void __cache_free(kmem_cache_t *cachep, void *objp) +{ + struct array_cache *ac = ac_data(cachep); + + check_irq_off(); + objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0)); + + if (likely(ac->avail < ac->limit)) { + STATS_INC_FREEHIT(cachep); + ac_entry(ac)[ac->avail++] = objp; + return; + } else { + STATS_INC_FREEMISS(cachep); + cache_flusharray(cachep, ac); + ac_entry(ac)[ac->avail++] = objp; + } +} + +/** + * kmem_cache_alloc - Allocate an object + * @cachep: The cache to allocate from. + * @flags: See kmalloc(). + * + * Allocate an object from this cache. The flags are only relevant + * if the cache has no available objects. + */ +void *kmem_cache_alloc(kmem_cache_t *cachep, unsigned int __nocast flags) +{ + return __cache_alloc(cachep, flags); +} +EXPORT_SYMBOL(kmem_cache_alloc); + +/** + * kmem_ptr_validate - check if an untrusted pointer might + * be a slab entry. + * @cachep: the cache we're checking against + * @ptr: pointer to validate + * + * This verifies that the untrusted pointer looks sane: + * it is _not_ a guarantee that the pointer is actually + * part of the slab cache in question, but it at least + * validates that the pointer can be dereferenced and + * looks half-way sane. + * + * Currently only used for dentry validation. + */ +int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr) +{ + unsigned long addr = (unsigned long) ptr; + unsigned long min_addr = PAGE_OFFSET; + unsigned long align_mask = BYTES_PER_WORD-1; + unsigned long size = cachep->objsize; + struct page *page; + + if (unlikely(addr < min_addr)) + goto out; + if (unlikely(addr > (unsigned long)high_memory - size)) + goto out; + if (unlikely(addr & align_mask)) + goto out; + if (unlikely(!kern_addr_valid(addr))) + goto out; + if (unlikely(!kern_addr_valid(addr + size - 1))) + goto out; + page = virt_to_page(ptr); + if (unlikely(!PageSlab(page))) + goto out; + if (unlikely(GET_PAGE_CACHE(page) != cachep)) + goto out; + return 1; +out: + return 0; +} + +#ifdef CONFIG_NUMA +/** + * kmem_cache_alloc_node - Allocate an object on the specified node + * @cachep: The cache to allocate from. + * @flags: See kmalloc(). + * @nodeid: node number of the target node. + * + * Identical to kmem_cache_alloc, except that this function is slow + * and can sleep. And it will allocate memory on the given node, which + * can improve the performance for cpu bound structures. + */ +void *kmem_cache_alloc_node(kmem_cache_t *cachep, int nodeid) +{ + int loop; + void *objp; + struct slab *slabp; + kmem_bufctl_t next; + + for (loop = 0;;loop++) { + struct list_head *q; + + objp = NULL; + check_irq_on(); + spin_lock_irq(&cachep->spinlock); + /* walk through all partial and empty slab and find one + * from the right node */ + list_for_each(q,&cachep->lists.slabs_partial) { + slabp = list_entry(q, struct slab, list); + + if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid || + loop > 2) + goto got_slabp; + } + list_for_each(q, &cachep->lists.slabs_free) { + slabp = list_entry(q, struct slab, list); + + if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid || + loop > 2) + goto got_slabp; + } + spin_unlock_irq(&cachep->spinlock); + + local_irq_disable(); + if (!cache_grow(cachep, GFP_KERNEL, nodeid)) { + local_irq_enable(); + return NULL; + } + local_irq_enable(); + } +got_slabp: + /* found one: allocate object */ + check_slabp(cachep, slabp); + check_spinlock_acquired(cachep); + + STATS_INC_ALLOCED(cachep); + STATS_INC_ACTIVE(cachep); + STATS_SET_HIGH(cachep); + STATS_INC_NODEALLOCS(cachep); + + objp = slabp->s_mem + slabp->free*cachep->objsize; + + slabp->inuse++; + next = slab_bufctl(slabp)[slabp->free]; +#if DEBUG + slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; +#endif + slabp->free = next; + check_slabp(cachep, slabp); + + /* move slabp to correct slabp list: */ + list_del(&slabp->list); + if (slabp->free == BUFCTL_END) + list_add(&slabp->list, &cachep->lists.slabs_full); + else + list_add(&slabp->list, &cachep->lists.slabs_partial); + + list3_data(cachep)->free_objects--; + spin_unlock_irq(&cachep->spinlock); + + objp = cache_alloc_debugcheck_after(cachep, GFP_KERNEL, objp, + __builtin_return_address(0)); + return objp; +} +EXPORT_SYMBOL(kmem_cache_alloc_node); + +#endif + +/** + * kmalloc - allocate memory + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate. + * + * kmalloc is the normal method of allocating memory + * in the kernel. + * + * The @flags argument may be one of: + * + * %GFP_USER - Allocate memory on behalf of user. May sleep. + * + * %GFP_KERNEL - Allocate normal kernel ram. May sleep. + * + * %GFP_ATOMIC - Allocation will not sleep. Use inside interrupt handlers. + * + * Additionally, the %GFP_DMA flag may be set to indicate the memory + * must be suitable for DMA. This can mean different things on different + * platforms. For example, on i386, it means that the memory must come + * from the first 16MB. + */ +void *__kmalloc(size_t size, unsigned int __nocast flags) +{ + kmem_cache_t *cachep; + + cachep = kmem_find_general_cachep(size, flags); + if (unlikely(cachep == NULL)) + return NULL; + return __cache_alloc(cachep, flags); +} +EXPORT_SYMBOL(__kmalloc); + +#ifdef CONFIG_SMP +/** + * __alloc_percpu - allocate one copy of the object for every present + * cpu in the system, zeroing them. + * Objects should be dereferenced using the per_cpu_ptr macro only. + * + * @size: how many bytes of memory are required. + * @align: the alignment, which can't be greater than SMP_CACHE_BYTES. + */ +void *__alloc_percpu(size_t size, size_t align) +{ + int i; + struct percpu_data *pdata = kmalloc(sizeof (*pdata), GFP_KERNEL); + + if (!pdata) + return NULL; + + for (i = 0; i < NR_CPUS; i++) { + if (!cpu_possible(i)) + continue; + pdata->ptrs[i] = kmem_cache_alloc_node( + kmem_find_general_cachep(size, GFP_KERNEL), + cpu_to_node(i)); + + if (!pdata->ptrs[i]) + goto unwind_oom; + memset(pdata->ptrs[i], 0, size); + } + + /* Catch derefs w/o wrappers */ + return (void *) (~(unsigned long) pdata); + +unwind_oom: + while (--i >= 0) { + if (!cpu_possible(i)) + continue; + kfree(pdata->ptrs[i]); + } + kfree(pdata); + return NULL; +} +EXPORT_SYMBOL(__alloc_percpu); +#endif + +/** + * kmem_cache_free - Deallocate an object + * @cachep: The cache the allocation was from. + * @objp: The previously allocated object. + * + * Free an object which was previously allocated from this + * cache. + */ +void kmem_cache_free(kmem_cache_t *cachep, void *objp) +{ + unsigned long flags; + + local_irq_save(flags); + __cache_free(cachep, objp); + local_irq_restore(flags); +} +EXPORT_SYMBOL(kmem_cache_free); + +/** + * kcalloc - allocate memory for an array. The memory is set to zero. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate. + */ +void *kcalloc(size_t n, size_t size, unsigned int __nocast flags) +{ + void *ret = NULL; + + if (n != 0 && size > INT_MAX / n) + return ret; + + ret = kmalloc(n * size, flags); + if (ret) + memset(ret, 0, n * size); + return ret; +} +EXPORT_SYMBOL(kcalloc); + +/** + * kfree - free previously allocated memory + * @objp: pointer returned by kmalloc. + * + * Don't free memory not originally allocated by kmalloc() + * or you will run into trouble. + */ +void kfree(const void *objp) +{ + kmem_cache_t *c; + unsigned long flags; + + if (unlikely(!objp)) + return; + local_irq_save(flags); + kfree_debugcheck(objp); + c = GET_PAGE_CACHE(virt_to_page(objp)); + __cache_free(c, (void*)objp); + local_irq_restore(flags); +} +EXPORT_SYMBOL(kfree); + +#ifdef CONFIG_SMP +/** + * free_percpu - free previously allocated percpu memory + * @objp: pointer returned by alloc_percpu. + * + * Don't free memory not originally allocated by alloc_percpu() + * The complemented objp is to check for that. + */ +void +free_percpu(const void *objp) +{ + int i; + struct percpu_data *p = (struct percpu_data *) (~(unsigned long) objp); + + for (i = 0; i < NR_CPUS; i++) { + if (!cpu_possible(i)) + continue; + kfree(p->ptrs[i]); + } + kfree(p); +} +EXPORT_SYMBOL(free_percpu); +#endif + +unsigned int kmem_cache_size(kmem_cache_t *cachep) +{ + return obj_reallen(cachep); +} +EXPORT_SYMBOL(kmem_cache_size); + +struct ccupdate_struct { + kmem_cache_t *cachep; + struct array_cache *new[NR_CPUS]; +}; + +static void do_ccupdate_local(void *info) +{ + struct ccupdate_struct *new = (struct ccupdate_struct *)info; + struct array_cache *old; + + check_irq_off(); + old = ac_data(new->cachep); + + new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()]; + new->new[smp_processor_id()] = old; +} + + +static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount, + int shared) +{ + struct ccupdate_struct new; + struct array_cache *new_shared; + int i; + + memset(&new.new,0,sizeof(new.new)); + for (i = 0; i < NR_CPUS; i++) { + if (cpu_online(i)) { + new.new[i] = alloc_arraycache(i, limit, batchcount); + if (!new.new[i]) { + for (i--; i >= 0; i--) kfree(new.new[i]); + return -ENOMEM; + } + } else { + new.new[i] = NULL; + } + } + new.cachep = cachep; + + smp_call_function_all_cpus(do_ccupdate_local, (void *)&new); + + check_irq_on(); + spin_lock_irq(&cachep->spinlock); + cachep->batchcount = batchcount; + cachep->limit = limit; + cachep->free_limit = (1+num_online_cpus())*cachep->batchcount + cachep->num; + spin_unlock_irq(&cachep->spinlock); + + for (i = 0; i < NR_CPUS; i++) { + struct array_cache *ccold = new.new[i]; + if (!ccold) + continue; + spin_lock_irq(&cachep->spinlock); + free_block(cachep, ac_entry(ccold), ccold->avail); + spin_unlock_irq(&cachep->spinlock); + kfree(ccold); + } + new_shared = alloc_arraycache(-1, batchcount*shared, 0xbaadf00d); + if (new_shared) { + struct array_cache *old; + + spin_lock_irq(&cachep->spinlock); + old = cachep->lists.shared; + cachep->lists.shared = new_shared; + if (old) + free_block(cachep, ac_entry(old), old->avail); + spin_unlock_irq(&cachep->spinlock); + kfree(old); + } + + return 0; +} + + +static void enable_cpucache(kmem_cache_t *cachep) +{ + int err; + int limit, shared; + + /* The head array serves three purposes: + * - create a LIFO ordering, i.e. return objects that are cache-warm + * - reduce the number of spinlock operations. + * - reduce the number of linked list operations on the slab and + * bufctl chains: array operations are cheaper. + * The numbers are guessed, we should auto-tune as described by + * Bonwick. + */ + if (cachep->objsize > 131072) + limit = 1; + else if (cachep->objsize > PAGE_SIZE) + limit = 8; + else if (cachep->objsize > 1024) + limit = 24; + else if (cachep->objsize > 256) + limit = 54; + else + limit = 120; + + /* Cpu bound tasks (e.g. network routing) can exhibit cpu bound + * allocation behaviour: Most allocs on one cpu, most free operations + * on another cpu. For these cases, an efficient object passing between + * cpus is necessary. This is provided by a shared array. The array + * replaces Bonwick's magazine layer. + * On uniprocessor, it's functionally equivalent (but less efficient) + * to a larger limit. Thus disabled by default. + */ + shared = 0; +#ifdef CONFIG_SMP + if (cachep->objsize <= PAGE_SIZE) + shared = 8; +#endif + +#if DEBUG + /* With debugging enabled, large batchcount lead to excessively + * long periods with disabled local interrupts. Limit the + * batchcount + */ + if (limit > 32) + limit = 32; +#endif + err = do_tune_cpucache(cachep, limit, (limit+1)/2, shared); + if (err) + printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", + cachep->name, -err); +} + +static void drain_array_locked(kmem_cache_t *cachep, + struct array_cache *ac, int force) +{ + int tofree; + + check_spinlock_acquired(cachep); + if (ac->touched && !force) { + ac->touched = 0; + } else if (ac->avail) { + tofree = force ? ac->avail : (ac->limit+4)/5; + if (tofree > ac->avail) { + tofree = (ac->avail+1)/2; + } + free_block(cachep, ac_entry(ac), tofree); + ac->avail -= tofree; + memmove(&ac_entry(ac)[0], &ac_entry(ac)[tofree], + sizeof(void*)*ac->avail); + } +} + +/** + * cache_reap - Reclaim memory from caches. + * + * Called from workqueue/eventd every few seconds. + * Purpose: + * - clear the per-cpu caches for this CPU. + * - return freeable pages to the main free memory pool. + * + * If we cannot acquire the cache chain semaphore then just give up - we'll + * try again on the next iteration. + */ +static void cache_reap(void *unused) +{ + struct list_head *walk; + + if (down_trylock(&cache_chain_sem)) { + /* Give up. Setup the next iteration. */ + schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC + smp_processor_id()); + return; + } + + list_for_each(walk, &cache_chain) { + kmem_cache_t *searchp; + struct list_head* p; + int tofree; + struct slab *slabp; + + searchp = list_entry(walk, kmem_cache_t, next); + + if (searchp->flags & SLAB_NO_REAP) + goto next; + + check_irq_on(); + + spin_lock_irq(&searchp->spinlock); + + drain_array_locked(searchp, ac_data(searchp), 0); + + if(time_after(searchp->lists.next_reap, jiffies)) + goto next_unlock; + + searchp->lists.next_reap = jiffies + REAPTIMEOUT_LIST3; + + if (searchp->lists.shared) + drain_array_locked(searchp, searchp->lists.shared, 0); + + if (searchp->lists.free_touched) { + searchp->lists.free_touched = 0; + goto next_unlock; + } + + tofree = (searchp->free_limit+5*searchp->num-1)/(5*searchp->num); + do { + p = list3_data(searchp)->slabs_free.next; + if (p == &(list3_data(searchp)->slabs_free)) + break; + + slabp = list_entry(p, struct slab, list); + BUG_ON(slabp->inuse); + list_del(&slabp->list); + STATS_INC_REAPED(searchp); + + /* Safe to drop the lock. The slab is no longer + * linked to the cache. + * searchp cannot disappear, we hold + * cache_chain_lock + */ + searchp->lists.free_objects -= searchp->num; + spin_unlock_irq(&searchp->spinlock); + slab_destroy(searchp, slabp); + spin_lock_irq(&searchp->spinlock); + } while(--tofree > 0); +next_unlock: + spin_unlock_irq(&searchp->spinlock); +next: + cond_resched(); + } + check_irq_on(); + up(&cache_chain_sem); + /* Setup the next iteration */ + schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC + smp_processor_id()); +} + +#ifdef CONFIG_PROC_FS + +static void *s_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + struct list_head *p; + + down(&cache_chain_sem); + if (!n) { + /* + * Output format version, so at least we can change it + * without _too_ many complaints. + */ +#if STATS + seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); +#else + seq_puts(m, "slabinfo - version: 2.1\n"); +#endif + seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>"); + seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); + seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); +#if STATS + seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped>" + " <error> <maxfreeable> <freelimit> <nodeallocs>"); + seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); +#endif + seq_putc(m, '\n'); + } + p = cache_chain.next; + while (n--) { + p = p->next; + if (p == &cache_chain) + return NULL; + } + return list_entry(p, kmem_cache_t, next); +} + +static void *s_next(struct seq_file *m, void *p, loff_t *pos) +{ + kmem_cache_t *cachep = p; + ++*pos; + return cachep->next.next == &cache_chain ? NULL + : list_entry(cachep->next.next, kmem_cache_t, next); +} + +static void s_stop(struct seq_file *m, void *p) +{ + up(&cache_chain_sem); +} + +static int s_show(struct seq_file *m, void *p) +{ + kmem_cache_t *cachep = p; + struct list_head *q; + struct slab *slabp; + unsigned long active_objs; + unsigned long num_objs; + unsigned long active_slabs = 0; + unsigned long num_slabs; + const char *name; + char *error = NULL; + + check_irq_on(); + spin_lock_irq(&cachep->spinlock); + active_objs = 0; + num_slabs = 0; + list_for_each(q,&cachep->lists.slabs_full) { + slabp = list_entry(q, struct slab, list); + if (slabp->inuse != cachep->num && !error) + error = "slabs_full accounting error"; + active_objs += cachep->num; + active_slabs++; + } + list_for_each(q,&cachep->lists.slabs_partial) { + slabp = list_entry(q, struct slab, list); + if (slabp->inuse == cachep->num && !error) + error = "slabs_partial inuse accounting error"; + if (!slabp->inuse && !error) + error = "slabs_partial/inuse accounting error"; + active_objs += slabp->inuse; + active_slabs++; + } + list_for_each(q,&cachep->lists.slabs_free) { + slabp = list_entry(q, struct slab, list); + if (slabp->inuse && !error) + error = "slabs_free/inuse accounting error"; + num_slabs++; + } + num_slabs+=active_slabs; + num_objs = num_slabs*cachep->num; + if (num_objs - active_objs != cachep->lists.free_objects && !error) + error = "free_objects accounting error"; + + name = cachep->name; + if (error) + printk(KERN_ERR "slab: cache %s error: %s\n", name, error); + + seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", + name, active_objs, num_objs, cachep->objsize, + cachep->num, (1<<cachep->gfporder)); + seq_printf(m, " : tunables %4u %4u %4u", + cachep->limit, cachep->batchcount, + cachep->lists.shared->limit/cachep->batchcount); + seq_printf(m, " : slabdata %6lu %6lu %6u", + active_slabs, num_slabs, cachep->lists.shared->avail); +#if STATS + { /* list3 stats */ + unsigned long high = cachep->high_mark; + unsigned long allocs = cachep->num_allocations; + unsigned long grown = cachep->grown; + unsigned long reaped = cachep->reaped; + unsigned long errors = cachep->errors; + unsigned long max_freeable = cachep->max_freeable; + unsigned long free_limit = cachep->free_limit; + unsigned long node_allocs = cachep->node_allocs; + + seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu", + allocs, high, grown, reaped, errors, + max_freeable, free_limit, node_allocs); + } + /* cpu stats */ + { + unsigned long allochit = atomic_read(&cachep->allochit); + unsigned long allocmiss = atomic_read(&cachep->allocmiss); + unsigned long freehit = atomic_read(&cachep->freehit); + unsigned long freemiss = atomic_read(&cachep->freemiss); + + seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", + allochit, allocmiss, freehit, freemiss); + } +#endif + seq_putc(m, '\n'); + spin_unlock_irq(&cachep->spinlock); + return 0; +} + +/* + * slabinfo_op - iterator that generates /proc/slabinfo + * + * Output layout: + * cache-name + * num-active-objs + * total-objs + * object size + * num-active-slabs + * total-slabs + * num-pages-per-slab + * + further values on SMP and with statistics enabled + */ + +struct seq_operations slabinfo_op = { + .start = s_start, + .next = s_next, + .stop = s_stop, + .show = s_show, +}; + +#define MAX_SLABINFO_WRITE 128 +/** + * slabinfo_write - Tuning for the slab allocator + * @file: unused + * @buffer: user buffer + * @count: data length + * @ppos: unused + */ +ssize_t slabinfo_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + char kbuf[MAX_SLABINFO_WRITE+1], *tmp; + int limit, batchcount, shared, res; + struct list_head *p; + + if (count > MAX_SLABINFO_WRITE) + return -EINVAL; + if (copy_from_user(&kbuf, buffer, count)) + return -EFAULT; + kbuf[MAX_SLABINFO_WRITE] = '\0'; + + tmp = strchr(kbuf, ' '); + if (!tmp) + return -EINVAL; + *tmp = '\0'; + tmp++; + if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3) + return -EINVAL; + + /* Find the cache in the chain of caches. */ + down(&cache_chain_sem); + res = -EINVAL; + list_for_each(p,&cache_chain) { + kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next); + + if (!strcmp(cachep->name, kbuf)) { + if (limit < 1 || + batchcount < 1 || + batchcount > limit || + shared < 0) { + res = -EINVAL; + } else { + res = do_tune_cpucache(cachep, limit, batchcount, shared); + } + break; + } + } + up(&cache_chain_sem); + if (res >= 0) + res = count; + return res; +} +#endif + +unsigned int ksize(const void *objp) +{ + kmem_cache_t *c; + unsigned long flags; + unsigned int size = 0; + + if (likely(objp != NULL)) { + local_irq_save(flags); + c = GET_PAGE_CACHE(virt_to_page(objp)); + size = kmem_cache_size(c); + local_irq_restore(flags); + } + + return size; +} diff --git a/mm/swap.c b/mm/swap.c new file mode 100644 index 000000000000..7771d2803f62 --- /dev/null +++ b/mm/swap.c @@ -0,0 +1,485 @@ +/* + * linux/mm/swap.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + */ + +/* + * This file contains the default values for the opereation of the + * Linux VM subsystem. Fine-tuning documentation can be found in + * Documentation/sysctl/vm.txt. + * Started 18.12.91 + * Swap aging added 23.2.95, Stephen Tweedie. + * Buffermem limits added 12.3.98, Rik van Riel. + */ + +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/kernel_stat.h> +#include <linux/swap.h> +#include <linux/mman.h> +#include <linux/pagemap.h> +#include <linux/pagevec.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm_inline.h> +#include <linux/buffer_head.h> /* for try_to_release_page() */ +#include <linux/module.h> +#include <linux/percpu_counter.h> +#include <linux/percpu.h> +#include <linux/cpu.h> +#include <linux/notifier.h> +#include <linux/init.h> + +/* How many pages do we try to swap or page in/out together? */ +int page_cluster; + +#ifdef CONFIG_HUGETLB_PAGE + +void put_page(struct page *page) +{ + if (unlikely(PageCompound(page))) { + page = (struct page *)page->private; + if (put_page_testzero(page)) { + void (*dtor)(struct page *page); + + dtor = (void (*)(struct page *))page[1].mapping; + (*dtor)(page); + } + return; + } + if (!PageReserved(page) && put_page_testzero(page)) + __page_cache_release(page); +} +EXPORT_SYMBOL(put_page); +#endif + +/* + * Writeback is about to end against a page which has been marked for immediate + * reclaim. If it still appears to be reclaimable, move it to the tail of the + * inactive list. The page still has PageWriteback set, which will pin it. + * + * We don't expect many pages to come through here, so don't bother batching + * things up. + * + * To avoid placing the page at the tail of the LRU while PG_writeback is still + * set, this function will clear PG_writeback before performing the page + * motion. Do that inside the lru lock because once PG_writeback is cleared + * we may not touch the page. + * + * Returns zero if it cleared PG_writeback. + */ +int rotate_reclaimable_page(struct page *page) +{ + struct zone *zone; + unsigned long flags; + + if (PageLocked(page)) + return 1; + if (PageDirty(page)) + return 1; + if (PageActive(page)) + return 1; + if (!PageLRU(page)) + return 1; + + zone = page_zone(page); + spin_lock_irqsave(&zone->lru_lock, flags); + if (PageLRU(page) && !PageActive(page)) { + list_del(&page->lru); + list_add_tail(&page->lru, &zone->inactive_list); + inc_page_state(pgrotated); + } + if (!test_clear_page_writeback(page)) + BUG(); + spin_unlock_irqrestore(&zone->lru_lock, flags); + return 0; +} + +/* + * FIXME: speed this up? + */ +void fastcall activate_page(struct page *page) +{ + struct zone *zone = page_zone(page); + + spin_lock_irq(&zone->lru_lock); + if (PageLRU(page) && !PageActive(page)) { + del_page_from_inactive_list(zone, page); + SetPageActive(page); + add_page_to_active_list(zone, page); + inc_page_state(pgactivate); + } + spin_unlock_irq(&zone->lru_lock); +} + +/* + * Mark a page as having seen activity. + * + * inactive,unreferenced -> inactive,referenced + * inactive,referenced -> active,unreferenced + * active,unreferenced -> active,referenced + */ +void fastcall mark_page_accessed(struct page *page) +{ + if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { + activate_page(page); + ClearPageReferenced(page); + } else if (!PageReferenced(page)) { + SetPageReferenced(page); + } +} + +EXPORT_SYMBOL(mark_page_accessed); + +/** + * lru_cache_add: add a page to the page lists + * @page: the page to add + */ +static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, }; +static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, }; + +void fastcall lru_cache_add(struct page *page) +{ + struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); + + page_cache_get(page); + if (!pagevec_add(pvec, page)) + __pagevec_lru_add(pvec); + put_cpu_var(lru_add_pvecs); +} + +void fastcall lru_cache_add_active(struct page *page) +{ + struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs); + + page_cache_get(page); + if (!pagevec_add(pvec, page)) + __pagevec_lru_add_active(pvec); + put_cpu_var(lru_add_active_pvecs); +} + +void lru_add_drain(void) +{ + struct pagevec *pvec = &get_cpu_var(lru_add_pvecs); + + if (pagevec_count(pvec)) + __pagevec_lru_add(pvec); + pvec = &__get_cpu_var(lru_add_active_pvecs); + if (pagevec_count(pvec)) + __pagevec_lru_add_active(pvec); + put_cpu_var(lru_add_pvecs); +} + +/* + * This path almost never happens for VM activity - pages are normally + * freed via pagevecs. But it gets used by networking. + */ +void fastcall __page_cache_release(struct page *page) +{ + unsigned long flags; + struct zone *zone = page_zone(page); + + spin_lock_irqsave(&zone->lru_lock, flags); + if (TestClearPageLRU(page)) + del_page_from_lru(zone, page); + if (page_count(page) != 0) + page = NULL; + spin_unlock_irqrestore(&zone->lru_lock, flags); + if (page) + free_hot_page(page); +} + +EXPORT_SYMBOL(__page_cache_release); + +/* + * Batched page_cache_release(). Decrement the reference count on all the + * passed pages. If it fell to zero then remove the page from the LRU and + * free it. + * + * Avoid taking zone->lru_lock if possible, but if it is taken, retain it + * for the remainder of the operation. + * + * The locking in this function is against shrink_cache(): we recheck the + * page count inside the lock to see whether shrink_cache grabbed the page + * via the LRU. If it did, give up: shrink_cache will free it. + */ +void release_pages(struct page **pages, int nr, int cold) +{ + int i; + struct pagevec pages_to_free; + struct zone *zone = NULL; + + pagevec_init(&pages_to_free, cold); + for (i = 0; i < nr; i++) { + struct page *page = pages[i]; + struct zone *pagezone; + + if (PageReserved(page) || !put_page_testzero(page)) + continue; + + pagezone = page_zone(page); + if (pagezone != zone) { + if (zone) + spin_unlock_irq(&zone->lru_lock); + zone = pagezone; + spin_lock_irq(&zone->lru_lock); + } + if (TestClearPageLRU(page)) + del_page_from_lru(zone, page); + if (page_count(page) == 0) { + if (!pagevec_add(&pages_to_free, page)) { + spin_unlock_irq(&zone->lru_lock); + __pagevec_free(&pages_to_free); + pagevec_reinit(&pages_to_free); + zone = NULL; /* No lock is held */ + } + } + } + if (zone) + spin_unlock_irq(&zone->lru_lock); + + pagevec_free(&pages_to_free); +} + +/* + * The pages which we're about to release may be in the deferred lru-addition + * queues. That would prevent them from really being freed right now. That's + * OK from a correctness point of view but is inefficient - those pages may be + * cache-warm and we want to give them back to the page allocator ASAP. + * + * So __pagevec_release() will drain those queues here. __pagevec_lru_add() + * and __pagevec_lru_add_active() call release_pages() directly to avoid + * mutual recursion. + */ +void __pagevec_release(struct pagevec *pvec) +{ + lru_add_drain(); + release_pages(pvec->pages, pagevec_count(pvec), pvec->cold); + pagevec_reinit(pvec); +} + +/* + * pagevec_release() for pages which are known to not be on the LRU + * + * This function reinitialises the caller's pagevec. + */ +void __pagevec_release_nonlru(struct pagevec *pvec) +{ + int i; + struct pagevec pages_to_free; + + pagevec_init(&pages_to_free, pvec->cold); + pages_to_free.cold = pvec->cold; + for (i = 0; i < pagevec_count(pvec); i++) { + struct page *page = pvec->pages[i]; + + BUG_ON(PageLRU(page)); + if (put_page_testzero(page)) + pagevec_add(&pages_to_free, page); + } + pagevec_free(&pages_to_free); + pagevec_reinit(pvec); +} + +/* + * Add the passed pages to the LRU, then drop the caller's refcount + * on them. Reinitialises the caller's pagevec. + */ +void __pagevec_lru_add(struct pagevec *pvec) +{ + int i; + struct zone *zone = NULL; + + for (i = 0; i < pagevec_count(pvec); i++) { + struct page *page = pvec->pages[i]; + struct zone *pagezone = page_zone(page); + + if (pagezone != zone) { + if (zone) + spin_unlock_irq(&zone->lru_lock); + zone = pagezone; + spin_lock_irq(&zone->lru_lock); + } + if (TestSetPageLRU(page)) + BUG(); + add_page_to_inactive_list(zone, page); + } + if (zone) + spin_unlock_irq(&zone->lru_lock); + release_pages(pvec->pages, pvec->nr, pvec->cold); + pagevec_reinit(pvec); +} + +EXPORT_SYMBOL(__pagevec_lru_add); + +void __pagevec_lru_add_active(struct pagevec *pvec) +{ + int i; + struct zone *zone = NULL; + + for (i = 0; i < pagevec_count(pvec); i++) { + struct page *page = pvec->pages[i]; + struct zone *pagezone = page_zone(page); + + if (pagezone != zone) { + if (zone) + spin_unlock_irq(&zone->lru_lock); + zone = pagezone; + spin_lock_irq(&zone->lru_lock); + } + if (TestSetPageLRU(page)) + BUG(); + if (TestSetPageActive(page)) + BUG(); + add_page_to_active_list(zone, page); + } + if (zone) + spin_unlock_irq(&zone->lru_lock); + release_pages(pvec->pages, pvec->nr, pvec->cold); + pagevec_reinit(pvec); +} + +/* + * Try to drop buffers from the pages in a pagevec + */ +void pagevec_strip(struct pagevec *pvec) +{ + int i; + + for (i = 0; i < pagevec_count(pvec); i++) { + struct page *page = pvec->pages[i]; + + if (PagePrivate(page) && !TestSetPageLocked(page)) { + try_to_release_page(page, 0); + unlock_page(page); + } + } +} + +/** + * pagevec_lookup - gang pagecache lookup + * @pvec: Where the resulting pages are placed + * @mapping: The address_space to search + * @start: The starting page index + * @nr_pages: The maximum number of pages + * + * pagevec_lookup() will search for and return a group of up to @nr_pages pages + * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a + * reference against the pages in @pvec. + * + * The search returns a group of mapping-contiguous pages with ascending + * indexes. There may be holes in the indices due to not-present pages. + * + * pagevec_lookup() returns the number of pages which were found. + */ +unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, + pgoff_t start, unsigned nr_pages) +{ + pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages); + return pagevec_count(pvec); +} + +unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping, + pgoff_t *index, int tag, unsigned nr_pages) +{ + pvec->nr = find_get_pages_tag(mapping, index, tag, + nr_pages, pvec->pages); + return pagevec_count(pvec); +} + + +#ifdef CONFIG_SMP +/* + * We tolerate a little inaccuracy to avoid ping-ponging the counter between + * CPUs + */ +#define ACCT_THRESHOLD max(16, NR_CPUS * 2) + +static DEFINE_PER_CPU(long, committed_space) = 0; + +void vm_acct_memory(long pages) +{ + long *local; + + preempt_disable(); + local = &__get_cpu_var(committed_space); + *local += pages; + if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { + atomic_add(*local, &vm_committed_space); + *local = 0; + } + preempt_enable(); +} +EXPORT_SYMBOL(vm_acct_memory); + +#ifdef CONFIG_HOTPLUG_CPU +static void lru_drain_cache(unsigned int cpu) +{ + struct pagevec *pvec = &per_cpu(lru_add_pvecs, cpu); + + /* CPU is dead, so no locking needed. */ + if (pagevec_count(pvec)) + __pagevec_lru_add(pvec); + pvec = &per_cpu(lru_add_active_pvecs, cpu); + if (pagevec_count(pvec)) + __pagevec_lru_add_active(pvec); +} + +/* Drop the CPU's cached committed space back into the central pool. */ +static int cpu_swap_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + long *committed; + + committed = &per_cpu(committed_space, (long)hcpu); + if (action == CPU_DEAD) { + atomic_add(*committed, &vm_committed_space); + *committed = 0; + lru_drain_cache((long)hcpu); + } + return NOTIFY_OK; +} +#endif /* CONFIG_HOTPLUG_CPU */ +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_SMP +void percpu_counter_mod(struct percpu_counter *fbc, long amount) +{ + long count; + long *pcount; + int cpu = get_cpu(); + + pcount = per_cpu_ptr(fbc->counters, cpu); + count = *pcount + amount; + if (count >= FBC_BATCH || count <= -FBC_BATCH) { + spin_lock(&fbc->lock); + fbc->count += count; + spin_unlock(&fbc->lock); + count = 0; + } + *pcount = count; + put_cpu(); +} +EXPORT_SYMBOL(percpu_counter_mod); +#endif + +/* + * Perform any setup for the swap system + */ +void __init swap_setup(void) +{ + unsigned long megs = num_physpages >> (20 - PAGE_SHIFT); + + /* Use a smaller cluster for small-memory machines */ + if (megs < 16) + page_cluster = 2; + else + page_cluster = 3; + /* + * Right now other parts of the system means that we + * _really_ don't want to cluster much more + */ + hotcpu_notifier(cpu_swap_callback, 0); +} diff --git a/mm/swap_state.c b/mm/swap_state.c new file mode 100644 index 000000000000..a063a902ed03 --- /dev/null +++ b/mm/swap_state.c @@ -0,0 +1,382 @@ +/* + * linux/mm/swap_state.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * Swap reorganised 29.12.95, Stephen Tweedie + * + * Rewritten to use page cache, (C) 1998 Stephen Tweedie + */ +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/kernel_stat.h> +#include <linux/swap.h> +#include <linux/init.h> +#include <linux/pagemap.h> +#include <linux/buffer_head.h> +#include <linux/backing-dev.h> + +#include <asm/pgtable.h> + +/* + * swapper_space is a fiction, retained to simplify the path through + * vmscan's shrink_list, to make sync_page look nicer, and to allow + * future use of radix_tree tags in the swap cache. + */ +static struct address_space_operations swap_aops = { + .writepage = swap_writepage, + .sync_page = block_sync_page, + .set_page_dirty = __set_page_dirty_nobuffers, +}; + +static struct backing_dev_info swap_backing_dev_info = { + .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK, + .unplug_io_fn = swap_unplug_io_fn, +}; + +struct address_space swapper_space = { + .page_tree = RADIX_TREE_INIT(GFP_ATOMIC|__GFP_NOWARN), + .tree_lock = RW_LOCK_UNLOCKED, + .a_ops = &swap_aops, + .i_mmap_nonlinear = LIST_HEAD_INIT(swapper_space.i_mmap_nonlinear), + .backing_dev_info = &swap_backing_dev_info, +}; +EXPORT_SYMBOL(swapper_space); + +#define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0) + +static struct { + unsigned long add_total; + unsigned long del_total; + unsigned long find_success; + unsigned long find_total; + unsigned long noent_race; + unsigned long exist_race; +} swap_cache_info; + +void show_swap_cache_info(void) +{ + printk("Swap cache: add %lu, delete %lu, find %lu/%lu, race %lu+%lu\n", + swap_cache_info.add_total, swap_cache_info.del_total, + swap_cache_info.find_success, swap_cache_info.find_total, + swap_cache_info.noent_race, swap_cache_info.exist_race); + printk("Free swap = %lukB\n", nr_swap_pages << (PAGE_SHIFT - 10)); + printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10)); +} + +/* + * __add_to_swap_cache resembles add_to_page_cache on swapper_space, + * but sets SwapCache flag and private instead of mapping and index. + */ +static int __add_to_swap_cache(struct page *page, + swp_entry_t entry, int gfp_mask) +{ + int error; + + BUG_ON(PageSwapCache(page)); + BUG_ON(PagePrivate(page)); + error = radix_tree_preload(gfp_mask); + if (!error) { + write_lock_irq(&swapper_space.tree_lock); + error = radix_tree_insert(&swapper_space.page_tree, + entry.val, page); + if (!error) { + page_cache_get(page); + SetPageLocked(page); + SetPageSwapCache(page); + page->private = entry.val; + total_swapcache_pages++; + pagecache_acct(1); + } + write_unlock_irq(&swapper_space.tree_lock); + radix_tree_preload_end(); + } + return error; +} + +static int add_to_swap_cache(struct page *page, swp_entry_t entry) +{ + int error; + + if (!swap_duplicate(entry)) { + INC_CACHE_INFO(noent_race); + return -ENOENT; + } + error = __add_to_swap_cache(page, entry, GFP_KERNEL); + /* + * Anon pages are already on the LRU, we don't run lru_cache_add here. + */ + if (error) { + swap_free(entry); + if (error == -EEXIST) + INC_CACHE_INFO(exist_race); + return error; + } + INC_CACHE_INFO(add_total); + return 0; +} + +/* + * This must be called only on pages that have + * been verified to be in the swap cache. + */ +void __delete_from_swap_cache(struct page *page) +{ + BUG_ON(!PageLocked(page)); + BUG_ON(!PageSwapCache(page)); + BUG_ON(PageWriteback(page)); + + radix_tree_delete(&swapper_space.page_tree, page->private); + page->private = 0; + ClearPageSwapCache(page); + total_swapcache_pages--; + pagecache_acct(-1); + INC_CACHE_INFO(del_total); +} + +/** + * add_to_swap - allocate swap space for a page + * @page: page we want to move to swap + * + * Allocate swap space for the page and add the page to the + * swap cache. Caller needs to hold the page lock. + */ +int add_to_swap(struct page * page) +{ + swp_entry_t entry; + int pf_flags; + int err; + + if (!PageLocked(page)) + BUG(); + + for (;;) { + entry = get_swap_page(); + if (!entry.val) + return 0; + + /* Radix-tree node allocations are performing + * GFP_ATOMIC allocations under PF_MEMALLOC. + * They can completely exhaust the page allocator. + * + * So PF_MEMALLOC is dropped here. This causes the slab + * allocations to fail earlier, so radix-tree nodes will + * then be allocated from the mempool reserves. + * + * We're still using __GFP_HIGH for radix-tree node + * allocations, so some of the emergency pools are available, + * just not all of them. + */ + + pf_flags = current->flags; + current->flags &= ~PF_MEMALLOC; + + /* + * Add it to the swap cache and mark it dirty + */ + err = __add_to_swap_cache(page, entry, GFP_ATOMIC|__GFP_NOWARN); + + if (pf_flags & PF_MEMALLOC) + current->flags |= PF_MEMALLOC; + + switch (err) { + case 0: /* Success */ + SetPageUptodate(page); + SetPageDirty(page); + INC_CACHE_INFO(add_total); + return 1; + case -EEXIST: + /* Raced with "speculative" read_swap_cache_async */ + INC_CACHE_INFO(exist_race); + swap_free(entry); + continue; + default: + /* -ENOMEM radix-tree allocation failure */ + swap_free(entry); + return 0; + } + } +} + +/* + * This must be called only on pages that have + * been verified to be in the swap cache and locked. + * It will never put the page into the free list, + * the caller has a reference on the page. + */ +void delete_from_swap_cache(struct page *page) +{ + swp_entry_t entry; + + BUG_ON(!PageSwapCache(page)); + BUG_ON(!PageLocked(page)); + BUG_ON(PageWriteback(page)); + BUG_ON(PagePrivate(page)); + + entry.val = page->private; + + write_lock_irq(&swapper_space.tree_lock); + __delete_from_swap_cache(page); + write_unlock_irq(&swapper_space.tree_lock); + + swap_free(entry); + page_cache_release(page); +} + +/* + * Strange swizzling function only for use by shmem_writepage + */ +int move_to_swap_cache(struct page *page, swp_entry_t entry) +{ + int err = __add_to_swap_cache(page, entry, GFP_ATOMIC); + if (!err) { + remove_from_page_cache(page); + page_cache_release(page); /* pagecache ref */ + if (!swap_duplicate(entry)) + BUG(); + SetPageDirty(page); + INC_CACHE_INFO(add_total); + } else if (err == -EEXIST) + INC_CACHE_INFO(exist_race); + return err; +} + +/* + * Strange swizzling function for shmem_getpage (and shmem_unuse) + */ +int move_from_swap_cache(struct page *page, unsigned long index, + struct address_space *mapping) +{ + int err = add_to_page_cache(page, mapping, index, GFP_ATOMIC); + if (!err) { + delete_from_swap_cache(page); + /* shift page from clean_pages to dirty_pages list */ + ClearPageDirty(page); + set_page_dirty(page); + } + return err; +} + +/* + * If we are the only user, then try to free up the swap cache. + * + * Its ok to check for PageSwapCache without the page lock + * here because we are going to recheck again inside + * exclusive_swap_page() _with_ the lock. + * - Marcelo + */ +static inline void free_swap_cache(struct page *page) +{ + if (PageSwapCache(page) && !TestSetPageLocked(page)) { + remove_exclusive_swap_page(page); + unlock_page(page); + } +} + +/* + * Perform a free_page(), also freeing any swap cache associated with + * this page if it is the last user of the page. Can not do a lock_page, + * as we are holding the page_table_lock spinlock. + */ +void free_page_and_swap_cache(struct page *page) +{ + free_swap_cache(page); + page_cache_release(page); +} + +/* + * Passed an array of pages, drop them all from swapcache and then release + * them. They are removed from the LRU and freed if this is their last use. + */ +void free_pages_and_swap_cache(struct page **pages, int nr) +{ + int chunk = 16; + struct page **pagep = pages; + + lru_add_drain(); + while (nr) { + int todo = min(chunk, nr); + int i; + + for (i = 0; i < todo; i++) + free_swap_cache(pagep[i]); + release_pages(pagep, todo, 0); + pagep += todo; + nr -= todo; + } +} + +/* + * Lookup a swap entry in the swap cache. A found page will be returned + * unlocked and with its refcount incremented - we rely on the kernel + * lock getting page table operations atomic even if we drop the page + * lock before returning. + */ +struct page * lookup_swap_cache(swp_entry_t entry) +{ + struct page *page; + + page = find_get_page(&swapper_space, entry.val); + + if (page) + INC_CACHE_INFO(find_success); + + INC_CACHE_INFO(find_total); + return page; +} + +/* + * Locate a page of swap in physical memory, reserving swap cache space + * and reading the disk if it is not already cached. + * A failure return means that either the page allocation failed or that + * the swap entry is no longer in use. + */ +struct page *read_swap_cache_async(swp_entry_t entry, + struct vm_area_struct *vma, unsigned long addr) +{ + struct page *found_page, *new_page = NULL; + int err; + + do { + /* + * First check the swap cache. Since this is normally + * called after lookup_swap_cache() failed, re-calling + * that would confuse statistics. + */ + found_page = find_get_page(&swapper_space, entry.val); + if (found_page) + break; + + /* + * Get a new page to read into from swap. + */ + if (!new_page) { + new_page = alloc_page_vma(GFP_HIGHUSER, vma, addr); + if (!new_page) + break; /* Out of memory */ + } + + /* + * Associate the page with swap entry in the swap cache. + * May fail (-ENOENT) if swap entry has been freed since + * our caller observed it. May fail (-EEXIST) if there + * is already a page associated with this entry in the + * swap cache: added by a racing read_swap_cache_async, + * or by try_to_swap_out (or shmem_writepage) re-using + * the just freed swap entry for an existing page. + * May fail (-ENOMEM) if radix-tree node allocation failed. + */ + err = add_to_swap_cache(new_page, entry); + if (!err) { + /* + * Initiate read into locked page and return. + */ + lru_cache_add_active(new_page); + swap_readpage(NULL, new_page); + return new_page; + } + } while (err != -ENOENT && err != -ENOMEM); + + if (new_page) + page_cache_release(new_page); + return found_page; +} diff --git a/mm/swapfile.c b/mm/swapfile.c new file mode 100644 index 000000000000..a60e0075d55b --- /dev/null +++ b/mm/swapfile.c @@ -0,0 +1,1672 @@ +/* + * linux/mm/swapfile.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * Swap reorganised 29.12.95, Stephen Tweedie + */ + +#include <linux/config.h> +#include <linux/mm.h> +#include <linux/hugetlb.h> +#include <linux/mman.h> +#include <linux/slab.h> +#include <linux/kernel_stat.h> +#include <linux/swap.h> +#include <linux/vmalloc.h> +#include <linux/pagemap.h> +#include <linux/namei.h> +#include <linux/shm.h> +#include <linux/blkdev.h> +#include <linux/writeback.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/rmap.h> +#include <linux/security.h> +#include <linux/backing-dev.h> +#include <linux/syscalls.h> + +#include <asm/pgtable.h> +#include <asm/tlbflush.h> +#include <linux/swapops.h> + +DEFINE_SPINLOCK(swaplock); +unsigned int nr_swapfiles; +long total_swap_pages; +static int swap_overflow; + +EXPORT_SYMBOL(total_swap_pages); + +static const char Bad_file[] = "Bad swap file entry "; +static const char Unused_file[] = "Unused swap file entry "; +static const char Bad_offset[] = "Bad swap offset entry "; +static const char Unused_offset[] = "Unused swap offset entry "; + +struct swap_list_t swap_list = {-1, -1}; + +struct swap_info_struct swap_info[MAX_SWAPFILES]; + +static DECLARE_MUTEX(swapon_sem); + +/* + * We need this because the bdev->unplug_fn can sleep and we cannot + * hold swap_list_lock while calling the unplug_fn. And swap_list_lock + * cannot be turned into a semaphore. + */ +static DECLARE_RWSEM(swap_unplug_sem); + +#define SWAPFILE_CLUSTER 256 + +void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) +{ + swp_entry_t entry; + + down_read(&swap_unplug_sem); + entry.val = page->private; + if (PageSwapCache(page)) { + struct block_device *bdev = swap_info[swp_type(entry)].bdev; + struct backing_dev_info *bdi; + + /* + * If the page is removed from swapcache from under us (with a + * racy try_to_unuse/swapoff) we need an additional reference + * count to avoid reading garbage from page->private above. If + * the WARN_ON triggers during a swapoff it maybe the race + * condition and it's harmless. However if it triggers without + * swapoff it signals a problem. + */ + WARN_ON(page_count(page) <= 1); + + bdi = bdev->bd_inode->i_mapping->backing_dev_info; + bdi->unplug_io_fn(bdi, page); + } + up_read(&swap_unplug_sem); +} + +static inline int scan_swap_map(struct swap_info_struct *si) +{ + unsigned long offset; + /* + * We try to cluster swap pages by allocating them + * sequentially in swap. Once we've allocated + * SWAPFILE_CLUSTER pages this way, however, we resort to + * first-free allocation, starting a new cluster. This + * prevents us from scattering swap pages all over the entire + * swap partition, so that we reduce overall disk seek times + * between swap pages. -- sct */ + if (si->cluster_nr) { + while (si->cluster_next <= si->highest_bit) { + offset = si->cluster_next++; + if (si->swap_map[offset]) + continue; + si->cluster_nr--; + goto got_page; + } + } + si->cluster_nr = SWAPFILE_CLUSTER; + + /* try to find an empty (even not aligned) cluster. */ + offset = si->lowest_bit; + check_next_cluster: + if (offset+SWAPFILE_CLUSTER-1 <= si->highest_bit) + { + unsigned long nr; + for (nr = offset; nr < offset+SWAPFILE_CLUSTER; nr++) + if (si->swap_map[nr]) + { + offset = nr+1; + goto check_next_cluster; + } + /* We found a completly empty cluster, so start + * using it. + */ + goto got_page; + } + /* No luck, so now go finegrined as usual. -Andrea */ + for (offset = si->lowest_bit; offset <= si->highest_bit ; offset++) { + if (si->swap_map[offset]) + continue; + si->lowest_bit = offset+1; + got_page: + if (offset == si->lowest_bit) + si->lowest_bit++; + if (offset == si->highest_bit) + si->highest_bit--; + if (si->lowest_bit > si->highest_bit) { + si->lowest_bit = si->max; + si->highest_bit = 0; + } + si->swap_map[offset] = 1; + si->inuse_pages++; + nr_swap_pages--; + si->cluster_next = offset+1; + return offset; + } + si->lowest_bit = si->max; + si->highest_bit = 0; + return 0; +} + +swp_entry_t get_swap_page(void) +{ + struct swap_info_struct * p; + unsigned long offset; + swp_entry_t entry; + int type, wrapped = 0; + + entry.val = 0; /* Out of memory */ + swap_list_lock(); + type = swap_list.next; + if (type < 0) + goto out; + if (nr_swap_pages <= 0) + goto out; + + while (1) { + p = &swap_info[type]; + if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) { + swap_device_lock(p); + offset = scan_swap_map(p); + swap_device_unlock(p); + if (offset) { + entry = swp_entry(type,offset); + type = swap_info[type].next; + if (type < 0 || + p->prio != swap_info[type].prio) { + swap_list.next = swap_list.head; + } else { + swap_list.next = type; + } + goto out; + } + } + type = p->next; + if (!wrapped) { + if (type < 0 || p->prio != swap_info[type].prio) { + type = swap_list.head; + wrapped = 1; + } + } else + if (type < 0) + goto out; /* out of swap space */ + } +out: + swap_list_unlock(); + return entry; +} + +static struct swap_info_struct * swap_info_get(swp_entry_t entry) +{ + struct swap_info_struct * p; + unsigned long offset, type; + + if (!entry.val) + goto out; + type = swp_type(entry); + if (type >= nr_swapfiles) + goto bad_nofile; + p = & swap_info[type]; + if (!(p->flags & SWP_USED)) + goto bad_device; + offset = swp_offset(entry); + if (offset >= p->max) + goto bad_offset; + if (!p->swap_map[offset]) + goto bad_free; + swap_list_lock(); + if (p->prio > swap_info[swap_list.next].prio) + swap_list.next = type; + swap_device_lock(p); + return p; + +bad_free: + printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val); + goto out; +bad_offset: + printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val); + goto out; +bad_device: + printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val); + goto out; +bad_nofile: + printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val); +out: + return NULL; +} + +static void swap_info_put(struct swap_info_struct * p) +{ + swap_device_unlock(p); + swap_list_unlock(); +} + +static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) +{ + int count = p->swap_map[offset]; + + if (count < SWAP_MAP_MAX) { + count--; + p->swap_map[offset] = count; + if (!count) { + if (offset < p->lowest_bit) + p->lowest_bit = offset; + if (offset > p->highest_bit) + p->highest_bit = offset; + nr_swap_pages++; + p->inuse_pages--; + } + } + return count; +} + +/* + * Caller has made sure that the swapdevice corresponding to entry + * is still around or has not been recycled. + */ +void swap_free(swp_entry_t entry) +{ + struct swap_info_struct * p; + + p = swap_info_get(entry); + if (p) { + swap_entry_free(p, swp_offset(entry)); + swap_info_put(p); + } +} + +/* + * Check if we're the only user of a swap page, + * when the page is locked. + */ +static int exclusive_swap_page(struct page *page) +{ + int retval = 0; + struct swap_info_struct * p; + swp_entry_t entry; + + entry.val = page->private; + p = swap_info_get(entry); + if (p) { + /* Is the only swap cache user the cache itself? */ + if (p->swap_map[swp_offset(entry)] == 1) { + /* Recheck the page count with the swapcache lock held.. */ + write_lock_irq(&swapper_space.tree_lock); + if (page_count(page) == 2) + retval = 1; + write_unlock_irq(&swapper_space.tree_lock); + } + swap_info_put(p); + } + return retval; +} + +/* + * We can use this swap cache entry directly + * if there are no other references to it. + * + * Here "exclusive_swap_page()" does the real + * work, but we opportunistically check whether + * we need to get all the locks first.. + */ +int can_share_swap_page(struct page *page) +{ + int retval = 0; + + if (!PageLocked(page)) + BUG(); + switch (page_count(page)) { + case 3: + if (!PagePrivate(page)) + break; + /* Fallthrough */ + case 2: + if (!PageSwapCache(page)) + break; + retval = exclusive_swap_page(page); + break; + case 1: + if (PageReserved(page)) + break; + retval = 1; + } + return retval; +} + +/* + * Work out if there are any other processes sharing this + * swap cache page. Free it if you can. Return success. + */ +int remove_exclusive_swap_page(struct page *page) +{ + int retval; + struct swap_info_struct * p; + swp_entry_t entry; + + BUG_ON(PagePrivate(page)); + BUG_ON(!PageLocked(page)); + + if (!PageSwapCache(page)) + return 0; + if (PageWriteback(page)) + return 0; + if (page_count(page) != 2) /* 2: us + cache */ + return 0; + + entry.val = page->private; + p = swap_info_get(entry); + if (!p) + return 0; + + /* Is the only swap cache user the cache itself? */ + retval = 0; + if (p->swap_map[swp_offset(entry)] == 1) { + /* Recheck the page count with the swapcache lock held.. */ + write_lock_irq(&swapper_space.tree_lock); + if ((page_count(page) == 2) && !PageWriteback(page)) { + __delete_from_swap_cache(page); + SetPageDirty(page); + retval = 1; + } + write_unlock_irq(&swapper_space.tree_lock); + } + swap_info_put(p); + + if (retval) { + swap_free(entry); + page_cache_release(page); + } + + return retval; +} + +/* + * Free the swap entry like above, but also try to + * free the page cache entry if it is the last user. + */ +void free_swap_and_cache(swp_entry_t entry) +{ + struct swap_info_struct * p; + struct page *page = NULL; + + p = swap_info_get(entry); + if (p) { + if (swap_entry_free(p, swp_offset(entry)) == 1) + page = find_trylock_page(&swapper_space, entry.val); + swap_info_put(p); + } + if (page) { + int one_user; + + BUG_ON(PagePrivate(page)); + page_cache_get(page); + one_user = (page_count(page) == 2); + /* Only cache user (+us), or swap space full? Free it! */ + if (!PageWriteback(page) && (one_user || vm_swap_full())) { + delete_from_swap_cache(page); + SetPageDirty(page); + } + unlock_page(page); + page_cache_release(page); + } +} + +/* + * Always set the resulting pte to be nowrite (the same as COW pages + * after one process has exited). We don't know just how many PTEs will + * share this swap entry, so be cautious and let do_wp_page work out + * what to do if a write is requested later. + * + * vma->vm_mm->page_table_lock is held. + */ +static void unuse_pte(struct vm_area_struct *vma, pte_t *pte, + unsigned long addr, swp_entry_t entry, struct page *page) +{ + inc_mm_counter(vma->vm_mm, rss); + get_page(page); + set_pte_at(vma->vm_mm, addr, pte, + pte_mkold(mk_pte(page, vma->vm_page_prot))); + page_add_anon_rmap(page, vma, addr); + swap_free(entry); + /* + * Move the page to the active list so it is not + * immediately swapped out again after swapon. + */ + activate_page(page); +} + +static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) +{ + pte_t *pte; + pte_t swp_pte = swp_entry_to_pte(entry); + + pte = pte_offset_map(pmd, addr); + do { + /* + * swapoff spends a _lot_ of time in this loop! + * Test inline before going to call unuse_pte. + */ + if (unlikely(pte_same(*pte, swp_pte))) { + unuse_pte(vma, pte, addr, entry, page); + pte_unmap(pte); + return 1; + } + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap(pte - 1); + return 0; +} + +static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + if (unuse_pte_range(vma, pmd, addr, next, entry, page)) + return 1; + } while (pmd++, addr = next, addr != end); + return 0; +} + +static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) +{ + pud_t *pud; + unsigned long next; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + if (unuse_pmd_range(vma, pud, addr, next, entry, page)) + return 1; + } while (pud++, addr = next, addr != end); + return 0; +} + +static int unuse_vma(struct vm_area_struct *vma, + swp_entry_t entry, struct page *page) +{ + pgd_t *pgd; + unsigned long addr, end, next; + + if (page->mapping) { + addr = page_address_in_vma(page, vma); + if (addr == -EFAULT) + return 0; + else + end = addr + PAGE_SIZE; + } else { + addr = vma->vm_start; + end = vma->vm_end; + } + + pgd = pgd_offset(vma->vm_mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + if (unuse_pud_range(vma, pgd, addr, next, entry, page)) + return 1; + } while (pgd++, addr = next, addr != end); + return 0; +} + +static int unuse_mm(struct mm_struct *mm, + swp_entry_t entry, struct page *page) +{ + struct vm_area_struct *vma; + + if (!down_read_trylock(&mm->mmap_sem)) { + /* + * Our reference to the page stops try_to_unmap_one from + * unmapping its ptes, so swapoff can make progress. + */ + unlock_page(page); + down_read(&mm->mmap_sem); + lock_page(page); + } + spin_lock(&mm->page_table_lock); + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (vma->anon_vma && unuse_vma(vma, entry, page)) + break; + } + spin_unlock(&mm->page_table_lock); + up_read(&mm->mmap_sem); + /* + * Currently unuse_mm cannot fail, but leave error handling + * at call sites for now, since we change it from time to time. + */ + return 0; +} + +/* + * Scan swap_map from current position to next entry still in use. + * Recycle to start on reaching the end, returning 0 when empty. + */ +static int find_next_to_unuse(struct swap_info_struct *si, int prev) +{ + int max = si->max; + int i = prev; + int count; + + /* + * No need for swap_device_lock(si) here: we're just looking + * for whether an entry is in use, not modifying it; false + * hits are okay, and sys_swapoff() has already prevented new + * allocations from this area (while holding swap_list_lock()). + */ + for (;;) { + if (++i >= max) { + if (!prev) { + i = 0; + break; + } + /* + * No entries in use at top of swap_map, + * loop back to start and recheck there. + */ + max = prev + 1; + prev = 0; + i = 1; + } + count = si->swap_map[i]; + if (count && count != SWAP_MAP_BAD) + break; + } + return i; +} + +/* + * We completely avoid races by reading each swap page in advance, + * and then search for the process using it. All the necessary + * page table adjustments can then be made atomically. + */ +static int try_to_unuse(unsigned int type) +{ + struct swap_info_struct * si = &swap_info[type]; + struct mm_struct *start_mm; + unsigned short *swap_map; + unsigned short swcount; + struct page *page; + swp_entry_t entry; + int i = 0; + int retval = 0; + int reset_overflow = 0; + int shmem; + + /* + * When searching mms for an entry, a good strategy is to + * start at the first mm we freed the previous entry from + * (though actually we don't notice whether we or coincidence + * freed the entry). Initialize this start_mm with a hold. + * + * A simpler strategy would be to start at the last mm we + * freed the previous entry from; but that would take less + * advantage of mmlist ordering, which clusters forked mms + * together, child after parent. If we race with dup_mmap(), we + * prefer to resolve parent before child, lest we miss entries + * duplicated after we scanned child: using last mm would invert + * that. Though it's only a serious concern when an overflowed + * swap count is reset from SWAP_MAP_MAX, preventing a rescan. + */ + start_mm = &init_mm; + atomic_inc(&init_mm.mm_users); + + /* + * Keep on scanning until all entries have gone. Usually, + * one pass through swap_map is enough, but not necessarily: + * there are races when an instance of an entry might be missed. + */ + while ((i = find_next_to_unuse(si, i)) != 0) { + if (signal_pending(current)) { + retval = -EINTR; + break; + } + + /* + * Get a page for the entry, using the existing swap + * cache page if there is one. Otherwise, get a clean + * page and read the swap into it. + */ + swap_map = &si->swap_map[i]; + entry = swp_entry(type, i); + page = read_swap_cache_async(entry, NULL, 0); + if (!page) { + /* + * Either swap_duplicate() failed because entry + * has been freed independently, and will not be + * reused since sys_swapoff() already disabled + * allocation from here, or alloc_page() failed. + */ + if (!*swap_map) + continue; + retval = -ENOMEM; + break; + } + + /* + * Don't hold on to start_mm if it looks like exiting. + */ + if (atomic_read(&start_mm->mm_users) == 1) { + mmput(start_mm); + start_mm = &init_mm; + atomic_inc(&init_mm.mm_users); + } + + /* + * Wait for and lock page. When do_swap_page races with + * try_to_unuse, do_swap_page can handle the fault much + * faster than try_to_unuse can locate the entry. This + * apparently redundant "wait_on_page_locked" lets try_to_unuse + * defer to do_swap_page in such a case - in some tests, + * do_swap_page and try_to_unuse repeatedly compete. + */ + wait_on_page_locked(page); + wait_on_page_writeback(page); + lock_page(page); + wait_on_page_writeback(page); + + /* + * Remove all references to entry. + * Whenever we reach init_mm, there's no address space + * to search, but use it as a reminder to search shmem. + */ + shmem = 0; + swcount = *swap_map; + if (swcount > 1) { + if (start_mm == &init_mm) + shmem = shmem_unuse(entry, page); + else + retval = unuse_mm(start_mm, entry, page); + } + if (*swap_map > 1) { + int set_start_mm = (*swap_map >= swcount); + struct list_head *p = &start_mm->mmlist; + struct mm_struct *new_start_mm = start_mm; + struct mm_struct *prev_mm = start_mm; + struct mm_struct *mm; + + atomic_inc(&new_start_mm->mm_users); + atomic_inc(&prev_mm->mm_users); + spin_lock(&mmlist_lock); + while (*swap_map > 1 && !retval && + (p = p->next) != &start_mm->mmlist) { + mm = list_entry(p, struct mm_struct, mmlist); + if (atomic_inc_return(&mm->mm_users) == 1) { + atomic_dec(&mm->mm_users); + continue; + } + spin_unlock(&mmlist_lock); + mmput(prev_mm); + prev_mm = mm; + + cond_resched(); + + swcount = *swap_map; + if (swcount <= 1) + ; + else if (mm == &init_mm) { + set_start_mm = 1; + shmem = shmem_unuse(entry, page); + } else + retval = unuse_mm(mm, entry, page); + if (set_start_mm && *swap_map < swcount) { + mmput(new_start_mm); + atomic_inc(&mm->mm_users); + new_start_mm = mm; + set_start_mm = 0; + } + spin_lock(&mmlist_lock); + } + spin_unlock(&mmlist_lock); + mmput(prev_mm); + mmput(start_mm); + start_mm = new_start_mm; + } + if (retval) { + unlock_page(page); + page_cache_release(page); + break; + } + + /* + * How could swap count reach 0x7fff when the maximum + * pid is 0x7fff, and there's no way to repeat a swap + * page within an mm (except in shmem, where it's the + * shared object which takes the reference count)? + * We believe SWAP_MAP_MAX cannot occur in Linux 2.4. + * + * If that's wrong, then we should worry more about + * exit_mmap() and do_munmap() cases described above: + * we might be resetting SWAP_MAP_MAX too early here. + * We know "Undead"s can happen, they're okay, so don't + * report them; but do report if we reset SWAP_MAP_MAX. + */ + if (*swap_map == SWAP_MAP_MAX) { + swap_device_lock(si); + *swap_map = 1; + swap_device_unlock(si); + reset_overflow = 1; + } + + /* + * If a reference remains (rare), we would like to leave + * the page in the swap cache; but try_to_unmap could + * then re-duplicate the entry once we drop page lock, + * so we might loop indefinitely; also, that page could + * not be swapped out to other storage meanwhile. So: + * delete from cache even if there's another reference, + * after ensuring that the data has been saved to disk - + * since if the reference remains (rarer), it will be + * read from disk into another page. Splitting into two + * pages would be incorrect if swap supported "shared + * private" pages, but they are handled by tmpfs files. + * + * Note shmem_unuse already deleted a swappage from + * the swap cache, unless the move to filepage failed: + * in which case it left swappage in cache, lowered its + * swap count to pass quickly through the loops above, + * and now we must reincrement count to try again later. + */ + if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + }; + + swap_writepage(page, &wbc); + lock_page(page); + wait_on_page_writeback(page); + } + if (PageSwapCache(page)) { + if (shmem) + swap_duplicate(entry); + else + delete_from_swap_cache(page); + } + + /* + * So we could skip searching mms once swap count went + * to 1, we did not mark any present ptes as dirty: must + * mark page dirty so shrink_list will preserve it. + */ + SetPageDirty(page); + unlock_page(page); + page_cache_release(page); + + /* + * Make sure that we aren't completely killing + * interactive performance. + */ + cond_resched(); + } + + mmput(start_mm); + if (reset_overflow) { + printk(KERN_WARNING "swapoff: cleared swap entry overflow\n"); + swap_overflow = 0; + } + return retval; +} + +/* + * After a successful try_to_unuse, if no swap is now in use, we know we + * can empty the mmlist. swap_list_lock must be held on entry and exit. + * Note that mmlist_lock nests inside swap_list_lock, and an mm must be + * added to the mmlist just after page_duplicate - before would be racy. + */ +static void drain_mmlist(void) +{ + struct list_head *p, *next; + unsigned int i; + + for (i = 0; i < nr_swapfiles; i++) + if (swap_info[i].inuse_pages) + return; + spin_lock(&mmlist_lock); + list_for_each_safe(p, next, &init_mm.mmlist) + list_del_init(p); + spin_unlock(&mmlist_lock); +} + +/* + * Use this swapdev's extent info to locate the (PAGE_SIZE) block which + * corresponds to page offset `offset'. + */ +sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset) +{ + struct swap_extent *se = sis->curr_swap_extent; + struct swap_extent *start_se = se; + + for ( ; ; ) { + struct list_head *lh; + + if (se->start_page <= offset && + offset < (se->start_page + se->nr_pages)) { + return se->start_block + (offset - se->start_page); + } + lh = se->list.prev; + if (lh == &sis->extent_list) + lh = lh->prev; + se = list_entry(lh, struct swap_extent, list); + sis->curr_swap_extent = se; + BUG_ON(se == start_se); /* It *must* be present */ + } +} + +/* + * Free all of a swapdev's extent information + */ +static void destroy_swap_extents(struct swap_info_struct *sis) +{ + while (!list_empty(&sis->extent_list)) { + struct swap_extent *se; + + se = list_entry(sis->extent_list.next, + struct swap_extent, list); + list_del(&se->list); + kfree(se); + } + sis->nr_extents = 0; +} + +/* + * Add a block range (and the corresponding page range) into this swapdev's + * extent list. The extent list is kept sorted in block order. + * + * This function rather assumes that it is called in ascending sector_t order. + * It doesn't look for extent coalescing opportunities. + */ +static int +add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, + unsigned long nr_pages, sector_t start_block) +{ + struct swap_extent *se; + struct swap_extent *new_se; + struct list_head *lh; + + lh = sis->extent_list.next; /* The highest-addressed block */ + while (lh != &sis->extent_list) { + se = list_entry(lh, struct swap_extent, list); + if (se->start_block + se->nr_pages == start_block && + se->start_page + se->nr_pages == start_page) { + /* Merge it */ + se->nr_pages += nr_pages; + return 0; + } + lh = lh->next; + } + + /* + * No merge. Insert a new extent, preserving ordering. + */ + new_se = kmalloc(sizeof(*se), GFP_KERNEL); + if (new_se == NULL) + return -ENOMEM; + new_se->start_page = start_page; + new_se->nr_pages = nr_pages; + new_se->start_block = start_block; + + lh = sis->extent_list.prev; /* The lowest block */ + while (lh != &sis->extent_list) { + se = list_entry(lh, struct swap_extent, list); + if (se->start_block > start_block) + break; + lh = lh->prev; + } + list_add_tail(&new_se->list, lh); + sis->nr_extents++; + return 0; +} + +/* + * A `swap extent' is a simple thing which maps a contiguous range of pages + * onto a contiguous range of disk blocks. An ordered list of swap extents + * is built at swapon time and is then used at swap_writepage/swap_readpage + * time for locating where on disk a page belongs. + * + * If the swapfile is an S_ISBLK block device, a single extent is installed. + * This is done so that the main operating code can treat S_ISBLK and S_ISREG + * swap files identically. + * + * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap + * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK + * swapfiles are handled *identically* after swapon time. + * + * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks + * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If + * some stray blocks are found which do not fall within the PAGE_SIZE alignment + * requirements, they are simply tossed out - we will never use those blocks + * for swapping. + * + * For S_ISREG swapfiles we hold i_sem across the life of the swapon. This + * prevents root from shooting her foot off by ftruncating an in-use swapfile, + * which will scribble on the fs. + * + * The amount of disk space which a single swap extent represents varies. + * Typically it is in the 1-4 megabyte range. So we can have hundreds of + * extents in the list. To avoid much list walking, we cache the previous + * search location in `curr_swap_extent', and start new searches from there. + * This is extremely effective. The average number of iterations in + * map_swap_page() has been measured at about 0.3 per page. - akpm. + */ +static int setup_swap_extents(struct swap_info_struct *sis) +{ + struct inode *inode; + unsigned blocks_per_page; + unsigned long page_no; + unsigned blkbits; + sector_t probe_block; + sector_t last_block; + int ret; + + inode = sis->swap_file->f_mapping->host; + if (S_ISBLK(inode->i_mode)) { + ret = add_swap_extent(sis, 0, sis->max, 0); + goto done; + } + + blkbits = inode->i_blkbits; + blocks_per_page = PAGE_SIZE >> blkbits; + + /* + * Map all the blocks into the extent list. This code doesn't try + * to be very smart. + */ + probe_block = 0; + page_no = 0; + last_block = i_size_read(inode) >> blkbits; + while ((probe_block + blocks_per_page) <= last_block && + page_no < sis->max) { + unsigned block_in_page; + sector_t first_block; + + first_block = bmap(inode, probe_block); + if (first_block == 0) + goto bad_bmap; + + /* + * It must be PAGE_SIZE aligned on-disk + */ + if (first_block & (blocks_per_page - 1)) { + probe_block++; + goto reprobe; + } + + for (block_in_page = 1; block_in_page < blocks_per_page; + block_in_page++) { + sector_t block; + + block = bmap(inode, probe_block + block_in_page); + if (block == 0) + goto bad_bmap; + if (block != first_block + block_in_page) { + /* Discontiguity */ + probe_block++; + goto reprobe; + } + } + + /* + * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks + */ + ret = add_swap_extent(sis, page_no, 1, + first_block >> (PAGE_SHIFT - blkbits)); + if (ret) + goto out; + page_no++; + probe_block += blocks_per_page; +reprobe: + continue; + } + ret = 0; + if (page_no == 0) + ret = -EINVAL; + sis->max = page_no; + sis->highest_bit = page_no - 1; +done: + sis->curr_swap_extent = list_entry(sis->extent_list.prev, + struct swap_extent, list); + goto out; +bad_bmap: + printk(KERN_ERR "swapon: swapfile has holes\n"); + ret = -EINVAL; +out: + return ret; +} + +#if 0 /* We don't need this yet */ +#include <linux/backing-dev.h> +int page_queue_congested(struct page *page) +{ + struct backing_dev_info *bdi; + + BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */ + + if (PageSwapCache(page)) { + swp_entry_t entry = { .val = page->private }; + struct swap_info_struct *sis; + + sis = get_swap_info_struct(swp_type(entry)); + bdi = sis->bdev->bd_inode->i_mapping->backing_dev_info; + } else + bdi = page->mapping->backing_dev_info; + return bdi_write_congested(bdi); +} +#endif + +asmlinkage long sys_swapoff(const char __user * specialfile) +{ + struct swap_info_struct * p = NULL; + unsigned short *swap_map; + struct file *swap_file, *victim; + struct address_space *mapping; + struct inode *inode; + char * pathname; + int i, type, prev; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + pathname = getname(specialfile); + err = PTR_ERR(pathname); + if (IS_ERR(pathname)) + goto out; + + victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0); + putname(pathname); + err = PTR_ERR(victim); + if (IS_ERR(victim)) + goto out; + + mapping = victim->f_mapping; + prev = -1; + swap_list_lock(); + for (type = swap_list.head; type >= 0; type = swap_info[type].next) { + p = swap_info + type; + if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) { + if (p->swap_file->f_mapping == mapping) + break; + } + prev = type; + } + if (type < 0) { + err = -EINVAL; + swap_list_unlock(); + goto out_dput; + } + if (!security_vm_enough_memory(p->pages)) + vm_unacct_memory(p->pages); + else { + err = -ENOMEM; + swap_list_unlock(); + goto out_dput; + } + if (prev < 0) { + swap_list.head = p->next; + } else { + swap_info[prev].next = p->next; + } + if (type == swap_list.next) { + /* just pick something that's safe... */ + swap_list.next = swap_list.head; + } + nr_swap_pages -= p->pages; + total_swap_pages -= p->pages; + p->flags &= ~SWP_WRITEOK; + swap_list_unlock(); + current->flags |= PF_SWAPOFF; + err = try_to_unuse(type); + current->flags &= ~PF_SWAPOFF; + + /* wait for any unplug function to finish */ + down_write(&swap_unplug_sem); + up_write(&swap_unplug_sem); + + if (err) { + /* re-insert swap space back into swap_list */ + swap_list_lock(); + for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next) + if (p->prio >= swap_info[i].prio) + break; + p->next = i; + if (prev < 0) + swap_list.head = swap_list.next = p - swap_info; + else + swap_info[prev].next = p - swap_info; + nr_swap_pages += p->pages; + total_swap_pages += p->pages; + p->flags |= SWP_WRITEOK; + swap_list_unlock(); + goto out_dput; + } + down(&swapon_sem); + swap_list_lock(); + drain_mmlist(); + swap_device_lock(p); + swap_file = p->swap_file; + p->swap_file = NULL; + p->max = 0; + swap_map = p->swap_map; + p->swap_map = NULL; + p->flags = 0; + destroy_swap_extents(p); + swap_device_unlock(p); + swap_list_unlock(); + up(&swapon_sem); + vfree(swap_map); + inode = mapping->host; + if (S_ISBLK(inode->i_mode)) { + struct block_device *bdev = I_BDEV(inode); + set_blocksize(bdev, p->old_block_size); + bd_release(bdev); + } else { + down(&inode->i_sem); + inode->i_flags &= ~S_SWAPFILE; + up(&inode->i_sem); + } + filp_close(swap_file, NULL); + err = 0; + +out_dput: + filp_close(victim, NULL); +out: + return err; +} + +#ifdef CONFIG_PROC_FS +/* iterator */ +static void *swap_start(struct seq_file *swap, loff_t *pos) +{ + struct swap_info_struct *ptr = swap_info; + int i; + loff_t l = *pos; + + down(&swapon_sem); + + for (i = 0; i < nr_swapfiles; i++, ptr++) { + if (!(ptr->flags & SWP_USED) || !ptr->swap_map) + continue; + if (!l--) + return ptr; + } + + return NULL; +} + +static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) +{ + struct swap_info_struct *ptr = v; + struct swap_info_struct *endptr = swap_info + nr_swapfiles; + + for (++ptr; ptr < endptr; ptr++) { + if (!(ptr->flags & SWP_USED) || !ptr->swap_map) + continue; + ++*pos; + return ptr; + } + + return NULL; +} + +static void swap_stop(struct seq_file *swap, void *v) +{ + up(&swapon_sem); +} + +static int swap_show(struct seq_file *swap, void *v) +{ + struct swap_info_struct *ptr = v; + struct file *file; + int len; + + if (v == swap_info) + seq_puts(swap, "Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n"); + + file = ptr->swap_file; + len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\"); + seq_printf(swap, "%*s%s\t%d\t%ld\t%d\n", + len < 40 ? 40 - len : 1, " ", + S_ISBLK(file->f_dentry->d_inode->i_mode) ? + "partition" : "file\t", + ptr->pages << (PAGE_SHIFT - 10), + ptr->inuse_pages << (PAGE_SHIFT - 10), + ptr->prio); + return 0; +} + +static struct seq_operations swaps_op = { + .start = swap_start, + .next = swap_next, + .stop = swap_stop, + .show = swap_show +}; + +static int swaps_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &swaps_op); +} + +static struct file_operations proc_swaps_operations = { + .open = swaps_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __init procswaps_init(void) +{ + struct proc_dir_entry *entry; + + entry = create_proc_entry("swaps", 0, NULL); + if (entry) + entry->proc_fops = &proc_swaps_operations; + return 0; +} +__initcall(procswaps_init); +#endif /* CONFIG_PROC_FS */ + +/* + * Written 01/25/92 by Simmule Turner, heavily changed by Linus. + * + * The swapon system call + */ +asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) +{ + struct swap_info_struct * p; + char *name = NULL; + struct block_device *bdev = NULL; + struct file *swap_file = NULL; + struct address_space *mapping; + unsigned int type; + int i, prev; + int error; + static int least_priority; + union swap_header *swap_header = NULL; + int swap_header_version; + int nr_good_pages = 0; + unsigned long maxpages = 1; + int swapfilesize; + unsigned short *swap_map; + struct page *page = NULL; + struct inode *inode = NULL; + int did_down = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + swap_list_lock(); + p = swap_info; + for (type = 0 ; type < nr_swapfiles ; type++,p++) + if (!(p->flags & SWP_USED)) + break; + error = -EPERM; + /* + * Test if adding another swap device is possible. There are + * two limiting factors: 1) the number of bits for the swap + * type swp_entry_t definition and 2) the number of bits for + * the swap type in the swap ptes as defined by the different + * architectures. To honor both limitations a swap entry + * with swap offset 0 and swap type ~0UL is created, encoded + * to a swap pte, decoded to a swp_entry_t again and finally + * the swap type part is extracted. This will mask all bits + * from the initial ~0UL that can't be encoded in either the + * swp_entry_t or the architecture definition of a swap pte. + */ + if (type > swp_type(pte_to_swp_entry(swp_entry_to_pte(swp_entry(~0UL,0))))) { + swap_list_unlock(); + goto out; + } + if (type >= nr_swapfiles) + nr_swapfiles = type+1; + INIT_LIST_HEAD(&p->extent_list); + p->flags = SWP_USED; + p->nr_extents = 0; + p->swap_file = NULL; + p->old_block_size = 0; + p->swap_map = NULL; + p->lowest_bit = 0; + p->highest_bit = 0; + p->cluster_nr = 0; + p->inuse_pages = 0; + spin_lock_init(&p->sdev_lock); + p->next = -1; + if (swap_flags & SWAP_FLAG_PREFER) { + p->prio = + (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT; + } else { + p->prio = --least_priority; + } + swap_list_unlock(); + name = getname(specialfile); + error = PTR_ERR(name); + if (IS_ERR(name)) { + name = NULL; + goto bad_swap_2; + } + swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0); + error = PTR_ERR(swap_file); + if (IS_ERR(swap_file)) { + swap_file = NULL; + goto bad_swap_2; + } + + p->swap_file = swap_file; + mapping = swap_file->f_mapping; + inode = mapping->host; + + error = -EBUSY; + for (i = 0; i < nr_swapfiles; i++) { + struct swap_info_struct *q = &swap_info[i]; + + if (i == type || !q->swap_file) + continue; + if (mapping == q->swap_file->f_mapping) + goto bad_swap; + } + + error = -EINVAL; + if (S_ISBLK(inode->i_mode)) { + bdev = I_BDEV(inode); + error = bd_claim(bdev, sys_swapon); + if (error < 0) { + bdev = NULL; + goto bad_swap; + } + p->old_block_size = block_size(bdev); + error = set_blocksize(bdev, PAGE_SIZE); + if (error < 0) + goto bad_swap; + p->bdev = bdev; + } else if (S_ISREG(inode->i_mode)) { + p->bdev = inode->i_sb->s_bdev; + down(&inode->i_sem); + did_down = 1; + if (IS_SWAPFILE(inode)) { + error = -EBUSY; + goto bad_swap; + } + } else { + goto bad_swap; + } + + swapfilesize = i_size_read(inode) >> PAGE_SHIFT; + + /* + * Read the swap header. + */ + if (!mapping->a_ops->readpage) { + error = -EINVAL; + goto bad_swap; + } + page = read_cache_page(mapping, 0, + (filler_t *)mapping->a_ops->readpage, swap_file); + if (IS_ERR(page)) { + error = PTR_ERR(page); + goto bad_swap; + } + wait_on_page_locked(page); + if (!PageUptodate(page)) + goto bad_swap; + kmap(page); + swap_header = page_address(page); + + if (!memcmp("SWAP-SPACE",swap_header->magic.magic,10)) + swap_header_version = 1; + else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10)) + swap_header_version = 2; + else { + printk("Unable to find swap-space signature\n"); + error = -EINVAL; + goto bad_swap; + } + + switch (swap_header_version) { + case 1: + printk(KERN_ERR "version 0 swap is no longer supported. " + "Use mkswap -v1 %s\n", name); + error = -EINVAL; + goto bad_swap; + case 2: + /* Check the swap header's sub-version and the size of + the swap file and bad block lists */ + if (swap_header->info.version != 1) { + printk(KERN_WARNING + "Unable to handle swap header version %d\n", + swap_header->info.version); + error = -EINVAL; + goto bad_swap; + } + + p->lowest_bit = 1; + /* + * Find out how many pages are allowed for a single swap + * device. There are two limiting factors: 1) the number of + * bits for the swap offset in the swp_entry_t type and + * 2) the number of bits in the a swap pte as defined by + * the different architectures. In order to find the + * largest possible bit mask a swap entry with swap type 0 + * and swap offset ~0UL is created, encoded to a swap pte, + * decoded to a swp_entry_t again and finally the swap + * offset is extracted. This will mask all the bits from + * the initial ~0UL mask that can't be encoded in either + * the swp_entry_t or the architecture definition of a + * swap pte. + */ + maxpages = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0,~0UL)))) - 1; + if (maxpages > swap_header->info.last_page) + maxpages = swap_header->info.last_page; + p->highest_bit = maxpages - 1; + + error = -EINVAL; + if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) + goto bad_swap; + + /* OK, set up the swap map and apply the bad block list */ + if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) { + error = -ENOMEM; + goto bad_swap; + } + + error = 0; + memset(p->swap_map, 0, maxpages * sizeof(short)); + for (i=0; i<swap_header->info.nr_badpages; i++) { + int page = swap_header->info.badpages[i]; + if (page <= 0 || page >= swap_header->info.last_page) + error = -EINVAL; + else + p->swap_map[page] = SWAP_MAP_BAD; + } + nr_good_pages = swap_header->info.last_page - + swap_header->info.nr_badpages - + 1 /* header page */; + if (error) + goto bad_swap; + } + + if (swapfilesize && maxpages > swapfilesize) { + printk(KERN_WARNING + "Swap area shorter than signature indicates\n"); + error = -EINVAL; + goto bad_swap; + } + if (!nr_good_pages) { + printk(KERN_WARNING "Empty swap-file\n"); + error = -EINVAL; + goto bad_swap; + } + p->swap_map[0] = SWAP_MAP_BAD; + p->max = maxpages; + p->pages = nr_good_pages; + + error = setup_swap_extents(p); + if (error) + goto bad_swap; + + down(&swapon_sem); + swap_list_lock(); + swap_device_lock(p); + p->flags = SWP_ACTIVE; + nr_swap_pages += nr_good_pages; + total_swap_pages += nr_good_pages; + printk(KERN_INFO "Adding %dk swap on %s. Priority:%d extents:%d\n", + nr_good_pages<<(PAGE_SHIFT-10), name, + p->prio, p->nr_extents); + + /* insert swap space into swap_list: */ + prev = -1; + for (i = swap_list.head; i >= 0; i = swap_info[i].next) { + if (p->prio >= swap_info[i].prio) { + break; + } + prev = i; + } + p->next = i; + if (prev < 0) { + swap_list.head = swap_list.next = p - swap_info; + } else { + swap_info[prev].next = p - swap_info; + } + swap_device_unlock(p); + swap_list_unlock(); + up(&swapon_sem); + error = 0; + goto out; +bad_swap: + if (bdev) { + set_blocksize(bdev, p->old_block_size); + bd_release(bdev); + } +bad_swap_2: + swap_list_lock(); + swap_map = p->swap_map; + p->swap_file = NULL; + p->swap_map = NULL; + p->flags = 0; + if (!(swap_flags & SWAP_FLAG_PREFER)) + ++least_priority; + swap_list_unlock(); + destroy_swap_extents(p); + vfree(swap_map); + if (swap_file) + filp_close(swap_file, NULL); +out: + if (page && !IS_ERR(page)) { + kunmap(page); + page_cache_release(page); + } + if (name) + putname(name); + if (did_down) { + if (!error) + inode->i_flags |= S_SWAPFILE; + up(&inode->i_sem); + } + return error; +} + +void si_swapinfo(struct sysinfo *val) +{ + unsigned int i; + unsigned long nr_to_be_unused = 0; + + swap_list_lock(); + for (i = 0; i < nr_swapfiles; i++) { + if (!(swap_info[i].flags & SWP_USED) || + (swap_info[i].flags & SWP_WRITEOK)) + continue; + nr_to_be_unused += swap_info[i].inuse_pages; + } + val->freeswap = nr_swap_pages + nr_to_be_unused; + val->totalswap = total_swap_pages + nr_to_be_unused; + swap_list_unlock(); +} + +/* + * Verify that a swap entry is valid and increment its swap map count. + * + * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as + * "permanent", but will be reclaimed by the next swapoff. + */ +int swap_duplicate(swp_entry_t entry) +{ + struct swap_info_struct * p; + unsigned long offset, type; + int result = 0; + + type = swp_type(entry); + if (type >= nr_swapfiles) + goto bad_file; + p = type + swap_info; + offset = swp_offset(entry); + + swap_device_lock(p); + if (offset < p->max && p->swap_map[offset]) { + if (p->swap_map[offset] < SWAP_MAP_MAX - 1) { + p->swap_map[offset]++; + result = 1; + } else if (p->swap_map[offset] <= SWAP_MAP_MAX) { + if (swap_overflow++ < 5) + printk(KERN_WARNING "swap_dup: swap entry overflow\n"); + p->swap_map[offset] = SWAP_MAP_MAX; + result = 1; + } + } + swap_device_unlock(p); +out: + return result; + +bad_file: + printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val); + goto out; +} + +struct swap_info_struct * +get_swap_info_struct(unsigned type) +{ + return &swap_info[type]; +} + +/* + * swap_device_lock prevents swap_map being freed. Don't grab an extra + * reference on the swaphandle, it doesn't matter if it becomes unused. + */ +int valid_swaphandles(swp_entry_t entry, unsigned long *offset) +{ + int ret = 0, i = 1 << page_cluster; + unsigned long toff; + struct swap_info_struct *swapdev = swp_type(entry) + swap_info; + + if (!page_cluster) /* no readahead */ + return 0; + toff = (swp_offset(entry) >> page_cluster) << page_cluster; + if (!toff) /* first page is swap header */ + toff++, i--; + *offset = toff; + + swap_device_lock(swapdev); + do { + /* Don't read-ahead past the end of the swap area */ + if (toff >= swapdev->max) + break; + /* Don't read in free or bad pages */ + if (!swapdev->swap_map[toff]) + break; + if (swapdev->swap_map[toff] == SWAP_MAP_BAD) + break; + toff++; + ret++; + } while (--i); + swap_device_unlock(swapdev); + return ret; +} diff --git a/mm/thrash.c b/mm/thrash.c new file mode 100644 index 000000000000..11461f7ad830 --- /dev/null +++ b/mm/thrash.c @@ -0,0 +1,102 @@ +/* + * mm/thrash.c + * + * Copyright (C) 2004, Red Hat, Inc. + * Copyright (C) 2004, Rik van Riel <riel@redhat.com> + * Released under the GPL, see the file COPYING for details. + * + * Simple token based thrashing protection, using the algorithm + * described in: http://www.cs.wm.edu/~sjiang/token.pdf + */ +#include <linux/jiffies.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/swap.h> + +static DEFINE_SPINLOCK(swap_token_lock); +static unsigned long swap_token_timeout; +static unsigned long swap_token_check; +struct mm_struct * swap_token_mm = &init_mm; + +#define SWAP_TOKEN_CHECK_INTERVAL (HZ * 2) +#define SWAP_TOKEN_TIMEOUT 0 +/* + * Currently disabled; Needs further code to work at HZ * 300. + */ +unsigned long swap_token_default_timeout = SWAP_TOKEN_TIMEOUT; + +/* + * Take the token away if the process had no page faults + * in the last interval, or if it has held the token for + * too long. + */ +#define SWAP_TOKEN_ENOUGH_RSS 1 +#define SWAP_TOKEN_TIMED_OUT 2 +static int should_release_swap_token(struct mm_struct *mm) +{ + int ret = 0; + if (!mm->recent_pagein) + ret = SWAP_TOKEN_ENOUGH_RSS; + else if (time_after(jiffies, swap_token_timeout)) + ret = SWAP_TOKEN_TIMED_OUT; + mm->recent_pagein = 0; + return ret; +} + +/* + * Try to grab the swapout protection token. We only try to + * grab it once every TOKEN_CHECK_INTERVAL, both to prevent + * SMP lock contention and to check that the process that held + * the token before is no longer thrashing. + */ +void grab_swap_token(void) +{ + struct mm_struct *mm; + int reason; + + /* We have the token. Let others know we still need it. */ + if (has_swap_token(current->mm)) { + current->mm->recent_pagein = 1; + return; + } + + if (time_after(jiffies, swap_token_check)) { + + /* Can't get swapout protection if we exceed our RSS limit. */ + // if (current->mm->rss > current->mm->rlimit_rss) + // return; + + /* ... or if we recently held the token. */ + if (time_before(jiffies, current->mm->swap_token_time)) + return; + + if (!spin_trylock(&swap_token_lock)) + return; + + swap_token_check = jiffies + SWAP_TOKEN_CHECK_INTERVAL; + + mm = swap_token_mm; + if ((reason = should_release_swap_token(mm))) { + unsigned long eligible = jiffies; + if (reason == SWAP_TOKEN_TIMED_OUT) { + eligible += swap_token_default_timeout; + } + mm->swap_token_time = eligible; + swap_token_timeout = jiffies + swap_token_default_timeout; + swap_token_mm = current->mm; + } + spin_unlock(&swap_token_lock); + } + return; +} + +/* Called on process exit. */ +void __put_swap_token(struct mm_struct *mm) +{ + spin_lock(&swap_token_lock); + if (likely(mm == swap_token_mm)) { + swap_token_mm = &init_mm; + swap_token_check = jiffies; + } + spin_unlock(&swap_token_lock); +} diff --git a/mm/tiny-shmem.c b/mm/tiny-shmem.c new file mode 100644 index 000000000000..c13a2161bca2 --- /dev/null +++ b/mm/tiny-shmem.c @@ -0,0 +1,122 @@ +/* + * tiny-shmem.c: simple shmemfs and tmpfs using ramfs code + * + * Matt Mackall <mpm@selenic.com> January, 2004 + * derived from mm/shmem.c and fs/ramfs/inode.c + * + * This is intended for small system where the benefits of the full + * shmem code (swap-backed and resource-limited) are outweighed by + * their complexity. On systems without swap this code should be + * effectively equivalent, but much lighter weight. + */ + +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/devfs_fs_kernel.h> +#include <linux/vfs.h> +#include <linux/mount.h> +#include <linux/file.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/swap.h> +#include <linux/ramfs.h> + +static struct file_system_type tmpfs_fs_type = { + .name = "tmpfs", + .get_sb = ramfs_get_sb, + .kill_sb = kill_litter_super, +}; + +static struct vfsmount *shm_mnt; + +static int __init init_tmpfs(void) +{ + register_filesystem(&tmpfs_fs_type); +#ifdef CONFIG_TMPFS + devfs_mk_dir("shm"); +#endif + shm_mnt = kern_mount(&tmpfs_fs_type); + return 0; +} +module_init(init_tmpfs) + +/* + * shmem_file_setup - get an unlinked file living in tmpfs + * + * @name: name for dentry (to be seen in /proc/<pid>/maps + * @size: size to be set for the file + * + */ +struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags) +{ + int error; + struct file *file; + struct inode *inode; + struct dentry *dentry, *root; + struct qstr this; + + if (IS_ERR(shm_mnt)) + return (void *)shm_mnt; + + error = -ENOMEM; + this.name = name; + this.len = strlen(name); + this.hash = 0; /* will go */ + root = shm_mnt->mnt_root; + dentry = d_alloc(root, &this); + if (!dentry) + goto put_memory; + + error = -ENFILE; + file = get_empty_filp(); + if (!file) + goto put_dentry; + + error = -ENOSPC; + inode = ramfs_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0); + if (!inode) + goto close_file; + + d_instantiate(dentry, inode); + inode->i_size = size; + inode->i_nlink = 0; /* It is unlinked */ + file->f_vfsmnt = mntget(shm_mnt); + file->f_dentry = dentry; + file->f_mapping = inode->i_mapping; + file->f_op = &ramfs_file_operations; + file->f_mode = FMODE_WRITE | FMODE_READ; + return file; + +close_file: + put_filp(file); +put_dentry: + dput(dentry); +put_memory: + return ERR_PTR(error); +} + +/* + * shmem_zero_setup - setup a shared anonymous mapping + * + * @vma: the vma to be mmapped is prepared by do_mmap_pgoff + */ +int shmem_zero_setup(struct vm_area_struct *vma) +{ + struct file *file; + loff_t size = vma->vm_end - vma->vm_start; + + file = shmem_file_setup("dev/zero", size, vma->vm_flags); + if (IS_ERR(file)) + return PTR_ERR(file); + + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = file; + vma->vm_ops = &generic_file_vm_ops; + return 0; +} + +int shmem_unuse(swp_entry_t entry, struct page *page) +{ + return 0; +} diff --git a/mm/truncate.c b/mm/truncate.c new file mode 100644 index 000000000000..c9a63f0b69a2 --- /dev/null +++ b/mm/truncate.c @@ -0,0 +1,336 @@ +/* + * mm/truncate.c - code for taking down pages from address_spaces + * + * Copyright (C) 2002, Linus Torvalds + * + * 10Sep2002 akpm@zip.com.au + * Initial version. + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/pagemap.h> +#include <linux/pagevec.h> +#include <linux/buffer_head.h> /* grr. try_to_release_page, + block_invalidatepage */ + + +static int do_invalidatepage(struct page *page, unsigned long offset) +{ + int (*invalidatepage)(struct page *, unsigned long); + invalidatepage = page->mapping->a_ops->invalidatepage; + if (invalidatepage == NULL) + invalidatepage = block_invalidatepage; + return (*invalidatepage)(page, offset); +} + +static inline void truncate_partial_page(struct page *page, unsigned partial) +{ + memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial); + if (PagePrivate(page)) + do_invalidatepage(page, partial); +} + +/* + * If truncate cannot remove the fs-private metadata from the page, the page + * becomes anonymous. It will be left on the LRU and may even be mapped into + * user pagetables if we're racing with filemap_nopage(). + * + * We need to bale out if page->mapping is no longer equal to the original + * mapping. This happens a) when the VM reclaimed the page while we waited on + * its lock, b) when a concurrent invalidate_inode_pages got there first and + * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space. + */ +static void +truncate_complete_page(struct address_space *mapping, struct page *page) +{ + if (page->mapping != mapping) + return; + + if (PagePrivate(page)) + do_invalidatepage(page, 0); + + clear_page_dirty(page); + ClearPageUptodate(page); + ClearPageMappedToDisk(page); + remove_from_page_cache(page); + page_cache_release(page); /* pagecache ref */ +} + +/* + * This is for invalidate_inode_pages(). That function can be called at + * any time, and is not supposed to throw away dirty pages. But pages can + * be marked dirty at any time too. So we re-check the dirtiness inside + * ->tree_lock. That provides exclusion against the __set_page_dirty + * functions. + * + * Returns non-zero if the page was successfully invalidated. + */ +static int +invalidate_complete_page(struct address_space *mapping, struct page *page) +{ + if (page->mapping != mapping) + return 0; + + if (PagePrivate(page) && !try_to_release_page(page, 0)) + return 0; + + write_lock_irq(&mapping->tree_lock); + if (PageDirty(page)) { + write_unlock_irq(&mapping->tree_lock); + return 0; + } + + BUG_ON(PagePrivate(page)); + __remove_from_page_cache(page); + write_unlock_irq(&mapping->tree_lock); + ClearPageUptodate(page); + page_cache_release(page); /* pagecache ref */ + return 1; +} + +/** + * truncate_inode_pages - truncate *all* the pages from an offset + * @mapping: mapping to truncate + * @lstart: offset from which to truncate + * + * Truncate the page cache at a set offset, removing the pages that are beyond + * that offset (and zeroing out partial pages). + * + * Truncate takes two passes - the first pass is nonblocking. It will not + * block on page locks and it will not block on writeback. The second pass + * will wait. This is to prevent as much IO as possible in the affected region. + * The first pass will remove most pages, so the search cost of the second pass + * is low. + * + * When looking at page->index outside the page lock we need to be careful to + * copy it into a local to avoid races (it could change at any time). + * + * We pass down the cache-hot hint to the page freeing code. Even if the + * mapping is large, it is probably the case that the final pages are the most + * recently touched, and freeing happens in ascending file offset order. + * + * Called under (and serialised by) inode->i_sem. + */ +void truncate_inode_pages(struct address_space *mapping, loff_t lstart) +{ + const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; + const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); + struct pagevec pvec; + pgoff_t next; + int i; + + if (mapping->nrpages == 0) + return; + + pagevec_init(&pvec, 0); + next = start; + while (pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { + for (i = 0; i < pagevec_count(&pvec); i++) { + struct page *page = pvec.pages[i]; + pgoff_t page_index = page->index; + + if (page_index > next) + next = page_index; + next++; + if (TestSetPageLocked(page)) + continue; + if (PageWriteback(page)) { + unlock_page(page); + continue; + } + truncate_complete_page(mapping, page); + unlock_page(page); + } + pagevec_release(&pvec); + cond_resched(); + } + + if (partial) { + struct page *page = find_lock_page(mapping, start - 1); + if (page) { + wait_on_page_writeback(page); + truncate_partial_page(page, partial); + unlock_page(page); + page_cache_release(page); + } + } + + next = start; + for ( ; ; ) { + cond_resched(); + if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { + if (next == start) + break; + next = start; + continue; + } + for (i = 0; i < pagevec_count(&pvec); i++) { + struct page *page = pvec.pages[i]; + + lock_page(page); + wait_on_page_writeback(page); + if (page->index > next) + next = page->index; + next++; + truncate_complete_page(mapping, page); + unlock_page(page); + } + pagevec_release(&pvec); + } +} + +EXPORT_SYMBOL(truncate_inode_pages); + +/** + * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode + * @mapping: the address_space which holds the pages to invalidate + * @start: the offset 'from' which to invalidate + * @end: the offset 'to' which to invalidate (inclusive) + * + * This function only removes the unlocked pages, if you want to + * remove all the pages of one inode, you must call truncate_inode_pages. + * + * invalidate_mapping_pages() will not block on IO activity. It will not + * invalidate pages which are dirty, locked, under writeback or mapped into + * pagetables. + */ +unsigned long invalidate_mapping_pages(struct address_space *mapping, + pgoff_t start, pgoff_t end) +{ + struct pagevec pvec; + pgoff_t next = start; + unsigned long ret = 0; + int i; + + pagevec_init(&pvec, 0); + while (next <= end && + pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { + for (i = 0; i < pagevec_count(&pvec); i++) { + struct page *page = pvec.pages[i]; + + if (TestSetPageLocked(page)) { + next++; + continue; + } + if (page->index > next) + next = page->index; + next++; + if (PageDirty(page) || PageWriteback(page)) + goto unlock; + if (page_mapped(page)) + goto unlock; + ret += invalidate_complete_page(mapping, page); +unlock: + unlock_page(page); + if (next > end) + break; + } + pagevec_release(&pvec); + cond_resched(); + } + return ret; +} + +unsigned long invalidate_inode_pages(struct address_space *mapping) +{ + return invalidate_mapping_pages(mapping, 0, ~0UL); +} + +EXPORT_SYMBOL(invalidate_inode_pages); + +/** + * invalidate_inode_pages2_range - remove range of pages from an address_space + * @mapping - the address_space + * @start: the page offset 'from' which to invalidate + * @end: the page offset 'to' which to invalidate (inclusive) + * + * Any pages which are found to be mapped into pagetables are unmapped prior to + * invalidation. + * + * Returns -EIO if any pages could not be invalidated. + */ +int invalidate_inode_pages2_range(struct address_space *mapping, + pgoff_t start, pgoff_t end) +{ + struct pagevec pvec; + pgoff_t next; + int i; + int ret = 0; + int did_range_unmap = 0; + int wrapped = 0; + + pagevec_init(&pvec, 0); + next = start; + while (next <= end && !ret && !wrapped && + pagevec_lookup(&pvec, mapping, next, + min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { + for (i = 0; !ret && i < pagevec_count(&pvec); i++) { + struct page *page = pvec.pages[i]; + pgoff_t page_index; + int was_dirty; + + lock_page(page); + if (page->mapping != mapping) { + unlock_page(page); + continue; + } + page_index = page->index; + next = page_index + 1; + if (next == 0) + wrapped = 1; + if (page_index > end) { + unlock_page(page); + break; + } + wait_on_page_writeback(page); + while (page_mapped(page)) { + if (!did_range_unmap) { + /* + * Zap the rest of the file in one hit. + */ + unmap_mapping_range(mapping, + page_index << PAGE_CACHE_SHIFT, + (end - page_index + 1) + << PAGE_CACHE_SHIFT, + 0); + did_range_unmap = 1; + } else { + /* + * Just zap this page + */ + unmap_mapping_range(mapping, + page_index << PAGE_CACHE_SHIFT, + PAGE_CACHE_SIZE, 0); + } + } + was_dirty = test_clear_page_dirty(page); + if (!invalidate_complete_page(mapping, page)) { + if (was_dirty) + set_page_dirty(page); + ret = -EIO; + } + unlock_page(page); + } + pagevec_release(&pvec); + cond_resched(); + } + return ret; +} +EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range); + +/** + * invalidate_inode_pages2 - remove all pages from an address_space + * @mapping - the address_space + * + * Any pages which are found to be mapped into pagetables are unmapped prior to + * invalidation. + * + * Returns -EIO if any pages could not be invalidated. + */ +int invalidate_inode_pages2(struct address_space *mapping) +{ + return invalidate_inode_pages2_range(mapping, 0, -1); +} +EXPORT_SYMBOL_GPL(invalidate_inode_pages2); diff --git a/mm/vmalloc.c b/mm/vmalloc.c new file mode 100644 index 000000000000..c6182f6f1305 --- /dev/null +++ b/mm/vmalloc.c @@ -0,0 +1,588 @@ +/* + * linux/mm/vmalloc.c + * + * Copyright (C) 1993 Linus Torvalds + * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 + * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 + * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 + */ + +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/highmem.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> + +#include <linux/vmalloc.h> + +#include <asm/uaccess.h> +#include <asm/tlbflush.h> + + +DEFINE_RWLOCK(vmlist_lock); +struct vm_struct *vmlist; + +static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) +{ + pte_t *pte; + + pte = pte_offset_kernel(pmd, addr); + do { + pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); + WARN_ON(!pte_none(ptent) && !pte_present(ptent)); + } while (pte++, addr += PAGE_SIZE, addr != end); +} + +static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr, + unsigned long end) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + vunmap_pte_range(pmd, addr, next); + } while (pmd++, addr = next, addr != end); +} + +static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr, + unsigned long end) +{ + pud_t *pud; + unsigned long next; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + vunmap_pmd_range(pud, addr, next); + } while (pud++, addr = next, addr != end); +} + +void unmap_vm_area(struct vm_struct *area) +{ + pgd_t *pgd; + unsigned long next; + unsigned long addr = (unsigned long) area->addr; + unsigned long end = addr + area->size; + + BUG_ON(addr >= end); + pgd = pgd_offset_k(addr); + flush_cache_vunmap(addr, end); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + vunmap_pud_range(pgd, addr, next); + } while (pgd++, addr = next, addr != end); + flush_tlb_kernel_range((unsigned long) area->addr, end); +} + +static int vmap_pte_range(pmd_t *pmd, unsigned long addr, + unsigned long end, pgprot_t prot, struct page ***pages) +{ + pte_t *pte; + + pte = pte_alloc_kernel(&init_mm, pmd, addr); + if (!pte) + return -ENOMEM; + do { + struct page *page = **pages; + WARN_ON(!pte_none(*pte)); + if (!page) + return -ENOMEM; + set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); + (*pages)++; + } while (pte++, addr += PAGE_SIZE, addr != end); + return 0; +} + +static inline int vmap_pmd_range(pud_t *pud, unsigned long addr, + unsigned long end, pgprot_t prot, struct page ***pages) +{ + pmd_t *pmd; + unsigned long next; + + pmd = pmd_alloc(&init_mm, pud, addr); + if (!pmd) + return -ENOMEM; + do { + next = pmd_addr_end(addr, end); + if (vmap_pte_range(pmd, addr, next, prot, pages)) + return -ENOMEM; + } while (pmd++, addr = next, addr != end); + return 0; +} + +static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr, + unsigned long end, pgprot_t prot, struct page ***pages) +{ + pud_t *pud; + unsigned long next; + + pud = pud_alloc(&init_mm, pgd, addr); + if (!pud) + return -ENOMEM; + do { + next = pud_addr_end(addr, end); + if (vmap_pmd_range(pud, addr, next, prot, pages)) + return -ENOMEM; + } while (pud++, addr = next, addr != end); + return 0; +} + +int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages) +{ + pgd_t *pgd; + unsigned long next; + unsigned long addr = (unsigned long) area->addr; + unsigned long end = addr + area->size - PAGE_SIZE; + int err; + + BUG_ON(addr >= end); + pgd = pgd_offset_k(addr); + spin_lock(&init_mm.page_table_lock); + do { + next = pgd_addr_end(addr, end); + err = vmap_pud_range(pgd, addr, next, prot, pages); + if (err) + break; + } while (pgd++, addr = next, addr != end); + spin_unlock(&init_mm.page_table_lock); + flush_cache_vmap((unsigned long) area->addr, end); + return err; +} + +#define IOREMAP_MAX_ORDER (7 + PAGE_SHIFT) /* 128 pages */ + +struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, + unsigned long start, unsigned long end) +{ + struct vm_struct **p, *tmp, *area; + unsigned long align = 1; + unsigned long addr; + + if (flags & VM_IOREMAP) { + int bit = fls(size); + + if (bit > IOREMAP_MAX_ORDER) + bit = IOREMAP_MAX_ORDER; + else if (bit < PAGE_SHIFT) + bit = PAGE_SHIFT; + + align = 1ul << bit; + } + addr = ALIGN(start, align); + size = PAGE_ALIGN(size); + + area = kmalloc(sizeof(*area), GFP_KERNEL); + if (unlikely(!area)) + return NULL; + + if (unlikely(!size)) { + kfree (area); + return NULL; + } + + /* + * We always allocate a guard page. + */ + size += PAGE_SIZE; + + write_lock(&vmlist_lock); + for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) { + if ((unsigned long)tmp->addr < addr) { + if((unsigned long)tmp->addr + tmp->size >= addr) + addr = ALIGN(tmp->size + + (unsigned long)tmp->addr, align); + continue; + } + if ((size + addr) < addr) + goto out; + if (size + addr <= (unsigned long)tmp->addr) + goto found; + addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align); + if (addr > end - size) + goto out; + } + +found: + area->next = *p; + *p = area; + + area->flags = flags; + area->addr = (void *)addr; + area->size = size; + area->pages = NULL; + area->nr_pages = 0; + area->phys_addr = 0; + write_unlock(&vmlist_lock); + + return area; + +out: + write_unlock(&vmlist_lock); + kfree(area); + if (printk_ratelimit()) + printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n"); + return NULL; +} + +/** + * get_vm_area - reserve a contingous kernel virtual area + * + * @size: size of the area + * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC + * + * Search an area of @size in the kernel virtual mapping area, + * and reserved it for out purposes. Returns the area descriptor + * on success or %NULL on failure. + */ +struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) +{ + return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END); +} + +/** + * remove_vm_area - find and remove a contingous kernel virtual area + * + * @addr: base address + * + * Search for the kernel VM area starting at @addr, and remove it. + * This function returns the found VM area, but using it is NOT safe + * on SMP machines. + */ +struct vm_struct *remove_vm_area(void *addr) +{ + struct vm_struct **p, *tmp; + + write_lock(&vmlist_lock); + for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) { + if (tmp->addr == addr) + goto found; + } + write_unlock(&vmlist_lock); + return NULL; + +found: + unmap_vm_area(tmp); + *p = tmp->next; + write_unlock(&vmlist_lock); + + /* + * Remove the guard page. + */ + tmp->size -= PAGE_SIZE; + return tmp; +} + +void __vunmap(void *addr, int deallocate_pages) +{ + struct vm_struct *area; + + if (!addr) + return; + + if ((PAGE_SIZE-1) & (unsigned long)addr) { + printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr); + WARN_ON(1); + return; + } + + area = remove_vm_area(addr); + if (unlikely(!area)) { + printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", + addr); + WARN_ON(1); + return; + } + + if (deallocate_pages) { + int i; + + for (i = 0; i < area->nr_pages; i++) { + if (unlikely(!area->pages[i])) + BUG(); + __free_page(area->pages[i]); + } + + if (area->nr_pages > PAGE_SIZE/sizeof(struct page *)) + vfree(area->pages); + else + kfree(area->pages); + } + + kfree(area); + return; +} + +/** + * vfree - release memory allocated by vmalloc() + * + * @addr: memory base address + * + * Free the virtually contiguous memory area starting at @addr, as + * obtained from vmalloc(), vmalloc_32() or __vmalloc(). + * + * May not be called in interrupt context. + */ +void vfree(void *addr) +{ + BUG_ON(in_interrupt()); + __vunmap(addr, 1); +} + +EXPORT_SYMBOL(vfree); + +/** + * vunmap - release virtual mapping obtained by vmap() + * + * @addr: memory base address + * + * Free the virtually contiguous memory area starting at @addr, + * which was created from the page array passed to vmap(). + * + * May not be called in interrupt context. + */ +void vunmap(void *addr) +{ + BUG_ON(in_interrupt()); + __vunmap(addr, 0); +} + +EXPORT_SYMBOL(vunmap); + +/** + * vmap - map an array of pages into virtually contiguous space + * + * @pages: array of page pointers + * @count: number of pages to map + * @flags: vm_area->flags + * @prot: page protection for the mapping + * + * Maps @count pages from @pages into contiguous kernel virtual + * space. + */ +void *vmap(struct page **pages, unsigned int count, + unsigned long flags, pgprot_t prot) +{ + struct vm_struct *area; + + if (count > num_physpages) + return NULL; + + area = get_vm_area((count << PAGE_SHIFT), flags); + if (!area) + return NULL; + if (map_vm_area(area, prot, &pages)) { + vunmap(area->addr); + return NULL; + } + + return area->addr; +} + +EXPORT_SYMBOL(vmap); + +void *__vmalloc_area(struct vm_struct *area, unsigned int __nocast gfp_mask, pgprot_t prot) +{ + struct page **pages; + unsigned int nr_pages, array_size, i; + + nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT; + array_size = (nr_pages * sizeof(struct page *)); + + area->nr_pages = nr_pages; + /* Please note that the recursion is strictly bounded. */ + if (array_size > PAGE_SIZE) + pages = __vmalloc(array_size, gfp_mask, PAGE_KERNEL); + else + pages = kmalloc(array_size, (gfp_mask & ~__GFP_HIGHMEM)); + area->pages = pages; + if (!area->pages) { + remove_vm_area(area->addr); + kfree(area); + return NULL; + } + memset(area->pages, 0, array_size); + + for (i = 0; i < area->nr_pages; i++) { + area->pages[i] = alloc_page(gfp_mask); + if (unlikely(!area->pages[i])) { + /* Successfully allocated i pages, free them in __vunmap() */ + area->nr_pages = i; + goto fail; + } + } + + if (map_vm_area(area, prot, &pages)) + goto fail; + return area->addr; + +fail: + vfree(area->addr); + return NULL; +} + +/** + * __vmalloc - allocate virtually contiguous memory + * + * @size: allocation size + * @gfp_mask: flags for the page level allocator + * @prot: protection mask for the allocated pages + * + * Allocate enough pages to cover @size from the page level + * allocator with @gfp_mask flags. Map them into contiguous + * kernel virtual space, using a pagetable protection of @prot. + */ +void *__vmalloc(unsigned long size, unsigned int __nocast gfp_mask, pgprot_t prot) +{ + struct vm_struct *area; + + size = PAGE_ALIGN(size); + if (!size || (size >> PAGE_SHIFT) > num_physpages) + return NULL; + + area = get_vm_area(size, VM_ALLOC); + if (!area) + return NULL; + + return __vmalloc_area(area, gfp_mask, prot); +} + +EXPORT_SYMBOL(__vmalloc); + +/** + * vmalloc - allocate virtually contiguous memory + * + * @size: allocation size + * + * Allocate enough pages to cover @size from the page level + * allocator and map them into contiguous kernel virtual space. + * + * For tight cotrol over page level allocator and protection flags + * use __vmalloc() instead. + */ +void *vmalloc(unsigned long size) +{ + return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); +} + +EXPORT_SYMBOL(vmalloc); + +/** + * vmalloc_exec - allocate virtually contiguous, executable memory + * + * @size: allocation size + * + * Kernel-internal function to allocate enough pages to cover @size + * the page level allocator and map them into contiguous and + * executable kernel virtual space. + * + * For tight cotrol over page level allocator and protection flags + * use __vmalloc() instead. + */ + +#ifndef PAGE_KERNEL_EXEC +# define PAGE_KERNEL_EXEC PAGE_KERNEL +#endif + +void *vmalloc_exec(unsigned long size) +{ + return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); +} + +/** + * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) + * + * @size: allocation size + * + * Allocate enough 32bit PA addressable pages to cover @size from the + * page level allocator and map them into contiguous kernel virtual space. + */ +void *vmalloc_32(unsigned long size) +{ + return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); +} + +EXPORT_SYMBOL(vmalloc_32); + +long vread(char *buf, char *addr, unsigned long count) +{ + struct vm_struct *tmp; + char *vaddr, *buf_start = buf; + unsigned long n; + + /* Don't allow overflow */ + if ((unsigned long) addr + count < count) + count = -(unsigned long) addr; + + read_lock(&vmlist_lock); + for (tmp = vmlist; tmp; tmp = tmp->next) { + vaddr = (char *) tmp->addr; + if (addr >= vaddr + tmp->size - PAGE_SIZE) + continue; + while (addr < vaddr) { + if (count == 0) + goto finished; + *buf = '\0'; + buf++; + addr++; + count--; + } + n = vaddr + tmp->size - PAGE_SIZE - addr; + do { + if (count == 0) + goto finished; + *buf = *addr; + buf++; + addr++; + count--; + } while (--n > 0); + } +finished: + read_unlock(&vmlist_lock); + return buf - buf_start; +} + +long vwrite(char *buf, char *addr, unsigned long count) +{ + struct vm_struct *tmp; + char *vaddr, *buf_start = buf; + unsigned long n; + + /* Don't allow overflow */ + if ((unsigned long) addr + count < count) + count = -(unsigned long) addr; + + read_lock(&vmlist_lock); + for (tmp = vmlist; tmp; tmp = tmp->next) { + vaddr = (char *) tmp->addr; + if (addr >= vaddr + tmp->size - PAGE_SIZE) + continue; + while (addr < vaddr) { + if (count == 0) + goto finished; + buf++; + addr++; + count--; + } + n = vaddr + tmp->size - PAGE_SIZE - addr; + do { + if (count == 0) + goto finished; + *addr = *buf; + buf++; + addr++; + count--; + } while (--n > 0); + } +finished: + read_unlock(&vmlist_lock); + return buf - buf_start; +} diff --git a/mm/vmscan.c b/mm/vmscan.c new file mode 100644 index 000000000000..4003c0518d28 --- /dev/null +++ b/mm/vmscan.c @@ -0,0 +1,1311 @@ +/* + * linux/mm/vmscan.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * + * Swap reorganised 29.12.95, Stephen Tweedie. + * kswapd added: 7.1.96 sct + * Removed kswapd_ctl limits, and swap out as many pages as needed + * to bring the system back to freepages.high: 2.4.97, Rik van Riel. + * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). + * Multiqueue VM started 5.8.00, Rik van Riel. + */ + +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/kernel_stat.h> +#include <linux/swap.h> +#include <linux/pagemap.h> +#include <linux/init.h> +#include <linux/highmem.h> +#include <linux/file.h> +#include <linux/writeback.h> +#include <linux/blkdev.h> +#include <linux/buffer_head.h> /* for try_to_release_page(), + buffer_heads_over_limit */ +#include <linux/mm_inline.h> +#include <linux/pagevec.h> +#include <linux/backing-dev.h> +#include <linux/rmap.h> +#include <linux/topology.h> +#include <linux/cpu.h> +#include <linux/cpuset.h> +#include <linux/notifier.h> +#include <linux/rwsem.h> + +#include <asm/tlbflush.h> +#include <asm/div64.h> + +#include <linux/swapops.h> + +/* possible outcome of pageout() */ +typedef enum { + /* failed to write page out, page is locked */ + PAGE_KEEP, + /* move page to the active list, page is locked */ + PAGE_ACTIVATE, + /* page has been sent to the disk successfully, page is unlocked */ + PAGE_SUCCESS, + /* page is clean and locked */ + PAGE_CLEAN, +} pageout_t; + +struct scan_control { + /* Ask refill_inactive_zone, or shrink_cache to scan this many pages */ + unsigned long nr_to_scan; + + /* Incremented by the number of inactive pages that were scanned */ + unsigned long nr_scanned; + + /* Incremented by the number of pages reclaimed */ + unsigned long nr_reclaimed; + + unsigned long nr_mapped; /* From page_state */ + + /* How many pages shrink_cache() should reclaim */ + int nr_to_reclaim; + + /* Ask shrink_caches, or shrink_zone to scan at this priority */ + unsigned int priority; + + /* This context's GFP mask */ + unsigned int gfp_mask; + + int may_writepage; + + /* This context's SWAP_CLUSTER_MAX. If freeing memory for + * suspend, we effectively ignore SWAP_CLUSTER_MAX. + * In this context, it doesn't matter that we scan the + * whole list at once. */ + int swap_cluster_max; +}; + +/* + * The list of shrinker callbacks used by to apply pressure to + * ageable caches. + */ +struct shrinker { + shrinker_t shrinker; + struct list_head list; + int seeks; /* seeks to recreate an obj */ + long nr; /* objs pending delete */ +}; + +#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) + +#ifdef ARCH_HAS_PREFETCH +#define prefetch_prev_lru_page(_page, _base, _field) \ + do { \ + if ((_page)->lru.prev != _base) { \ + struct page *prev; \ + \ + prev = lru_to_page(&(_page->lru)); \ + prefetch(&prev->_field); \ + } \ + } while (0) +#else +#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0) +#endif + +#ifdef ARCH_HAS_PREFETCHW +#define prefetchw_prev_lru_page(_page, _base, _field) \ + do { \ + if ((_page)->lru.prev != _base) { \ + struct page *prev; \ + \ + prev = lru_to_page(&(_page->lru)); \ + prefetchw(&prev->_field); \ + } \ + } while (0) +#else +#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0) +#endif + +/* + * From 0 .. 100. Higher means more swappy. + */ +int vm_swappiness = 60; +static long total_memory; + +static LIST_HEAD(shrinker_list); +static DECLARE_RWSEM(shrinker_rwsem); + +/* + * Add a shrinker callback to be called from the vm + */ +struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker) +{ + struct shrinker *shrinker; + + shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL); + if (shrinker) { + shrinker->shrinker = theshrinker; + shrinker->seeks = seeks; + shrinker->nr = 0; + down_write(&shrinker_rwsem); + list_add_tail(&shrinker->list, &shrinker_list); + up_write(&shrinker_rwsem); + } + return shrinker; +} +EXPORT_SYMBOL(set_shrinker); + +/* + * Remove one + */ +void remove_shrinker(struct shrinker *shrinker) +{ + down_write(&shrinker_rwsem); + list_del(&shrinker->list); + up_write(&shrinker_rwsem); + kfree(shrinker); +} +EXPORT_SYMBOL(remove_shrinker); + +#define SHRINK_BATCH 128 +/* + * Call the shrink functions to age shrinkable caches + * + * Here we assume it costs one seek to replace a lru page and that it also + * takes a seek to recreate a cache object. With this in mind we age equal + * percentages of the lru and ageable caches. This should balance the seeks + * generated by these structures. + * + * If the vm encounted mapped pages on the LRU it increase the pressure on + * slab to avoid swapping. + * + * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits. + * + * `lru_pages' represents the number of on-LRU pages in all the zones which + * are eligible for the caller's allocation attempt. It is used for balancing + * slab reclaim versus page reclaim. + */ +static int shrink_slab(unsigned long scanned, unsigned int gfp_mask, + unsigned long lru_pages) +{ + struct shrinker *shrinker; + + if (scanned == 0) + scanned = SWAP_CLUSTER_MAX; + + if (!down_read_trylock(&shrinker_rwsem)) + return 0; + + list_for_each_entry(shrinker, &shrinker_list, list) { + unsigned long long delta; + unsigned long total_scan; + + delta = (4 * scanned) / shrinker->seeks; + delta *= (*shrinker->shrinker)(0, gfp_mask); + do_div(delta, lru_pages + 1); + shrinker->nr += delta; + if (shrinker->nr < 0) + shrinker->nr = LONG_MAX; /* It wrapped! */ + + total_scan = shrinker->nr; + shrinker->nr = 0; + + while (total_scan >= SHRINK_BATCH) { + long this_scan = SHRINK_BATCH; + int shrink_ret; + + shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask); + if (shrink_ret == -1) + break; + mod_page_state(slabs_scanned, this_scan); + total_scan -= this_scan; + + cond_resched(); + } + + shrinker->nr += total_scan; + } + up_read(&shrinker_rwsem); + return 0; +} + +/* Called without lock on whether page is mapped, so answer is unstable */ +static inline int page_mapping_inuse(struct page *page) +{ + struct address_space *mapping; + + /* Page is in somebody's page tables. */ + if (page_mapped(page)) + return 1; + + /* Be more reluctant to reclaim swapcache than pagecache */ + if (PageSwapCache(page)) + return 1; + + mapping = page_mapping(page); + if (!mapping) + return 0; + + /* File is mmap'd by somebody? */ + return mapping_mapped(mapping); +} + +static inline int is_page_cache_freeable(struct page *page) +{ + return page_count(page) - !!PagePrivate(page) == 2; +} + +static int may_write_to_queue(struct backing_dev_info *bdi) +{ + if (current_is_kswapd()) + return 1; + if (current_is_pdflush()) /* This is unlikely, but why not... */ + return 1; + if (!bdi_write_congested(bdi)) + return 1; + if (bdi == current->backing_dev_info) + return 1; + return 0; +} + +/* + * We detected a synchronous write error writing a page out. Probably + * -ENOSPC. We need to propagate that into the address_space for a subsequent + * fsync(), msync() or close(). + * + * The tricky part is that after writepage we cannot touch the mapping: nothing + * prevents it from being freed up. But we have a ref on the page and once + * that page is locked, the mapping is pinned. + * + * We're allowed to run sleeping lock_page() here because we know the caller has + * __GFP_FS. + */ +static void handle_write_error(struct address_space *mapping, + struct page *page, int error) +{ + lock_page(page); + if (page_mapping(page) == mapping) { + if (error == -ENOSPC) + set_bit(AS_ENOSPC, &mapping->flags); + else + set_bit(AS_EIO, &mapping->flags); + } + unlock_page(page); +} + +/* + * pageout is called by shrink_list() for each dirty page. Calls ->writepage(). + */ +static pageout_t pageout(struct page *page, struct address_space *mapping) +{ + /* + * If the page is dirty, only perform writeback if that write + * will be non-blocking. To prevent this allocation from being + * stalled by pagecache activity. But note that there may be + * stalls if we need to run get_block(). We could test + * PagePrivate for that. + * + * If this process is currently in generic_file_write() against + * this page's queue, we can perform writeback even if that + * will block. + * + * If the page is swapcache, write it back even if that would + * block, for some throttling. This happens by accident, because + * swap_backing_dev_info is bust: it doesn't reflect the + * congestion state of the swapdevs. Easy to fix, if needed. + * See swapfile.c:page_queue_congested(). + */ + if (!is_page_cache_freeable(page)) + return PAGE_KEEP; + if (!mapping) { + /* + * Some data journaling orphaned pages can have + * page->mapping == NULL while being dirty with clean buffers. + */ + if (PageDirty(page) && PagePrivate(page)) { + if (try_to_free_buffers(page)) { + ClearPageDirty(page); + printk("%s: orphaned page\n", __FUNCTION__); + return PAGE_CLEAN; + } + } + return PAGE_KEEP; + } + if (mapping->a_ops->writepage == NULL) + return PAGE_ACTIVATE; + if (!may_write_to_queue(mapping->backing_dev_info)) + return PAGE_KEEP; + + if (clear_page_dirty_for_io(page)) { + int res; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = SWAP_CLUSTER_MAX, + .nonblocking = 1, + .for_reclaim = 1, + }; + + SetPageReclaim(page); + res = mapping->a_ops->writepage(page, &wbc); + if (res < 0) + handle_write_error(mapping, page, res); + if (res == WRITEPAGE_ACTIVATE) { + ClearPageReclaim(page); + return PAGE_ACTIVATE; + } + if (!PageWriteback(page)) { + /* synchronous write or broken a_ops? */ + ClearPageReclaim(page); + } + + return PAGE_SUCCESS; + } + + return PAGE_CLEAN; +} + +/* + * shrink_list adds the number of reclaimed pages to sc->nr_reclaimed + */ +static int shrink_list(struct list_head *page_list, struct scan_control *sc) +{ + LIST_HEAD(ret_pages); + struct pagevec freed_pvec; + int pgactivate = 0; + int reclaimed = 0; + + cond_resched(); + + pagevec_init(&freed_pvec, 1); + while (!list_empty(page_list)) { + struct address_space *mapping; + struct page *page; + int may_enter_fs; + int referenced; + + cond_resched(); + + page = lru_to_page(page_list); + list_del(&page->lru); + + if (TestSetPageLocked(page)) + goto keep; + + BUG_ON(PageActive(page)); + + sc->nr_scanned++; + /* Double the slab pressure for mapped and swapcache pages */ + if (page_mapped(page) || PageSwapCache(page)) + sc->nr_scanned++; + + if (PageWriteback(page)) + goto keep_locked; + + referenced = page_referenced(page, 1, sc->priority <= 0); + /* In active use or really unfreeable? Activate it. */ + if (referenced && page_mapping_inuse(page)) + goto activate_locked; + +#ifdef CONFIG_SWAP + /* + * Anonymous process memory has backing store? + * Try to allocate it some swap space here. + */ + if (PageAnon(page) && !PageSwapCache(page)) { + if (!add_to_swap(page)) + goto activate_locked; + } +#endif /* CONFIG_SWAP */ + + mapping = page_mapping(page); + may_enter_fs = (sc->gfp_mask & __GFP_FS) || + (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); + + /* + * The page is mapped into the page tables of one or more + * processes. Try to unmap it here. + */ + if (page_mapped(page) && mapping) { + switch (try_to_unmap(page)) { + case SWAP_FAIL: + goto activate_locked; + case SWAP_AGAIN: + goto keep_locked; + case SWAP_SUCCESS: + ; /* try to free the page below */ + } + } + + if (PageDirty(page)) { + if (referenced) + goto keep_locked; + if (!may_enter_fs) + goto keep_locked; + if (laptop_mode && !sc->may_writepage) + goto keep_locked; + + /* Page is dirty, try to write it out here */ + switch(pageout(page, mapping)) { + case PAGE_KEEP: + goto keep_locked; + case PAGE_ACTIVATE: + goto activate_locked; + case PAGE_SUCCESS: + if (PageWriteback(page) || PageDirty(page)) + goto keep; + /* + * A synchronous write - probably a ramdisk. Go + * ahead and try to reclaim the page. + */ + if (TestSetPageLocked(page)) + goto keep; + if (PageDirty(page) || PageWriteback(page)) + goto keep_locked; + mapping = page_mapping(page); + case PAGE_CLEAN: + ; /* try to free the page below */ + } + } + + /* + * If the page has buffers, try to free the buffer mappings + * associated with this page. If we succeed we try to free + * the page as well. + * + * We do this even if the page is PageDirty(). + * try_to_release_page() does not perform I/O, but it is + * possible for a page to have PageDirty set, but it is actually + * clean (all its buffers are clean). This happens if the + * buffers were written out directly, with submit_bh(). ext3 + * will do this, as well as the blockdev mapping. + * try_to_release_page() will discover that cleanness and will + * drop the buffers and mark the page clean - it can be freed. + * + * Rarely, pages can have buffers and no ->mapping. These are + * the pages which were not successfully invalidated in + * truncate_complete_page(). We try to drop those buffers here + * and if that worked, and the page is no longer mapped into + * process address space (page_count == 1) it can be freed. + * Otherwise, leave the page on the LRU so it is swappable. + */ + if (PagePrivate(page)) { + if (!try_to_release_page(page, sc->gfp_mask)) + goto activate_locked; + if (!mapping && page_count(page) == 1) + goto free_it; + } + + if (!mapping) + goto keep_locked; /* truncate got there first */ + + write_lock_irq(&mapping->tree_lock); + + /* + * The non-racy check for busy page. It is critical to check + * PageDirty _after_ making sure that the page is freeable and + * not in use by anybody. (pagecache + us == 2) + */ + if (page_count(page) != 2 || PageDirty(page)) { + write_unlock_irq(&mapping->tree_lock); + goto keep_locked; + } + +#ifdef CONFIG_SWAP + if (PageSwapCache(page)) { + swp_entry_t swap = { .val = page->private }; + __delete_from_swap_cache(page); + write_unlock_irq(&mapping->tree_lock); + swap_free(swap); + __put_page(page); /* The pagecache ref */ + goto free_it; + } +#endif /* CONFIG_SWAP */ + + __remove_from_page_cache(page); + write_unlock_irq(&mapping->tree_lock); + __put_page(page); + +free_it: + unlock_page(page); + reclaimed++; + if (!pagevec_add(&freed_pvec, page)) + __pagevec_release_nonlru(&freed_pvec); + continue; + +activate_locked: + SetPageActive(page); + pgactivate++; +keep_locked: + unlock_page(page); +keep: + list_add(&page->lru, &ret_pages); + BUG_ON(PageLRU(page)); + } + list_splice(&ret_pages, page_list); + if (pagevec_count(&freed_pvec)) + __pagevec_release_nonlru(&freed_pvec); + mod_page_state(pgactivate, pgactivate); + sc->nr_reclaimed += reclaimed; + return reclaimed; +} + +/* + * zone->lru_lock is heavily contended. Some of the functions that + * shrink the lists perform better by taking out a batch of pages + * and working on them outside the LRU lock. + * + * For pagecache intensive workloads, this function is the hottest + * spot in the kernel (apart from copy_*_user functions). + * + * Appropriate locks must be held before calling this function. + * + * @nr_to_scan: The number of pages to look through on the list. + * @src: The LRU list to pull pages off. + * @dst: The temp list to put pages on to. + * @scanned: The number of pages that were scanned. + * + * returns how many pages were moved onto *@dst. + */ +static int isolate_lru_pages(int nr_to_scan, struct list_head *src, + struct list_head *dst, int *scanned) +{ + int nr_taken = 0; + struct page *page; + int scan = 0; + + while (scan++ < nr_to_scan && !list_empty(src)) { + page = lru_to_page(src); + prefetchw_prev_lru_page(page, src, flags); + + if (!TestClearPageLRU(page)) + BUG(); + list_del(&page->lru); + if (get_page_testone(page)) { + /* + * It is being freed elsewhere + */ + __put_page(page); + SetPageLRU(page); + list_add(&page->lru, src); + continue; + } else { + list_add(&page->lru, dst); + nr_taken++; + } + } + + *scanned = scan; + return nr_taken; +} + +/* + * shrink_cache() adds the number of pages reclaimed to sc->nr_reclaimed + */ +static void shrink_cache(struct zone *zone, struct scan_control *sc) +{ + LIST_HEAD(page_list); + struct pagevec pvec; + int max_scan = sc->nr_to_scan; + + pagevec_init(&pvec, 1); + + lru_add_drain(); + spin_lock_irq(&zone->lru_lock); + while (max_scan > 0) { + struct page *page; + int nr_taken; + int nr_scan; + int nr_freed; + + nr_taken = isolate_lru_pages(sc->swap_cluster_max, + &zone->inactive_list, + &page_list, &nr_scan); + zone->nr_inactive -= nr_taken; + zone->pages_scanned += nr_scan; + spin_unlock_irq(&zone->lru_lock); + + if (nr_taken == 0) + goto done; + + max_scan -= nr_scan; + if (current_is_kswapd()) + mod_page_state_zone(zone, pgscan_kswapd, nr_scan); + else + mod_page_state_zone(zone, pgscan_direct, nr_scan); + nr_freed = shrink_list(&page_list, sc); + if (current_is_kswapd()) + mod_page_state(kswapd_steal, nr_freed); + mod_page_state_zone(zone, pgsteal, nr_freed); + sc->nr_to_reclaim -= nr_freed; + + spin_lock_irq(&zone->lru_lock); + /* + * Put back any unfreeable pages. + */ + while (!list_empty(&page_list)) { + page = lru_to_page(&page_list); + if (TestSetPageLRU(page)) + BUG(); + list_del(&page->lru); + if (PageActive(page)) + add_page_to_active_list(zone, page); + else + add_page_to_inactive_list(zone, page); + if (!pagevec_add(&pvec, page)) { + spin_unlock_irq(&zone->lru_lock); + __pagevec_release(&pvec); + spin_lock_irq(&zone->lru_lock); + } + } + } + spin_unlock_irq(&zone->lru_lock); +done: + pagevec_release(&pvec); +} + +/* + * This moves pages from the active list to the inactive list. + * + * We move them the other way if the page is referenced by one or more + * processes, from rmap. + * + * If the pages are mostly unmapped, the processing is fast and it is + * appropriate to hold zone->lru_lock across the whole operation. But if + * the pages are mapped, the processing is slow (page_referenced()) so we + * should drop zone->lru_lock around each page. It's impossible to balance + * this, so instead we remove the pages from the LRU while processing them. + * It is safe to rely on PG_active against the non-LRU pages in here because + * nobody will play with that bit on a non-LRU page. + * + * The downside is that we have to touch page->_count against each page. + * But we had to alter page->flags anyway. + */ +static void +refill_inactive_zone(struct zone *zone, struct scan_control *sc) +{ + int pgmoved; + int pgdeactivate = 0; + int pgscanned; + int nr_pages = sc->nr_to_scan; + LIST_HEAD(l_hold); /* The pages which were snipped off */ + LIST_HEAD(l_inactive); /* Pages to go onto the inactive_list */ + LIST_HEAD(l_active); /* Pages to go onto the active_list */ + struct page *page; + struct pagevec pvec; + int reclaim_mapped = 0; + long mapped_ratio; + long distress; + long swap_tendency; + + lru_add_drain(); + spin_lock_irq(&zone->lru_lock); + pgmoved = isolate_lru_pages(nr_pages, &zone->active_list, + &l_hold, &pgscanned); + zone->pages_scanned += pgscanned; + zone->nr_active -= pgmoved; + spin_unlock_irq(&zone->lru_lock); + + /* + * `distress' is a measure of how much trouble we're having reclaiming + * pages. 0 -> no problems. 100 -> great trouble. + */ + distress = 100 >> zone->prev_priority; + + /* + * The point of this algorithm is to decide when to start reclaiming + * mapped memory instead of just pagecache. Work out how much memory + * is mapped. + */ + mapped_ratio = (sc->nr_mapped * 100) / total_memory; + + /* + * Now decide how much we really want to unmap some pages. The mapped + * ratio is downgraded - just because there's a lot of mapped memory + * doesn't necessarily mean that page reclaim isn't succeeding. + * + * The distress ratio is important - we don't want to start going oom. + * + * A 100% value of vm_swappiness overrides this algorithm altogether. + */ + swap_tendency = mapped_ratio / 2 + distress + vm_swappiness; + + /* + * Now use this metric to decide whether to start moving mapped memory + * onto the inactive list. + */ + if (swap_tendency >= 100) + reclaim_mapped = 1; + + while (!list_empty(&l_hold)) { + cond_resched(); + page = lru_to_page(&l_hold); + list_del(&page->lru); + if (page_mapped(page)) { + if (!reclaim_mapped || + (total_swap_pages == 0 && PageAnon(page)) || + page_referenced(page, 0, sc->priority <= 0)) { + list_add(&page->lru, &l_active); + continue; + } + } + list_add(&page->lru, &l_inactive); + } + + pagevec_init(&pvec, 1); + pgmoved = 0; + spin_lock_irq(&zone->lru_lock); + while (!list_empty(&l_inactive)) { + page = lru_to_page(&l_inactive); + prefetchw_prev_lru_page(page, &l_inactive, flags); + if (TestSetPageLRU(page)) + BUG(); + if (!TestClearPageActive(page)) + BUG(); + list_move(&page->lru, &zone->inactive_list); + pgmoved++; + if (!pagevec_add(&pvec, page)) { + zone->nr_inactive += pgmoved; + spin_unlock_irq(&zone->lru_lock); + pgdeactivate += pgmoved; + pgmoved = 0; + if (buffer_heads_over_limit) + pagevec_strip(&pvec); + __pagevec_release(&pvec); + spin_lock_irq(&zone->lru_lock); + } + } + zone->nr_inactive += pgmoved; + pgdeactivate += pgmoved; + if (buffer_heads_over_limit) { + spin_unlock_irq(&zone->lru_lock); + pagevec_strip(&pvec); + spin_lock_irq(&zone->lru_lock); + } + + pgmoved = 0; + while (!list_empty(&l_active)) { + page = lru_to_page(&l_active); + prefetchw_prev_lru_page(page, &l_active, flags); + if (TestSetPageLRU(page)) + BUG(); + BUG_ON(!PageActive(page)); + list_move(&page->lru, &zone->active_list); + pgmoved++; + if (!pagevec_add(&pvec, page)) { + zone->nr_active += pgmoved; + pgmoved = 0; + spin_unlock_irq(&zone->lru_lock); + __pagevec_release(&pvec); + spin_lock_irq(&zone->lru_lock); + } + } + zone->nr_active += pgmoved; + spin_unlock_irq(&zone->lru_lock); + pagevec_release(&pvec); + + mod_page_state_zone(zone, pgrefill, pgscanned); + mod_page_state(pgdeactivate, pgdeactivate); +} + +/* + * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. + */ +static void +shrink_zone(struct zone *zone, struct scan_control *sc) +{ + unsigned long nr_active; + unsigned long nr_inactive; + + /* + * Add one to `nr_to_scan' just to make sure that the kernel will + * slowly sift through the active list. + */ + zone->nr_scan_active += (zone->nr_active >> sc->priority) + 1; + nr_active = zone->nr_scan_active; + if (nr_active >= sc->swap_cluster_max) + zone->nr_scan_active = 0; + else + nr_active = 0; + + zone->nr_scan_inactive += (zone->nr_inactive >> sc->priority) + 1; + nr_inactive = zone->nr_scan_inactive; + if (nr_inactive >= sc->swap_cluster_max) + zone->nr_scan_inactive = 0; + else + nr_inactive = 0; + + sc->nr_to_reclaim = sc->swap_cluster_max; + + while (nr_active || nr_inactive) { + if (nr_active) { + sc->nr_to_scan = min(nr_active, + (unsigned long)sc->swap_cluster_max); + nr_active -= sc->nr_to_scan; + refill_inactive_zone(zone, sc); + } + + if (nr_inactive) { + sc->nr_to_scan = min(nr_inactive, + (unsigned long)sc->swap_cluster_max); + nr_inactive -= sc->nr_to_scan; + shrink_cache(zone, sc); + if (sc->nr_to_reclaim <= 0) + break; + } + } + + throttle_vm_writeout(); +} + +/* + * This is the direct reclaim path, for page-allocating processes. We only + * try to reclaim pages from zones which will satisfy the caller's allocation + * request. + * + * We reclaim from a zone even if that zone is over pages_high. Because: + * a) The caller may be trying to free *extra* pages to satisfy a higher-order + * allocation or + * b) The zones may be over pages_high but they must go *over* pages_high to + * satisfy the `incremental min' zone defense algorithm. + * + * Returns the number of reclaimed pages. + * + * If a zone is deemed to be full of pinned pages then just give it a light + * scan then give up on it. + */ +static void +shrink_caches(struct zone **zones, struct scan_control *sc) +{ + int i; + + for (i = 0; zones[i] != NULL; i++) { + struct zone *zone = zones[i]; + + if (zone->present_pages == 0) + continue; + + if (!cpuset_zone_allowed(zone)) + continue; + + zone->temp_priority = sc->priority; + if (zone->prev_priority > sc->priority) + zone->prev_priority = sc->priority; + + if (zone->all_unreclaimable && sc->priority != DEF_PRIORITY) + continue; /* Let kswapd poll it */ + + shrink_zone(zone, sc); + } +} + +/* + * This is the main entry point to direct page reclaim. + * + * If a full scan of the inactive list fails to free enough memory then we + * are "out of memory" and something needs to be killed. + * + * If the caller is !__GFP_FS then the probability of a failure is reasonably + * high - the zone may be full of dirty or under-writeback pages, which this + * caller can't do much about. We kick pdflush and take explicit naps in the + * hope that some of these pages can be written. But if the allocating task + * holds filesystem locks which prevent writeout this might not work, and the + * allocation attempt will fail. + */ +int try_to_free_pages(struct zone **zones, + unsigned int gfp_mask, unsigned int order) +{ + int priority; + int ret = 0; + int total_scanned = 0, total_reclaimed = 0; + struct reclaim_state *reclaim_state = current->reclaim_state; + struct scan_control sc; + unsigned long lru_pages = 0; + int i; + + sc.gfp_mask = gfp_mask; + sc.may_writepage = 0; + + inc_page_state(allocstall); + + for (i = 0; zones[i] != NULL; i++) { + struct zone *zone = zones[i]; + + if (!cpuset_zone_allowed(zone)) + continue; + + zone->temp_priority = DEF_PRIORITY; + lru_pages += zone->nr_active + zone->nr_inactive; + } + + for (priority = DEF_PRIORITY; priority >= 0; priority--) { + sc.nr_mapped = read_page_state(nr_mapped); + sc.nr_scanned = 0; + sc.nr_reclaimed = 0; + sc.priority = priority; + sc.swap_cluster_max = SWAP_CLUSTER_MAX; + shrink_caches(zones, &sc); + shrink_slab(sc.nr_scanned, gfp_mask, lru_pages); + if (reclaim_state) { + sc.nr_reclaimed += reclaim_state->reclaimed_slab; + reclaim_state->reclaimed_slab = 0; + } + total_scanned += sc.nr_scanned; + total_reclaimed += sc.nr_reclaimed; + if (total_reclaimed >= sc.swap_cluster_max) { + ret = 1; + goto out; + } + + /* + * Try to write back as many pages as we just scanned. This + * tends to cause slow streaming writers to write data to the + * disk smoothly, at the dirtying rate, which is nice. But + * that's undesirable in laptop mode, where we *want* lumpy + * writeout. So in laptop mode, write out the whole world. + */ + if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) { + wakeup_bdflush(laptop_mode ? 0 : total_scanned); + sc.may_writepage = 1; + } + + /* Take a nap, wait for some writeback to complete */ + if (sc.nr_scanned && priority < DEF_PRIORITY - 2) + blk_congestion_wait(WRITE, HZ/10); + } +out: + for (i = 0; zones[i] != 0; i++) { + struct zone *zone = zones[i]; + + if (!cpuset_zone_allowed(zone)) + continue; + + zone->prev_priority = zone->temp_priority; + } + return ret; +} + +/* + * For kswapd, balance_pgdat() will work across all this node's zones until + * they are all at pages_high. + * + * If `nr_pages' is non-zero then it is the number of pages which are to be + * reclaimed, regardless of the zone occupancies. This is a software suspend + * special. + * + * Returns the number of pages which were actually freed. + * + * There is special handling here for zones which are full of pinned pages. + * This can happen if the pages are all mlocked, or if they are all used by + * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb. + * What we do is to detect the case where all pages in the zone have been + * scanned twice and there has been zero successful reclaim. Mark the zone as + * dead and from now on, only perform a short scan. Basically we're polling + * the zone for when the problem goes away. + * + * kswapd scans the zones in the highmem->normal->dma direction. It skips + * zones which have free_pages > pages_high, but once a zone is found to have + * free_pages <= pages_high, we scan that zone and the lower zones regardless + * of the number of free pages in the lower zones. This interoperates with + * the page allocator fallback scheme to ensure that aging of pages is balanced + * across the zones. + */ +static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order) +{ + int to_free = nr_pages; + int all_zones_ok; + int priority; + int i; + int total_scanned, total_reclaimed; + struct reclaim_state *reclaim_state = current->reclaim_state; + struct scan_control sc; + +loop_again: + total_scanned = 0; + total_reclaimed = 0; + sc.gfp_mask = GFP_KERNEL; + sc.may_writepage = 0; + sc.nr_mapped = read_page_state(nr_mapped); + + inc_page_state(pageoutrun); + + for (i = 0; i < pgdat->nr_zones; i++) { + struct zone *zone = pgdat->node_zones + i; + + zone->temp_priority = DEF_PRIORITY; + } + + for (priority = DEF_PRIORITY; priority >= 0; priority--) { + int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */ + unsigned long lru_pages = 0; + + all_zones_ok = 1; + + if (nr_pages == 0) { + /* + * Scan in the highmem->dma direction for the highest + * zone which needs scanning + */ + for (i = pgdat->nr_zones - 1; i >= 0; i--) { + struct zone *zone = pgdat->node_zones + i; + + if (zone->present_pages == 0) + continue; + + if (zone->all_unreclaimable && + priority != DEF_PRIORITY) + continue; + + if (!zone_watermark_ok(zone, order, + zone->pages_high, 0, 0, 0)) { + end_zone = i; + goto scan; + } + } + goto out; + } else { + end_zone = pgdat->nr_zones - 1; + } +scan: + for (i = 0; i <= end_zone; i++) { + struct zone *zone = pgdat->node_zones + i; + + lru_pages += zone->nr_active + zone->nr_inactive; + } + + /* + * Now scan the zone in the dma->highmem direction, stopping + * at the last zone which needs scanning. + * + * We do this because the page allocator works in the opposite + * direction. This prevents the page allocator from allocating + * pages behind kswapd's direction of progress, which would + * cause too much scanning of the lower zones. + */ + for (i = 0; i <= end_zone; i++) { + struct zone *zone = pgdat->node_zones + i; + + if (zone->present_pages == 0) + continue; + + if (zone->all_unreclaimable && priority != DEF_PRIORITY) + continue; + + if (nr_pages == 0) { /* Not software suspend */ + if (!zone_watermark_ok(zone, order, + zone->pages_high, end_zone, 0, 0)) + all_zones_ok = 0; + } + zone->temp_priority = priority; + if (zone->prev_priority > priority) + zone->prev_priority = priority; + sc.nr_scanned = 0; + sc.nr_reclaimed = 0; + sc.priority = priority; + sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX; + shrink_zone(zone, &sc); + reclaim_state->reclaimed_slab = 0; + shrink_slab(sc.nr_scanned, GFP_KERNEL, lru_pages); + sc.nr_reclaimed += reclaim_state->reclaimed_slab; + total_reclaimed += sc.nr_reclaimed; + total_scanned += sc.nr_scanned; + if (zone->all_unreclaimable) + continue; + if (zone->pages_scanned >= (zone->nr_active + + zone->nr_inactive) * 4) + zone->all_unreclaimable = 1; + /* + * If we've done a decent amount of scanning and + * the reclaim ratio is low, start doing writepage + * even in laptop mode + */ + if (total_scanned > SWAP_CLUSTER_MAX * 2 && + total_scanned > total_reclaimed+total_reclaimed/2) + sc.may_writepage = 1; + } + if (nr_pages && to_free > total_reclaimed) + continue; /* swsusp: need to do more work */ + if (all_zones_ok) + break; /* kswapd: all done */ + /* + * OK, kswapd is getting into trouble. Take a nap, then take + * another pass across the zones. + */ + if (total_scanned && priority < DEF_PRIORITY - 2) + blk_congestion_wait(WRITE, HZ/10); + + /* + * We do this so kswapd doesn't build up large priorities for + * example when it is freeing in parallel with allocators. It + * matches the direct reclaim path behaviour in terms of impact + * on zone->*_priority. + */ + if ((total_reclaimed >= SWAP_CLUSTER_MAX) && (!nr_pages)) + break; + } +out: + for (i = 0; i < pgdat->nr_zones; i++) { + struct zone *zone = pgdat->node_zones + i; + + zone->prev_priority = zone->temp_priority; + } + if (!all_zones_ok) { + cond_resched(); + goto loop_again; + } + + return total_reclaimed; +} + +/* + * The background pageout daemon, started as a kernel thread + * from the init process. + * + * This basically trickles out pages so that we have _some_ + * free memory available even if there is no other activity + * that frees anything up. This is needed for things like routing + * etc, where we otherwise might have all activity going on in + * asynchronous contexts that cannot page things out. + * + * If there are applications that are active memory-allocators + * (most normal use), this basically shouldn't matter. + */ +static int kswapd(void *p) +{ + unsigned long order; + pg_data_t *pgdat = (pg_data_t*)p; + struct task_struct *tsk = current; + DEFINE_WAIT(wait); + struct reclaim_state reclaim_state = { + .reclaimed_slab = 0, + }; + cpumask_t cpumask; + + daemonize("kswapd%d", pgdat->node_id); + cpumask = node_to_cpumask(pgdat->node_id); + if (!cpus_empty(cpumask)) + set_cpus_allowed(tsk, cpumask); + current->reclaim_state = &reclaim_state; + + /* + * Tell the memory management that we're a "memory allocator", + * and that if we need more memory we should get access to it + * regardless (see "__alloc_pages()"). "kswapd" should + * never get caught in the normal page freeing logic. + * + * (Kswapd normally doesn't need memory anyway, but sometimes + * you need a small amount of memory in order to be able to + * page out something else, and this flag essentially protects + * us from recursively trying to free more memory as we're + * trying to free the first piece of memory in the first place). + */ + tsk->flags |= PF_MEMALLOC|PF_KSWAPD; + + order = 0; + for ( ; ; ) { + unsigned long new_order; + if (current->flags & PF_FREEZE) + refrigerator(PF_FREEZE); + + prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); + new_order = pgdat->kswapd_max_order; + pgdat->kswapd_max_order = 0; + if (order < new_order) { + /* + * Don't sleep if someone wants a larger 'order' + * allocation + */ + order = new_order; + } else { + schedule(); + order = pgdat->kswapd_max_order; + } + finish_wait(&pgdat->kswapd_wait, &wait); + + balance_pgdat(pgdat, 0, order); + } + return 0; +} + +/* + * A zone is low on free memory, so wake its kswapd task to service it. + */ +void wakeup_kswapd(struct zone *zone, int order) +{ + pg_data_t *pgdat; + + if (zone->present_pages == 0) + return; + + pgdat = zone->zone_pgdat; + if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0, 0)) + return; + if (pgdat->kswapd_max_order < order) + pgdat->kswapd_max_order = order; + if (!cpuset_zone_allowed(zone)) + return; + if (!waitqueue_active(&zone->zone_pgdat->kswapd_wait)) + return; + wake_up_interruptible(&zone->zone_pgdat->kswapd_wait); +} + +#ifdef CONFIG_PM +/* + * Try to free `nr_pages' of memory, system-wide. Returns the number of freed + * pages. + */ +int shrink_all_memory(int nr_pages) +{ + pg_data_t *pgdat; + int nr_to_free = nr_pages; + int ret = 0; + struct reclaim_state reclaim_state = { + .reclaimed_slab = 0, + }; + + current->reclaim_state = &reclaim_state; + for_each_pgdat(pgdat) { + int freed; + freed = balance_pgdat(pgdat, nr_to_free, 0); + ret += freed; + nr_to_free -= freed; + if (nr_to_free <= 0) + break; + } + current->reclaim_state = NULL; + return ret; +} +#endif + +#ifdef CONFIG_HOTPLUG_CPU +/* It's optimal to keep kswapds on the same CPUs as their memory, but + not required for correctness. So if the last cpu in a node goes + away, we get changed to run anywhere: as the first one comes back, + restore their cpu bindings. */ +static int __devinit cpu_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + pg_data_t *pgdat; + cpumask_t mask; + + if (action == CPU_ONLINE) { + for_each_pgdat(pgdat) { + mask = node_to_cpumask(pgdat->node_id); + if (any_online_cpu(mask) != NR_CPUS) + /* One of our CPUs online: restore mask */ + set_cpus_allowed(pgdat->kswapd, mask); + } + } + return NOTIFY_OK; +} +#endif /* CONFIG_HOTPLUG_CPU */ + +static int __init kswapd_init(void) +{ + pg_data_t *pgdat; + swap_setup(); + for_each_pgdat(pgdat) + pgdat->kswapd + = find_task_by_pid(kernel_thread(kswapd, pgdat, CLONE_KERNEL)); + total_memory = nr_free_pagecache_pages(); + hotcpu_notifier(cpu_callback, 0); + return 0; +} + +module_init(kswapd_init) |