diff options
| author | Linus Torvalds | 2019-12-05 09:46:26 -0800 |
|---|---|---|
| committer | Linus Torvalds | 2019-12-05 09:46:26 -0800 |
| commit | 5ecc9d15f73b82c748526350d5602c94fdd65cac (patch) | |
| tree | 686ae55a0de4be8bad9e2248b019e459525cb109 /arch | |
| parent | 2f13437b8917627119d163d62f73e7a78a92303a (diff) | |
| parent | f949286c668aed5aa24acdb5838be9cfd9513bd3 (diff) | |
Merge branch 'akpm' (patches from Andrew)
Merge more updates from Andrew Morton:
"Most of the rest of MM and various other things. Some Kconfig rework
still awaits merges of dependent trees from linux-next.
Subsystems affected by this patch series: mm/hotfixes, mm/memcg,
mm/vmstat, mm/thp, procfs, sysctl, misc, notifiers, core-kernel,
bitops, lib, checkpatch, epoll, binfmt, init, rapidio, uaccess, kcov,
ubsan, ipc, bitmap, mm/pagemap"
* akpm: (86 commits)
mm: remove __ARCH_HAS_4LEVEL_HACK and include/asm-generic/4level-fixup.h
um: add support for folded p4d page tables
um: remove unused pxx_offset_proc() and addr_pte() functions
sparc32: use pgtable-nopud instead of 4level-fixup
parisc/hugetlb: use pgtable-nopXd instead of 4level-fixup
parisc: use pgtable-nopXd instead of 4level-fixup
nds32: use pgtable-nopmd instead of 4level-fixup
microblaze: use pgtable-nopmd instead of 4level-fixup
m68k: mm: use pgtable-nopXd instead of 4level-fixup
m68k: nommu: use pgtable-nopud instead of 4level-fixup
c6x: use pgtable-nopud instead of 4level-fixup
arm: nommu: use pgtable-nopud instead of 4level-fixup
alpha: use pgtable-nopud instead of 4level-fixup
gpio: pca953x: tighten up indentation
gpio: pca953x: convert to use bitmap API
gpio: pca953x: use input from regs structure in pca953x_irq_pending()
gpio: pca953x: remove redundant variable and check in IRQ handler
lib/bitmap: introduce bitmap_replace() helper
lib/test_bitmap: fix comment about this file
lib/test_bitmap: move exp1 and exp2 upper for others to use
...
Diffstat (limited to 'arch')
77 files changed, 517 insertions, 375 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 7b861fe3f900..48b5e103bdb0 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -72,11 +72,11 @@ config KPROBES If in doubt, say "N". config JUMP_LABEL - bool "Optimize very unlikely/likely branches" - depends on HAVE_ARCH_JUMP_LABEL - depends on CC_HAS_ASM_GOTO - help - This option enables a transparent branch optimization that + bool "Optimize very unlikely/likely branches" + depends on HAVE_ARCH_JUMP_LABEL + depends on CC_HAS_ASM_GOTO + help + This option enables a transparent branch optimization that makes certain almost-always-true or almost-always-false branch conditions even cheaper to execute within the kernel. @@ -84,7 +84,7 @@ config JUMP_LABEL scheduler functionality, networking code and KVM have such branches and include support for this optimization technique. - If it is detected that the compiler has support for "asm goto", + If it is detected that the compiler has support for "asm goto", the kernel will compile such branches with just a nop instruction. When the condition flag is toggled to true, the nop will be converted to a jump instruction to execute the @@ -151,8 +151,8 @@ config HAVE_EFFICIENT_UNALIGNED_ACCESS information on the topic of unaligned memory accesses. config ARCH_USE_BUILTIN_BSWAP - bool - help + bool + help Modern versions of GCC (since 4.4) have builtin functions for handling byte-swapping. Using these, instead of the old inline assembler that the architecture code provides in the @@ -221,10 +221,10 @@ config HAVE_DMA_CONTIGUOUS bool config GENERIC_SMP_IDLE_THREAD - bool + bool config GENERIC_IDLE_POLL_SETUP - bool + bool config ARCH_HAS_FORTIFY_SOURCE bool @@ -257,7 +257,7 @@ config ARCH_HAS_UNCACHED_SEGMENT # Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK - bool + bool # Select if arch has its private alloc_task_struct() function config ARCH_TASK_STRUCT_ALLOCATOR diff --git a/arch/alpha/include/asm/mmzone.h b/arch/alpha/include/asm/mmzone.h index 889b5d3ad825..7ee144f484f1 100644 --- a/arch/alpha/include/asm/mmzone.h +++ b/arch/alpha/include/asm/mmzone.h @@ -73,7 +73,6 @@ PLAT_NODE_DATA_LOCALNR(unsigned long p, int n) #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> 32)) -#define pgd_page(pgd) (pfn_to_page(pgd_val(pgd) >> 32)) #define pte_pfn(pte) (pte_val(pte) >> 32) #define mk_pte(page, pgprot) \ diff --git a/arch/alpha/include/asm/pgalloc.h b/arch/alpha/include/asm/pgalloc.h index eb91f1e85629..a1a29f60934c 100644 --- a/arch/alpha/include/asm/pgalloc.h +++ b/arch/alpha/include/asm/pgalloc.h @@ -27,9 +27,9 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) } static inline void -pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) +pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { - pgd_set(pgd, pmd); + pud_set(pud, pmd); } extern pgd_t *pgd_alloc(struct mm_struct *mm); diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 065b57f408c3..299791ce14b6 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -2,7 +2,7 @@ #ifndef _ALPHA_PGTABLE_H #define _ALPHA_PGTABLE_H -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopud.h> /* * This file contains the functions and defines necessary to modify and use @@ -226,8 +226,8 @@ extern inline pte_t pte_modify(pte_t pte, pgprot_t newprot) extern inline void pmd_set(pmd_t * pmdp, pte_t * ptep) { pmd_val(*pmdp) = _PAGE_TABLE | ((((unsigned long) ptep) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } -extern inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) -{ pgd_val(*pgdp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } +extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) +{ pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } extern inline unsigned long @@ -238,11 +238,11 @@ pmd_page_vaddr(pmd_t pmd) #ifndef CONFIG_DISCONTIGMEM #define pmd_page(pmd) (mem_map + ((pmd_val(pmd) & _PFN_MASK) >> 32)) -#define pgd_page(pgd) (mem_map + ((pgd_val(pgd) & _PFN_MASK) >> 32)) +#define pud_page(pud) (mem_map + ((pud_val(pud) & _PFN_MASK) >> 32)) #endif -extern inline unsigned long pgd_page_vaddr(pgd_t pgd) -{ return PAGE_OFFSET + ((pgd_val(pgd) & _PFN_MASK) >> (32-PAGE_SHIFT)); } +extern inline unsigned long pud_page_vaddr(pud_t pgd) +{ return PAGE_OFFSET + ((pud_val(pgd) & _PFN_MASK) >> (32-PAGE_SHIFT)); } extern inline int pte_none(pte_t pte) { return !pte_val(pte); } extern inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_VALID; } @@ -256,10 +256,10 @@ extern inline int pmd_bad(pmd_t pmd) { return (pmd_val(pmd) & ~_PFN_MASK) != _P extern inline int pmd_present(pmd_t pmd) { return pmd_val(pmd) & _PAGE_VALID; } extern inline void pmd_clear(pmd_t * pmdp) { pmd_val(*pmdp) = 0; } -extern inline int pgd_none(pgd_t pgd) { return !pgd_val(pgd); } -extern inline int pgd_bad(pgd_t pgd) { return (pgd_val(pgd) & ~_PFN_MASK) != _PAGE_TABLE; } -extern inline int pgd_present(pgd_t pgd) { return pgd_val(pgd) & _PAGE_VALID; } -extern inline void pgd_clear(pgd_t * pgdp) { pgd_val(*pgdp) = 0; } +extern inline int pud_none(pud_t pud) { return !pud_val(pud); } +extern inline int pud_bad(pud_t pud) { return (pud_val(pud) & ~_PFN_MASK) != _PAGE_TABLE; } +extern inline int pud_present(pud_t pud) { return pud_val(pud) & _PAGE_VALID; } +extern inline void pud_clear(pud_t * pudp) { pud_val(*pudp) = 0; } /* * The following only work if pte_present() is true. @@ -301,9 +301,9 @@ extern inline pte_t pte_mkspecial(pte_t pte) { return pte; } */ /* Find an entry in the second-level page table.. */ -extern inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address) +extern inline pmd_t * pmd_offset(pud_t * dir, unsigned long address) { - pmd_t *ret = (pmd_t *) pgd_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PAGE - 1)); + pmd_t *ret = (pmd_t *) pud_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PAGE - 1)); smp_read_barrier_depends(); /* see above */ return ret; } diff --git a/arch/alpha/mm/init.c b/arch/alpha/mm/init.c index e2cbec3789e8..12e218d3792a 100644 --- a/arch/alpha/mm/init.c +++ b/arch/alpha/mm/init.c @@ -146,6 +146,8 @@ callback_init(void * kernel_end) { struct crb_struct * crb; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; void *two_pages; @@ -184,8 +186,10 @@ callback_init(void * kernel_end) memset(two_pages, 0, 2*PAGE_SIZE); pgd = pgd_offset_k(VMALLOC_START); - pgd_set(pgd, (pmd_t *)two_pages); - pmd = pmd_offset(pgd, VMALLOC_START); + p4d = p4d_offset(pgd, VMALLOC_START); + pud = pud_offset(p4d, VMALLOC_START); + pud_set(pud, (pmd_t *)two_pages); + pmd = pmd_offset(pud, VMALLOC_START); pmd_set(pmd, (pte_t *)(two_pages + PAGE_SIZE)); if (alpha_using_srm) { @@ -214,9 +218,9 @@ callback_init(void * kernel_end) /* Newer consoles (especially on larger systems) may require more pages of PTEs. Grab additional pages as needed. */ - if (pmd != pmd_offset(pgd, vaddr)) { + if (pmd != pmd_offset(pud, vaddr)) { memset(kernel_end, 0, PAGE_SIZE); - pmd = pmd_offset(pgd, vaddr); + pmd = pmd_offset(pud, vaddr); pmd_set(pmd, (pte_t *)kernel_end); kernel_end += PAGE_SIZE; } diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 3ae120cd1715..eabcb48a7840 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -12,7 +12,7 @@ #ifndef CONFIG_MMU -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopud.h> #include <asm/pgtable-nommu.h> #else diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 1df6eb42f22e..e822af0d9219 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -529,7 +529,7 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) static bool __in_atomic_pool(void *start, size_t size) { - return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); + return gen_pool_has_addr(atomic_pool, (unsigned long)start, size); } static int __free_from_pool(void *start, size_t size) diff --git a/arch/c6x/include/asm/pgtable.h b/arch/c6x/include/asm/pgtable.h index 0b6919c00413..197c473b796a 100644 --- a/arch/c6x/include/asm/pgtable.h +++ b/arch/c6x/include/asm/pgtable.h @@ -8,7 +8,7 @@ #ifndef _ASM_C6X_PGTABLE_H #define _ASM_C6X_PGTABLE_H -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopud.h> #include <asm/setup.h> #include <asm/page.h> diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index b34d44d666a4..82ec54c2eaa4 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -28,9 +28,6 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address) return (pmd_t *) pgd; } -#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); }) -#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); }) - #define pmd_populate(mm, pmd, page) (pmd_val(*pmd) = \ (unsigned long)(page_address(page))) @@ -45,8 +42,6 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, __free_page(page); } -#define __pmd_free_tlb(tlb, pmd, address) do { } while (0) - static inline struct page *pte_alloc_one(struct mm_struct *mm) { struct page *page = alloc_pages(GFP_DMA, 0); @@ -100,6 +95,4 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) return new_pgd; } -#define pgd_populate(mm, pmd, pte) BUG() - #endif /* M68K_MCF_PGALLOC_H */ diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h index 5d5502cb2b2d..b9f45aeded25 100644 --- a/arch/m68k/include/asm/mcf_pgtable.h +++ b/arch/m68k/include/asm/mcf_pgtable.h @@ -198,17 +198,9 @@ static inline int pmd_bad2(pmd_t *pmd) { return 0; } #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } -static inline int pgd_none(pgd_t pgd) { return 0; } -static inline int pgd_bad(pgd_t pgd) { return 0; } -static inline int pgd_present(pgd_t pgd) { return 1; } -static inline void pgd_clear(pgd_t *pgdp) {} - #define pte_ERROR(e) \ printk(KERN_ERR "%s:%d: bad pte %08lx.\n", \ __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pmd %08lx.\n", \ - __FILE__, __LINE__, pmd_val(e)) #define pgd_ERROR(e) \ printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ __FILE__, __LINE__, pgd_val(e)) @@ -340,14 +332,6 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; #define pgd_offset_k(address) pgd_offset(&init_mm, address) /* - * Find an entry in the second-level pagetable. - */ -static inline pmd_t *pmd_offset(pgd_t *pgd, unsigned long address) -{ - return (pmd_t *) pgd; -} - -/* * Find an entry in the third-level pagetable. */ #define __pte_offset(address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) @@ -360,12 +344,16 @@ static inline pmd_t *pmd_offset(pgd_t *pgd, unsigned long address) static inline void nocache_page(void *vaddr) { pgd_t *dir; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long addr = (unsigned long) vaddr; dir = pgd_offset_k(addr); - pmdp = pmd_offset(dir, addr); + p4dp = p4d_offset(dir, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_kernel(pmdp, addr); *ptep = pte_mknocache(*ptep); } @@ -376,12 +364,16 @@ static inline void nocache_page(void *vaddr) static inline void cache_page(void *vaddr) { pgd_t *dir; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long addr = (unsigned long) vaddr; dir = pgd_offset_k(addr); - pmdp = pmd_offset(dir, addr); + p4dp = p4d_offset(dir, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_kernel(pmdp, addr); *ptep = pte_mkcache(*ptep); } diff --git a/arch/m68k/include/asm/mmu_context.h b/arch/m68k/include/asm/mmu_context.h index f5b1852b4663..cac9f289d1f6 100644 --- a/arch/m68k/include/asm/mmu_context.h +++ b/arch/m68k/include/asm/mmu_context.h @@ -100,6 +100,8 @@ static inline void load_ksp_mmu(struct task_struct *task) struct mm_struct *mm; int asid; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long mmuar; @@ -127,7 +129,15 @@ static inline void load_ksp_mmu(struct task_struct *task) if (pgd_none(*pgd)) goto bug; - pmd = pmd_offset(pgd, mmuar); + p4d = p4d_offset(pgd, mmuar); + if (p4d_none(*p4d)) + goto bug; + + pud = pud_offset(p4d, mmuar); + if (pud_none(*pud)) + goto bug; + + pmd = pmd_offset(pud, mmuar); if (pmd_none(*pmd)) goto bug; diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index acab315c851f..ff9cc401ffd1 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -106,9 +106,9 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page } #define pmd_pgtable(pmd) pmd_page(pmd) -static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { - pgd_set(pgd, pmd); + pud_set(pud, pmd); } #endif /* _MOTOROLA_PGALLOC_H */ diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index 7f66a7bad7a5..62bedc61f110 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -117,14 +117,14 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) } } -static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) +static inline void pud_set(pud_t *pudp, pmd_t *pmdp) { - pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); + pud_val(*pudp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); } #define __pte_page(pte) ((unsigned long)__va(pte_val(pte) & PAGE_MASK)) #define __pmd_page(pmd) ((unsigned long)__va(pmd_val(pmd) & _TABLE_MASK)) -#define __pgd_page(pgd) ((unsigned long)__va(pgd_val(pgd) & _TABLE_MASK)) +#define pud_page_vaddr(pud) ((unsigned long)__va(pud_val(pud) & _TABLE_MASK)) #define pte_none(pte) (!pte_val(pte)) @@ -147,11 +147,11 @@ static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) #define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd))) -#define pgd_none(pgd) (!pgd_val(pgd)) -#define pgd_bad(pgd) ((pgd_val(pgd) & _DESCTYPE_MASK) != _PAGE_TABLE) -#define pgd_present(pgd) (pgd_val(pgd) & _PAGE_TABLE) -#define pgd_clear(pgdp) ({ pgd_val(*pgdp) = 0; }) -#define pgd_page(pgd) (mem_map + ((unsigned long)(__va(pgd_val(pgd)) - PAGE_OFFSET) >> PAGE_SHIFT)) +#define pud_none(pud) (!pud_val(pud)) +#define pud_bad(pud) ((pud_val(pud) & _DESCTYPE_MASK) != _PAGE_TABLE) +#define pud_present(pud) (pud_val(pud) & _PAGE_TABLE) +#define pud_clear(pudp) ({ pud_val(*pudp) = 0; }) +#define pud_page(pud) (mem_map + ((unsigned long)(__va(pud_val(pud)) - PAGE_OFFSET) >> PAGE_SHIFT)) #define pte_ERROR(e) \ printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) @@ -209,9 +209,9 @@ static inline pgd_t *pgd_offset_k(unsigned long address) /* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pgd_t *dir, unsigned long address) +static inline pmd_t *pmd_offset(pud_t *dir, unsigned long address) { - return (pmd_t *)__pgd_page(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PMD-1)); + return (pmd_t *)pud_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PMD-1)); } /* Find an entry in the third-level page table.. */ @@ -239,11 +239,15 @@ static inline void nocache_page(void *vaddr) if (CPU_IS_040_OR_060) { pgd_t *dir; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; dir = pgd_offset_k(addr); - pmdp = pmd_offset(dir, addr); + p4dp = p4d_offset(dir, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_kernel(pmdp, addr); *ptep = pte_mknocache(*ptep); } @@ -255,11 +259,15 @@ static inline void cache_page(void *vaddr) if (CPU_IS_040_OR_060) { pgd_t *dir; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; dir = pgd_offset_k(addr); - pmdp = pmd_offset(dir, addr); + p4dp = p4d_offset(dir, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_kernel(pmdp, addr); *ptep = pte_mkcache(*ptep); } diff --git a/arch/m68k/include/asm/page.h b/arch/m68k/include/asm/page.h index 700d8195880c..05e1e1e77a9a 100644 --- a/arch/m68k/include/asm/page.h +++ b/arch/m68k/include/asm/page.h @@ -21,19 +21,22 @@ /* * These are used to make use of C type-checking.. */ -typedef struct { unsigned long pte; } pte_t; +#if !defined(CONFIG_MMU) || CONFIG_PGTABLE_LEVELS == 3 typedef struct { unsigned long pmd[16]; } pmd_t; +#define pmd_val(x) ((&x)->pmd[0]) +#define __pmd(x) ((pmd_t) { { (x) }, }) +#endif + +typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; typedef struct page *pgtable_t; #define pte_val(x) ((x).pte) -#define pmd_val(x) ((&x)->pmd[0]) #define pgd_val(x) ((x).pgd) #define pgprot_val(x) ((x).pgprot) #define __pte(x) ((pte_t) { (x) } ) -#define __pmd(x) ((pmd_t) { { (x) }, }) #define __pgd(x) ((pgd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) diff --git a/arch/m68k/include/asm/pgtable_mm.h b/arch/m68k/include/asm/pgtable_mm.h index 646c174fff99..2bf5c3501e78 100644 --- a/arch/m68k/include/asm/pgtable_mm.h +++ b/arch/m68k/include/asm/pgtable_mm.h @@ -2,7 +2,12 @@ #ifndef _M68K_PGTABLE_H #define _M68K_PGTABLE_H -#include <asm-generic/4level-fixup.h> + +#if defined(CONFIG_SUN3) || defined(CONFIG_COLDFIRE) +#include <asm-generic/pgtable-nopmd.h> +#else +#include <asm-generic/pgtable-nopud.h> +#endif #include <asm/setup.h> @@ -30,9 +35,7 @@ /* PMD_SHIFT determines the size of the area a second-level page table can map */ -#ifdef CONFIG_SUN3 -#define PMD_SHIFT 17 -#else +#if CONFIG_PGTABLE_LEVELS == 3 #define PMD_SHIFT 22 #endif #define PMD_SIZE (1UL << PMD_SHIFT) diff --git a/arch/m68k/include/asm/pgtable_no.h b/arch/m68k/include/asm/pgtable_no.h index c18165b0d904..ccc4568299e5 100644 --- a/arch/m68k/include/asm/pgtable_no.h +++ b/arch/m68k/include/asm/pgtable_no.h @@ -2,7 +2,7 @@ #ifndef _M68KNOMMU_PGTABLE_H #define _M68KNOMMU_PGTABLE_H -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopud.h> /* * (C) Copyright 2000-2002, Greg Ungerer <gerg@snapgear.com> diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h index 856121122b91..11b95dadf7c0 100644 --- a/arch/m68k/include/asm/sun3_pgalloc.h +++ b/arch/m68k/include/asm/sun3_pgalloc.h @@ -17,8 +17,6 @@ extern const char bad_pmd_string[]; -#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) - #define __pte_free_tlb(tlb,pte,addr) \ do { \ pgtable_pte_page_dtor(pte); \ @@ -41,7 +39,6 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page * inside the pgd, so has no extra memory associated with it. */ #define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb, x, addr) do { } while (0) static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { @@ -58,6 +55,4 @@ static inline pgd_t * pgd_alloc(struct mm_struct *mm) return new_pgd; } -#define pgd_populate(mm, pmd, pte) BUG() - #endif /* SUN3_PGALLOC_H */ diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h index c987d50866b4..bc4155264810 100644 --- a/arch/m68k/include/asm/sun3_pgtable.h +++ b/arch/m68k/include/asm/sun3_pgtable.h @@ -110,11 +110,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pmd_set(pmdp,ptep) do {} while (0) -static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) -{ - pgd_val(*pgdp) = virt_to_phys(pmdp); -} - #define __pte_page(pte) \ ((unsigned long) __va ((pte_val (pte) & SUN3_PAGE_PGNUM_MASK) << PAGE_SHIFT)) #define __pmd_page(pmd) \ @@ -145,16 +140,9 @@ static inline int pmd_present2 (pmd_t *pmd) { return pmd_val (*pmd) & SUN3_PMD_V #define pmd_present(pmd) (!pmd_none2(&(pmd))) static inline void pmd_clear (pmd_t *pmdp) { pmd_val (*pmdp) = 0; } -static inline int pgd_none (pgd_t pgd) { return 0; } -static inline int pgd_bad (pgd_t pgd) { return 0; } -static inline int pgd_present (pgd_t pgd) { return 1; } -static inline void pgd_clear (pgd_t *pgdp) {} - #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -194,12 +182,6 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; /* Find an entry in a kernel pagetable directory. */ #define pgd_offset_k(address) pgd_offset(&init_mm, address) -/* Find an entry in the second-level pagetable. */ -static inline pmd_t *pmd_offset (pgd_t *pgd, unsigned long address) -{ - return (pmd_t *) pgd; -} - /* Find an entry in the third-level pagetable. */ #define pte_index(address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) #define pte_offset_kernel(pmd, address) ((pte_t *) __pmd_page(*pmd) + pte_index(address)) diff --git a/arch/m68k/kernel/sys_m68k.c b/arch/m68k/kernel/sys_m68k.c index 6363ec83a290..18a4de7d5934 100644 --- a/arch/m68k/kernel/sys_m68k.c +++ b/arch/m68k/kernel/sys_m68k.c @@ -465,6 +465,8 @@ sys_atomic_cmpxchg_32(unsigned long newval, int oldval, int d3, int d4, int d5, for (;;) { struct mm_struct *mm = current->mm; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; spinlock_t *ptl; @@ -474,7 +476,13 @@ sys_atomic_cmpxchg_32(unsigned long newval, int oldval, int d3, int d4, int d5, pgd = pgd_offset(mm, (unsigned long)mem); if (!pgd_present(*pgd)) goto bad_access; - pmd = pmd_offset(pgd, (unsigned long)mem); + p4d = p4d_offset(pgd, (unsigned long)mem); + if (!p4d_present(*p4d)) + goto bad_access; + pud = pud_offset(p4d, (unsigned long)mem); + if (!pud_present(*pud)) + goto bad_access; + pmd = pmd_offset(pud, (unsigned long)mem); if (!pmd_present(*pmd)) goto bad_access; pte = pte_offset_map_lock(mm, pmd, (unsigned long)mem, &ptl); diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c index 778cacb7d57b..27c453f4fffe 100644 --- a/arch/m68k/mm/init.c +++ b/arch/m68k/mm/init.c @@ -130,8 +130,10 @@ static inline void init_pointer_tables(void) /* insert pointer tables allocated so far into the tablelist */ init_pointer_table((unsigned long)kernel_pg_dir); for (i = 0; i < PTRS_PER_PGD; i++) { - if (pgd_present(kernel_pg_dir[i])) - init_pointer_table(__pgd_page(kernel_pg_dir[i])); + pud_t *pud = (pud_t *)(&kernel_pg_dir[i]); + + if (pud_present(*pud)) + init_pointer_table(pgd_page_vaddr(kernel_pg_dir[i])); } /* insert also pointer table that we used to unmap the zero page */ diff --git a/arch/m68k/mm/kmap.c b/arch/m68k/mm/kmap.c index 23f9466aabb5..120030ad8dc4 100644 --- a/arch/m68k/mm/kmap.c +++ b/arch/m68k/mm/kmap.c @@ -63,18 +63,23 @@ static void __free_io_area(void *addr, unsigned long size) { unsigned long virtaddr = (unsigned long)addr; pgd_t *pgd_dir; + p4d_t *p4d_dir; + pud_t *pud_dir; pmd_t *pmd_dir; pte_t *pte_dir; while ((long)size > 0) { pgd_dir = pgd_offset_k(virtaddr); - if (pgd_bad(*pgd_dir)) { - printk("iounmap: bad pgd(%08lx)\n", pgd_val(*pgd_dir)); - pgd_clear(pgd_dir); + p4d_dir = p4d_offset(pgd_dir, virtaddr); + pud_dir = pud_offset(p4d_dir, virtaddr); + if (pud_bad(*pud_dir)) { + printk("iounmap: bad pud(%08lx)\n", pud_val(*pud_dir)); + pud_clear(pud_dir); return; } - pmd_dir = pmd_offset(pgd_dir, virtaddr); + pmd_dir = pmd_offset(pud_dir, virtaddr); +#if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { int pmd_off = (virtaddr/PTRTREESIZE) & 15; int pmd_type = pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK; @@ -87,6 +92,7 @@ static void __free_io_area(void *addr, unsigned long size) } else if (pmd_type == 0) continue; } +#endif if (pmd_bad(*pmd_dir)) { printk("iounmap: bad pmd (%08lx)\n", pmd_val(*pmd_dir)); @@ -159,6 +165,8 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla unsigned long virtaddr, retaddr; long offset; pgd_t *pgd_dir; + p4d_t *p4d_dir; + pud_t *pud_dir; pmd_t *pmd_dir; pte_t *pte_dir; @@ -245,18 +253,23 @@ void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cachefla printk ("\npa=%#lx va=%#lx ", physaddr, virtaddr); #endif pgd_dir = pgd_offset_k(virtaddr); - pmd_dir = pmd_alloc(&init_mm, pgd_dir, virtaddr); + p4d_dir = p4d_offset(pgd_dir, virtaddr); + pud_dir = pud_offset(p4d_dir, virtaddr); + pmd_dir = pmd_alloc(&init_mm, pud_dir, virtaddr); if (!pmd_dir) { printk("ioremap: no mem for pmd_dir\n"); return NULL; } +#if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { pmd_dir->pmd[(virtaddr/PTRTREESIZE) & 15] = physaddr; physaddr += PTRTREESIZE; virtaddr += PTRTREESIZE; size -= PTRTREESIZE; - } else { + } else +#endif + { pte_dir = pte_alloc_kernel(pmd_dir, virtaddr); if (!pte_dir) { printk("ioremap: no mem for pte_dir\n"); @@ -307,6 +320,8 @@ void kernel_set_cachemode(void *addr, unsigned long size, int cmode) { unsigned long virtaddr = (unsigned long)addr; pgd_t *pgd_dir; + p4d_t *p4d_dir; + pud_t *pud_dir; pmd_t *pmd_dir; pte_t *pte_dir; @@ -341,13 +356,16 @@ void kernel_set_cachemode(void *addr, unsigned long size, int cmode) while ((long)size > 0) { pgd_dir = pgd_offset_k(virtaddr); - if (pgd_bad(*pgd_dir)) { - printk("iocachemode: bad pgd(%08lx)\n", pgd_val(*pgd_dir)); - pgd_clear(pgd_dir); + p4d_dir = p4d_offset(pgd_dir, virtaddr); + pud_dir = pud_offset(p4d_dir, virtaddr); + if (pud_bad(*pud_dir)) { + printk("iocachemode: bad pud(%08lx)\n", pud_val(*pud_dir)); + pud_clear(pud_dir); return; } - pmd_dir = pmd_offset(pgd_dir, virtaddr); + pmd_dir = pmd_offset(pud_dir, virtaddr); +#if CONFIG_PGTABLE_LEVELS == 3 if (CPU_IS_020_OR_030) { int pmd_off = (virtaddr/PTRTREESIZE) & 15; @@ -359,6 +377,7 @@ void kernel_set_cachemode(void *addr, unsigned long size, int cmode) continue; } } +#endif if (pmd_bad(*pmd_dir)) { printk("iocachemode: bad pmd (%08lx)\n", pmd_val(*pmd_dir)); diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c index 6cb1e41d58d0..0ea375607767 100644 --- a/arch/m68k/mm/mcfmmu.c +++ b/arch/m68k/mm/mcfmmu.c @@ -92,6 +92,8 @@ int cf_tlb_miss(struct pt_regs *regs, int write, int dtlb, int extension_word) unsigned long flags, mmuar, mmutr; struct mm_struct *mm; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; int asid; @@ -113,7 +115,19 @@ int cf_tlb_miss(struct pt_regs *regs, int write, int dtlb, int extension_word) return -1; } - pmd = pmd_offset(pgd, mmuar); + p4d = p4d_offset(pgd, mmuar); + if (p4d_none(*p4d)) { + local_irq_restore(flags); + return -1; + } + + pud = pud_offset(p4d, mmuar); + if (pud_none(*pud)) { + local_irq_restore(flags); + return -1; + } + + pmd = pmd_offset(pud, mmuar); if (pmd_none(*pmd)) { local_irq_restore(flags); return -1; diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index 356601bf96d9..4857985b8080 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -82,9 +82,11 @@ static pmd_t * __init kernel_ptr_table(void) */ last = (unsigned long)kernel_pg_dir; for (i = 0; i < PTRS_PER_PGD; i++) { - if (!pgd_present(kernel_pg_dir[i])) + pud_t *pud = (pud_t *)(&kernel_pg_dir[i]); + + if (!pud_present(*pud)) continue; - pmd = __pgd_page(kernel_pg_dir[i]); + pmd = pgd_page_vaddr(kernel_pg_dir[i]); if (pmd > last) last = pmd; } @@ -118,6 +120,8 @@ static void __init map_node(int node) #define ROOTTREESIZE (32*1024*1024) unsigned long physaddr, virtaddr, size; pgd_t *pgd_dir; + p4d_t *p4d_dir; + pud_t *pud_dir; pmd_t *pmd_dir; pte_t *pte_dir; @@ -149,14 +153,16 @@ static void __init map_node(int node) continue; } } - if (!pgd_present(*pgd_dir)) { + p4d_dir = p4d_offset(pgd_dir, virtaddr); + pud_dir = pud_offset(p4d_dir, virtaddr); + if (!pud_present(*pud_dir)) { pmd_dir = kernel_ptr_table(); #ifdef DEBUG printk ("[new pointer %p]", pmd_dir); #endif - pgd_set(pgd_dir, pmd_dir); + pud_set(pud_dir, pmd_dir); } else - pmd_dir = pmd_offset(pgd_dir, virtaddr); + pmd_dir = pmd_offset(pud_dir, virtaddr); if (CPU_IS_020_OR_030) { if (virtaddr) { @@ -304,4 +310,3 @@ void __init paging_init(void) node_set_state(i, N_NORMAL_MEMORY); } } - diff --git a/arch/m68k/sun3x/dvma.c b/arch/m68k/sun3x/dvma.c index 89e630e66555..c4b8aa1d80f4 100644 --- a/arch/m68k/sun3x/dvma.c +++ b/arch/m68k/sun3x/dvma.c @@ -80,6 +80,8 @@ inline int dvma_map_cpu(unsigned long kaddr, unsigned long vaddr, int len) { pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; unsigned long end; int ret = 0; @@ -90,12 +92,14 @@ inline int dvma_map_cpu(unsigned long kaddr, pr_debug("dvma: mapping kern %08lx to virt %08lx\n", kaddr, vaddr); pgd = pgd_offset_k(vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); do { pmd_t *pmd; unsigned long end2; - if((pmd = pmd_alloc(&init_mm, pgd, vaddr)) == NULL) { + if((pmd = pmd_alloc(&init_mm, pud, vaddr)) == NULL) { ret = -ENOMEM; goto out; } @@ -196,4 +200,3 @@ void dvma_unmap_iommu(unsigned long baddr, int len) } } - diff --git a/arch/microblaze/include/asm/page.h b/arch/microblaze/include/asm/page.h index d506bb0893f9..f4b44b24b02e 100644 --- a/arch/microblaze/include/asm/page.h +++ b/arch/microblaze/include/asm/page.h @@ -90,7 +90,6 @@ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long pgprot; } pgprot_t; /* FIXME this can depend on linux kernel version */ # ifdef CONFIG_MMU -typedef struct { unsigned long pmd; } pmd_t; typedef struct { unsigned long pgd; } pgd_t; # else /* CONFIG_MMU */ typedef struct { unsigned long ste[64]; } pmd_t; @@ -103,7 +102,6 @@ typedef struct { p4d_t pge[1]; } pgd_t; # define pgprot_val(x) ((x).pgprot) # ifdef CONFIG_MMU -# define pmd_val(x) ((x).pmd) # define pgd_val(x) ((x).pgd) # else /* CONFIG_MMU */ # define pmd_val(x) ((x).ste[0]) @@ -112,7 +110,6 @@ typedef struct { p4d_t pge[1]; } pgd_t; # endif /* CONFIG_MMU */ # define __pte(x) ((pte_t) { (x) }) -# define __pmd(x) ((pmd_t) { (x) }) # define __pgd(x) ((pgd_t) { (x) }) # define __pgprot(x) ((pgprot_t) { (x) }) diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h index 7ecb05baa601..fcf1e23f2e0a 100644 --- a/arch/microblaze/include/asm/pgalloc.h +++ b/arch/microblaze/include/asm/pgalloc.h @@ -41,13 +41,6 @@ static inline void free_pgd(pgd_t *pgd) #define pmd_pgtable(pmd) pmd_page(pmd) -/* - * We don't have any real pmd's, and this code never triggers because - * the pgd will always be present.. - */ -#define pmd_alloc_one_fast(mm, address) ({ BUG(); ((pmd_t *)1); }) -#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); }) - extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm); #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, (pte)) @@ -58,15 +51,6 @@ extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm); #define pmd_populate_kernel(mm, pmd, pte) \ (pmd_val(*(pmd)) = (unsigned long) (pte)) -/* - * We don't have any real pmd's, and this code never triggers because - * the pgd will always be present.. - */ -#define pmd_alloc_one(mm, address) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(mm, x) do { } while (0) -#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) -#define pgd_populate(mm, pmd, pte) BUG() - #endif /* CONFIG_MMU */ #endif /* _ASM_MICROBLAZE_PGALLOC_H */ diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index 954b69af451f..2def331f9e2c 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -59,9 +59,7 @@ extern int mem_init_done; #else /* CONFIG_MMU */ -#include <asm-generic/4level-fixup.h> - -#define __PAGETABLE_PMD_FOLDED 1 +#include <asm-generic/pgtable-nopmd.h> #ifdef __KERNEL__ #ifndef __ASSEMBLY__ @@ -138,13 +136,8 @@ static inline pte_t pte_mkspecial(pte_t pte) { return pte; } * */ -/* PMD_SHIFT determines the size of the area mapped by the PTE pages */ -#define PMD_SHIFT (PAGE_SHIFT + PTE_SHIFT) -#define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) - /* PGDIR_SHIFT determines what a top-level page table entry can map */ -#define PGDIR_SHIFT PMD_SHIFT +#define PGDIR_SHIFT (PAGE_SHIFT + PTE_SHIFT) #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) @@ -165,9 +158,6 @@ static inline pte_t pte_mkspecial(pte_t pte) { return pte; } #define pte_ERROR(e) \ printk(KERN_ERR "%s:%d: bad pte "PTE_FMT".\n", \ __FILE__, __LINE__, pte_val(e)) -#define pmd_ERROR(e) \ - printk(KERN_ERR "%s:%d: bad pmd %08lx.\n", \ - __FILE__, __LINE__, pmd_val(e)) #define pgd_ERROR(e) \ printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \ __FILE__, __LINE__, pgd_val(e)) @@ -314,18 +304,6 @@ extern unsigned long empty_zero_page[1024]; #ifndef __ASSEMBLY__ /* - * The "pgd_xxx()" functions here are trivial for a folded two-level - * setup: the pgd is never bad, and a pmd always exists (as it's folded - * into the pgd entry) - */ -static inline int pgd_none(pgd_t pgd) { return 0; } -static inline int pgd_bad(pgd_t pgd) { return 0; } -static inline int pgd_present(pgd_t pgd) { return 1; } -#define pgd_clear(xp) do { } while (0) -#define pgd_page(pgd) \ - ((unsigned long) __va(pgd_val(pgd) & PAGE_MASK)) - -/* * The following only work if pte_present() is true. * Undefined behaviour if not.. */ @@ -479,12 +457,6 @@ static inline void ptep_mkdirty(struct mm_struct *mm, #define pgd_index(address) ((address) >> PGDIR_SHIFT) #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pgd_t *dir, unsigned long address) -{ - return (pmd_t *) dir; -} - /* Find an entry in the third-level page table.. */ #define pte_index(address) \ (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c index cdd4feb279c5..c9125c328949 100644 --- a/arch/microblaze/kernel/signal.c +++ b/arch/microblaze/kernel/signal.c @@ -160,6 +160,9 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, int err = 0, sig = ksig->sig; unsigned long address = 0; #ifdef CONFIG_MMU + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; #endif @@ -195,9 +198,10 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, address = ((unsigned long)frame->tramp); #ifdef CONFIG_MMU - pmdp = pmd_offset(pud_offset( - pgd_offset(current->mm, address), - address), address); + pgdp = pgd_offset(current->mm, address); + p4dp = p4d_offset(pgdp, address); + pudp = pud_offset(p4dp, address); + pmdp = pmd_offset(pudp, address); preempt_disable(); ptep = pte_offset_map(pmdp, address); diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c index a015a951c8b7..050fc621c920 100644 --- a/arch/microblaze/mm/init.c +++ b/arch/microblaze/mm/init.c @@ -53,8 +53,11 @@ EXPORT_SYMBOL(kmap_prot); static inline pte_t *virt_to_kpte(unsigned long vaddr) { - return pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), - vaddr), vaddr); + pgd_t *pgd = pgd_offset_k(vaddr); + p4d_t *p4d = p4d_offset(pgd, vaddr); + pud_t *pud = pud_offset(p4d, vaddr); + + return pte_offset_kernel(pmd_offset(pud, vaddr), vaddr); } static void __init highmem_init(void) diff --git a/arch/microblaze/mm/pgtable.c b/arch/microblaze/mm/pgtable.c index 010bb9cee2e4..68c26cacd930 100644 --- a/arch/microblaze/mm/pgtable.c +++ b/arch/microblaze/mm/pgtable.c @@ -134,11 +134,16 @@ EXPORT_SYMBOL(iounmap); int map_page(unsigned long va, phys_addr_t pa, int flags) { + p4d_t *p4d; + pud_t *pud; pmd_t *pd; pte_t *pg; int err = -ENOMEM; + /* Use upper 10 bits of VA to index the first level map */ - pd = pmd_offset(pgd_offset_k(va), va); + p4d = p4d_offset(pgd_offset_k(va), va); + pud = pud_offset(p4d, va); + pd = pmd_offset(pud, va); /* Use middle 10 bits of VA to index the second-level map */ pg = pte_alloc_kernel(pd, va); /* from powerpc - pgtable.c */ /* pg = pte_alloc_kernel(&init_mm, pd, va); */ @@ -188,13 +193,17 @@ void __init mapin_ram(void) static int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep) { pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; int retval = 0; pgd = pgd_offset(mm, addr & PAGE_MASK); if (pgd) { - pmd = pmd_offset(pgd, addr & PAGE_MASK); + p4d = p4d_offset(pgd, addr & PAGE_MASK); + pud = pud_offset(p4d, addr & PAGE_MASK); + pmd = pmd_offset(pud, addr & PAGE_MASK); if (pmd_present(*pmd)) { pte = pte_offset_kernel(pmd, addr & PAGE_MASK); if (pte) { diff --git a/arch/mips/include/uapi/asm/msgbuf.h b/arch/mips/include/uapi/asm/msgbuf.h index 9e0c2e230274..128af72f2dfe 100644 --- a/arch/mips/include/uapi/asm/msgbuf.h +++ b/arch/mips/include/uapi/asm/msgbuf.h @@ -2,6 +2,7 @@ #ifndef _ASM_MSGBUF_H #define _ASM_MSGBUF_H +#include <asm/ipcbuf.h> /* * The msqid64_ds structure for the MIPS architecture. diff --git a/arch/mips/include/uapi/asm/sembuf.h b/arch/mips/include/uapi/asm/sembuf.h index 43e1b4a2f68a..ba7fe0c89e7d 100644 --- a/arch/mips/include/uapi/asm/sembuf.h +++ b/arch/mips/include/uapi/asm/sembuf.h @@ -2,6 +2,8 @@ #ifndef _ASM_SEMBUF_H #define _ASM_SEMBUF_H +#include <asm/ipcbuf.h> + /* * The semid64_ds structure for the MIPS architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/nds32/include/asm/page.h b/arch/nds32/include/asm/page.h index 8feb1fa12f01..86b32014c5f9 100644 --- a/arch/nds32/include/asm/page.h +++ b/arch/nds32/include/asm/page.h @@ -41,17 +41,14 @@ void clear_page(void *page); void copy_page(void *to, void *from); typedef unsigned long pte_t; -typedef unsigned long pmd_t; typedef unsigned long pgd_t; typedef unsigned long pgprot_t; #define pte_val(x) (x) -#define pmd_val(x) (x) #define pgd_val(x) (x) #define pgprot_val(x) (x) #define __pte(x) (x) -#define __pmd(x) (x) #define __pgd(x) (x) #define __pgprot(x) (x) diff --git a/arch/nds32/include/asm/pgalloc.h b/arch/nds32/include/asm/pgalloc.h index 37125e6884d7..85c117347c86 100644 --- a/arch/nds32/include/asm/pgalloc.h +++ b/arch/nds32/include/asm/pgalloc.h @@ -15,9 +15,6 @@ /* * Since we have only two-level page tables, these are trivial */ -#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(mm, pmd) do { } while (0) -#define pgd_populate(mm, pmd, pte) BUG() #define pmd_pgtable(pmd) pmd_page(pmd) extern pgd_t *pgd_alloc(struct mm_struct *mm); diff --git a/arch/nds32/include/asm/pgtable.h b/arch/nds32/include/asm/pgtable.h index 6fbf251cfc26..0214e4150539 100644 --- a/arch/nds32/include/asm/pgtable.h +++ b/arch/nds32/include/asm/pgtable.h @@ -4,8 +4,7 @@ #ifndef _ASMNDS32_PGTABLE_H #define _ASMNDS32_PGTABLE_H -#define __PAGETABLE_PMD_FOLDED 1 -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopmd.h> #include <linux/sizes.h> #include <asm/memory.h> @@ -18,26 +17,20 @@ #ifdef CONFIG_ANDES_PAGE_SIZE_4KB #define PGDIR_SHIFT 22 #define PTRS_PER_PGD 1024 -#define PMD_SHIFT 22 -#define PTRS_PER_PMD 1 #define PTRS_PER_PTE 1024 #endif #ifdef CONFIG_ANDES_PAGE_SIZE_8KB #define PGDIR_SHIFT 24 #define PTRS_PER_PGD 256 -#define PMD_SHIFT 24 -#define PTRS_PER_PMD 1 #define PTRS_PER_PTE 2048 #endif #ifndef __ASSEMBLY__ extern void __pte_error(const char *file, int line, unsigned long val); -extern void __pmd_error(const char *file, int line, unsigned long val); extern void __pgd_error(const char *file, int line, unsigned long val); #define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte)) -#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd)) #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) #endif /* !__ASSEMBLY__ */ @@ -368,9 +361,6 @@ static inline pmd_t __mk_pmd(pte_t * ptep, unsigned long prot) /* to find an entry in a kernel page-table-directory */ #define pgd_offset_k(addr) pgd_offset(&init_mm, addr) -/* Find an entry in the second-level page table.. */ -#define pmd_offset(dir, addr) ((pmd_t *)(dir)) - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const unsigned long mask = 0xfff; diff --git a/arch/nds32/include/asm/tlb.h b/arch/nds32/include/asm/tlb.h index a8aff1c8b4f4..672603804a3b 100644 --- a/arch/nds32/include/asm/tlb.h +++ b/arch/nds32/include/asm/tlb.h @@ -7,6 +7,5 @@ #include <asm-generic/tlb.h> #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) -#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tln)->mm, pmd) #endif diff --git a/arch/nds32/kernel/pm.c b/arch/nds32/kernel/pm.c index ffa8040d8be7..e25700e125d8 100644 --- a/arch/nds32/kernel/pm.c +++ b/arch/nds32/kernel/pm.c @@ -14,6 +14,7 @@ unsigned int *phy_addr_sp_tmp; static void nds32_suspend2ram(void) { pgd_t *pgdv; + p4d_t *p4dv; pud_t *pudv; pmd_t *pmdv; pte_t *ptev; @@ -21,7 +22,8 @@ static void nds32_suspend2ram(void) pgdv = (pgd_t *)__va((__nds32__mfsr(NDS32_SR_L1_PPTB) & L1_PPTB_mskBASE)) + pgd_index((unsigned int)cpu_resume); - pudv = pud_offset(pgdv, (unsigned int)cpu_resume); + p4dv = p4d_offset(pgdv, (unsigned int)cpu_resume); + pudv = pud_offset(p4dv, (unsigned int)cpu_resume); pmdv = pmd_offset(pudv, (unsigned int)cpu_resume); ptev = pte_offset_map(pmdv, (unsigned int)cpu_resume); diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c index 064ae5d2159d..906dfb25353c 100644 --- a/arch/nds32/mm/fault.c +++ b/arch/nds32/mm/fault.c @@ -31,6 +31,8 @@ void show_pte(struct mm_struct *mm, unsigned long addr) pr_alert("[%08lx] *pgd=%08lx", addr, pgd_val(*pgd)); do { + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; if (pgd_none(*pgd)) @@ -41,7 +43,9 @@ void show_pte(struct mm_struct *mm, unsigned long addr) break; } - pmd = pmd_offset(pgd, addr); + p4d = p4d_offset(pgd, addr); + pud = pud_offset(p4d, addr); + pmd = pmd_offset(pud, addr); #if PTRS_PER_PMD != 1 pr_alert(", *pmd=%08lx", pmd_val(*pmd)); #endif @@ -359,6 +363,7 @@ vmalloc_fault: unsigned int index = pgd_index(addr); pgd_t *pgd, *pgd_k; + p4d_t *p4d, *p4d_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; @@ -369,8 +374,13 @@ vmalloc_fault: if (!pgd_present(*pgd_k)) goto no_context; - pud = pud_offset(pgd, addr); - pud_k = pud_offset(pgd_k, addr); + p4d = p4d_offset(pgd, addr); + p4d_k = p4d_offset(pgd_k, addr); + if (!p4d_present(*p4d_k)) + goto no_context; + + pud = pud_offset(p4d, addr); + pud_k = pud_offset(p4d_k, addr); if (!pud_present(*pud_k)) goto no_context; diff --git a/arch/nds32/mm/init.c b/arch/nds32/mm/init.c index 55703b03d172..0be3833f6814 100644 --- a/arch/nds32/mm/init.c +++ b/arch/nds32/mm/init.c @@ -54,6 +54,7 @@ static void __init map_ram(void) { unsigned long v, p, e; pgd_t *pge; + p4d_t *p4e; pud_t *pue; pmd_t *pme; pte_t *pte; @@ -69,7 +70,8 @@ static void __init map_ram(void) while (p < e) { int j; - pue = pud_offset(pge, v); + p4e = p4d_offset(pge, v); + pue = pud_offset(p4e, v); pme = pmd_offset(pue, v); if ((u32) pue != (u32) pge || (u32) pme != (u32) pge) { @@ -100,6 +102,7 @@ static void __init fixedrange_init(void) { unsigned long vaddr; pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; #ifdef CONFIG_HIGHMEM @@ -111,7 +114,8 @@ static void __init fixedrange_init(void) */ vaddr = __fix_to_virt(__end_of_fixed_addresses - 1); pgd = swapper_pg_dir + pgd_index(vaddr); - pud = pud_offset(pgd, vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); pmd = pmd_offset(pud, vaddr); fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!fixmap_pmd_p) @@ -126,7 +130,8 @@ static void __init fixedrange_init(void) vaddr = PKMAP_BASE; pgd = swapper_pg_dir + pgd_index(vaddr); - pud = pud_offset(pgd, vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); pmd = pmd_offset(pud, vaddr); pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!pte) diff --git a/arch/nds32/mm/mm-nds32.c b/arch/nds32/mm/mm-nds32.c index 3b43798d754f..8503bee882d1 100644 --- a/arch/nds32/mm/mm-nds32.c +++ b/arch/nds32/mm/mm-nds32.c @@ -74,6 +74,8 @@ void setup_mm_for_reboot(char mode) { unsigned long pmdval; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; int i; @@ -84,7 +86,9 @@ void setup_mm_for_reboot(char mode) for (i = 0; i < USER_PTRS_PER_PGD; i++) { pmdval = (i << PGDIR_SHIFT); - pmd = pmd_offset(pgd + i, i << PGDIR_SHIFT); + p4d = p4d_offset(pgd, i << PGDIR_SHIFT); + pud = pud_offset(p4d, i << PGDIR_SHIFT); + pmd = pmd_offset(pud + i, i << PGDIR_SHIFT); set_pmd(pmd, __pmd(pmdval)); } } diff --git a/arch/nds32/mm/proc.c b/arch/nds32/mm/proc.c index ba80992d13a2..837ae7728830 100644 --- a/arch/nds32/mm/proc.c +++ b/arch/nds32/mm/proc.c @@ -16,10 +16,14 @@ extern struct cache_info L1_cache_info[2]; int va_kernel_present(unsigned long addr) { + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *ptep, pte; - pmd = pmd_offset(pgd_offset_k(addr), addr); + p4d = p4d_offset(pgd_offset_k(addr), addr); + pud = pud_offset(p4d, addr); + pmd = pmd_offset(pud, addr); if (!pmd_none(*pmd)) { ptep = pte_offset_map(pmd, addr); pte = *ptep; @@ -32,20 +36,24 @@ int va_kernel_present(unsigned long addr) pte_t va_present(struct mm_struct * mm, unsigned long addr) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *ptep, pte; pgd = pgd_offset(mm, addr); if (!pgd_none(*pgd)) { - pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) { - ptep = pte_offset_map(pmd, addr); - pte = *ptep; - if (pte_present(pte)) - return pte; + p4d = p4d_offset(pgd, addr); + if (!p4d_none(*p4d)) { + pud = pud_offset(p4d, addr); + if (!pud_none(*pud)) { + pmd = pmd_offset(pud, addr); + if (!pmd_none(*pmd)) { + ptep = pte_offset_map(pmd, addr); + pte = *ptep; + if (pte_present(pte)) + return pte; + } } } } diff --git a/arch/parisc/include/asm/page.h b/arch/parisc/include/asm/page.h index 93caf17ac5e2..796ae29e9b9a 100644 --- a/arch/parisc/include/asm/page.h +++ b/arch/parisc/include/asm/page.h @@ -42,48 +42,54 @@ typedef struct { unsigned long pte; } pte_t; /* either 32 or 64bit */ /* NOTE: even on 64 bits, these entries are __u32 because we allocate * the pmd and pgd in ZONE_DMA (i.e. under 4GB) */ -typedef struct { __u32 pmd; } pmd_t; typedef struct { __u32 pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; -#define pte_val(x) ((x).pte) -/* These do not work lvalues, so make sure we don't use them as such. */ +#if CONFIG_PGTABLE_LEVELS == 3 +typedef struct { __u32 pmd; } pmd_t; +#define __pmd(x) ((pmd_t) { (x) } ) +/* pXd_val() do not work as lvalues, so make sure we don't use them as such. */ #define pmd_val(x) ((x).pmd + 0) +#endif + +#define pte_val(x) ((x).pte) #define pgd_val(x) ((x).pgd + 0) #define pgprot_val(x) ((x).pgprot) #define __pte(x) ((pte_t) { (x) } ) -#define __pmd(x) ((pmd_t) { (x) } ) #define __pgd(x) ((pgd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) -#define __pmd_val_set(x,n) (x).pmd = (n) -#define __pgd_val_set(x,n) (x).pgd = (n) - #else /* * .. while these make it easier on the compiler */ typedef unsigned long pte_t; + +#if CONFIG_PGTABLE_LEVELS == 3 typedef __u32 pmd_t; +#define pmd_val(x) (x) +#define __pmd(x) (x) +#endif + typedef __u32 pgd_t; typedef unsigned long pgprot_t; #define pte_val(x) (x) -#define pmd_val(x) (x) #define pgd_val(x) (x) #define pgprot_val(x) (x) #define __pte(x) (x) -#define __pmd(x) (x) #define __pgd(x) (x) #define __pgprot(x) (x) -#define __pmd_val_set(x,n) (x) = (n) -#define __pgd_val_set(x,n) (x) = (n) - #endif /* STRICT_MM_TYPECHECKS */ +#define set_pmd(pmdptr, pmdval) (*(pmdptr) = (pmdval)) +#if CONFIG_PGTABLE_LEVELS == 3 +#define set_pud(pudptr, pudval) (*(pudptr) = (pudval)) +#endif + typedef struct page *pgtable_t; typedef struct __physmem_range { diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h index d98647c29b74..9ac74da256b8 100644 --- a/arch/parisc/include/asm/pgalloc.h +++ b/arch/parisc/include/asm/pgalloc.h @@ -34,13 +34,13 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) /* Populate first pmd with allocated memory. We mark it * with PxD_FLAG_ATTACHED as a signal to the system that this * pmd entry may not be cleared. */ - __pgd_val_set(*actual_pgd, (PxD_FLAG_PRESENT | - PxD_FLAG_VALID | - PxD_FLAG_ATTACHED) - + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); + set_pgd(actual_pgd, __pgd((PxD_FLAG_PRESENT | + PxD_FLAG_VALID | + PxD_FLAG_ATTACHED) + + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT))); /* The first pmd entry also is marked with PxD_FLAG_ATTACHED as * a signal that this pmd may not be freed */ - __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); + set_pgd(pgd, __pgd(PxD_FLAG_ATTACHED)); #endif } spin_lock_init(pgd_spinlock(actual_pgd)); @@ -59,10 +59,10 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) /* Three Level Page Table Support for pmd's */ -static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { - __pgd_val_set(*pgd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) + - (__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT)); + set_pud(pud, __pud((PxD_FLAG_PRESENT | PxD_FLAG_VALID) + + (__u32)(__pa((unsigned long)pmd) >> PxD_VALUE_SHIFT))); } static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) @@ -88,19 +88,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) free_pages((unsigned long)pmd, PMD_ORDER); } -#else - -/* Two Level Page Table Support for pmd's */ - -/* - * allocating and freeing a pmd is trivial: the 1-entry pmd is - * inside the pgd, so has no extra memory associated with it. - */ - -#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(mm, x) do { } while (0) -#define pgd_populate(mm, pmd, pte) BUG() - #endif static inline void @@ -110,14 +97,14 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) /* preserve the gateway marker if this is the beginning of * the permanent pmd */ if(pmd_flag(*pmd) & PxD_FLAG_ATTACHED) - __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | - PxD_FLAG_VALID | - PxD_FLAG_ATTACHED) - + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); + set_pmd(pmd, __pmd((PxD_FLAG_PRESENT | + PxD_FLAG_VALID | + PxD_FLAG_ATTACHED) + + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT))); else #endif - __pmd_val_set(*pmd, (PxD_FLAG_PRESENT | PxD_FLAG_VALID) - + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT)); + set_pmd(pmd, __pmd((PxD_FLAG_PRESENT | PxD_FLAG_VALID) + + (__u32)(__pa((unsigned long)pte) >> PxD_VALUE_SHIFT))); } #define pmd_populate(mm, pmd, pte_page) \ diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 4ac374b3a99f..f0a365950536 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -3,7 +3,12 @@ #define _PARISC_PGTABLE_H #include <asm/page.h> -#include <asm-generic/4level-fixup.h> + +#if CONFIG_PGTABLE_LEVELS == 3 +#include <asm-generic/pgtable-nopud.h> +#elif CONFIG_PGTABLE_LEVELS == 2 +#include <asm-generic/pgtable-nopmd.h> +#endif #include <asm/fixmap.h> @@ -101,8 +106,10 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) #define pte_ERROR(e) \ printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) +#if CONFIG_PGTABLE_LEVELS == 3 #define pmd_ERROR(e) \ printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, (unsigned long)pmd_val(e)) +#endif #define pgd_ERROR(e) \ printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) @@ -132,19 +139,18 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) #define PTRS_PER_PTE (1UL << BITS_PER_PTE) /* Definitions for 2nd level */ +#if CONFIG_PGTABLE_LEVELS == 3 #define PMD_SHIFT (PLD_SHIFT + BITS_PER_PTE) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -#if CONFIG_PGTABLE_LEVELS == 3 #define BITS_PER_PMD (PAGE_SHIFT + PMD_ORDER - BITS_PER_PMD_ENTRY) +#define PTRS_PER_PMD (1UL << BITS_PER_PMD) #else -#define __PAGETABLE_PMD_FOLDED 1 #define BITS_PER_PMD 0 #endif -#define PTRS_PER_PMD (1UL << BITS_PER_PMD) /* Definitions for 1st level */ -#define PGDIR_SHIFT (PMD_SHIFT + BITS_PER_PMD) +#define PGDIR_SHIFT (PLD_SHIFT + BITS_PER_PTE + BITS_PER_PMD) #if (PGDIR_SHIFT + PAGE_SHIFT + PGD_ORDER - BITS_PER_PGD_ENTRY) > BITS_PER_LONG #define BITS_PER_PGD (BITS_PER_LONG - PGDIR_SHIFT) #else @@ -317,6 +323,8 @@ extern unsigned long *empty_zero_page; #define pmd_flag(x) (pmd_val(x) & PxD_FLAG_MASK) #define pmd_address(x) ((unsigned long)(pmd_val(x) &~ PxD_FLAG_MASK) << PxD_VALUE_SHIFT) +#define pud_flag(x) (pud_val(x) & PxD_FLAG_MASK) +#define pud_address(x) ((unsigned long)(pud_val(x) &~ PxD_FLAG_MASK) << PxD_VALUE_SHIFT) #define pgd_flag(x) (pgd_val(x) & PxD_FLAG_MASK) #define pgd_address(x) ((unsigned long)(pgd_val(x) &~ PxD_FLAG_MASK) << PxD_VALUE_SHIFT) @@ -334,42 +342,32 @@ static inline void pmd_clear(pmd_t *pmd) { if (pmd_flag(*pmd) & PxD_FLAG_ATTACHED) /* This is the entry pointing to the permanent pmd * attached to the pgd; cannot clear it */ - __pmd_val_set(*pmd, PxD_FLAG_ATTACHED); + set_pmd(pmd, __pmd(PxD_FLAG_ATTACHED)); else #endif - __pmd_val_set(*pmd, 0); + set_pmd(pmd, __pmd(0)); } #if CONFIG_PGTABLE_LEVELS == 3 -#define pgd_page_vaddr(pgd) ((unsigned long) __va(pgd_address(pgd))) -#define pgd_page(pgd) virt_to_page((void *)pgd_page_vaddr(pgd)) +#define pud_page_vaddr(pud) ((unsigned long) __va(pud_address(pud))) +#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) /* For 64 bit we have three level tables */ -#define pgd_none(x) (!pgd_val(x)) -#define pgd_bad(x) (!(pgd_flag(x) & PxD_FLAG_VALID)) -#define pgd_present(x) (pgd_flag(x) & PxD_FLAG_PRESENT) -static inline void pgd_clear(pgd_t *pgd) { +#define pud_none(x) (!pud_val(x)) +#define pud_bad(x) (!(pud_flag(x) & PxD_FLAG_VALID)) +#define pud_present(x) (pud_flag(x) & PxD_FLAG_PRESENT) +static inline void pud_clear(pud_t *pud) { #if CONFIG_PGTABLE_LEVELS == 3 - if(pgd_flag(*pgd) & PxD_FLAG_ATTACHED) - /* This is the permanent pmd attached to the pgd; cannot + if(pud_flag(*pud) & PxD_FLAG_ATTACHED) + /* This is the permanent pmd attached to the pud; cannot * free it */ return; #endif - __pgd_val_set(*pgd, 0); + set_pud(pud, __pud(0)); } -#else -/* - * The "pgd_xxx()" functions here are trivial for a folded two-level - * setup: the pgd is never bad, and a pmd always exists (as it's folded - * into the pgd entry) - */ -static inline int pgd_none(pgd_t pgd) { return 0; } -static inline int pgd_bad(pgd_t pgd) { return 0; } -static inline int pgd_present(pgd_t pgd) { return 1; } -static inline void pgd_clear(pgd_t * pgdp) { } #endif /* @@ -452,7 +450,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #if CONFIG_PGTABLE_LEVELS == 3 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) #define pmd_offset(dir,address) \ -((pmd_t *) pgd_page_vaddr(*(dir)) + pmd_index(address)) +((pmd_t *) pud_page_vaddr(*(dir)) + pmd_index(address)) #else #define pmd_offset(dir,addr) ((pmd_t *) dir) #endif diff --git a/arch/parisc/include/asm/tlb.h b/arch/parisc/include/asm/tlb.h index 8c0446b04c9e..44235f367674 100644 --- a/arch/parisc/include/asm/tlb.h +++ b/arch/parisc/include/asm/tlb.h @@ -4,7 +4,9 @@ #include <asm-generic/tlb.h> +#if CONFIG_PGTABLE_LEVELS == 3 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#endif #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #endif diff --git a/arch/parisc/include/uapi/asm/msgbuf.h b/arch/parisc/include/uapi/asm/msgbuf.h index 3b877335da38..3b4de5b668c3 100644 --- a/arch/parisc/include/uapi/asm/msgbuf.h +++ b/arch/parisc/include/uapi/asm/msgbuf.h @@ -3,6 +3,7 @@ #define _PARISC_MSGBUF_H #include <asm/bitsperlong.h> +#include <asm/ipcbuf.h> /* * The msqid64_ds structure for parisc architecture, copied from sparc. diff --git a/arch/parisc/include/uapi/asm/sembuf.h b/arch/parisc/include/uapi/asm/sembuf.h index 8241cf126018..e2ca4301c7fb 100644 --- a/arch/parisc/include/uapi/asm/sembuf.h +++ b/arch/parisc/include/uapi/asm/sembuf.h @@ -3,6 +3,7 @@ #define _PARISC_SEMBUF_H #include <asm/bitsperlong.h> +#include <asm/ipcbuf.h> /* * The semid64_ds structure for parisc architecture. diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 2407b0b789d3..1eedfecc5137 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -534,11 +534,14 @@ static inline pte_t *get_ptep(pgd_t *pgd, unsigned long addr) pte_t *ptep = NULL; if (!pgd_none(*pgd)) { - pud_t *pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd_t *pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) - ptep = pte_offset_map(pmd, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + if (!p4d_none(*p4d)) { + pud_t *pud = pud_offset(p4d, addr); + if (!pud_none(*pud)) { + pmd_t *pmd = pmd_offset(pud, addr); + if (!pmd_none(*pmd)) + ptep = pte_offset_map(pmd, addr); + } } } return ptep; diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index a60d47fd4d55..0f1b460ee715 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -133,9 +133,14 @@ static inline int map_uncached_pages(unsigned long vaddr, unsigned long size, dir = pgd_offset_k(vaddr); do { + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; - - pmd = pmd_alloc(NULL, dir, vaddr); + + p4d = p4d_offset(dir, vaddr); + pud = pud_offset(p4d, vaddr); + pmd = pmd_alloc(NULL, pud, vaddr); + if (!pmd) return -ENOMEM; if (map_pmd_uncached(pmd, vaddr, end - vaddr, &paddr)) diff --git a/arch/parisc/mm/fixmap.c b/arch/parisc/mm/fixmap.c index 474cd241c150..e2d8b0a857ee 100644 --- a/arch/parisc/mm/fixmap.c +++ b/arch/parisc/mm/fixmap.c @@ -14,11 +14,13 @@ void notrace set_fixmap(enum fixed_addresses idx, phys_addr_t phys) { unsigned long vaddr = __fix_to_virt(idx); pgd_t *pgd = pgd_offset_k(vaddr); - pmd_t *pmd = pmd_offset(pgd, vaddr); + p4d_t *p4d = p4d_offset(pgd, vaddr); + pud_t *pud = pud_offset(p4d, vaddr); + pmd_t *pmd = pmd_offset(pud, vaddr); pte_t *pte; if (pmd_none(*pmd)) - pmd = pmd_alloc(NULL, pgd, vaddr); + pmd = pmd_alloc(NULL, pud, vaddr); pte = pte_offset_kernel(pmd, vaddr); if (pte_none(*pte)) @@ -32,7 +34,9 @@ void notrace clear_fixmap(enum fixed_addresses idx) { unsigned long vaddr = __fix_to_virt(idx); pgd_t *pgd = pgd_offset_k(vaddr); - pmd_t *pmd = pmd_offset(pgd, vaddr); + p4d_t *p4d = p4d_offset(pgd, vaddr); + pud_t *pud = pud_offset(p4d, vaddr); + pmd_t *pmd = pmd_offset(pud, vaddr); pte_t *pte = pte_offset_kernel(pmd, vaddr); if (WARN_ON(pte_none(*pte))) diff --git a/arch/parisc/mm/hugetlbpage.c b/arch/parisc/mm/hugetlbpage.c index d578809e55cf..0e1e212f1c96 100644 --- a/arch/parisc/mm/hugetlbpage.c +++ b/arch/parisc/mm/hugetlbpage.c @@ -49,6 +49,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; @@ -61,7 +62,8 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); - pud = pud_alloc(mm, pgd, addr); + p4d = p4d_offset(pgd, addr); + pud = pud_alloc(mm, p4d, addr); if (pud) { pmd = pmd_alloc(mm, pud, addr); if (pmd) @@ -74,6 +76,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; @@ -82,11 +85,14 @@ pte_t *huge_pte_offset(struct mm_struct *mm, pgd = pgd_offset(mm, addr); if (!pgd_none(*pgd)) { - pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) - pte = pte_offset_map(pmd, addr); + p4d = p4d_offset(pgd, addr); + if (!p4d_none(*p4d)) { + pud = pud_offset(p4d, addr); + if (!pud_none(*pud)) { + pmd = pmd_offset(pud, addr); + if (!pmd_none(*pmd)) + pte = pte_offset_map(pmd, addr); + } } } return pte; diff --git a/arch/powerpc/include/uapi/asm/msgbuf.h b/arch/powerpc/include/uapi/asm/msgbuf.h index 969bd83e4d3d..7919b2ba41b5 100644 --- a/arch/powerpc/include/uapi/asm/msgbuf.h +++ b/arch/powerpc/include/uapi/asm/msgbuf.h @@ -2,6 +2,8 @@ #ifndef _ASM_POWERPC_MSGBUF_H #define _ASM_POWERPC_MSGBUF_H +#include <asm/ipcbuf.h> + /* * The msqid64_ds structure for the PowerPC architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/powerpc/include/uapi/asm/sembuf.h b/arch/powerpc/include/uapi/asm/sembuf.h index 008ae77c6746..85e96ccb5f0f 100644 --- a/arch/powerpc/include/uapi/asm/sembuf.h +++ b/arch/powerpc/include/uapi/asm/sembuf.h @@ -2,6 +2,8 @@ #ifndef _ASM_POWERPC_SEMBUF_H #define _ASM_POWERPC_SEMBUF_H +#include <asm/ipcbuf.h> + /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License diff --git a/arch/s390/include/uapi/asm/ipcbuf.h b/arch/s390/include/uapi/asm/ipcbuf.h index 5b1c4f47c656..1030cd186899 100644 --- a/arch/s390/include/uapi/asm/ipcbuf.h +++ b/arch/s390/include/uapi/asm/ipcbuf.h @@ -2,6 +2,8 @@ #ifndef __S390_IPCBUF_H__ #define __S390_IPCBUF_H__ +#include <linux/posix_types.h> + /* * The user_ipc_perm structure for S/390 architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index 10538a4d1a1e..eae0c92ec422 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -26,14 +26,14 @@ static inline void free_pgd_fast(pgd_t *pgd) #define pgd_free(mm, pgd) free_pgd_fast(pgd) #define pgd_alloc(mm) get_pgd_fast() -static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) +static inline void pud_set(pud_t * pudp, pmd_t * pmdp) { unsigned long pa = __nocache_pa(pmdp); - set_pte((pte_t *)pgdp, __pte((SRMMU_ET_PTD | (pa >> 4)))); + set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4)))); } -#define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD) +#define pud_populate(MM, PGD, PMD) pud_set(PGD, PMD) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index 31da44826645..6d6f44c0cad9 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -12,7 +12,7 @@ #include <linux/const.h> #ifndef __ASSEMBLY__ -#include <asm-generic/4level-fixup.h> +#include <asm-generic/pgtable-nopud.h> #include <linux/spinlock.h> #include <linux/mm_types.h> @@ -132,12 +132,12 @@ static inline struct page *pmd_page(pmd_t pmd) return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); } -static inline unsigned long pgd_page_vaddr(pgd_t pgd) +static inline unsigned long pud_page_vaddr(pud_t pud) { - if (srmmu_device_memory(pgd_val(pgd))) { + if (srmmu_device_memory(pud_val(pud))) { return ~0; } else { - unsigned long v = pgd_val(pgd) & SRMMU_PTD_PMASK; + unsigned long v = pud_val(pud) & SRMMU_PTD_PMASK; return (unsigned long)__nocache_va(v << 4); } } @@ -184,24 +184,24 @@ static inline void pmd_clear(pmd_t *pmdp) set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); } -static inline int pgd_none(pgd_t pgd) +static inline int pud_none(pud_t pud) { - return !(pgd_val(pgd) & 0xFFFFFFF); + return !(pud_val(pud) & 0xFFFFFFF); } -static inline int pgd_bad(pgd_t pgd) +static inline int pud_bad(pud_t pud) { - return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; + return (pud_val(pud) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } -static inline int pgd_present(pgd_t pgd) +static inline int pud_present(pud_t pud) { - return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); + return ((pud_val(pud) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } -static inline void pgd_clear(pgd_t *pgdp) +static inline void pud_clear(pud_t *pudp) { - set_pte((pte_t *)pgdp, __pte(0)); + set_pte((pte_t *)pudp, __pte(0)); } /* @@ -319,9 +319,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pgd_offset_k(address) pgd_offset(&init_mm, address) /* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pgd_t * dir, unsigned long address) +static inline pmd_t *pmd_offset(pud_t * dir, unsigned long address) { - return (pmd_t *) pgd_page_vaddr(*dir) + + return (pmd_t *) pud_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); } diff --git a/arch/sparc/include/uapi/asm/ipcbuf.h b/arch/sparc/include/uapi/asm/ipcbuf.h index 9d0d125500e2..5b933a598a33 100644 --- a/arch/sparc/include/uapi/asm/ipcbuf.h +++ b/arch/sparc/include/uapi/asm/ipcbuf.h @@ -2,6 +2,8 @@ #ifndef __SPARC_IPCBUF_H #define __SPARC_IPCBUF_H +#include <linux/posix_types.h> + /* * The ipc64_perm structure for sparc/sparc64 architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/sparc/include/uapi/asm/msgbuf.h b/arch/sparc/include/uapi/asm/msgbuf.h index eeeb91933280..0954552da188 100644 --- a/arch/sparc/include/uapi/asm/msgbuf.h +++ b/arch/sparc/include/uapi/asm/msgbuf.h @@ -2,6 +2,8 @@ #ifndef _SPARC_MSGBUF_H #define _SPARC_MSGBUF_H +#include <asm/ipcbuf.h> + /* * The msqid64_ds structure for sparc64 architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/sparc/include/uapi/asm/sembuf.h b/arch/sparc/include/uapi/asm/sembuf.h index cbcbaa4e7128..10621d066d79 100644 --- a/arch/sparc/include/uapi/asm/sembuf.h +++ b/arch/sparc/include/uapi/asm/sembuf.h @@ -2,6 +2,8 @@ #ifndef _SPARC_SEMBUF_H #define _SPARC_SEMBUF_H +#include <asm/ipcbuf.h> + /* * The semid64_ds structure for sparc architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 8d69de111470..89976c9b936c 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -351,6 +351,8 @@ vmalloc_fault: */ int offset = pgd_index(address); pgd_t *pgd, *pgd_k; + p4d_t *p4d, *p4d_k; + pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pgd = tsk->active_mm->pgd + offset; @@ -363,8 +365,13 @@ vmalloc_fault: return; } - pmd = pmd_offset(pgd, address); - pmd_k = pmd_offset(pgd_k, address); + p4d = p4d_offset(pgd, address); + pud = pud_offset(p4d, address); + pmd = pmd_offset(pud, address); + + p4d_k = p4d_offset(pgd_k, address); + pud_k = pud_offset(p4d_k, address); + pmd_k = pmd_offset(pud_k, address); if (pmd_present(*pmd) || !pmd_present(*pmd_k)) goto bad_area_nosemaphore; diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 86bc2a58d26c..d4a80adea7e5 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -39,10 +39,14 @@ static pte_t *kmap_pte; void __init kmap_init(void) { unsigned long address; + p4d_t *p4d; + pud_t *pud; pmd_t *dir; address = __fix_to_virt(FIX_KMAP_BEGIN); - dir = pmd_offset(pgd_offset_k(address), address); + p4d = p4d_offset(pgd_offset_k(address), address); + pud = pud_offset(p4d, address); + dir = pmd_offset(pud, address); /* cache the first kmap pte */ kmap_pte = pte_offset_kernel(dir, address); diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index f770ee7229d8..33a0facd9eb5 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -239,12 +239,16 @@ static void *iounit_alloc(struct device *dev, size_t len, page = va; { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; long i; pgdp = pgd_offset(&init_mm, addr); - pmdp = pmd_offset(pgdp, addr); + p4dp = p4d_offset(pgdp, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_map(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index 71ac353032b6..4d3c6991f0ae 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -343,6 +343,8 @@ static void *sbus_iommu_alloc(struct device *dev, size_t len, page = va; { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; @@ -354,7 +356,9 @@ static void *sbus_iommu_alloc(struct device *dev, size_t len, __flush_page_to_ram(page); pgdp = pgd_offset(&init_mm, addr); - pmdp = pmd_offset(pgdp, addr); + p4dp = p4d_offset(pgdp, addr); + pudp = pud_offset(p4dp, addr); + pmdp = pmd_offset(pudp, addr); ptep = pte_offset_map(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index cc3ad64479ac..f56c3c9a9793 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -296,6 +296,8 @@ static void __init srmmu_nocache_init(void) void *srmmu_nocache_bitmap; unsigned int bitmap_bits; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long paddr, vaddr; @@ -329,6 +331,8 @@ static void __init srmmu_nocache_init(void) while (vaddr < srmmu_nocache_end) { pgd = pgd_offset_k(vaddr); + p4d = p4d_offset(__nocache_fix(pgd), vaddr); + pud = pud_offset(__nocache_fix(p4d), vaddr); pmd = pmd_offset(__nocache_fix(pgd), vaddr); pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); @@ -516,13 +520,17 @@ static inline void srmmu_mapioaddr(unsigned long physaddr, unsigned long virt_addr, int bus_type) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long tmp; physaddr &= PAGE_MASK; pgdp = pgd_offset_k(virt_addr); - pmdp = pmd_offset(pgdp, virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); ptep = pte_offset_kernel(pmdp, virt_addr); tmp = (physaddr >> 4) | SRMMU_ET_PTE; @@ -551,11 +559,16 @@ void srmmu_mapiorange(unsigned int bus, unsigned long xpa, static inline void srmmu_unmapioaddr(unsigned long virt_addr) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; + pgdp = pgd_offset_k(virt_addr); - pmdp = pmd_offset(pgdp, virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); ptep = pte_offset_kernel(pmdp, virt_addr); /* No need to flush uncacheable page. */ @@ -693,20 +706,24 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; while (start < end) { pgdp = pgd_offset_k(start); - if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*(pud_t *)__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(__nocache_fix(pgdp), pmdp); + pud_set(__nocache_fix(pudp), pmdp); } - pmdp = pmd_offset(__nocache_fix(pgdp), start); + pmdp = pmd_offset(__nocache_fix(pudp), start); if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) @@ -724,19 +741,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, unsigned long end) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; while (start < end) { pgdp = pgd_offset_k(start); - if (pgd_none(*pgdp)) { + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*pudp)) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(pgdp, pmdp); + pud_set((pud_t *)pgdp, pmdp); } - pmdp = pmd_offset(pgdp, start); + pmdp = pmd_offset(pudp, start); if (srmmu_pmd_none(*pmdp)) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); @@ -779,6 +800,8 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, unsigned long probed; unsigned long addr; pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ @@ -810,18 +833,20 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, } pgdp = pgd_offset_k(start); + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); if (what == 2) { *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); start += SRMMU_PGDIR_SIZE; continue; } - if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + if (pud_none(*(pud_t *)__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(__nocache_fix(pgdp), pmdp); + pud_set(__nocache_fix(pudp), pmdp); } pmdp = pmd_offset(__nocache_fix(pgdp), start); if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { @@ -906,6 +931,8 @@ void __init srmmu_paging_init(void) phandle cpunode; char node_str[128]; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long pages_avail; @@ -967,7 +994,9 @@ void __init srmmu_paging_init(void) srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); pgd = pgd_offset_k(PKMAP_BASE); - pmd = pmd_offset(pgd, PKMAP_BASE); + p4d = p4d_offset(pgd, PKMAP_BASE); + pud = pud_offset(p4d, PKMAP_BASE); + pmd = pmd_offset(pud, PKMAP_BASE); pte = pte_offset_kernel(pmd, PKMAP_BASE); pkmap_page_table = pte; diff --git a/arch/um/include/asm/pgtable-2level.h b/arch/um/include/asm/pgtable-2level.h index 32b3d26a7109..32106d31e4ab 100644 --- a/arch/um/include/asm/pgtable-2level.h +++ b/arch/um/include/asm/pgtable-2level.h @@ -8,7 +8,6 @@ #ifndef __UM_PGTABLE_2LEVEL_H #define __UM_PGTABLE_2LEVEL_H -#define __ARCH_USE_5LEVEL_HACK #include <asm-generic/pgtable-nopmd.h> /* PGDIR_SHIFT determines what a third-level page table entry can map */ diff --git a/arch/um/include/asm/pgtable-3level.h b/arch/um/include/asm/pgtable-3level.h index 9812269fefc9..8a3b689e0f86 100644 --- a/arch/um/include/asm/pgtable-3level.h +++ b/arch/um/include/asm/pgtable-3level.h @@ -7,7 +7,6 @@ #ifndef __UM_PGTABLE_3LEVEL_H #define __UM_PGTABLE_3LEVEL_H -#define __ARCH_USE_5LEVEL_HACK #include <asm-generic/pgtable-nopud.h> /* PGDIR_SHIFT determines what a third-level page table entry can map */ diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h index 36a44d58f373..2daa58df2190 100644 --- a/arch/um/include/asm/pgtable.h +++ b/arch/um/include/asm/pgtable.h @@ -106,6 +106,9 @@ extern unsigned long end_iomem; #define pud_newpage(x) (pud_val(x) & _PAGE_NEWPAGE) #define pud_mkuptodate(x) (pud_val(x) &= ~_PAGE_NEWPAGE) +#define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE) +#define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE) + #define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK) #define pte_page(x) pfn_to_page(pte_pfn(x)) diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 417ff647fb37..30885d0b94ac 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -96,6 +96,7 @@ static void __init fixrange_init(unsigned long start, unsigned long end, pgd_t *pgd_base) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; int i, j; @@ -107,7 +108,8 @@ static void __init fixrange_init(unsigned long start, unsigned long end, pgd = pgd_base + i; for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { - pud = pud_offset(pgd, vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); if (pud_none(*pud)) one_md_table_init(pud); pmd = pmd_offset(pud, vaddr); @@ -124,6 +126,7 @@ static void __init fixaddr_user_init( void) #ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA long size = FIXADDR_USER_END - FIXADDR_USER_START; pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -144,7 +147,8 @@ static void __init fixaddr_user_init( void) for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE, p += PAGE_SIZE) { pgd = swapper_pg_dir + pgd_index(vaddr); - pud = pud_offset(pgd, vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); pmd = pmd_offset(pud, vaddr); pte = pte_offset_kernel(pmd, vaddr); pte_set_val(*pte, p, PAGE_READONLY); diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c index b5e3d91fc9c2..3f0d9a573fd6 100644 --- a/arch/um/kernel/skas/mmu.c +++ b/arch/um/kernel/skas/mmu.c @@ -19,15 +19,21 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, unsigned long kernel) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; pgd = pgd_offset(mm, proc); - pud = pud_alloc(mm, pgd, proc); - if (!pud) + + p4d = p4d_alloc(mm, pgd, proc); + if (!p4d) goto out; + pud = pud_alloc(mm, p4d, proc); + if (!pud) + goto out_pud; + pmd = pmd_alloc(mm, pud, proc); if (!pmd) goto out_pmd; @@ -44,6 +50,8 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, pmd_free(mm, pmd); out_pmd: pud_free(mm, pud); + out_pud: + p4d_free(mm, p4d); out: return -ENOMEM; } diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index 3236052f20e6..d617f8dc9c19 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -17,6 +17,7 @@ pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; @@ -27,7 +28,11 @@ pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr) if (!pgd_present(*pgd)) return NULL; - pud = pud_offset(pgd, addr); + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) + return NULL; + + pud = pud_offset(p4d, addr); if (!pud_present(*pud)) return NULL; diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index b7eaf655635c..80a358c6d652 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -277,7 +277,7 @@ static inline int update_pmd_range(pud_t *pud, unsigned long addr, return ret; } -static inline int update_pud_range(pgd_t *pgd, unsigned long addr, +static inline int update_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, struct host_vm_change *hvc) { @@ -285,7 +285,7 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr, unsigned long next; int ret = 0; - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); if (!pud_present(*pud)) { @@ -299,6 +299,28 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr, return ret; } +static inline int update_p4d_range(pgd_t *pgd, unsigned long addr, + unsigned long end, + struct host_vm_change *hvc) +{ + p4d_t *p4d; + unsigned long next; + int ret = 0; + + p4d = p4d_offset(pgd, addr); + do { + next = p4d_addr_end(addr, end); + if (!p4d_present(*p4d)) { + if (hvc->force || p4d_newpage(*p4d)) { + ret = add_munmap(addr, next - addr, hvc); + p4d_mkuptodate(*p4d); + } + } else + ret = update_pud_range(p4d, addr, next, hvc); + } while (p4d++, addr = next, ((addr < end) && !ret)); + return ret; +} + void fix_range_common(struct mm_struct *mm, unsigned long start_addr, unsigned long end_addr, int force) { @@ -316,8 +338,8 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr, ret = add_munmap(addr, next - addr, &hvc); pgd_mkuptodate(*pgd); } - } - else ret = update_pud_range(pgd, addr, next, &hvc); + } else + ret = update_p4d_range(pgd, addr, next, &hvc); } while (pgd++, addr = next, ((addr < end_addr) && !ret)); if (!ret) @@ -338,6 +360,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) { struct mm_struct *mm; pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -364,7 +387,23 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) continue; } - pud = pud_offset(pgd, addr); + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) { + last = ADD_ROUND(addr, P4D_SIZE); + if (last > end) + last = end; + if (p4d_newpage(*p4d)) { + updated = 1; + err = add_munmap(addr, last - addr, &hvc); + if (err < 0) + panic("munmap failed, errno = %d\n", + -err); + } + addr = last; + continue; + } + + pud = pud_offset(p4d, addr); if (!pud_present(*pud)) { last = ADD_ROUND(addr, PUD_SIZE); if (last > end) @@ -424,6 +463,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -437,7 +477,11 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) if (!pgd_present(*pgd)) goto kill; - pud = pud_offset(pgd, address); + p4d = p4d_offset(pgd, address); + if (!p4d_present(*p4d)) + goto kill; + + pud = pud_offset(p4d, address); if (!pud_present(*pud)) goto kill; @@ -490,35 +534,6 @@ kill: force_sig(SIGKILL); } -pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address) -{ - return pgd_offset(mm, address); -} - -pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address) -{ - return pud_offset(pgd, address); -} - -pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address) -{ - return pmd_offset(pud, address); -} - -pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address) -{ - return pte_offset_kernel(pmd, address); -} - -pte_t *addr_pte(struct task_struct *task, unsigned long addr) -{ - pgd_t *pgd = pgd_offset(task->mm, addr); - pud_t *pud = pud_offset(pgd, addr); - pmd_t *pmd = pmd_offset(pud, addr); - - return pte_offset_map(pmd, addr); -} - void flush_tlb_all(void) { /* diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index e62296c66c95..818553064f04 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -28,6 +28,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -104,7 +105,8 @@ good_area: } pgd = pgd_offset(mm, address); - pud = pud_offset(pgd, address); + p4d = p4d_offset(pgd, address); + pud = pud_offset(p4d, address); pmd = pmd_offset(pud, address); pte = pte_offset_kernel(pmd, address); } while (!pte_present(*pte)); diff --git a/arch/x86/include/uapi/asm/msgbuf.h b/arch/x86/include/uapi/asm/msgbuf.h index 7c5bb43ed8af..b3d0664fadc9 100644 --- a/arch/x86/include/uapi/asm/msgbuf.h +++ b/arch/x86/include/uapi/asm/msgbuf.h @@ -5,6 +5,9 @@ #if !defined(__x86_64__) || !defined(__ILP32__) #include <asm-generic/msgbuf.h> #else + +#include <asm/ipcbuf.h> + /* * The msqid64_ds structure for x86 architecture with x32 ABI. * diff --git a/arch/x86/include/uapi/asm/sembuf.h b/arch/x86/include/uapi/asm/sembuf.h index 93030e97269a..71205b02a191 100644 --- a/arch/x86/include/uapi/asm/sembuf.h +++ b/arch/x86/include/uapi/asm/sembuf.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_SEMBUF_H #define _ASM_X86_SEMBUF_H +#include <asm/ipcbuf.h> + /* * The semid64_ds structure for x86 architecture. * Note extra padding because this structure is passed back and forth diff --git a/arch/xtensa/include/uapi/asm/ipcbuf.h b/arch/xtensa/include/uapi/asm/ipcbuf.h index a57afa0b606f..3bd0642f6660 100644 --- a/arch/xtensa/include/uapi/asm/ipcbuf.h +++ b/arch/xtensa/include/uapi/asm/ipcbuf.h @@ -12,6 +12,8 @@ #ifndef _XTENSA_IPCBUF_H #define _XTENSA_IPCBUF_H +#include <linux/posix_types.h> + /* * Pad space is left for: * - 32-bit mode_t and seq diff --git a/arch/xtensa/include/uapi/asm/msgbuf.h b/arch/xtensa/include/uapi/asm/msgbuf.h index d6915e9f071c..1ed2c85b693a 100644 --- a/arch/xtensa/include/uapi/asm/msgbuf.h +++ b/arch/xtensa/include/uapi/asm/msgbuf.h @@ -17,6 +17,8 @@ #ifndef _XTENSA_MSGBUF_H #define _XTENSA_MSGBUF_H +#include <asm/ipcbuf.h> + struct msqid64_ds { struct ipc64_perm msg_perm; #ifdef __XTENSA_EB__ diff --git a/arch/xtensa/include/uapi/asm/sembuf.h b/arch/xtensa/include/uapi/asm/sembuf.h index 09f348d643f1..3b9cdd406dfe 100644 --- a/arch/xtensa/include/uapi/asm/sembuf.h +++ b/arch/xtensa/include/uapi/asm/sembuf.h @@ -22,6 +22,7 @@ #define _XTENSA_SEMBUF_H #include <asm/byteorder.h> +#include <asm/ipcbuf.h> struct semid64_ds { struct ipc64_perm sem_perm; /* permissions .. see ipc.h */ |