diff options
| -rw-r--r-- | arch/riscv/Kconfig | 6 | ||||
| -rw-r--r-- | arch/riscv/include/asm/fixmap.h | 44 | ||||
| -rw-r--r-- | arch/riscv/include/asm/pgtable.h | 1 | ||||
| -rw-r--r-- | arch/riscv/include/asm/smp.h | 18 | ||||
| -rw-r--r-- | arch/riscv/kernel/cpu.c | 30 | ||||
| -rw-r--r-- | arch/riscv/kernel/cpufeature.c | 48 | ||||
| -rw-r--r-- | arch/riscv/kernel/ftrace.c | 2 | ||||
| -rw-r--r-- | arch/riscv/kernel/setup.c | 141 | ||||
| -rw-r--r-- | arch/riscv/kernel/smp.c | 10 | ||||
| -rw-r--r-- | arch/riscv/kernel/smpboot.c | 24 | ||||
| -rw-r--r-- | arch/riscv/mm/init.c | 156 |
11 files changed, 287 insertions, 193 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index bd149905a5b5..b41311f6a94f 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -90,14 +90,14 @@ config GENERIC_CSUM config GENERIC_HWEIGHT def_bool y +config FIX_EARLYCON_MEM + def_bool y + config PGTABLE_LEVELS int default 3 if 64BIT default 2 -config HAVE_KPROBES - def_bool n - menu "Platform type" choice diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h new file mode 100644 index 000000000000..57afe604b495 --- /dev/null +++ b/arch/riscv/include/asm/fixmap.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + */ + +#ifndef _ASM_RISCV_FIXMAP_H +#define _ASM_RISCV_FIXMAP_H + +#include <linux/kernel.h> +#include <linux/sizes.h> +#include <asm/page.h> +#include <asm/pgtable.h> + +/* + * Here we define all the compile-time 'special' virtual addresses. + * The point is to have a constant address at compile time, but to + * set the physical address only in the boot process. + * + * These 'compile-time allocated' memory buffers are page-sized. Use + * set_fixmap(idx,phys) to associate physical memory with fixmap indices. + */ +enum fixed_addresses { + FIX_HOLE, + FIX_EARLYCON_MEM_BASE, + __end_of_fixed_addresses +}; + +#define FIXADDR_SIZE (__end_of_fixed_addresses * PAGE_SIZE) +#define FIXADDR_TOP (PAGE_OFFSET) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#define FIXMAP_PAGE_IO PAGE_KERNEL + +#define __early_set_fixmap __set_fixmap + +#define __late_set_fixmap __set_fixmap +#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) + +extern void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t prot); + +#include <asm-generic/fixmap.h> + +#endif /* _ASM_RISCV_FIXMAP_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index a8179a8c1491..1141364d990e 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -404,6 +404,7 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, #define kern_addr_valid(addr) (1) /* FIXME */ #endif +extern void setup_bootmem(void); extern void paging_init(void); static inline void pgtable_cache_init(void) diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 41aa73b476f4..636a934f013a 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -19,16 +19,17 @@ #include <linux/thread_info.h> #define INVALID_HARTID ULONG_MAX + +struct seq_file; +extern unsigned long boot_cpu_hartid; + +#ifdef CONFIG_SMP /* * Mapping between linux logical cpu index and hartid. */ extern unsigned long __cpuid_to_hartid_map[NR_CPUS]; #define cpuid_to_hartid_map(cpu) __cpuid_to_hartid_map[cpu] -struct seq_file; - -#ifdef CONFIG_SMP - /* print IPI stats */ void show_ipi_stats(struct seq_file *p, int prec); @@ -58,7 +59,14 @@ static inline void show_ipi_stats(struct seq_file *p, int prec) static inline int riscv_hartid_to_cpuid(int hartid) { - return 0; + if (hartid == boot_cpu_hartid) + return 0; + + return -1; +} +static inline unsigned long cpuid_to_hartid_map(int cpu) +{ + return boot_cpu_hartid; } static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in, diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index f8fa2c63aa89..cf2fca12414a 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -17,44 +17,36 @@ #include <asm/smp.h> /* - * Returns the hart ID of the given device tree node, or -1 if the device tree - * node isn't a RISC-V hart. + * Returns the hart ID of the given device tree node, or -ENODEV if the node + * isn't an enabled and valid RISC-V hart node. */ int riscv_of_processor_hartid(struct device_node *node) { - const char *isa, *status; + const char *isa; u32 hart; if (!of_device_is_compatible(node, "riscv")) { pr_warn("Found incompatible CPU\n"); - return -(ENODEV); + return -ENODEV; } if (of_property_read_u32(node, "reg", &hart)) { pr_warn("Found CPU without hart ID\n"); - return -(ENODEV); - } - if (hart >= NR_CPUS) { - pr_info("Found hart ID %d, which is above NR_CPUs. Disabling this hart\n", hart); - return -(ENODEV); + return -ENODEV; } - if (of_property_read_string(node, "status", &status)) { - pr_warn("CPU with hartid=%d has no \"status\" property\n", hart); - return -(ENODEV); - } - if (strcmp(status, "okay")) { - pr_info("CPU with hartid=%d has a non-okay status of \"%s\"\n", hart, status); - return -(ENODEV); + if (!of_device_is_available(node)) { + pr_info("CPU with hartid=%d is not available\n", hart); + return -ENODEV; } if (of_property_read_string(node, "riscv,isa", &isa)) { pr_warn("CPU with hartid=%d has no \"riscv,isa\" property\n", hart); - return -(ENODEV); + return -ENODEV; } if (isa[0] != 'r' || isa[1] != 'v') { pr_warn("CPU with hartid=%d has an invalid ISA of \"%s\"\n", hart, isa); - return -(ENODEV); + return -ENODEV; } return hart; @@ -106,7 +98,7 @@ static void print_isa(struct seq_file *f, const char *orig_isa) * a bit of info describing what went wrong. */ if (isa[0] != '\0') - pr_info("unsupported ISA \"%s\" in device tree", orig_isa); + pr_info("unsupported ISA \"%s\" in device tree\n", orig_isa); } static void print_mmu(struct seq_file *f, const char *mmu_type) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index a6e369edbbd7..bc29b010b722 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -20,6 +20,7 @@ #include <linux/of.h> #include <asm/processor.h> #include <asm/hwcap.h> +#include <asm/smp.h> unsigned long elf_hwcap __read_mostly; #ifdef CONFIG_FPU @@ -28,7 +29,7 @@ bool has_fpu __read_mostly; void riscv_fill_hwcap(void) { - struct device_node *node = NULL; + struct device_node *node; const char *isa; size_t i; static unsigned long isa2hwcap[256] = {0}; @@ -42,36 +43,39 @@ void riscv_fill_hwcap(void) elf_hwcap = 0; - /* - * We don't support running Linux on hertergenous ISA systems. For - * now, we just check the ISA of the first "okay" processor. - */ - while ((node = of_find_node_by_type(node, "cpu"))) - if (riscv_of_processor_hartid(node) >= 0) - break; - if (!node) { - pr_warning("Unable to find \"cpu\" devicetree entry"); - return; - } + for_each_of_cpu_node(node) { + unsigned long this_hwcap = 0; - if (of_property_read_string(node, "riscv,isa", &isa)) { - pr_warning("Unable to find \"riscv,isa\" devicetree entry"); - of_node_put(node); - return; - } - of_node_put(node); + if (riscv_of_processor_hartid(node) < 0) + continue; + + if (of_property_read_string(node, "riscv,isa", &isa)) { + pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); + continue; + } - for (i = 0; i < strlen(isa); ++i) - elf_hwcap |= isa2hwcap[(unsigned char)(isa[i])]; + for (i = 0; i < strlen(isa); ++i) + this_hwcap |= isa2hwcap[(unsigned char)(isa[i])]; + + /* + * All "okay" hart should have same isa. Set HWCAP based on + * common capabilities of every "okay" hart, in case they don't + * have. + */ + if (elf_hwcap) + elf_hwcap &= this_hwcap; + else + elf_hwcap = this_hwcap; + } /* We don't support systems with F but without D, so mask those out * here. */ if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { - pr_info("This kernel does not support systems with F but not D"); + pr_info("This kernel does not support systems with F but not D\n"); elf_hwcap &= ~COMPAT_HWCAP_ISA_F; } - pr_info("elf_hwcap is 0x%lx", elf_hwcap); + pr_info("elf_hwcap is 0x%lx\n", elf_hwcap); #ifdef CONFIG_FPU if (elf_hwcap & (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)) diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index a840b7d074f7..b94d8db5ddcc 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -32,7 +32,7 @@ static int ftrace_check_current_call(unsigned long hook_pos, * return must be -EINVAL on failed comparison */ if (memcmp(expected, replaced, sizeof(replaced))) { - pr_err("%p: expected (%08x %08x) but get (%08x %08x)", + pr_err("%p: expected (%08x %08x) but got (%08x %08x)\n", (void *)hook_pos, expected[0], expected[1], replaced[0], replaced[1]); return -EINVAL; diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 77564310235f..ecb654f6a79e 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -23,7 +23,6 @@ #include <linux/mm.h> #include <linux/memblock.h> #include <linux/sched.h> -#include <linux/initrd.h> #include <linux/console.h> #include <linux/screen_info.h> #include <linux/of_fdt.h> @@ -61,95 +60,9 @@ EXPORT_SYMBOL(empty_zero_page); atomic_t hart_lottery; unsigned long boot_cpu_hartid; -unsigned long __cpuid_to_hartid_map[NR_CPUS] = { - [0 ... NR_CPUS-1] = INVALID_HARTID -}; - -void __init smp_setup_processor_id(void) -{ - cpuid_to_hartid_map(0) = boot_cpu_hartid; -} - -#ifdef CONFIG_BLK_DEV_INITRD -static void __init setup_initrd(void) -{ - unsigned long size; - - if (initrd_start >= initrd_end) { - printk(KERN_INFO "initrd not found or empty"); - goto disable; - } - if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { - printk(KERN_ERR "initrd extends beyond end of memory"); - goto disable; - } - - size = initrd_end - initrd_start; - memblock_reserve(__pa(initrd_start), size); - initrd_below_start_ok = 1; - - printk(KERN_INFO "Initial ramdisk at: 0x%p (%lu bytes)\n", - (void *)(initrd_start), size); - return; -disable: - pr_cont(" - disabling initrd\n"); - initrd_start = 0; - initrd_end = 0; -} -#endif /* CONFIG_BLK_DEV_INITRD */ - -pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; -pgd_t trampoline_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); - -#ifndef __PAGETABLE_PMD_FOLDED -#define NUM_SWAPPER_PMDS ((uintptr_t)-PAGE_OFFSET >> PGDIR_SHIFT) -pmd_t swapper_pmd[PTRS_PER_PMD*((-PAGE_OFFSET)/PGDIR_SIZE)] __page_aligned_bss; -pmd_t trampoline_pmd[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); -#endif - -asmlinkage void __init setup_vm(void) -{ - extern char _start; - uintptr_t i; - uintptr_t pa = (uintptr_t) &_start; - pgprot_t prot = __pgprot(pgprot_val(PAGE_KERNEL) | _PAGE_EXEC); - - va_pa_offset = PAGE_OFFSET - pa; - pfn_base = PFN_DOWN(pa); - - /* Sanity check alignment and size */ - BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); - BUG_ON((pa % (PAGE_SIZE * PTRS_PER_PTE)) != 0); - -#ifndef __PAGETABLE_PMD_FOLDED - trampoline_pg_dir[(PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD] = - pfn_pgd(PFN_DOWN((uintptr_t)trampoline_pmd), - __pgprot(_PAGE_TABLE)); - trampoline_pmd[0] = pfn_pmd(PFN_DOWN(pa), prot); - - for (i = 0; i < (-PAGE_OFFSET)/PGDIR_SIZE; ++i) { - size_t o = (PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD + i; - swapper_pg_dir[o] = - pfn_pgd(PFN_DOWN((uintptr_t)swapper_pmd) + i, - __pgprot(_PAGE_TABLE)); - } - for (i = 0; i < ARRAY_SIZE(swapper_pmd); i++) - swapper_pmd[i] = pfn_pmd(PFN_DOWN(pa + i * PMD_SIZE), prot); -#else - trampoline_pg_dir[(PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD] = - pfn_pgd(PFN_DOWN(pa), prot); - - for (i = 0; i < (-PAGE_OFFSET)/PGDIR_SIZE; ++i) { - size_t o = (PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD + i; - swapper_pg_dir[o] = - pfn_pgd(PFN_DOWN(pa + i * PGDIR_SIZE), prot); - } -#endif -} - void __init parse_dtb(unsigned int hartid, void *dtb) { - if (!early_init_dt_scan(__va(dtb))) + if (early_init_dt_scan(__va(dtb))) return; pr_err("No DTB passed to the kernel\n"); @@ -159,60 +72,17 @@ void __init parse_dtb(unsigned int hartid, void *dtb) #endif } -static void __init setup_bootmem(void) -{ - struct memblock_region *reg; - phys_addr_t mem_size = 0; - - /* Find the memory region containing the kernel */ - for_each_memblock(memory, reg) { - phys_addr_t vmlinux_end = __pa(_end); - phys_addr_t end = reg->base + reg->size; - - if (reg->base <= vmlinux_end && vmlinux_end <= end) { - /* - * Reserve from the start of the region to the end of - * the kernel - */ - memblock_reserve(reg->base, vmlinux_end - reg->base); - mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); - } - } - BUG_ON(mem_size == 0); - - set_max_mapnr(PFN_DOWN(mem_size)); - max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); - -#ifdef CONFIG_BLK_DEV_INITRD - setup_initrd(); -#endif /* CONFIG_BLK_DEV_INITRD */ - - early_init_fdt_reserve_self(); - early_init_fdt_scan_reserved_mem(); - memblock_allow_resize(); - memblock_dump_all(); - - for_each_memblock(memory, reg) { - unsigned long start_pfn = memblock_region_memory_base_pfn(reg); - unsigned long end_pfn = memblock_region_memory_end_pfn(reg); - - memblock_set_node(PFN_PHYS(start_pfn), - PFN_PHYS(end_pfn - start_pfn), - &memblock.memory, 0); - } -} - void __init setup_arch(char **cmdline_p) { - *cmdline_p = boot_command_line; - - parse_early_param(); - init_mm.start_code = (unsigned long) _stext; init_mm.end_code = (unsigned long) _etext; init_mm.end_data = (unsigned long) _edata; init_mm.brk = (unsigned long) _end; + *cmdline_p = boot_command_line; + + parse_early_param(); + setup_bootmem(); paging_init(); unflatten_device_tree(); @@ -231,4 +101,3 @@ void __init setup_arch(char **cmdline_p) riscv_fill_hwcap(); } - diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 246635eac7bb..0c41d07ec281 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -36,6 +36,15 @@ enum ipi_message_type { IPI_MAX }; +unsigned long __cpuid_to_hartid_map[NR_CPUS] = { + [0 ... NR_CPUS-1] = INVALID_HARTID +}; + +void __init smp_setup_processor_id(void) +{ + cpuid_to_hartid_map(0) = boot_cpu_hartid; +} + /* A collection of single bit ipi messages. */ static struct { unsigned long stats[IPI_MAX] ____cacheline_aligned; @@ -51,7 +60,6 @@ int riscv_hartid_to_cpuid(int hartid) return i; pr_err("Couldn't find cpu id for hartid [%d]\n", hartid); - BUG(); return i; } diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 18cda0e8cf94..eb533b5c2c8c 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -39,6 +39,7 @@ void *__cpu_up_stack_pointer[NR_CPUS]; void *__cpu_up_task_pointer[NR_CPUS]; +static DECLARE_COMPLETION(cpu_running); void __init smp_prepare_boot_cpu(void) { @@ -50,12 +51,12 @@ void __init smp_prepare_cpus(unsigned int max_cpus) void __init setup_smp(void) { - struct device_node *dn = NULL; + struct device_node *dn; int hart; bool found_boot_cpu = false; int cpuid = 1; - while ((dn = of_find_node_by_type(dn, "cpu"))) { + for_each_of_cpu_node(dn) { hart = riscv_of_processor_hartid(dn); if (hart < 0) continue; @@ -65,6 +66,11 @@ void __init setup_smp(void) found_boot_cpu = 1; continue; } + if (cpuid >= NR_CPUS) { + pr_warn("Invalid cpuid [%d] for hartid [%d]\n", + cpuid, hart); + break; + } cpuid_to_hartid_map(cpuid) = hart; set_cpu_possible(cpuid, true); @@ -77,6 +83,7 @@ void __init setup_smp(void) int __cpu_up(unsigned int cpu, struct task_struct *tidle) { + int ret = 0; int hartid = cpuid_to_hartid_map(cpu); tidle->thread_info.cpu = cpu; @@ -92,10 +99,16 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) task_stack_page(tidle) + THREAD_SIZE); WRITE_ONCE(__cpu_up_task_pointer[hartid], tidle); - while (!cpu_online(cpu)) - cpu_relax(); + lockdep_assert_held(&cpu_running); + wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(1000)); + + if (!cpu_online(cpu)) { + pr_crit("CPU%u: failed to come online\n", cpu); + ret = -EIO; + } - return 0; + return ret; } void __init smp_cpus_done(unsigned int max_cpus) @@ -121,6 +134,7 @@ asmlinkage void __init smp_callin(void) * a local TLB flush right now just in case. */ local_flush_tlb_all(); + complete(&cpu_running); /* * Disable preemption before enabling interrupts, so we don't try to * schedule a CPU that hasn't actually started yet. diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 658ebf645f42..b379a75ac6a6 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -17,7 +17,9 @@ #include <linux/initrd.h> #include <linux/swap.h> #include <linux/sizes.h> +#include <linux/of_fdt.h> +#include <asm/fixmap.h> #include <asm/tlbflush.h> #include <asm/sections.h> #include <asm/pgtable.h> @@ -66,7 +68,159 @@ void free_initmem(void) } #ifdef CONFIG_BLK_DEV_INITRD -void free_initrd_mem(unsigned long start, unsigned long end) +static void __init setup_initrd(void) { + unsigned long size; + + if (initrd_start >= initrd_end) { + pr_info("initrd not found or empty"); + goto disable; + } + if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { + pr_err("initrd extends beyond end of memory"); + goto disable; + } + + size = initrd_end - initrd_start; + memblock_reserve(__pa(initrd_start), size); + initrd_below_start_ok = 1; + + pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", + (void *)(initrd_start), size); + return; +disable: + pr_cont(" - disabling initrd\n"); + initrd_start = 0; + initrd_end = 0; +} + +void __init free_initrd_mem(unsigned long start, unsigned long end) +{ + free_reserved_area((void *)start, (void *)end, -1, "initrd"); } #endif /* CONFIG_BLK_DEV_INITRD */ + +void __init setup_bootmem(void) +{ + struct memblock_region *reg; + phys_addr_t mem_size = 0; + + /* Find the memory region containing the kernel */ + for_each_memblock(memory, reg) { + phys_addr_t vmlinux_end = __pa(_end); + phys_addr_t end = reg->base + reg->size; + + if (reg->base <= vmlinux_end && vmlinux_end <= end) { + /* + * Reserve from the start of the region to the end of + * the kernel + */ + memblock_reserve(reg->base, vmlinux_end - reg->base); + mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); + } + } + BUG_ON(mem_size == 0); + + set_max_mapnr(PFN_DOWN(mem_size)); + max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); + +#ifdef CONFIG_BLK_DEV_INITRD + setup_initrd(); +#endif /* CONFIG_BLK_DEV_INITRD */ + + early_init_fdt_reserve_self(); + early_init_fdt_scan_reserved_mem(); + memblock_allow_resize(); + memblock_dump_all(); + + for_each_memblock(memory, reg) { + unsigned long start_pfn = memblock_region_memory_base_pfn(reg); + unsigned long end_pfn = memblock_region_memory_end_pfn(reg); + + memblock_set_node(PFN_PHYS(start_pfn), + PFN_PHYS(end_pfn - start_pfn), + &memblock.memory, 0); + } +} + +pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; +pgd_t trampoline_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); + +#ifndef __PAGETABLE_PMD_FOLDED +#define NUM_SWAPPER_PMDS ((uintptr_t)-PAGE_OFFSET >> PGDIR_SHIFT) +pmd_t swapper_pmd[PTRS_PER_PMD*((-PAGE_OFFSET)/PGDIR_SIZE)] __page_aligned_bss; +pmd_t trampoline_pmd[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); +pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; +#endif + +pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; + +void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *ptep; + + BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + ptep = &fixmap_pte[pte_index(addr)]; + + if (pgprot_val(prot)) { + set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); + } else { + pte_clear(&init_mm, addr, ptep); + local_flush_tlb_page(addr); + } +} + +asmlinkage void __init setup_vm(void) +{ + extern char _start; + uintptr_t i; + uintptr_t pa = (uintptr_t) &_start; + pgprot_t prot = __pgprot(pgprot_val(PAGE_KERNEL) | _PAGE_EXEC); + + va_pa_offset = PAGE_OFFSET - pa; + pfn_base = PFN_DOWN(pa); + + /* Sanity check alignment and size */ + BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); + BUG_ON((pa % (PAGE_SIZE * PTRS_PER_PTE)) != 0); + +#ifndef __PAGETABLE_PMD_FOLDED + trampoline_pg_dir[(PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD] = + pfn_pgd(PFN_DOWN((uintptr_t)trampoline_pmd), + __pgprot(_PAGE_TABLE)); + trampoline_pmd[0] = pfn_pmd(PFN_DOWN(pa), prot); + + for (i = 0; i < (-PAGE_OFFSET)/PGDIR_SIZE; ++i) { + size_t o = (PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD + i; + + swapper_pg_dir[o] = + pfn_pgd(PFN_DOWN((uintptr_t)swapper_pmd) + i, + __pgprot(_PAGE_TABLE)); + } + for (i = 0; i < ARRAY_SIZE(swapper_pmd); i++) + swapper_pmd[i] = pfn_pmd(PFN_DOWN(pa + i * PMD_SIZE), prot); + + swapper_pg_dir[(FIXADDR_START >> PGDIR_SHIFT) % PTRS_PER_PGD] = + pfn_pgd(PFN_DOWN((uintptr_t)fixmap_pmd), + __pgprot(_PAGE_TABLE)); + fixmap_pmd[(FIXADDR_START >> PMD_SHIFT) % PTRS_PER_PMD] = + pfn_pmd(PFN_DOWN((uintptr_t)fixmap_pte), + __pgprot(_PAGE_TABLE)); +#else + trampoline_pg_dir[(PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD] = + pfn_pgd(PFN_DOWN(pa), prot); + + for (i = 0; i < (-PAGE_OFFSET)/PGDIR_SIZE; ++i) { + size_t o = (PAGE_OFFSET >> PGDIR_SHIFT) % PTRS_PER_PGD + i; + + swapper_pg_dir[o] = + pfn_pgd(PFN_DOWN(pa + i * PGDIR_SIZE), prot); + } + + swapper_pg_dir[(FIXADDR_START >> PGDIR_SHIFT) % PTRS_PER_PGD] = + pfn_pgd(PFN_DOWN((uintptr_t)fixmap_pte), + __pgprot(_PAGE_TABLE)); +#endif +} |