diff options
| -rw-r--r-- | arch/x86/include/asm/processor.h | 2 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/common.c | 14 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/intel.c | 34 | ||||
| -rw-r--r-- | arch/x86/mm/tlb.c | 7 | ||||
| -rw-r--r-- | include/asm-generic/tlb.h | 3 |
5 files changed, 53 insertions, 7 deletions
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 39b2bd48dfbc..d048cad9bcad 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -72,6 +72,8 @@ extern u16 __read_mostly tlb_lli_4m[NR_INFO]; extern u16 __read_mostly tlb_lld_4k[NR_INFO]; extern u16 __read_mostly tlb_lld_2m[NR_INFO]; extern u16 __read_mostly tlb_lld_4m[NR_INFO]; +extern s8 __read_mostly tlb_flushall_shift; + /* * CPU type and hardware bug flags. Kept separately for each CPU. * Members of this structure are referenced in head.S, so think twice diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index b2016df00813..7595552600b8 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -459,16 +459,26 @@ u16 __read_mostly tlb_lld_4k[NR_INFO]; u16 __read_mostly tlb_lld_2m[NR_INFO]; u16 __read_mostly tlb_lld_4m[NR_INFO]; +/* + * tlb_flushall_shift shows the balance point in replacing cr3 write + * with multiple 'invlpg'. It will do this replacement when + * flush_tlb_lines <= active_lines/2^tlb_flushall_shift. + * If tlb_flushall_shift is -1, means the replacement will be disabled. + */ +s8 __read_mostly tlb_flushall_shift = -1; + void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) { if (this_cpu->c_detect_tlb) this_cpu->c_detect_tlb(c); printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ - "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n", + "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ + "tlb_flushall_shift is 0x%x\n", tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES], tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES], - tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]); + tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES], + tlb_flushall_shift); } void __cpuinit detect_ht(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index ed0d512cf51b..0a4ce2980a5a 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -610,6 +610,39 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc) } } +static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) +{ + if (!cpu_has_invlpg) { + tlb_flushall_shift = -1; + return; + } + switch ((c->x86 << 8) + c->x86_model) { + case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */ + case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */ + case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */ + case 0x61d: /* six-core 45 nm xeon "Dunnington" */ + tlb_flushall_shift = -1; + break; + case 0x61a: /* 45 nm nehalem, "Bloomfield" */ + case 0x61e: /* 45 nm nehalem, "Lynnfield" */ + case 0x625: /* 32 nm nehalem, "Clarkdale" */ + case 0x62c: /* 32 nm nehalem, "Gulftown" */ + case 0x62e: /* 45 nm nehalem-ex, "Beckton" */ + case 0x62f: /* 32 nm Xeon E7 */ + tlb_flushall_shift = 6; + break; + case 0x62a: /* SandyBridge */ + case 0x62d: /* SandyBridge, "Romely-EP" */ + tlb_flushall_shift = 5; + break; + case 0x63a: /* Ivybridge */ + tlb_flushall_shift = 1; + break; + default: + tlb_flushall_shift = 6; + } +} + static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) { int i, j, n; @@ -630,6 +663,7 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) for (j = 1 ; j < 16 ; j++) intel_tlb_lookup(desc[j]); } + intel_tlb_flushall_shift_set(c); } static const struct cpu_dev __cpuinitconst intel_cpu_dev = { diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 184a02a4d871..2939f2f9edbb 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -316,8 +316,6 @@ void flush_tlb_mm(struct mm_struct *mm) preempt_enable(); } -#define FLUSHALL_BAR 16 - #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline unsigned long has_large_page(struct mm_struct *mm, unsigned long start, unsigned long end) @@ -352,7 +350,7 @@ void flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm; - if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) { + if (vma->vm_flags & VM_HUGETLB || tlb_flushall_shift == -1) { flush_all: flush_tlb_mm(vma->vm_mm); return; @@ -373,7 +371,8 @@ flush_all: act_entries = tlb_entries > mm->total_vm ? mm->total_vm : tlb_entries; - if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR) + if ((end - start) >> PAGE_SHIFT > + act_entries >> tlb_flushall_shift) local_flush_tlb(); else { if (has_large_page(mm, start, end)) { diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index f96a5b58a975..75e888b3cfd2 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -113,7 +113,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb) void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm); void tlb_flush_mmu(struct mmu_gather *tlb); -void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end); +void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, + unsigned long end); int __tlb_remove_page(struct mmu_gather *tlb, struct page *page); /* tlb_remove_page |