aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/mm/nohash/tlb_low.S
blob: 6914bc8e4eada339960a8e277da517c52249bbfa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * This file contains low-level functions for performing various
 * types of TLB invalidations on various processors with no hash
 * table.
 *
 * This file implements the following functions for all no-hash
 * processors. Some aren't implemented for some variants. Some
 * are inline in tlbflush.h
 *
 *	- tlbil_va
 *	- tlbil_pid
 *	- tlbil_all
 *	- tlbivax_bcast
 *
 * Code mostly moved over from misc_32.S
 *
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 * Partially rewritten by Cort Dougan (cort@cs.nmt.edu)
 * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt.
 */

#include <asm/reg.h>
#include <asm/page.h>
#include <asm/cputable.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/processor.h>
#include <asm/bug.h>
#include <asm/asm-compat.h>
#include <asm/feature-fixups.h>

#if defined(CONFIG_40x)

/*
 * 40x implementation needs only tlbil_va
 */
_GLOBAL(__tlbil_va)
	/* We run the search with interrupts disabled because we have to change
	 * the PID and I don't want to preempt when that happens.
	 */
	mfmsr	r5
	mfspr	r6,SPRN_PID
	wrteei	0
	mtspr	SPRN_PID,r4
	tlbsx.	r3, 0, r3
	mtspr	SPRN_PID,r6
	wrtee	r5
	bne	1f
	sync
	/* There are only 64 TLB entries, so r3 < 64, which means bit 25 is
	 * clear. Since 25 is the V bit in the TLB_TAG, loading this value
	 * will invalidate the TLB entry. */
	tlbwe	r3, r3, TLB_TAG
	isync
1:	blr

#elif defined(CONFIG_PPC_8xx)

/*
 * Nothing to do for 8xx, everything is inline
 */

#elif defined(CONFIG_44x) /* Includes 47x */

/*
 * 440 implementation uses tlbsx/we for tlbil_va and a full sweep
 * of the TLB for everything else.
 */
_GLOBAL(__tlbil_va)
	mfspr	r5,SPRN_MMUCR
	mfmsr   r10

	/*
	 * We write 16 bits of STID since 47x supports that much, we
	 * will never be passed out of bounds values on 440 (hopefully)
	 */
	rlwimi  r5,r4,0,16,31

	/* We have to run the search with interrupts disabled, otherwise
	 * an interrupt which causes a TLB miss can clobber the MMUCR
	 * between the mtspr and the tlbsx.
	 *
	 * Critical and Machine Check interrupts take care of saving
	 * and restoring MMUCR, so only normal interrupts have to be
	 * taken care of.
	 */
	wrteei	0
	mtspr	SPRN_MMUCR,r5
	tlbsx.	r6,0,r3
	bne	10f
	sync
#ifndef CONFIG_PPC_47x
	/* On 440 There are only 64 TLB entries, so r3 < 64, which means bit
	 * 22, is clear.  Since 22 is the V bit in the TLB_PAGEID, loading this
	 * value will invalidate the TLB entry.
	 */
	tlbwe	r6,r6,PPC44x_TLB_PAGEID
#else
	oris	r7,r6,0x8000	/* specify way explicitly */
	clrrwi	r4,r3,12	/* get an EPN for the hashing with V = 0 */
	ori	r4,r4,PPC47x_TLBE_SIZE
	tlbwe   r4,r7,0		/* write it */
#endif /* !CONFIG_PPC_47x */
	isync
10:	wrtee	r10
	blr

_GLOBAL(_tlbil_all)
_GLOBAL(_tlbil_pid)
#ifndef CONFIG_PPC_47x
	li	r3,0
	sync

	/* Load high watermark */
	lis	r4,tlb_44x_hwater@ha
	lwz	r5,tlb_44x_hwater@l(r4)

1:	tlbwe	r3,r3,PPC44x_TLB_PAGEID
	addi	r3,r3,1
	cmpw	0,r3,r5
	ble	1b

	isync
	blr
#else
	/* 476 variant. There's not simple way to do this, hopefully we'll
	 * try to limit the amount of such full invalidates
	 */
	mfmsr	r11		/* Interrupts off */
	wrteei	0
	li	r3,-1		/* Current set */
	lis	r10,tlb_47x_boltmap@h
	ori	r10,r10,tlb_47x_boltmap@l
	lis	r7,0x8000	/* Specify way explicitly */

	b	9f		/* For each set */

1:	li	r9,4		/* Number of ways */
	li	r4,0		/* Current way */
	li	r6,0		/* Default entry value 0 */
	andi.	r0,r8,1		/* Check if way 0 is bolted */
	mtctr	r9		/* Load way counter */
	bne-	3f		/* Bolted, skip loading it */

2:	/* For each way */
	or	r5,r3,r4	/* Make way|index for tlbre */
	rlwimi	r5,r5,16,8,15	/* Copy index into position */
	tlbre	r6,r5,0		/* Read entry */
3:	addis	r4,r4,0x2000	/* Next way */
	andi.	r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */
	beq	4f		/* Nope, skip it */
	rlwimi	r7,r5,0,1,2	/* Insert way number */
	rlwinm	r6,r6,0,21,19	/* Clear V */
	tlbwe   r6,r7,0		/* Write it */
4:	bdnz	2b		/* Loop for each way */
	srwi	r8,r8,1		/* Next boltmap bit */
9:	cmpwi	cr1,r3,255	/* Last set done ? */
	addi	r3,r3,1		/* Next set */
	beq	cr1,1f		/* End of loop */
	andi.	r0,r3,0x1f	/* Need to load a new boltmap word ? */
	bne	1b		/* No, loop */
	lwz	r8,0(r10)	/* Load boltmap entry */
	addi	r10,r10,4	/* Next word */
	b	1b		/* Then loop */
1:	isync			/* Sync shadows */
	wrtee	r11
	blr
#endif /* !CONFIG_PPC_47x */

#ifdef CONFIG_PPC_47x

/*
 * _tlbivax_bcast is only on 47x. We don't bother doing a runtime
 * check though, it will blow up soon enough if we mistakenly try
 * to use it on a 440.
 */
_GLOBAL(_tlbivax_bcast)
	mfspr	r5,SPRN_MMUCR
	mfmsr	r10
	rlwimi	r5,r4,0,16,31
	wrteei	0
	mtspr	SPRN_MMUCR,r5
	isync
	PPC_TLBIVAX(0, R3)
	isync
	mbar
	tlbsync
BEGIN_FTR_SECTION
	b	1f
END_FTR_SECTION_IFSET(CPU_FTR_476_DD2)
	sync
	wrtee	r10
	blr
/*
 * DD2 HW could hang if in instruction fetch happens before msync completes.
 * Touch enough instruction cache lines to ensure cache hits
 */
1:	mflr	r9
	bcl	20,31,$+4
2:	mflr	r6
	li	r7,32
	PPC_ICBT(0,R6,R7)		/* touch next cache line */
	add	r6,r6,r7
	PPC_ICBT(0,R6,R7)		/* touch next cache line */
	add	r6,r6,r7
	PPC_ICBT(0,R6,R7)		/* touch next cache line */
	sync
	nop
	nop
	nop
	nop
	nop
	nop
	nop
	nop
	mtlr	r9
	wrtee	r10
	blr
#endif /* CONFIG_PPC_47x */

#elif defined(CONFIG_PPC_85xx)
/*
 * FSL BookE implementations.
 *
 * Since feature sections are using _SECTION_ELSE we need
 * to have the larger code path before the _SECTION_ELSE
 */

/*
 * Flush MMU TLB on the local processor
 */
_GLOBAL(_tlbil_all)
BEGIN_MMU_FTR_SECTION
	li	r3,(MMUCSR0_TLBFI)@l
	mtspr	SPRN_MMUCSR0, r3
1:
	mfspr	r3,SPRN_MMUCSR0
	andi.	r3,r3,MMUCSR0_TLBFI@l
	bne	1b
MMU_FTR_SECTION_ELSE
	PPC_TLBILX_ALL(0,R0)
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
	msync
	isync
	blr

_GLOBAL(_tlbil_pid)
BEGIN_MMU_FTR_SECTION
	slwi	r3,r3,16
	mfmsr	r10
	wrteei	0
	mfspr	r4,SPRN_MAS6	/* save MAS6 */
	mtspr	SPRN_MAS6,r3
	PPC_TLBILX_PID(0,R0)
	mtspr	SPRN_MAS6,r4	/* restore MAS6 */
	wrtee	r10
MMU_FTR_SECTION_ELSE
	li	r3,(MMUCSR0_TLBFI)@l
	mtspr	SPRN_MMUCSR0, r3
1:
	mfspr	r3,SPRN_MMUCSR0
	andi.	r3,r3,MMUCSR0_TLBFI@l
	bne	1b
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX)
	msync
	isync
	blr

/*
 * Flush MMU TLB for a particular address, but only on the local processor
 * (no broadcast)
 */
_GLOBAL(__tlbil_va)
	mfmsr	r10
	wrteei	0
	slwi	r4,r4,16
	ori	r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l
	mtspr	SPRN_MAS6,r4		/* assume AS=0 for now */
BEGIN_MMU_FTR_SECTION
	tlbsx	0,r3
	mfspr	r4,SPRN_MAS1		/* check valid */
	andis.	r3,r4,MAS1_VALID@h
	beq	1f
	rlwinm	r4,r4,0,1,31
	mtspr	SPRN_MAS1,r4
	tlbwe
MMU_FTR_SECTION_ELSE
	PPC_TLBILX_VA(0,R3)
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
	msync
	isync
1:	wrtee	r10
	blr
#elif defined(CONFIG_PPC_BOOK3E_64)
/*
 * New Book3E (>= 2.06) implementation
 *
 * Note: We may be able to get away without the interrupt masking stuff
 * if we save/restore MAS6 on exceptions that might modify it
 */
_GLOBAL(_tlbil_pid)
	slwi	r4,r3,MAS6_SPID_SHIFT
	mfmsr	r10
	wrteei	0
	mtspr	SPRN_MAS6,r4
	PPC_TLBILX_PID(0,R0)
	wrtee	r10
	msync
	isync
	blr

_GLOBAL(_tlbil_pid_noind)
	slwi	r4,r3,MAS6_SPID_SHIFT
	mfmsr	r10
	ori	r4,r4,MAS6_SIND
	wrteei	0
	mtspr	SPRN_MAS6,r4
	PPC_TLBILX_PID(0,R0)
	wrtee	r10
	msync
	isync
	blr

_GLOBAL(_tlbil_all)
	PPC_TLBILX_ALL(0,R0)
	msync
	isync
	blr

_GLOBAL(_tlbil_va)
	mfmsr	r10
	wrteei	0
	cmpwi	cr0,r6,0
	slwi	r4,r4,MAS6_SPID_SHIFT
	rlwimi	r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
	beq	1f
	rlwimi	r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
1:	mtspr	SPRN_MAS6,r4		/* assume AS=0 for now */
	PPC_TLBILX_VA(0,R3)
	msync
	isync
	wrtee	r10
	blr

_GLOBAL(_tlbivax_bcast)
	mfmsr	r10
	wrteei	0
	cmpwi	cr0,r6,0
	slwi	r4,r4,MAS6_SPID_SHIFT
	rlwimi	r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
	beq	1f
	rlwimi	r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
1:	mtspr	SPRN_MAS6,r4		/* assume AS=0 for now */
	PPC_TLBIVAX(0,R3)
	mbar
	tlbsync
	sync
	wrtee	r10
	blr
#else
#error Unsupported processor type !
#endif

#if defined(CONFIG_PPC_FSL_BOOK3E)
/*
 * extern void loadcam_entry(unsigned int index)
 *
 * Load TLBCAM[index] entry in to the L2 CAM MMU
 * Must preserve r7, r8, r9, r10, r11, r12
 */
_GLOBAL(loadcam_entry)
	mflr	r5
	LOAD_REG_ADDR_PIC(r4, TLBCAM)
	mtlr	r5
	mulli	r5,r3,TLBCAM_SIZE
	add	r3,r5,r4
	lwz	r4,TLBCAM_MAS0(r3)
	mtspr	SPRN_MAS0,r4
	lwz	r4,TLBCAM_MAS1(r3)
	mtspr	SPRN_MAS1,r4
	PPC_LL	r4,TLBCAM_MAS2(r3)
	mtspr	SPRN_MAS2,r4
	lwz	r4,TLBCAM_MAS3(r3)
	mtspr	SPRN_MAS3,r4
BEGIN_MMU_FTR_SECTION
	lwz	r4,TLBCAM_MAS7(r3)
	mtspr	SPRN_MAS7,r4
END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS)
	isync
	tlbwe
	isync
	blr

/*
 * Load multiple TLB entries at once, using an alternate-space
 * trampoline so that we don't have to care about whether the same
 * TLB entry maps us before and after.
 *
 * r3 = first entry to write
 * r4 = number of entries to write
 * r5 = temporary tlb entry (0 means no switch to AS1)
 */
_GLOBAL(loadcam_multi)
	mflr	r8
	/* Don't switch to AS=1 if already there */
	mfmsr	r11
	andi.	r11,r11,MSR_IS
	bne	10f
	mr.	r12, r5
	beq	10f

	/*
	 * Set up temporary TLB entry that is the same as what we're
	 * running from, but in AS=1.
	 */
	bcl	20,31,$+4
1:	mflr	r6
	tlbsx	0,r8
	mfspr	r6,SPRN_MAS1
	ori	r6,r6,MAS1_TS
	mtspr	SPRN_MAS1,r6
	mfspr	r6,SPRN_MAS0
	rlwimi	r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
	mr	r7,r5
	mtspr	SPRN_MAS0,r6
	isync
	tlbwe
	isync

	/* Switch to AS=1 */
	mfmsr	r6
	ori	r6,r6,MSR_IS|MSR_DS
	mtmsr	r6
	isync

10:
	mr	r9,r3
	add	r10,r3,r4
2:	bl	loadcam_entry
	addi	r9,r9,1
	cmpw	r9,r10
	mr	r3,r9
	blt	2b

	/* Don't return to AS=0 if we were in AS=1 at function start */
	andi.	r11,r11,MSR_IS
	bne	3f
	cmpwi	r12, 0
	beq	3f

	/* Return to AS=0 and clear the temporary entry */
	mfmsr	r6
	rlwinm.	r6,r6,0,~(MSR_IS|MSR_DS)
	mtmsr	r6
	isync

	li	r6,0
	mtspr	SPRN_MAS1,r6
	rlwinm	r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
	oris	r6,r6,MAS0_TLBSEL(1)@h
	mtspr	SPRN_MAS0,r6
	isync
	tlbwe
	isync

3:
	mtlr	r8
	blr
#endif