1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2017 Steven Rostedt, VMware Inc.
*/
#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/segment.h>
#include <asm/export.h>
#include <asm/ftrace.h>
#include <asm/nospec-branch.h>
#include <asm/frame.h>
#include <asm/asm-offsets.h>
#ifdef CONFIG_FRAME_POINTER
# define MCOUNT_FRAME 1 /* using frame = true */
#else
# define MCOUNT_FRAME 0 /* using frame = false */
#endif
SYM_FUNC_START(__fentry__)
ret
SYM_FUNC_END(__fentry__)
EXPORT_SYMBOL(__fentry__)
SYM_CODE_START(ftrace_caller)
#ifdef CONFIG_FRAME_POINTER
/*
* Frame pointers are of ip followed by bp.
* Since fentry is an immediate jump, we are left with
* parent-ip, function-ip. We need to add a frame with
* parent-ip followed by ebp.
*/
pushl 4(%esp) /* parent ip */
pushl %ebp
movl %esp, %ebp
pushl 2*4(%esp) /* function ip */
/* For mcount, the function ip is directly above */
pushl %ebp
movl %esp, %ebp
#endif
pushl %eax
pushl %ecx
pushl %edx
pushl $0 /* Pass NULL as regs pointer */
#ifdef CONFIG_FRAME_POINTER
/* Load parent ebp into edx */
movl 4*4(%esp), %edx
#else
/* There's no frame pointer, load the appropriate stack addr instead */
lea 4*4(%esp), %edx
#endif
movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
/* Get the parent ip */
movl 4(%edx), %edx /* edx has ebp */
movl function_trace_op, %ecx
subl $MCOUNT_INSN_SIZE, %eax
.globl ftrace_call
ftrace_call:
call ftrace_stub
addl $4, %esp /* skip NULL pointer */
popl %edx
popl %ecx
popl %eax
#ifdef CONFIG_FRAME_POINTER
popl %ebp
addl $4,%esp /* skip function ip */
popl %ebp /* this is the orig bp */
addl $4, %esp /* skip parent ip */
#endif
.Lftrace_ret:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call
ftrace_graph_call:
jmp ftrace_stub
#endif
/* This is weak to keep gas from relaxing the jumps */
SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
ret
SYM_CODE_END(ftrace_caller)
SYM_CODE_START(ftrace_regs_caller)
/*
* We're here from an mcount/fentry CALL, and the stack frame looks like:
*
* <previous context>
* RET-IP
*
* The purpose of this function is to call out in an emulated INT3
* environment with a stack frame like:
*
* <previous context>
* gap / RET-IP
* gap
* gap
* gap
* pt_regs
*
* We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
*/
subl $3*4, %esp # RET-IP + 3 gaps
pushl %ss # ss
pushl %esp # points at ss
addl $5*4, (%esp) # make it point at <previous context>
pushfl # flags
pushl $__KERNEL_CS # cs
pushl 7*4(%esp) # ip <- RET-IP
pushl $0 # orig_eax
pushl %gs
pushl %fs
pushl %es
pushl %ds
pushl %eax
pushl %ebp
pushl %edi
pushl %esi
pushl %edx
pushl %ecx
pushl %ebx
ENCODE_FRAME_POINTER
movl PT_EIP(%esp), %eax # 1st argument: IP
subl $MCOUNT_INSN_SIZE, %eax
movl 21*4(%esp), %edx # 2nd argument: parent ip
movl function_trace_op, %ecx # 3rd argument: ftrace_pos
pushl %esp # 4th argument: pt_regs
SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
call ftrace_stub
addl $4, %esp # skip 4th argument
/* place IP below the new SP */
movl PT_OLDESP(%esp), %eax
movl PT_EIP(%esp), %ecx
movl %ecx, -4(%eax)
/* place EAX below that */
movl PT_EAX(%esp), %ecx
movl %ecx, -8(%eax)
popl %ebx
popl %ecx
popl %edx
popl %esi
popl %edi
popl %ebp
lea -8(%eax), %esp
popl %eax
jmp .Lftrace_ret
SYM_CODE_END(ftrace_regs_caller)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_CODE_START(ftrace_graph_caller)
pushl %eax
pushl %ecx
pushl %edx
movl 3*4(%esp), %eax
/* Even with frame pointers, fentry doesn't have one here */
lea 4*4(%esp), %edx
movl $0, %ecx
subl $MCOUNT_INSN_SIZE, %eax
call prepare_ftrace_return
popl %edx
popl %ecx
popl %eax
ret
SYM_CODE_END(ftrace_graph_caller)
.globl return_to_handler
return_to_handler:
pushl %eax
pushl %edx
movl $0, %eax
call ftrace_return_to_handler
movl %eax, %ecx
popl %edx
popl %eax
JMP_NOSPEC %ecx
#endif
|
