Initial commit of some oprofile support
[akaros.git] / kern / src / oprofile / cpu_buffer.c
1 /**
2  * @file cpu_buffer.c
3  *
4  * @remark Copyright 2002-2009 OProfile authors
5  * @remark Read the file COPYING
6  *
7  * @author John Levon <levon@movementarian.org>
8  * @author Barry Kasindorf <barry.kasindorf@amd.com>
9  * @author Robert Richter <robert.richter@amd.com>
10  *
11  * Each CPU has a local buffer that stores PC value/event
12  * pairs. We also log context switches when we notice them.
13  * Eventually each CPU's buffer is processed into the global
14  * event buffer by sync_buffer().
15  *
16  * We use a local buffer for two reasons: an NMI or similar
17  * interrupt cannot synchronise, and high sampling rates
18  * would lead to catastrophic global synchronisation if
19  * a global buffer was used.
20  */
21
22 #include "event_buffer.h"
23 #include "cpu_buffer.h"
24 #include "buffer_sync.h"
25 #include "oprof.h"
26
27 #define OP_BUFFER_FLAGS 0
28
29 static struct ring_buffer *op_ring_buffer;
30 DEFINE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer);
31
32 static void wq_sync_buffer(struct work_struct *work);
33
34 #define DEFAULT_TIMER_EXPIRE (HZ / 10)
35 static int work_enabled;
36
37 unsigned long oprofile_get_cpu_buffer_size(void)
38 {
39         return oprofile_cpu_buffer_size;
40 }
41
42 void oprofile_cpu_buffer_inc_smpl_lost(void)
43 {
44         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
45
46         cpu_buf->sample_lost_overflow++;
47 }
48
49 void free_cpu_buffers(void)
50 {
51         if (op_ring_buffer)
52                 ring_buffer_free(op_ring_buffer);
53         op_ring_buffer = NULL;
54 }
55
56 #define RB_EVENT_HDR_SIZE 4
57
58 int alloc_cpu_buffers(void)
59 {
60         int i;
61
62         unsigned long buffer_size = oprofile_cpu_buffer_size;
63         unsigned long byte_size = buffer_size * (sizeof(struct op_sample) +
64                                                  RB_EVENT_HDR_SIZE);
65
66         op_ring_buffer = ring_buffer_alloc(byte_size, OP_BUFFER_FLAGS);
67         if (!op_ring_buffer)
68                 goto fail;
69
70         for_each_possible_cpu(i) {
71                 struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
72
73                 b->last_task = NULL;
74                 b->last_is_kernel = -1;
75                 b->tracing = 0;
76                 b->buffer_size = buffer_size;
77                 b->sample_received = 0;
78                 b->sample_lost_overflow = 0;
79                 b->backtrace_aborted = 0;
80                 b->sample_invalid_eip = 0;
81                 b->cpu = i;
82                 INIT_DELAYED_WORK(&b->work, wq_sync_buffer);
83         }
84         return 0;
85
86 fail:
87         free_cpu_buffers();
88         return -ENOMEM;
89 }
90
91 void start_cpu_work(void)
92 {
93         int i;
94
95         work_enabled = 1;
96
97         for_each_online_cpu(i) {
98                 struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
99
100                 /*
101                  * Spread the work by 1 jiffy per cpu so they dont all
102                  * fire at once.
103                  */
104                 schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i);
105         }
106 }
107
108 void end_cpu_work(void)
109 {
110         work_enabled = 0;
111 }
112
113 void flush_cpu_work(void)
114 {
115         int i;
116
117         for_each_online_cpu(i) {
118                 struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i);
119
120                 /* these works are per-cpu, no need for flush_sync */
121                 flush_delayed_work(&b->work);
122         }
123 }
124
125 /*
126  * This function prepares the cpu buffer to write a sample.
127  *
128  * Struct op_entry is used during operations on the ring buffer while
129  * struct op_sample contains the data that is stored in the ring
130  * buffer. Struct entry can be uninitialized. The function reserves a
131  * data array that is specified by size. Use
132  * op_cpu_buffer_write_commit() after preparing the sample. In case of
133  * errors a null pointer is returned, otherwise the pointer to the
134  * sample.
135  *
136  */
137 struct op_sample
138 *op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size)
139 {
140         entry->event = ring_buffer_lock_reserve
141                 (op_ring_buffer, sizeof(struct op_sample) +
142                  size * sizeof(entry->sample->data[0]));
143         if (!entry->event)
144                 return NULL;
145         entry->sample = ring_buffer_event_data(entry->event);
146         entry->size = size;
147         entry->data = entry->sample->data;
148
149         return entry->sample;
150 }
151
152 int op_cpu_buffer_write_commit(struct op_entry *entry)
153 {
154         return ring_buffer_unlock_commit(op_ring_buffer, entry->event);
155 }
156
157 struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu)
158 {
159         struct ring_buffer_event *e;
160         e = ring_buffer_consume(op_ring_buffer, cpu, NULL, NULL);
161         if (!e)
162                 return NULL;
163
164         entry->event = e;
165         entry->sample = ring_buffer_event_data(e);
166         entry->size = (ring_buffer_event_length(e) - sizeof(struct op_sample))
167                 / sizeof(entry->sample->data[0]);
168         entry->data = entry->sample->data;
169         return entry->sample;
170 }
171
172 unsigned long op_cpu_buffer_entries(int cpu)
173 {
174         return ring_buffer_entries_cpu(op_ring_buffer, cpu);
175 }
176
177 static int
178 op_add_code(struct oprofile_cpu_buffer *cpu_buf, unsigned long backtrace,
179             int is_kernel, struct task_struct *task)
180 {
181         struct op_entry entry;
182         struct op_sample *sample;
183         unsigned long flags;
184         int size;
185
186         flags = 0;
187
188         if (backtrace)
189                 flags |= TRACE_BEGIN;
190
191         /* notice a switch from user->kernel or vice versa */
192         is_kernel = !!is_kernel;
193         if (cpu_buf->last_is_kernel != is_kernel) {
194                 cpu_buf->last_is_kernel = is_kernel;
195                 flags |= KERNEL_CTX_SWITCH;
196                 if (is_kernel)
197                         flags |= IS_KERNEL;
198         }
199
200         /* notice a task switch */
201         if (cpu_buf->last_task != task) {
202                 cpu_buf->last_task = task;
203                 flags |= USER_CTX_SWITCH;
204         }
205
206         if (!flags)
207                 /* nothing to do */
208                 return 0;
209
210         if (flags & USER_CTX_SWITCH)
211                 size = 1;
212         else
213                 size = 0;
214
215         sample = op_cpu_buffer_write_reserve(&entry, size);
216         if (!sample)
217                 return -ENOMEM;
218
219         sample->eip = ESCAPE_CODE;
220         sample->event = flags;
221
222         if (size)
223                 op_cpu_buffer_add_data(&entry, (unsigned long)task);
224
225         op_cpu_buffer_write_commit(&entry);
226
227         return 0;
228 }
229
230 static inline int
231 op_add_sample(struct oprofile_cpu_buffer *cpu_buf,
232               unsigned long pc, unsigned long event)
233 {
234         struct op_entry entry;
235         struct op_sample *sample;
236
237         sample = op_cpu_buffer_write_reserve(&entry, 0);
238         if (!sample)
239                 return -ENOMEM;
240
241         sample->eip = pc;
242         sample->event = event;
243
244         return op_cpu_buffer_write_commit(&entry);
245 }
246
247 /*
248  * This must be safe from any context.
249  *
250  * is_kernel is needed because on some architectures you cannot
251  * tell if you are in kernel or user space simply by looking at
252  * pc. We tag this in the buffer by generating kernel enter/exit
253  * events whenever is_kernel changes
254  */
255 static int
256 log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
257            unsigned long backtrace, int is_kernel, unsigned long event,
258            struct task_struct *task)
259 {
260         struct task_struct *tsk = task ? task : current;
261         cpu_buf->sample_received++;
262
263         if (pc == ESCAPE_CODE) {
264                 cpu_buf->sample_invalid_eip++;
265                 return 0;
266         }
267
268         if (op_add_code(cpu_buf, backtrace, is_kernel, tsk))
269                 goto fail;
270
271         if (op_add_sample(cpu_buf, pc, event))
272                 goto fail;
273
274         return 1;
275
276 fail:
277         cpu_buf->sample_lost_overflow++;
278         return 0;
279 }
280
281 static inline void oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
282 {
283         cpu_buf->tracing = 1;
284 }
285
286 static inline void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
287 {
288         cpu_buf->tracing = 0;
289 }
290
291 static inline void
292 __oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
293                           unsigned long event, int is_kernel,
294                           struct task_struct *task)
295 {
296         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
297         unsigned long backtrace = oprofile_backtrace_depth;
298
299         /*
300          * if log_sample() fail we can't backtrace since we lost the
301          * source of this event
302          */
303         if (!log_sample(cpu_buf, pc, backtrace, is_kernel, event, task))
304                 /* failed */
305                 return;
306
307         if (!backtrace)
308                 return;
309
310         oprofile_begin_trace(cpu_buf);
311         oprofile_ops.backtrace(regs, backtrace);
312         oprofile_end_trace(cpu_buf);
313 }
314
315 void oprofile_add_ext_hw_sample(unsigned long pc, struct pt_regs * const regs,
316                                 unsigned long event, int is_kernel,
317                                 struct task_struct *task)
318 {
319         __oprofile_add_ext_sample(pc, regs, event, is_kernel, task);
320 }
321
322 void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
323                              unsigned long event, int is_kernel)
324 {
325         __oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL);
326 }
327
328 void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
329 {
330         int is_kernel;
331         unsigned long pc;
332
333         if (likely(regs)) {
334                 is_kernel = !user_mode(regs);
335                 pc = profile_pc(regs);
336         } else {
337                 is_kernel = 0;    /* This value will not be used */
338                 pc = ESCAPE_CODE; /* as this causes an early return. */
339         }
340
341         __oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL);
342 }
343
344 /*
345  * Add samples with data to the ring buffer.
346  *
347  * Use oprofile_add_data(&entry, val) to add data and
348  * oprofile_write_commit(&entry) to commit the sample.
349  */
350 void
351 oprofile_write_reserve(struct op_entry *entry, struct pt_regs * const regs,
352                        unsigned long pc, int code, int size)
353 {
354         struct op_sample *sample;
355         int is_kernel = !user_mode(regs);
356         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
357
358         cpu_buf->sample_received++;
359
360         /* no backtraces for samples with data */
361         if (op_add_code(cpu_buf, 0, is_kernel, current))
362                 goto fail;
363
364         sample = op_cpu_buffer_write_reserve(entry, size + 2);
365         if (!sample)
366                 goto fail;
367         sample->eip = ESCAPE_CODE;
368         sample->event = 0;              /* no flags */
369
370         op_cpu_buffer_add_data(entry, code);
371         op_cpu_buffer_add_data(entry, pc);
372
373         return;
374
375 fail:
376         entry->event = NULL;
377         cpu_buf->sample_lost_overflow++;
378 }
379
380 int oprofile_add_data(struct op_entry *entry, unsigned long val)
381 {
382         if (!entry->event)
383                 return 0;
384         return op_cpu_buffer_add_data(entry, val);
385 }
386
387 int oprofile_add_data64(struct op_entry *entry, u64 val)
388 {
389         if (!entry->event)
390                 return 0;
391         if (op_cpu_buffer_get_size(entry) < 2)
392                 /*
393                  * the function returns 0 to indicate a too small
394                  * buffer, even if there is some space left
395                  */
396                 return 0;
397         if (!op_cpu_buffer_add_data(entry, (u32)val))
398                 return 0;
399         return op_cpu_buffer_add_data(entry, (u32)(val >> 32));
400 }
401
402 int oprofile_write_commit(struct op_entry *entry)
403 {
404         if (!entry->event)
405                 return -EINVAL;
406         return op_cpu_buffer_write_commit(entry);
407 }
408
409 void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
410 {
411         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
412         log_sample(cpu_buf, pc, 0, is_kernel, event, NULL);
413 }
414
415 void oprofile_add_trace(unsigned long pc)
416 {
417         struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(op_cpu_buffer);
418
419         if (!cpu_buf->tracing)
420                 return;
421
422         /*
423          * broken frame can give an eip with the same value as an
424          * escape code, abort the trace if we get it
425          */
426         if (pc == ESCAPE_CODE)
427                 goto fail;
428
429         if (op_add_sample(cpu_buf, pc, 0))
430                 goto fail;
431
432         return;
433 fail:
434         cpu_buf->tracing = 0;
435         cpu_buf->backtrace_aborted++;
436         return;
437 }
438
439 /*
440  * This serves to avoid cpu buffer overflow, and makes sure
441  * the task mortuary progresses
442  *
443  * By using schedule_delayed_work_on and then schedule_delayed_work
444  * we guarantee this will stay on the correct cpu
445  */
446 static void wq_sync_buffer(struct work_struct *work)
447 {
448         struct oprofile_cpu_buffer *b =
449                 container_of(work, struct oprofile_cpu_buffer, work.work);
450         if (b->cpu != smp_processor_id() && !cpu_online(b->cpu)) {
451                 cancel_delayed_work(&b->work);
452                 return;
453         }
454         sync_buffer(b->cpu);
455
456         /* don't re-add the work if we're shutting down */
457         if (work_enabled)
458                 schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE);
459 }