1 /*
2 * linux/kernel/profile.c
3 * Simple profiling. Manages a direct-mapped profile hit count buffer,
4 * with configurable resolution, support for restricting the cpus on
5 * which profiling is done, and switching between cpu time and
6 * schedule() calls via kernel command line parameters passed at boot.
7 *
8 * Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
9 * Red Hat, July 2004
10 * Consolidation of architecture support code for profiling,
11 * William Irwin, Oracle, July 2004
12 * Amortized hit count accounting via per-cpu open-addressed hashtables
13 * to resolve timer interrupt livelocks, William Irwin, Oracle, 2004
14 */
15
16 #include <linux/module.h>
17 #include <linux/profile.h>
18 #include <linux/bootmem.h>
19 #include <linux/notifier.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/cpu.h>
23 #include <linux/highmem.h>
24 #include <linux/mutex.h>
25 #include <linux/sched.h>
26 #include <asm/sections.h>
27 #include <asm/semaphore.h>
28 #include <asm/irq_regs.h>
29 #include <asm/ptrace.h>
30
31 struct profile_hit {
32 u32 pc, hits;
33 };
34 #define PROFILE_GRPSHIFT 3
35 #define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
36 #define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
37 #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
38
39 /* Oprofile timer tick hook */
40 static int (*timer_hook)(struct pt_regs *) __read_mostly;
41
42 static atomic_t *prof_buffer;
43 static unsigned long prof_len, prof_shift;
44
45 int prof_on __read_mostly;
46 EXPORT_SYMBOL_GPL(prof_on);
47
48 static cpumask_t prof_cpu_mask = CPU_MASK_ALL;
49 int prof_pid = -1;
50 #ifdef CONFIG_SMP
51 static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
52 static DEFINE_PER_CPU(int, cpu_profile_flip);
53 static DEFINE_MUTEX(profile_flip_mutex);
54 #endif /* CONFIG_SMP */
55
56 static int __init profile_setup(char *str)
57 {
58 static char __initdata schedstr[] = "schedule";
59 static char __initdata sleepstr[] = "sleep";
60 static char __initdata kvmstr[] = "kvm";
61 int par;
62
63 if (!strncmp(str, sleepstr, strlen(sleepstr))) {
64 #ifdef CONFIG_SCHEDSTATS
65 prof_on = SLEEP_PROFILING;
66 if (str[strlen(sleepstr)] == ',')
67 str += strlen(sleepstr) + 1;
68 if (get_option(&str, &par))
69 prof_shift = par;
70 printk(KERN_INFO
71 "kernel sleep profiling enabled (shift: %ld)\n",
72 prof_shift);
73 #else
74 printk(KERN_WARNING
75 "kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
76 #endif /* CONFIG_SCHEDSTATS */
77 } else if (!strncmp(str, schedstr, strlen(schedstr))) {
78 prof_on = SCHED_PROFILING;
79 if (str[strlen(schedstr)] == ',')
80 str += strlen(schedstr) + 1;
81 if (get_option(&str, &par))
82 prof_shift = par;
83 printk(KERN_INFO
84 "kernel schedule profiling enabled (shift: %ld)\n",
85 prof_shift);
86 } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
87 prof_on = KVM_PROFILING;
88 if (str[strlen(kvmstr)] == ',')
89 str += strlen(kvmstr) + 1;
90 if (get_option(&str, &par))
91 prof_shift = par;
92 printk(KERN_INFO
93 "kernel KVM profiling enabled (shift: %ld)\n",
94 prof_shift);
95 } else if (get_option(&str, &par)) {
96 prof_shift = par;
97 prof_on = CPU_PROFILING;
98 printk(KERN_INFO "kernel profiling enabled (shift: %ld)\n",
99 prof_shift);
100 }
101 return 1;
102 }
103 __setup("profile=", profile_setup);
104
105
106 void __init profile_init(void)
107 {
108 if (!prof_on)
109 return;
110
111 /* only text is profiled */
112 prof_len = (_etext - _stext) >> prof_shift;
113 prof_buffer = alloc_bootmem(prof_len*sizeof(atomic_t));
114 }
115
116 /* Profile event notifications */
117
118 #ifdef CONFIG_PROFILING
119
120 static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
121 static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
122 static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
123
124 void profile_task_exit(struct task_struct *task)
125 {
126 blocking_notifier_call_chain(&task_exit_notifier, 0, task);
127 }
128
129 int profile_handoff_task(struct task_struct *task)
130 {
131 int ret;
132 ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
133 return (ret == NOTIFY_OK) ? 1 : 0;
134 }
135
136 void profile_munmap(unsigned long addr)
137 {
138 blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
139 }
140
141 int task_handoff_register(struct notifier_block *n)
142 {
143 return atomic_notifier_chain_register(&task_free_notifier, n);
144 }
145 EXPORT_SYMBOL_GPL(task_handoff_register);
146
147 int task_handoff_unregister(struct notifier_block *n)
148 {
149 return atomic_notifier_chain_unregister(&task_free_notifier, n);
150 }
151 EXPORT_SYMBOL_GPL(task_handoff_unregister);
152
153 int profile_event_register(enum profile_type type, struct notifier_block *n)
154 {
155 int err = -EINVAL;
156
157 switch (type) {
158 case PROFILE_TASK_EXIT:
159 err = blocking_notifier_chain_register(
160 &task_exit_notifier, n);
161 break;
162 case PROFILE_MUNMAP:
163 err = blocking_notifier_chain_register(
164 &munmap_notifier, n);
165 break;
166 }
167
168 return err;
169 }
170 EXPORT_SYMBOL_GPL(profile_event_register);
171
172 int profile_event_unregister(enum profile_type type, struct notifier_block *n)
173 {
174 int err = -EINVAL;
175
176 switch (type) {
177 case PROFILE_TASK_EXIT:
178 err = blocking_notifier_chain_unregister(
179 &task_exit_notifier, n);
180 break;
181 case PROFILE_MUNMAP:
182 err = blocking_notifier_chain_unregister(
183 &munmap_notifier, n);
184 break;
185 }
186
187 return err;
188 }
189 EXPORT_SYMBOL_GPL(profile_event_unregister);
190
191 int register_timer_hook(int (*hook)(struct pt_regs *))
192 {
193 if (timer_hook)
194 return -EBUSY;
195 timer_hook = hook;
196 return 0;
197 }
198 EXPORT_SYMBOL_GPL(register_timer_hook);
199
200 void unregister_timer_hook(int (*hook)(struct pt_regs *))
201 {
202 WARN_ON(hook != timer_hook);
203 timer_hook = NULL;
204 /* make sure all CPUs see the NULL hook */
205 synchronize_sched(); /* Allow ongoing interrupts to complete. */
206 }
207 EXPORT_SYMBOL_GPL(unregister_timer_hook);
208
209 #endif /* CONFIG_PROFILING */
210
211
212 #ifdef CONFIG_SMP
213 /*
214 * Each cpu has a pair of open-addressed hashtables for pending
215 * profile hits. read_profile() IPI's all cpus to request them
216 * to flip buffers and flushes their contents to prof_buffer itself.
217 * Flip requests are serialized by the profile_flip_mutex. The sole
218 * use of having a second hashtable is for avoiding cacheline
219 * contention that would otherwise happen during flushes of pending
220 * profile hits required for the accuracy of reported profile hits
221 * and so resurrect the interrupt livelock issue.
222 *
223 * The open-addressed hashtables are indexed by profile buffer slot
224 * and hold the number of pending hits to that profile buffer slot on
225 * a cpu in an entry. When the hashtable overflows, all pending hits
226 * are accounted to their corresponding profile buffer slots with
227 * atomic_add() and the hashtable emptied. As numerous pending hits
228 * may be accounted to a profile buffer slot in a hashtable entry,
229 * this amortizes a number of atomic profile buffer increments likely
230 * to be far larger than the number of entries in the hashtable,
231 * particularly given that the number of distinct profile buffer
232 * positions to which hits are accounted during short intervals (e.g.
233 * several seconds) is usually very small. Exclusion from buffer
234 * flipping is provided by interrupt disablement (note that for
235 * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
236 * process context).
237 * The hash function is meant to be lightweight as opposed to strong,
238 * and was vaguely inspired by ppc64 firmware-supported inverted
239 * pagetable hash functions, but uses a full hashtable full of finite
240 * collision chains, not just pairs of them.
241 *
242 * -- wli
243 */
244 static void __profile_flip_buffers(void *unused)
245 {
246 int cpu = smp_processor_id();
247
248 per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
249 }
250
251 static void profile_flip_buffers(void)
252 {
253 int i, j, cpu;
254
255 mutex_lock(&profile_flip_mutex);
256 j = per_cpu(cpu_profile_flip, get_cpu());
257 put_cpu();
258 on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
259 for_each_online_cpu(cpu) {
260 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
261 for (i = 0; i < NR_PROFILE_HIT; ++i) {
262 if (!hits[i].hits) {
263 if (hits[i].pc)
264 hits[i].pc = 0;
265 continue;
266 }
267 atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
268 hits[i].hits = hits[i].pc = 0;
269 }
270 }
271 mutex_unlock(&profile_flip_mutex);
272 }
273
274 static void profile_discard_flip_buffers(void)
275 {
276 int i, cpu;
277
278 mutex_lock(&profile_flip_mutex);
279 i = per_cpu(cpu_profile_flip, get_cpu());
280 put_cpu();
281 on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
282 for_each_online_cpu(cpu) {
283 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
284 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
285 }
286 mutex_unlock(&profile_flip_mutex);
287 }
288
289 void profile_hits(int type, void *__pc, unsigned int nr_hits)
290 {
291 unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
292 int i, j, cpu;
293 struct profile_hit *hits;
294
295 if (prof_on != type || !prof_buffer)
296 return;
297 pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
298 i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
299 secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
300 cpu = get_cpu();
301 hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
302 if (!hits) {
303 put_cpu();
304 return;
305 }
306 /*
307 * We buffer the global profiler buffer into a per-CPU
308 * queue and thus reduce the number of global (and possibly
309 * NUMA-alien) accesses. The write-queue is self-coalescing:
310 */
311 local_irq_save(flags);
312 do {
313 for (j = 0; j < PROFILE_GRPSZ; ++j) {
314 if (hits[i + j].pc == pc) {
315 hits[i + j].hits += nr_hits;
316 goto out;
317 } else if (!hits[i + j].hits) {
318 hits[i + j].pc = pc;
319 hits[i + j].hits = nr_hits;
320 goto out;
321 }
322 }
323 i = (i + secondary) & (NR_PROFILE_HIT - 1);
324 } while (i != primary);
325
326 /*
327 * Add the current hit(s) and flush the write-queue out
328 * to the global buffer:
329 */
330 atomic_add(nr_hits, &prof_buffer[pc]);
331 for (i = 0; i < NR_PROFILE_HIT; ++i) {
332 atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
333 hits[i].pc = hits[i].hits = 0;
334 }
335 out:
336 local_irq_restore(flags);
337 put_cpu();
338 }
339
340 static int __devinit profile_cpu_callback(struct notifier_block *info,
341 unsigned long action, void *__cpu)
342 {
343 int node, cpu = (unsigned long)__cpu;
344 struct page *page;
345
346 switch (action) {
347 case CPU_UP_PREPARE:
348 case CPU_UP_PREPARE_FROZEN:
349 node = cpu_to_node(cpu);
350 per_cpu(cpu_profile_flip, cpu) = 0;
351 if (!per_cpu(cpu_profile_hits, cpu)[1]) {
352 page = alloc_pages_node(node,
353 GFP_KERNEL | __GFP_ZERO,
354 0);
355 if (!page)
356 return NOTIFY_BAD;
357 per_cpu(cpu_profile_hits, cpu)[1] = page_address(page);
358 }
359 if (!per_cpu(cpu_profile_hits, cpu)[0]) {
360 page = alloc_pages_node(node,
361 GFP_KERNEL | __GFP_ZERO,
362 0);
363 if (!page)
364 goto out_free;
365 per_cpu(cpu_profile_hits, cpu)[0] = page_address(page);
366 }
367 break;
368 out_free:
369 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
370 per_cpu(cpu_profile_hits, cpu)[1] = NULL;
371 __free_page(page);
372 return NOTIFY_BAD;
373 case CPU_ONLINE:
374 case CPU_ONLINE_FROZEN:
375 cpu_set(cpu, prof_cpu_mask);
376 break;
377 case CPU_UP_CANCELED:
378 case CPU_UP_CANCELED_FROZEN:
379 case CPU_DEAD:
380 case CPU_DEAD_FROZEN:
381 cpu_clear(cpu, prof_cpu_mask);
382 if (per_cpu(cpu_profile_hits, cpu)[0]) {
383 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
384 per_cpu(cpu_profile_hits, cpu)[0] = NULL;
385 __free_page(page);
386 }
387 if (per_cpu(cpu_profile_hits, cpu)[1]) {
388 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
389 per_cpu(cpu_profile_hits, cpu)[1] = NULL;
390 __free_page(page);
391 }
392 break;
393 }
394 return NOTIFY_OK;
395 }
396 #else /* !CONFIG_SMP */
397 #define profile_flip_buffers() do { } while (0)
398 #define profile_discard_flip_buffers() do { } while (0)
399 #define profile_cpu_callback NULL
400
401 void profile_hits(int type, void *__pc, unsigned int nr_hits)
402 {
403 unsigned long pc;
404
405 if (prof_on != type || !prof_buffer)
406 return;
407 pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
408 atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
409 }
410 #endif /* !CONFIG_SMP */
411 EXPORT_SYMBOL_GPL(profile_hits);
412
413 void __profile_tick(int type, struct pt_regs *regs)
414 {
415 if (type == CPU_PROFILING && timer_hook)
416 timer_hook(regs);
417 if (!user_mode(regs) && cpu_isset(smp_processor_id(), prof_cpu_mask) &&
418 (prof_pid == -1 || prof_pid == current->pid))
419 profile_hit(type, (void *)profile_pc(regs));
420 }
421
422 void profile_tick(int type)
423 {
424 return __profile_tick(type, get_irq_regs());
425 }
426
427 #ifdef CONFIG_PROC_FS
428 #include <linux/proc_fs.h>
429 #include <asm/uaccess.h>
430 #include <asm/ptrace.h>
431
432 static int prof_cpu_mask_read_proc(char *page, char **start, off_t off,
433 int count, int *eof, void *data)
434 {
435 int len = cpumask_scnprintf(page, count, *(cpumask_t *)data);
436 if (count - len < 2)
437 return -EINVAL;
438 len += sprintf(page + len, "\n");
439 return len;
440 }
441
442 static int prof_cpu_mask_write_proc(struct file *file,
443 const char __user *buffer, unsigned long count, void *data)
444 {
445 cpumask_t *mask = (cpumask_t *)data;
446 unsigned long full_count = count, err;
447 cpumask_t new_value;
448
449 err = cpumask_parse_user(buffer, count, new_value);
450 if (err)
451 return err;
452
453 *mask = new_value;
454 return full_count;
455 }
456
457 void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir)
458 {
459 struct proc_dir_entry *entry;
460
461 /* create /proc/irq/prof_cpu_mask */
462 entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir);
463 if (!entry)
464 return;
465 entry->data = (void *)&prof_cpu_mask;
466 entry->read_proc = prof_cpu_mask_read_proc;
467 entry->write_proc = prof_cpu_mask_write_proc;
468 }
469
470 /*
471 * This function accesses profiling information. The returned data is
472 * binary: the sampling step and the actual contents of the profile
473 * buffer. Use of the program readprofile is recommended in order to
474 * get meaningful info out of these data.
475 */
476 static ssize_t
477 read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
478 {
479 unsigned long p = *ppos;
480 ssize_t read;
481 char *pnt;
482 unsigned int sample_step = 1 << prof_shift;
483
484 profile_flip_buffers();
485 if (p >= (prof_len+1)*sizeof(unsigned int))
486 return 0;
487 if (count > (prof_len+1)*sizeof(unsigned int) - p)
488 count = (prof_len+1)*sizeof(unsigned int) - p;
489 read = 0;
490
491 while (p < sizeof(unsigned int) && count > 0) {
492 if (put_user(*((char *)(&sample_step)+p), buf))
493 return -EFAULT;
494 buf++; p++; count--; read++;
495 }
496 pnt = (char *)prof_buffer + p - sizeof(atomic_t);
497 if (copy_to_user(buf, (void *)pnt, count))
498 return -EFAULT;
499 read += count;
500 *ppos += read;
501 return read;
502 }
503
504 /*
505 * Writing to /proc/profile resets the counters
506 *
507 * Writing a 'profiling multiplier' value into it also re-sets the profiling
508 * interrupt frequency, on architectures that support this.
509 */
510 static ssize_t write_profile(struct file *file, const char __user *buf,
511 size_t count, loff_t *ppos)
512 {
513 #ifdef CONFIG_SMP
514 extern int setup_profiling_timer(unsigned int multiplier);
515
516 if (count == sizeof(int)) {
517 unsigned int multiplier;
518
519 if (copy_from_user(&multiplier, buf, sizeof(int)))
520 return -EFAULT;
521
522 if (setup_profiling_timer(multiplier))
523 return -EINVAL;
524 }
525 #endif
526 profile_discard_flip_buffers();
527 memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
528 return count;
529 }
530
531 static const struct file_operations proc_profile_operations = {
532 .read = read_profile,
533 .write = write_profile,
534 };
535
536 #ifdef CONFIG_SMP
537 static void __init profile_nop(void *unused)
538 {
539 }
540
541 static int __init create_hash_tables(void)
542 {
543 int cpu;
544
545 for_each_online_cpu(cpu) {
546 int node = cpu_to_node(cpu);
547 struct page *page;
548
549 page = alloc_pages_node(node,
550 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
551 0);
552 if (!page)
553 goto out_cleanup;
554 per_cpu(cpu_profile_hits, cpu)[1]
555 = (struct profile_hit *)page_address(page);
556 page = alloc_pages_node(node,
557 GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
558 0);
559 if (!page)
560 goto out_cleanup;
561 per_cpu(cpu_profile_hits, cpu)[0]
562 = (struct profile_hit *)page_address(page);
563 }
564 return 0;
565 out_cleanup:
566 prof_on = 0;
567 smp_mb();
568 on_each_cpu(profile_nop, NULL, 0, 1);
569 for_each_online_cpu(cpu) {
570 struct page *page;
571
572 if (per_cpu(cpu_profile_hits, cpu)[0]) {
573 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
574 per_cpu(cpu_profile_hits, cpu)[0] = NULL;
575 __free_page(page);
576 }
577 if (per_cpu(cpu_profile_hits, cpu)[1]) {
578 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
579 per_cpu(cpu_profile_hits, cpu)[1] = NULL;
580 __free_page(page);
581 }
582 }
583 return -1;
584 }
585 #else
586 #define create_hash_tables() ({ 0; })
587 #endif
588
589 static int __init create_proc_profile(void)
590 {
591 struct proc_dir_entry *entry;
592
593 if (!prof_on)
594 return 0;
595 if (create_hash_tables())
596 return -1;
597 entry = create_proc_entry("profile", S_IWUSR | S_IRUGO, NULL);
598 if (!entry)
599 return 0;
600 entry->proc_fops = &proc_profile_operations;
601 entry->size = (1+prof_len) * sizeof(atomic_t);
602 hotcpu_notifier(profile_cpu_callback, 0);
603 return 0;
604 }
605 module_init(create_proc_profile);
606 #endif /* CONFIG_PROC_FS */
607
|
This page was automatically generated by the
LXR engine.
|