From ba361c92e73c771fcbbbd24c2c03c322e2de2e31 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 17 Oct 2012 18:50:13 -0300 Subject: perf tools: Don't stop synthesizing threads when one vanishes The perf_event__synthesize_threads routine synthesizes all the existing threads in the system, because we don't have any kernel facilities to ask for PERF_RECORD_{FORK,MMAP,COMM} for existing threads. It was returning an error as soon as one thread couldn't be synthesized, which is a bit extreme when, for instance, a forkish workload is running, like a kernel compile. Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Mike Galbraith Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lkml.kernel.org/n/tip-i7oas1eodpoer2bx38fwyasv@git.kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/event.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c index 0ae444ef142..ca9ca285406 100644 --- a/tools/perf/util/event.c +++ b/tools/perf/util/event.c @@ -405,16 +405,15 @@ int perf_event__synthesize_threads(struct perf_tool *tool, if (*end) /* only interested in proper numerical dirents */ continue; - - if (__event__synthesize_thread(comm_event, mmap_event, pid, 1, - process, tool, machine) != 0) { - err = -1; - goto out_closedir; - } + /* + * We may race with exiting thread, so don't stop just because + * one thread couldn't be synthesized. + */ + __event__synthesize_thread(comm_event, mmap_event, pid, 1, + process, tool, machine); } err = 0; -out_closedir: closedir(proc); out_free_mmap: free(mmap_event); -- cgit v1.2.3 From efd5745e43f3aabd95d521289e0caa0e30668cf4 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 17 Oct 2012 17:09:46 -0300 Subject: perf trace: Count number of events for each thread and globally The nr_events in trace__run was local, but we will need it in other trace methods, move it to struct trace. We'll also need the number of events per thread, so introduce a nr_events method for that in struct thread_trace. Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Mike Galbraith Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Link: http://lkml.kernel.org/n/tip-ksutaz0mtejnf7e6az3ca1td@git.kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-trace.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index ba055103b52..c95a3e9b033 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -67,6 +67,7 @@ struct thread_trace { u64 entry_time; u64 exit_time; bool entry_pending; + unsigned long nr_events; char *entry_str; }; @@ -77,16 +78,21 @@ static struct thread_trace *thread_trace__new(void) static struct thread_trace *thread__trace(struct thread *thread) { + struct thread_trace *ttrace; + if (thread == NULL) goto fail; if (thread->priv == NULL) thread->priv = thread_trace__new(); - + if (thread->priv == NULL) goto fail; - return thread->priv; + ttrace = thread->priv; + ++ttrace->nr_events; + + return ttrace; fail: color_fprintf(stdout, PERF_COLOR_RED, "WARNING: not enough memory, dropping samples!\n"); @@ -102,6 +108,7 @@ struct trace { struct perf_record_opts opts; struct machine host; u64 base_time; + unsigned long nr_events; bool multiple_threads; double duration_filter; }; @@ -386,7 +393,8 @@ static int trace__run(struct trace *trace, int argc, const char **argv) { struct perf_evlist *evlist = perf_evlist__new(NULL, NULL); struct perf_evsel *evsel; - int err = -1, i, nr_events = 0, before; + int err = -1, i; + unsigned long before; const bool forks = argc > 0; if (evlist == NULL) { @@ -444,7 +452,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv) trace->multiple_threads = evlist->threads->map[0] == -1 || evlist->threads->nr > 1; again: - before = nr_events; + before = trace->nr_events; for (i = 0; i < evlist->nr_mmaps; i++) { union perf_event *event; @@ -454,7 +462,7 @@ again: tracepoint_handler handler; struct perf_sample sample; - ++nr_events; + ++trace->nr_events; err = perf_evlist__parse_sample(evlist, event, &sample); if (err) { @@ -495,7 +503,7 @@ again: } } - if (nr_events == before) { + if (trace->nr_events == before) { if (done) goto out_delete_evlist; -- cgit v1.2.3 From 1302d88e66f12a7b46a5598e641d93f0713007e0 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 17 Oct 2012 17:13:12 -0300 Subject: perf trace: Use sched:sched_stat_runtime to provide a thread summary [root@sandy ~]# perf trace --sched --duration 0.100 --pid `pidof firefox` 17079.847 ( 0.009 ms): 17643 poll(ufds: 140037623086496, nfds: 11, timeout_msecs: 0) = 0 Timeout 17079.892 ( 0.010 ms): 17643 read(fd: 4, buf: 140038178943092, count: 4096 ) = -1 EAGAIN Resource temporarily unavailable 17079.921 ( 0.013 ms): 17643 poll(ufds: 140037623086496, nfds: 11, timeout_msecs: 0) = 0 Timeout 17079.949 ( 0.009 ms): 17643 read(fd: 4, buf: 140038178943092, count: 4096 ) = -1 EAGAIN Resource temporarily unavailable ^C _____________________________________________________________________ __) Summary of events (__ [ task - pid ] [ events ] [ ratio ] [ runtime ] _____________________________________________________________________ firefox - 17643 : 18013 [ 72.2% ] 359.110 ms firefox - 17663 : 41 [ 0.2% ] 21.439 ms firefox - 17664 : 6840 [ 27.4% ] 133.642 ms firefox - 17667 : 46 [ 0.2% ] 0.682 ms [root@sandy ~]# This is equivalent to the 'perf trace summary' subcomand in the tmp.perf/trace2 branch. Another example, setting a huge duration filter to get just a system wide summary: [root@sandy ~]# perf trace --duration 10000.0 --sched ^C _____________________________________________________________________ __) Summary of events (__ [ task - pid ] [ events ] [ ratio ] [ runtime ] _____________________________________________________________________ scsi_eh_1 - 258 : 15 [ 0.0% ] 0.133 ms kworker/0:1H - 322 : 13 [ 0.0% ] 0.032 ms jbd2/dm-0-8 - 384 : 4 [ 0.0% ] 0.115 ms flush-253:0 - 470 : 1 [ 0.0% ] 0.027 ms firefox - 950 : 4783 [ 0.1% ] 24.863 ms firefox - 992 : 1883 [ 0.1% ] 6.808 ms firefox - 995 : 35 [ 0.0% ] 0.111 ms ksoftirqd/6 - 4362 : 2 [ 0.0% ] 0.005 ms ksoftirqd/7 - 4365 : 1 [ 0.0% ] 0.007 ms Xorg - 4671 : 148 [ 0.0% ] 0.912 ms gnome-settings- - 4846 : 14 [ 0.0% ] 0.086 ms seahorse-daemon - 4847 : 14 [ 0.0% ] 0.092 ms gnome-panel - 4875 : 46 [ 0.0% ] 0.159 ms gnome-power-man - 4918 : 16 [ 0.0% ] 0.065 ms gvfs-afc-volume - 4992 : 77 [ 0.0% ] 0.136 ms gnome-screensav - 5114 : 24 [ 0.0% ] 0.128 ms xchat - 8082 : 466 [ 0.0% ] 2.019 ms synergyc - 8369 : 941 [ 0.0% ] 3.291 ms synergyc - 8371 : 85 [ 0.0% ] 1.817 ms jbd2/dm-4-8 - 9352 : 4 [ 0.0% ] 0.109 ms rpcbind - 9786 : 3 [ 0.0% ] 0.017 ms rtkit-daemon - 12802 : 10 [ 0.0% ] 0.038 ms rtkit-daemon - 12803 : 8 [ 0.0% ] 0.000 ms udisks-daemon - 13020 : 27 [ 0.0% ] 0.240 ms kworker/7:0 - 14651 : 669 [ 0.0% ] 2.616 ms kworker/5:1 - 16220 : 2 [ 0.0% ] 0.069 ms kworker/4:0 - 19776 : 13 [ 0.0% ] 0.176 ms openvpn - 20131 : 133 [ 0.0% ] 0.762 ms plugin-containe - 20508 : 60658 [ 1.7% ] 131.153 ms npviewer.bin - 20520 : 72208 [ 2.0% ] 138.945 ms npviewer.bin - 20542 : 35 [ 0.0% ] 0.074 ms npviewer.bin - 20543 : 30 [ 0.0% ] 0.074 ms npviewer.bin - 20547 : 35 [ 0.0% ] 0.092 ms npviewer.bin - 20552 : 35 [ 0.0% ] 0.093 ms sshd - 20645 : 32 [ 0.0% ] 0.071 ms npviewer.bin - 21053 : 35 [ 0.0% ] 0.074 ms npviewer.bin - 21054 : 35 [ 0.0% ] 0.097 ms kworker/0:2 - 21169 : 149 [ 0.0% ] 1.143 ms kworker/3:0 - 22171 : 113 [ 0.0% ] 96.892 ms flush-253:4 - 22410 : 1 [ 0.0% ] 0.028 ms kworker/6:0 - 24581 : 25 [ 0.0% ] 0.275 ms kworker/1:0 - 25572 : 4 [ 0.0% ] 0.103 ms kworker/2:1 - 26299 : 138 [ 0.0% ] 1.440 ms kworker/0:0 - 26325 : 1 [ 0.0% ] 0.003 ms perf - 26330 : 3506967 [ 96.1% ] 6648.310 ms [root@sandy ~]# Cc: David Ahern Cc: Frederic Weisbecker Cc: Jiri Olsa Cc: Mike Galbraith Cc: Namhyung Kim Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Stephane Eranian Cc: Thomas Gleixner Link: http://lkml.kernel.org/n/tip-mzuli0srnxyi1o029py6537x@git.kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-trace.txt | 3 ++ tools/perf/builtin-trace.c | 88 ++++++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/tools/perf/Documentation/perf-trace.txt b/tools/perf/Documentation/perf-trace.txt index 38d4b682af0..68718ccdd17 100644 --- a/tools/perf/Documentation/perf-trace.txt +++ b/tools/perf/Documentation/perf-trace.txt @@ -51,6 +51,9 @@ the thread executes on the designated CPUs. Default is to monitor all CPUs. --duration: Show only events that had a duration greater than N.M ms. +--sched: + Accrue thread runtime and provide a summary at the end of the session. + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-script[1] diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index c95a3e9b033..7932ffa2988 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -69,6 +69,7 @@ struct thread_trace { bool entry_pending; unsigned long nr_events; char *entry_str; + double runtime_ms; }; static struct thread_trace *thread_trace__new(void) @@ -109,8 +110,10 @@ struct trace { struct machine host; u64 base_time; unsigned long nr_events; + bool sched; bool multiple_threads; double duration_filter; + double runtime_ms; }; static bool trace__filter_duration(struct trace *trace, double t) @@ -389,6 +392,31 @@ out: return 0; } +static int trace__sched_stat_runtime(struct trace *trace, struct perf_evsel *evsel, + struct perf_sample *sample) +{ + u64 runtime = perf_evsel__intval(evsel, sample, "runtime"); + double runtime_ms = (double)runtime / NSEC_PER_MSEC; + struct thread *thread = machine__findnew_thread(&trace->host, sample->tid); + struct thread_trace *ttrace = thread__trace(thread); + + if (ttrace == NULL) + goto out_dump; + + ttrace->runtime_ms += runtime_ms; + trace->runtime_ms += runtime_ms; + return 0; + +out_dump: + printf("%s: comm=%s,pid=%u,runtime=%" PRIu64 ",vruntime=%" PRIu64 ")\n", + evsel->name, + perf_evsel__strval(evsel, sample, "comm"), + (pid_t)perf_evsel__intval(evsel, sample, "pid"), + runtime, + perf_evsel__intval(evsel, sample, "vruntime")); + return 0; +} + static int trace__run(struct trace *trace, int argc, const char **argv) { struct perf_evlist *evlist = perf_evlist__new(NULL, NULL); @@ -408,6 +436,13 @@ static int trace__run(struct trace *trace, int argc, const char **argv) goto out_delete_evlist; } + if (trace->sched && + perf_evlist__add_newtp(evlist, "sched", "sched_stat_runtime", + trace__sched_stat_runtime)) { + printf("Couldn't read the sched_stat_runtime tracepoint information!\n"); + goto out_delete_evlist; + } + err = perf_evlist__create_maps(evlist, &trace->opts.target); if (err < 0) { printf("Problems parsing the target to trace, check your options!\n"); @@ -521,6 +556,51 @@ out: return err; } +static size_t trace__fprintf_threads_header(FILE *fp) +{ + size_t printed; + + printed = fprintf(fp, "\n _____________________________________________________________________\n"); + printed += fprintf(fp," __) Summary of events (__\n\n"); + printed += fprintf(fp," [ task - pid ] [ events ] [ ratio ] [ runtime ]\n"); + printed += fprintf(fp," _____________________________________________________________________\n\n"); + + return printed; +} + +static size_t trace__fprintf_thread_summary(struct trace *trace, FILE *fp) +{ + size_t printed = trace__fprintf_threads_header(fp); + struct rb_node *nd; + + for (nd = rb_first(&trace->host.threads); nd; nd = rb_next(nd)) { + struct thread *thread = rb_entry(nd, struct thread, rb_node); + struct thread_trace *ttrace = thread->priv; + const char *color; + double ratio; + + if (ttrace == NULL) + continue; + + ratio = (double)ttrace->nr_events / trace->nr_events * 100.0; + + color = PERF_COLOR_NORMAL; + if (ratio > 50.0) + color = PERF_COLOR_RED; + else if (ratio > 25.0) + color = PERF_COLOR_GREEN; + else if (ratio > 5.0) + color = PERF_COLOR_YELLOW; + + printed += color_fprintf(fp, color, "%20s", thread->comm); + printed += fprintf(fp, " - %-5d :%11lu [", thread->pid, ttrace->nr_events); + printed += color_fprintf(fp, color, "%5.1f%%", ratio); + printed += fprintf(fp, " ] %10.3f ms\n", ttrace->runtime_ms); + } + + return printed; +} + static int trace__set_duration(const struct option *opt, const char *str, int unset __maybe_unused) { @@ -571,6 +651,7 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) OPT_CALLBACK(0, "duration", &trace, "float", "show only events with duration > N.M ms", trace__set_duration), + OPT_BOOLEAN(0, "sched", &trace.sched, "show blocking scheduler events"), OPT_END() }; int err; @@ -595,5 +676,10 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused) if (!argc && perf_target__none(&trace.opts.target)) trace.opts.target.system_wide = true; - return trace__run(&trace, argc, argv); + err = trace__run(&trace, argc, argv); + + if (trace.sched && !err) + trace__fprintf_thread_summary(&trace, stdout); + + return err; } -- cgit v1.2.3