From bfb3170e2481b76a4f8aae94176e45d681a37f3e Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Fri, 12 Apr 2019 14:38:26 +0300 Subject: perf scripts python: export-to-sqlite.py: Add support for pyside2 pyside2 is the future for pyside support. Note pyside use Qt4 whereas pyside2 uses Qt5. Signed-off-by: Adrian Hunter Cc: Jiri Olsa Link: http://lkml.kernel.org/r/20190412113830.4126-5-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-sqlite.py | 44 +++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) (limited to 'tools/perf/scripts/python/export-to-sqlite.py') diff --git a/tools/perf/scripts/python/export-to-sqlite.py b/tools/perf/scripts/python/export-to-sqlite.py index bf271fbc3a88..f617e518332f 100644 --- a/tools/perf/scripts/python/export-to-sqlite.py +++ b/tools/perf/scripts/python/export-to-sqlite.py @@ -21,6 +21,26 @@ import datetime # provides LGPL-licensed Python bindings for Qt. You will also need the package # libqt4-sql-sqlite for Qt sqlite3 support. # +# Examples of installing pyside: +# +# ubuntu: +# +# $ sudo apt-get install python-pyside.qtsql libqt4-sql-psql +# +# Alternately, to use Python3 and/or pyside 2, one of the following: +# +# $ sudo apt-get install python3-pyside.qtsql libqt4-sql-psql +# $ sudo apt-get install python-pyside2.qtsql libqt5sql5-psql +# $ sudo apt-get install python3-pyside2.qtsql libqt5sql5-psql +# fedora: +# +# $ sudo yum install python-pyside +# +# Alternately, to use Python3 and/or pyside 2, one of the following: +# $ sudo yum install python3-pyside +# $ pip install --user PySide2 +# $ pip3 install --user PySide2 +# # An example of using this script with Intel PT: # # $ perf record -e intel_pt//u ls @@ -49,7 +69,16 @@ import datetime # difference is the 'transaction' column of the 'samples' table which is # renamed 'transaction_' in sqlite because 'transaction' is a reserved word. -from PySide.QtSql import * +pyside_version_1 = True +if not "pyside-version-1" in sys.argv: + try: + from PySide2.QtSql import * + pyside_version_1 = False + except: + pass + +if pyside_version_1: + from PySide.QtSql import * sys.path.append(os.environ['PERF_EXEC_PATH'] + \ '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') @@ -69,11 +98,12 @@ def printdate(*args, **kw_args): print(datetime.datetime.today(), *args, sep=' ', **kw_args) def usage(): - printerr("Usage is: export-to-sqlite.py [] [] []"); - printerr("where: columns 'all' or 'branches'"); - printerr(" calls 'calls' => create calls and call_paths table"); - printerr(" callchains 'callchains' => create call_paths table"); - raise Exception("Too few arguments") + printerr("Usage is: export-to-sqlite.py [] [] [] []"); + printerr("where: columns 'all' or 'branches'"); + printerr(" calls 'calls' => create calls and call_paths table"); + printerr(" callchains 'callchains' => create call_paths table"); + printerr(" pyside-version-1 'pyside-version-1' => use pyside version 1"); + raise Exception("Too few or bad arguments") if (len(sys.argv) < 2): usage() @@ -95,6 +125,8 @@ for i in range(3,len(sys.argv)): perf_db_export_calls = True elif (sys.argv[i] == "callchains"): perf_db_export_callchains = True + elif (sys.argv[i] == "pyside-version-1"): + pass else: usage() -- cgit v1.2.3 From 64adadb3f9dbaaae3d14ea75fa71a3b877cbe82e Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 20 May 2019 14:37:22 +0300 Subject: perf scripts python: export-to-sqlite.py: Export IPC information Export cycle and instruction counts on samples and calls tables. Committer testing: First runs some workload collecting intel_pt with the 'cyc' ter just for userspace: [root@quaco adrian.hunter]# perf record -o simple-retpoline.perf.data -e intel_pt/cyc/u ./simple-retpoline [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.035 MB simple-retpoline.perf.data ] [root@quaco adrian.hunter]# Then use the export-to-sqlite.py script to see if the changes in this cset don't make it to break and if the changes in the db schema are the ones expected: [root@quaco adrian.hunter]# perf script -i simple-retpoline.perf.data --itrace=be -s ~acme/libexec/perf-core/scripts/python/export-to-sqlite.py simple-retpoline.db branches calls 2019-05-31 11:50:46.942710 Creating database ... 2019-05-31 11:50:46.949663 Writing records... 2019-05-31 11:50:47.224033 Adding indexes 2019-05-31 11:50:47.231599 Done [root@quaco adrian.hunter]# Now lets use the db: [root@quaco adrian.hunter]# sqlite3 simple-retpoline.db SQLite version 3.26.0 2018-12-01 12:34:55 Enter ".help" for usage hints. sqlite> .schema samples CREATE TABLE samples (id integer NOT NULL PRIMARY KEY,evsel_id bigint,machine_id bigint,thread_id bigint,comm_id bigint,dso_id bigint,symbol_id bigint,sym_offset bigint,ip bigint,time bigint,cpuinteger,to_dso_id bigint,to_symbol_id bigint,to_sym_offset bigint,to_ip bigint,branch_type integer,in_tx boolean,call_path_id bigint,insn_count bigint,cyc_count bigint); sqlite> Cool, the 'insn_count' and 'cyc_count' are there, now lets see if we can use them in a query: sqlite> select insn_count,cyc_count from samples where cyc_count > 1500 and insn_count < 10; 6|1507 sqlite> select insn_count,cyc_count from samples where cyc_count > 1500; 118|2210 140|1516 3783|1861 132|1521 6|1507 sqlite> Seems to work :-) Signed-off-by: Adrian Hunter Tested-by: Arnaldo Carvalho de Melo Cc: Jiri Olsa Link: http://lkml.kernel.org/r/20190520113728.14389-17-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-sqlite.py | 36 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) (limited to 'tools/perf/scripts/python/export-to-sqlite.py') diff --git a/tools/perf/scripts/python/export-to-sqlite.py b/tools/perf/scripts/python/export-to-sqlite.py index f617e518332f..4542ce89034b 100644 --- a/tools/perf/scripts/python/export-to-sqlite.py +++ b/tools/perf/scripts/python/export-to-sqlite.py @@ -218,7 +218,9 @@ if branches: 'to_ip bigint,' 'branch_type integer,' 'in_tx boolean,' - 'call_path_id bigint)') + 'call_path_id bigint,' + 'insn_count bigint,' + 'cyc_count bigint)') else: do_query(query, 'CREATE TABLE samples (' 'id integer NOT NULL PRIMARY KEY,' @@ -242,7 +244,9 @@ else: 'data_src bigint,' 'branch_type integer,' 'in_tx boolean,' - 'call_path_id bigint)') + 'call_path_id bigint,' + 'insn_count bigint,' + 'cyc_count bigint)') if perf_db_export_calls or perf_db_export_callchains: do_query(query, 'CREATE TABLE call_paths (' @@ -263,7 +267,9 @@ if perf_db_export_calls: 'return_id bigint,' 'parent_call_path_id bigint,' 'flags integer,' - 'parent_id bigint)') + 'parent_id bigint,' + 'insn_count bigint,' + 'cyc_count bigint)') # printf was added to sqlite in version 3.8.3 sqlite_has_printf = False @@ -359,6 +365,9 @@ if perf_db_export_calls: 'return_time,' 'return_time - call_time AS elapsed_time,' 'branch_count,' + 'insn_count,' + 'cyc_count,' + 'CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) ELSE ROUND(CAST(insn_count AS FLOAT) / cyc_count, 2) END AS IPC,' 'call_id,' 'return_id,' 'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE flags END AS flags,' @@ -384,7 +393,10 @@ do_query(query, 'CREATE VIEW samples_view AS ' 'to_sym_offset,' '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,' '(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,' - 'in_tx' + 'in_tx,' + 'insn_count,' + 'cyc_count,' + 'CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) ELSE ROUND(CAST(insn_count AS FLOAT) / cyc_count, 2) END AS IPC' ' FROM samples') do_query(query, 'END TRANSACTION') @@ -407,15 +419,15 @@ branch_type_query = QSqlQuery(db) branch_type_query.prepare("INSERT INTO branch_types VALUES (?, ?)") sample_query = QSqlQuery(db) if branches: - sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") else: - sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") if perf_db_export_calls or perf_db_export_callchains: call_path_query = QSqlQuery(db) call_path_query.prepare("INSERT INTO call_paths VALUES (?, ?, ?, ?)") if perf_db_export_calls: call_query = QSqlQuery(db) - call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") def trace_begin(): printdate("Writing records...") @@ -427,10 +439,10 @@ def trace_begin(): comm_table(0, "unknown") dso_table(0, 0, "unknown", "unknown", "") symbol_table(0, 0, 0, 0, 0, "unknown") - sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) if perf_db_export_calls or perf_db_export_callchains: call_path_table(0, 0, 0, 0) - call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) unhandled_count = 0 @@ -486,14 +498,14 @@ def sample_table(*x): if branches: for xx in x[0:15]: sample_query.addBindValue(str(xx)) - for xx in x[19:22]: + for xx in x[19:24]: sample_query.addBindValue(str(xx)) do_query_(sample_query) else: - bind_exec(sample_query, 22, x) + bind_exec(sample_query, 24, x) def call_path_table(*x): bind_exec(call_path_query, 4, x) def call_return_table(*x): - bind_exec(call_query, 12, x) + bind_exec(call_query, 14, x) -- cgit v1.2.3 From 5130c6e55531b9bbcdeb8b327711ff204bc4835f Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Sat, 22 Jun 2019 12:32:47 +0300 Subject: perf scripts python: export-to-sqlite.py: Export Intel PT power and ptwrite events The format of synthesized events is determined by the attribute config. For the formats for Intel PT power and ptwrite events, create tables and populate them when the synth_data handler is called. If the tables remain empty, drop them at the end. The tables and views, including a combined power_events_view, will display automatically from the tables menu of the exported exported-sql-viewer.py script. Note, currently only Atoms since Gemini Lake have support for ptwrite and mwait, pwre, exstop and pwrx, but all Intel PT implementations support cbr. Signed-off-by: Adrian Hunter Cc: Jiri Olsa Link: http://lkml.kernel.org/r/20190622093248.581-7-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/export-to-sqlite.py | 239 ++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) (limited to 'tools/perf/scripts/python/export-to-sqlite.py') diff --git a/tools/perf/scripts/python/export-to-sqlite.py b/tools/perf/scripts/python/export-to-sqlite.py index 4542ce89034b..3222a83f4184 100644 --- a/tools/perf/scripts/python/export-to-sqlite.py +++ b/tools/perf/scripts/python/export-to-sqlite.py @@ -271,6 +271,38 @@ if perf_db_export_calls: 'insn_count bigint,' 'cyc_count bigint)') +do_query(query, 'CREATE TABLE ptwrite (' + 'id integer NOT NULL PRIMARY KEY,' + 'payload bigint,' + 'exact_ip integer)') + +do_query(query, 'CREATE TABLE cbr (' + 'id integer NOT NULL PRIMARY KEY,' + 'cbr integer,' + 'mhz integer,' + 'percent integer)') + +do_query(query, 'CREATE TABLE mwait (' + 'id integer NOT NULL PRIMARY KEY,' + 'hints integer,' + 'extensions integer)') + +do_query(query, 'CREATE TABLE pwre (' + 'id integer NOT NULL PRIMARY KEY,' + 'cstate integer,' + 'subcstate integer,' + 'hw integer)') + +do_query(query, 'CREATE TABLE exstop (' + 'id integer NOT NULL PRIMARY KEY,' + 'exact_ip integer)') + +do_query(query, 'CREATE TABLE pwrx (' + 'id integer NOT NULL PRIMARY KEY,' + 'deepest_cstate integer,' + 'last_cstate integer,' + 'wake_reason integer)') + # printf was added to sqlite in version 3.8.3 sqlite_has_printf = False try: @@ -399,6 +431,102 @@ do_query(query, 'CREATE VIEW samples_view AS ' 'CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) ELSE ROUND(CAST(insn_count AS FLOAT) / cyc_count, 2) END AS IPC' ' FROM samples') +do_query(query, 'CREATE VIEW ptwrite_view AS ' + 'SELECT ' + 'ptwrite.id,' + 'time,' + 'cpu,' + + emit_to_hex('payload') + ' AS payload_hex,' + 'CASE WHEN exact_ip=0 THEN \'False\' ELSE \'True\' END AS exact_ip' + ' FROM ptwrite' + ' INNER JOIN samples ON samples.id = ptwrite.id') + +do_query(query, 'CREATE VIEW cbr_view AS ' + 'SELECT ' + 'cbr.id,' + 'time,' + 'cpu,' + 'cbr,' + 'mhz,' + 'percent' + ' FROM cbr' + ' INNER JOIN samples ON samples.id = cbr.id') + +do_query(query, 'CREATE VIEW mwait_view AS ' + 'SELECT ' + 'mwait.id,' + 'time,' + 'cpu,' + + emit_to_hex('hints') + ' AS hints_hex,' + + emit_to_hex('extensions') + ' AS extensions_hex' + ' FROM mwait' + ' INNER JOIN samples ON samples.id = mwait.id') + +do_query(query, 'CREATE VIEW pwre_view AS ' + 'SELECT ' + 'pwre.id,' + 'time,' + 'cpu,' + 'cstate,' + 'subcstate,' + 'CASE WHEN hw=0 THEN \'False\' ELSE \'True\' END AS hw' + ' FROM pwre' + ' INNER JOIN samples ON samples.id = pwre.id') + +do_query(query, 'CREATE VIEW exstop_view AS ' + 'SELECT ' + 'exstop.id,' + 'time,' + 'cpu,' + 'CASE WHEN exact_ip=0 THEN \'False\' ELSE \'True\' END AS exact_ip' + ' FROM exstop' + ' INNER JOIN samples ON samples.id = exstop.id') + +do_query(query, 'CREATE VIEW pwrx_view AS ' + 'SELECT ' + 'pwrx.id,' + 'time,' + 'cpu,' + 'deepest_cstate,' + 'last_cstate,' + 'CASE WHEN wake_reason=1 THEN \'Interrupt\'' + ' WHEN wake_reason=2 THEN \'Timer Deadline\'' + ' WHEN wake_reason=4 THEN \'Monitored Address\'' + ' WHEN wake_reason=8 THEN \'HW\'' + ' ELSE wake_reason ' + 'END AS wake_reason' + ' FROM pwrx' + ' INNER JOIN samples ON samples.id = pwrx.id') + +do_query(query, 'CREATE VIEW power_events_view AS ' + 'SELECT ' + 'samples.id,' + 'time,' + 'cpu,' + 'selected_events.name AS event,' + 'CASE WHEN selected_events.name=\'cbr\' THEN (SELECT cbr FROM cbr WHERE cbr.id = samples.id) ELSE "" END AS cbr,' + 'CASE WHEN selected_events.name=\'cbr\' THEN (SELECT mhz FROM cbr WHERE cbr.id = samples.id) ELSE "" END AS mhz,' + 'CASE WHEN selected_events.name=\'cbr\' THEN (SELECT percent FROM cbr WHERE cbr.id = samples.id) ELSE "" END AS percent,' + 'CASE WHEN selected_events.name=\'mwait\' THEN (SELECT ' + emit_to_hex('hints') + ' FROM mwait WHERE mwait.id = samples.id) ELSE "" END AS hints_hex,' + 'CASE WHEN selected_events.name=\'mwait\' THEN (SELECT ' + emit_to_hex('extensions') + ' FROM mwait WHERE mwait.id = samples.id) ELSE "" END AS extensions_hex,' + 'CASE WHEN selected_events.name=\'pwre\' THEN (SELECT cstate FROM pwre WHERE pwre.id = samples.id) ELSE "" END AS cstate,' + 'CASE WHEN selected_events.name=\'pwre\' THEN (SELECT subcstate FROM pwre WHERE pwre.id = samples.id) ELSE "" END AS subcstate,' + 'CASE WHEN selected_events.name=\'pwre\' THEN (SELECT hw FROM pwre WHERE pwre.id = samples.id) ELSE "" END AS hw,' + 'CASE WHEN selected_events.name=\'exstop\' THEN (SELECT exact_ip FROM exstop WHERE exstop.id = samples.id) ELSE "" END AS exact_ip,' + 'CASE WHEN selected_events.name=\'pwrx\' THEN (SELECT deepest_cstate FROM pwrx WHERE pwrx.id = samples.id) ELSE "" END AS deepest_cstate,' + 'CASE WHEN selected_events.name=\'pwrx\' THEN (SELECT last_cstate FROM pwrx WHERE pwrx.id = samples.id) ELSE "" END AS last_cstate,' + 'CASE WHEN selected_events.name=\'pwrx\' THEN (SELECT ' + 'CASE WHEN wake_reason=1 THEN \'Interrupt\'' + ' WHEN wake_reason=2 THEN \'Timer Deadline\'' + ' WHEN wake_reason=4 THEN \'Monitored Address\'' + ' WHEN wake_reason=8 THEN \'HW\'' + ' ELSE wake_reason ' + 'END' + ' FROM pwrx WHERE pwrx.id = samples.id) ELSE "" END AS wake_reason' + ' FROM samples' + ' INNER JOIN selected_events ON selected_events.id = evsel_id' + ' WHERE selected_events.name IN (\'cbr\',\'mwait\',\'exstop\',\'pwre\',\'pwrx\')') + do_query(query, 'END TRANSACTION') evsel_query = QSqlQuery(db) @@ -428,6 +556,18 @@ if perf_db_export_calls or perf_db_export_callchains: if perf_db_export_calls: call_query = QSqlQuery(db) call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") +ptwrite_query = QSqlQuery(db) +ptwrite_query.prepare("INSERT INTO ptwrite VALUES (?, ?, ?)") +cbr_query = QSqlQuery(db) +cbr_query.prepare("INSERT INTO cbr VALUES (?, ?, ?, ?)") +mwait_query = QSqlQuery(db) +mwait_query.prepare("INSERT INTO mwait VALUES (?, ?, ?)") +pwre_query = QSqlQuery(db) +pwre_query.prepare("INSERT INTO pwre VALUES (?, ?, ?, ?)") +exstop_query = QSqlQuery(db) +exstop_query.prepare("INSERT INTO exstop VALUES (?, ?)") +pwrx_query = QSqlQuery(db) +pwrx_query.prepare("INSERT INTO pwrx VALUES (?, ?, ?, ?)") def trace_begin(): printdate("Writing records...") @@ -446,6 +586,16 @@ def trace_begin(): unhandled_count = 0 +def is_table_empty(table_name): + do_query(query, 'SELECT * FROM ' + table_name + ' LIMIT 1'); + if query.next(): + return False + return True + +def drop(table_name): + do_query(query, 'DROP VIEW ' + table_name + '_view'); + do_query(query, 'DROP TABLE ' + table_name); + def trace_end(): do_query(query, 'END TRANSACTION') @@ -454,6 +604,18 @@ def trace_end(): do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)') do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)') + printdate("Dropping unused tables") + if is_table_empty("ptwrite"): + drop("ptwrite") + if is_table_empty("mwait") and is_table_empty("pwre") and is_table_empty("exstop") and is_table_empty("pwrx"): + drop("mwait") + drop("pwre") + drop("exstop") + drop("pwrx") + do_query(query, 'DROP VIEW power_events_view'); + if is_table_empty("cbr"): + drop("cbr") + if (unhandled_count): printdate("Warning: ", unhandled_count, " unhandled events") printdate("Done") @@ -509,3 +671,80 @@ def call_path_table(*x): def call_return_table(*x): bind_exec(call_query, 14, x) + +def ptwrite(id, raw_buf): + data = struct.unpack_from("> 32) & 0x3 + mwait_query.addBindValue(str(id)) + mwait_query.addBindValue(str(hints)) + mwait_query.addBindValue(str(extensions)) + do_query_(mwait_query) + +def pwre(id, raw_buf): + data = struct.unpack_from("> 7) & 1 + cstate = (payload >> 12) & 0xf + subcstate = (payload >> 8) & 0xf + pwre_query.addBindValue(str(id)) + pwre_query.addBindValue(str(cstate)) + pwre_query.addBindValue(str(subcstate)) + pwre_query.addBindValue(str(hw)) + do_query_(pwre_query) + +def exstop(id, raw_buf): + data = struct.unpack_from("> 4) & 0xf + wake_reason = (payload >> 8) & 0xf + pwrx_query.addBindValue(str(id)) + pwrx_query.addBindValue(str(deepest_cstate)) + pwrx_query.addBindValue(str(last_cstate)) + pwrx_query.addBindValue(str(wake_reason)) + do_query_(pwrx_query) + +def synth_data(id, config, raw_buf, *x): + if config == 0: + ptwrite(id, raw_buf) + elif config == 1: + mwait(id, raw_buf) + elif config == 2: + pwre(id, raw_buf) + elif config == 3: + exstop(id, raw_buf) + elif config == 4: + pwrx(id, raw_buf) + elif config == 5: + cbr(id, raw_buf) -- cgit v1.2.3