From aa7f23e83e51c481077d46b4d11854fa8fd95c03 Mon Sep 17 00:00:00 2001 From: Ilya Churaev Date: Thu, 10 Dec 2020 10:15:39 +0300 Subject: [PATCH 1/8] Updated sea_itt_lib, fixed buildall for python3 --- .gitignore | 8 +- CMakeLists.txt | 55 +- InstrumentationExample.cpp | 12 +- buildall.py | 104 ++- isea.bat | 1 + isea.sh | 4 + isea_test.h | 4 + itt_notify.hpp | 393 +++++++--- main.cpp | 567 +++++++------- memory.cpp | 10 +- runtool/collectors/android.py | 8 +- runtool/collectors/ftrace.py | 8 +- runtool/collectors/osx.py | 797 +++++++++++++++++--- runtool/collectors/win.py | 36 +- runtool/decoders/Adreno.py | 11 +- runtool/decoders/MSNT_SystemTrace.py | 20 +- runtool/decoders/PVR.py | 3 + runtool/exporters/BestTraceFormat.py | 16 +- runtool/exporters/ChromeTracing.py | 220 +++--- runtool/exporters/DGML.py | 69 +- runtool/exporters/GraphViz.py | 47 +- runtool/exporters/QtCreatorProfiler.py | 53 +- runtool/exporters/SQLite.py | 210 ++++++ runtool/exporters/Stat.py | 35 +- runtool/exporters/memory.py | 6 +- runtool/importers/csv.py | 59 -- runtool/importers/etw.py | 241 +++--- runtool/importers/json.py | 96 +++ runtool/importers/mac_log.py | 12 +- runtool/importers/osx.py | 319 +++++--- runtool/importers/pprof_importer.py | 4 +- runtool/importers/qnx.py | 7 +- runtool/python_compat.py | 26 + runtool/sea.py | 241 +++--- runtool/sea_runtool.py | 989 ++++++++++++++++++------- runtool/strings.py | 1 - sea_itt_lib/CMakeLists.txt | 6 - sea_itt_lib/IttNotifyStdSrc.cpp | 180 ++++- sea_itt_lib/IttNotifyStdSrc.h | 2 + sea_itt_lib/Recorder.cpp | 30 +- sea_itt_lib/Recorder.h | 1 + sea_itt_lib/Utils.cpp | 45 ++ sea_itt_lib/Utils.h | 26 +- sea_itt_lib/sea_itt_lib.cpp | 5 + test_osx.sh | 2 +- 45 files changed, 3549 insertions(+), 1440 deletions(-) create mode 100644 isea.bat create mode 100755 isea.sh create mode 100644 isea_test.h create mode 100644 runtool/exporters/SQLite.py delete mode 100644 runtool/importers/csv.py create mode 100644 runtool/importers/json.py create mode 100644 runtool/python_compat.py diff --git a/.gitignore b/.gitignore index 3972efb..654c887 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ -build_win/ +build_*/ *.pyc *.pyo bin/ .vs/ obj/ *.user - +runtool/SEARunTool.pyproj +runtool/.idea/ +.idea/ +tags +cmake-build* diff --git a/CMakeLists.txt b/CMakeLists.txt index 479c443..37f1a3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,8 @@ set(CMAKE_MACOSX_RPATH 1) OPTION(JDK "Enable Java build") OPTION(CO_PILOT "Enable co-pilot build") OPTION(FORCE_32 "Force a 32bit compile on 64bit" OFF) +OPTION(FAT_BINARY "Sets CMAKE_OSX_ARCHITECTURES" ON) +OPTION(INSTALLER "Sets INSTALLER (NSIS/ZIP)" NSIS) if (ANDROID_ARCH_NAME MATCHES "arm\\.*") set(ARM 1) @@ -106,7 +108,6 @@ if (EXISTS "${PROJECT_SOURCE_DIR}/RadTelemetry") set (RAD_TELEMETRY_DIR ${PROJECT_SOURCE_DIR}/RadTelemetry) endif() - if (WIN32) TARGET_LINK_LIBRARIES(IntelSEAPI) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /MP") @@ -117,11 +118,8 @@ if (WIN32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /GS /sdl") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /NXCompat /DynamicBase") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NXCompat /DynamicBase") - if(NOT ARCH_64) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SafeSEH") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SafeSEH") - endif() - + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SAFESEH:NO") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SAFESEH:NO") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") #in release as well set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") #in release as well elseif (APPLE) @@ -129,7 +127,9 @@ elseif (APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -std=c++11 -fPIC -Wno-unused-parameter -fstack-protector-strong -fPIE -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fPIE -Wl") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIE -Wl") - set(CMAKE_OSX_ARCHITECTURES "i386;x86_64") + if (FAT_BINARY) + set(CMAKE_OSX_ARCHITECTURES "i386;x86_64") + endif() elseif (ANDROID_NDK) TARGET_LINK_LIBRARIES(IntelSEAPI dl) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIE -pthread -fstack-protector-strong -fPIE -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security") @@ -147,9 +147,10 @@ endif() add_subdirectory(ittnotify) add_subdirectory(sea_itt_lib) + if (WIN32) if(NOT ARCH_64 AND CMAKE_BUILD_TYPE STREQUAL "Debug") #Compiler Automated Instrumentation - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Gh /GH") + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Gh /GH") FIXME: stopped working endif() else() #SET_TARGET_PROPERTIES(IntelSEAPI PROPERTIES COMPILE_FLAGS "-finstrument-functions") @@ -177,7 +178,9 @@ endif(DOXYGEN_FOUND) set(CPACK_PACKAGE_NAME "IntelSEAPI") set(CPACK_PACKAGE_VENDOR "Intel") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Intel(R) Single Event API") -set(CPACK_PACKAGE_VERSION "1.0.0") +# string(TIMESTAMP TODAY "%y.%m.%d") +#set(CPACK_PACKAGE_VERSION "${TODAY}") +set(CPACK_PACKAGE_VERSION "delete_me") set(CPACK_PACKAGE_INSTALL_DIRECTORY "IntelSEAPI") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Intel(R) Single Event API") set(CPACK_PACKAGE_DESCRIPTION_FILE "${PROJECT_SOURCE_DIR}/README.txt") @@ -188,9 +191,23 @@ set(CPACK_RESOURCE_FILE_WELCOME "${PROJECT_SOURCE_DIR}/README.txt") install( FILES ${PROJECT_SOURCE_DIR}/README.txt + ${PROJECT_SOURCE_DIR}/isea.sh + ${PROJECT_SOURCE_DIR}/isea.htm DESTINATION . ) +if (WIN32) + install( + FILES + ${PROJECT_SOURCE_DIR}/isea.bat + DESTINATION bin + ) + install( + DIRECTORY "${PROJECT_SOURCE_DIR}/pypy" + DESTINATION bin + ) +endif() + install( DIRECTORY "${PROJECT_SOURCE_DIR}/ittnotify/include" DESTINATION . @@ -246,17 +263,21 @@ if (WIN32) DESTINATION ETW ) - set(CPACK_GENERATOR NSIS) - - #run at exit: - set(CPACK_NSIS_EXECUTABLES_DIRECTORY "ETW") - set(CPACK_NSIS_MUI_FINISHPAGE_RUN "register.bat") - set(CPACK_NSIS_DISPLAY_NAME "Intel(R) Single Event API") + if(INSTALLER STREQUAL "NSIS") + set(CPACK_GENERATOR NSIS) - set(CPACK_NSIS_CONTACT "alexander.a.raud@intel.com") + #run at exit: + set(CPACK_NSIS_EXECUTABLES_DIRECTORY "ETW") + set(CPACK_NSIS_MUI_FINISHPAGE_RUN "register.bat") + set(CPACK_NSIS_DISPLAY_NAME "Intel(R) Single Event API") - SET(CPACK_NSIS_INSTALL_ROOT "c:\\\\Intel") + set(CPACK_NSIS_CONTACT "alexander.a.raud@intel.com") + SET(CPACK_NSIS_INSTALL_ROOT "c:\\\\Intel") + SET(CPACK_NSIS_MODIFY_PATH ON) + else() + set(CPACK_GENERATOR ${INSTALLER}) + endif() else() set(CPACK_BUNDLE_NAME "IntelSEAPI") diff --git a/InstrumentationExample.cpp b/InstrumentationExample.cpp index 1509ff1..f8a1548 100644 --- a/InstrumentationExample.cpp +++ b/InstrumentationExample.cpp @@ -90,6 +90,8 @@ void ITTAPI get_clock_info(__itt_clock_info* clock_info, void*) __itt_clock_domain* clock_domain = nullptr; __itt_string_handle* handle_stacked = __itt_string_handle_create("stacked"); +__itt_counter manual_counter = __itt_counter_create("manual", "counter"); + #ifndef _WIN32 #define sprintf_s sprintf #endif @@ -116,18 +118,20 @@ void workerthread(int data) TaskStack(5); __itt_id id = __itt_id_make(threadname, data); __itt_id_create(g_domain, id); + __itt_event event = __itt_event_create("event_test", 0); // Each worker thread does some number of "work" tasks uint64_t counter = 0; while (!g_done) { __itt_sync_acquired((void*)&workerthread); - + __itt_event_start(event); ITT_COUNTER("random", rand()); bool bOverlapped = !(rand() % 2); unsigned long long start = TClock::now().time_since_epoch().count(); __itt_task_begin(g_domain, id, __itt_null, handle_work); std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 10)); __itt_sync_releasing((void*)&workerthread); + __itt_event_end(event); if (rand() % 5 == 1) { @@ -150,6 +154,12 @@ void workerthread(int data) { __itt_task_end_overlapped(g_domain, id); } + + __itt_counter_set_value(manual_counter, &counter); + + uint64_t data[3] = {counter++, counter*2, counter*3}; + __itt_metadata_add(g_domain, __itt_null, handle_stacked, __itt_metadata_u64, 3, data); + } TaskStack(5); __itt_id_destroy(g_domain, id); diff --git a/buildall.py b/buildall.py index 30eaed8..32020b4 100644 --- a/buildall.py +++ b/buildall.py @@ -21,6 +21,7 @@ import sys import shutil import fnmatch +import datetime import subprocess @@ -28,8 +29,7 @@ def get_share_folder(): - import datetime - folder_name = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + folder_name = datetime.datetime.now().strftime("%Y%m%d") return os.path.join(install_dest, folder_name) @@ -39,12 +39,19 @@ def run_shell(cmd): def replace_in_file(file, what_by): - import fileinput - for line in fileinput.input(file, inplace=True): - for (what, by) in what_by: - if what in line: - line = line.replace(what, by) - sys.stdout.write(line) + from tempfile import mkstemp + fd, tmp_path = mkstemp() + with open(tmp_path,'wb') as dst: + with open(file, mode='rb') as src: + for line in src: + for (what, by) in what_by: + if what in line: + line = line.replace(what, by) + dst.write(line) + shutil.copymode(file, tmp_path) + os.remove(file) + shutil.move(tmp_path, file) + os.close(fd) def get_yocto(): @@ -114,6 +121,7 @@ def locate_exact(what): return [] return [item for item in items if item.endswith(what)] + def find_in(locations, what): try: items = subprocess.check_output(['find'] + locations + ['-name', what]).decode("utf-8").split('\n') @@ -121,6 +129,9 @@ def find_in(locations, what): return [] return [item for item in items if item.endswith(what)] +def run(cmd): + return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + def GetJDKPath(): if sys.platform == 'win32': bush = read_registry(r'HKLM\SOFTWARE\JavaSoft\Java Development Kit') @@ -129,7 +140,7 @@ def GetJDKPath(): return bush[subkeys[-1]]['JavaHome'] return None else: - path, err = subprocess.Popen("which javah", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + path, err = run("which javah") if err or not path: return None if sys.platform == 'darwin': @@ -189,11 +200,13 @@ def get_vs_versions(): # https://www.mztools.com/articles/2008/MZ2008003.aspx return sorted(versions) -def detect_cmake(): +def detect_cmake(args): + if args.cmake: + return args.cmake if sys.platform == 'darwin': - path, err = subprocess.Popen("which cmake", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + path, err = run("which cmake") if not path.strip(): - path, err = subprocess.Popen("which xcrun", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + path, err = run("which xcrun") if not path.strip(): print("No cmake and no XCode found...") return None @@ -201,6 +214,10 @@ def detect_cmake(): return 'cmake' +def detect_nsis(): + return os.path.exists(r'c:\Program Files (x86)\NSIS\NSIS.exe') + + def main(): import argparse parser = argparse.ArgumentParser() @@ -213,6 +230,7 @@ def main(): parser.add_argument("-c", "--clean", action="store_true") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--no_java", action="store_true") + parser.add_argument("--cmake") if sys.platform == 'win32' and vs_versions: parser.add_argument("--vs", choices=vs_versions, default=vs_versions[0]) args = parser.parse_args() @@ -236,6 +254,14 @@ def main(): perf_co_pilot = find_in(['/usr/lib', '/usr/local/lib'], 'libpcp_mmv.a') if sys.platform != 'win32' else None print("Found co-pilot:", perf_co_pilot) + perf_co_pilot = False # Enable when fully supported + + fat_binary = False # Mac only and only before xcode 10 + xcode_version, err = run('xcodebuild -version') + if not err: + xcode_version = int(xcode_version.split()[1].split('.')[0]) + fat_binary = xcode_version < 10 + work_dir = os.getcwd() print(work_dir) if args.clean: @@ -253,7 +279,7 @@ def main(): print("work_folder: ", work_folder) os.chdir(work_folder) - cmake = detect_cmake() + cmake = detect_cmake(args) if not cmake: print("Error: cmake is not found") return @@ -279,20 +305,28 @@ def main(): else: print("Set ANDROID_NDK environment to build Android!") continue + suffix = '' if sys.platform == 'win32': if vs_versions: - generator = ('Visual Studio %s' % args.vs) + (' Win64' if bits == '64' else '') + if int(args.vs) >= 16: + generator = ('Visual Studio %s' % args.vs) + suffix += ' -A x64' if bits == '64' else '-A Win32' + else: + generator = ('Visual Studio %s' % args.vs) + (' Win64' if bits == '64' else '') else: generator = 'Ninja' else: generator = 'Unix Makefiles' run_shell('%s "%s" -G"%s" %s' % (cmake, work_dir, generator, " ".join([ ("-DFORCE_32=ON" if bits == '32' else ""), + ("-DFAT_BINARY=OFF" if not fat_binary else ""), ("-DCMAKE_BUILD_TYPE=Debug" if args.debug else ""), ("-DYOCTO=1" if yocto else ""), (('-DJDK="%s"' % jdk_path) if jdk_path else ""), ('-DCO_PILOT=1' if perf_co_pilot else ""), - ('-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' if args.verbose else '') + ('-DINSTALLER=ZIP' if not detect_nsis() else "-DINSTALLER=NSIS"), + ('-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' if args.verbose else ''), + suffix ]))) if sys.platform == 'win32': install = args.install and bits == target_bits[-1] @@ -308,18 +342,32 @@ def main(): run_shell('%s --build . --config %s --target package' % (cmake, ('Debug' if args.debug else 'Release'))) installer = glob.glob(os.path.join(work_folder, "IntelSEAPI*.sh"))[0] - print(installer) - if sys.platform == 'darwin': - replace_in_file(installer, [ - ('toplevel="`pwd`"', 'toplevel="/Applications"'), - ('exit 0', 'open "${toplevel}/ReadMe.txt"; mkdir -p ~/"Library/Application Support/Instruments/PlugIns/Instruments"; ln -F -s "${toplevel}/dtrace/IntelSEAPI.instrument" ~/"Library/Application Support/Instruments/PlugIns/Instruments/IntelSEAPI.instrument"; exit 0') - ]) - elif 'linux' in sys.platform: - replace_in_file(installer, [ - ('toplevel="`pwd`"', 'toplevel="/opt/intel"'), - ('exit 0', 'open "${toplevel}/ReadMe.txt"; exit 0') - ]) - -if __name__== "__main__": + name = os.path.join(work_folder, 'IntelSEAPI-%s.sh' % str(datetime.date.today())[2:].replace('-', '.')) + os.rename(installer, os.path.join(work_folder, name)) + installer = name + print('Installer:', installer) + + to_replace = [] + if sys.platform != 'win32': + if sys.platform == 'darwin': + to_replace += [ + (b'toplevel="`pwd`"', b'toplevel="/Applications"'), + (b'IntelSEAPI-delete_me-Darwin', b'IntelSEAPI'), + ] + elif 'linux' in sys.platform: + to_replace += [ + (b'toplevel="`pwd`"', b'toplevel="/opt/intel"'), + (b'IntelSEAPI-delete_me-Linux', b'IntelSEAPI'), + ] + to_replace += [ + (b'exit 0', b'python ${toplevel}/runtool/sea_runtool.py install;exit 0'), + (b'delete_me', b'') + ] + if to_replace: + replace_in_file(installer, to_replace) + shutil.rmtree(glob.glob(os.path.join(work_folder, '_CPack_Packages'))[0]) + + +if __name__ == "__main__": main() diff --git a/isea.bat b/isea.bat new file mode 100644 index 0000000..d9a5b1a --- /dev/null +++ b/isea.bat @@ -0,0 +1 @@ +python %~dp0..\runtool\sea_runtool.py %* diff --git a/isea.sh b/isea.sh new file mode 100755 index 0000000..43cc5fb --- /dev/null +++ b/isea.sh @@ -0,0 +1,4 @@ +#!/bin/sh +SCRIPT_NAME=$(perl -e 'use Cwd "abs_path"; print abs_path(@ARGV[0])' -- "$0") +BASEDIR=$(dirname ${SCRIPT_NAME}) +python ${BASEDIR}/runtool/sea_runtool.py "$@" diff --git a/isea_test.h b/isea_test.h new file mode 100644 index 0000000..ddc651d --- /dev/null +++ b/isea_test.h @@ -0,0 +1,4 @@ +#pragma once +#include + +std::string get_environ_value(const std::string& name); diff --git a/itt_notify.hpp b/itt_notify.hpp index 1c2c13b..a2bee7e 100644 --- a/itt_notify.hpp +++ b/itt_notify.hpp @@ -1,142 +1,329 @@ -#pragma once -#include -#include -#include -#include +#ifndef SEA_ITT_NOTIFY_HEADER +#define SEA_ITT_NOTIFY_HEADER -#define INTEL_ITTNOTIFY_API_PRIVATE -#include "ittnotify.h" +#if defined(SEA_USE_DTRACE) || defined(SEA_USE_DTRACE_IMPL) +#ifdef KERNEL + #include +#else + #include +#endif + +void dtSEAHookScope(int type, const char* domain, const char* name) __attribute__ ((noinline)); +void dtSEAHookArgInt(const char* name, int value) __attribute__ ((noinline)); +void dtSEAHookArgStr(const char* name, const char* value) __attribute__ ((noinline)); +void dtSEAHookArgAddr(const char* name, const void* value) __attribute__ ((noinline)); + +void dtSEAHookArgBlobStart(int size, const char* name) /*never use directly*/ __attribute__ ((noinline)); +void dtSEAHookArgBlob1024(const void* ptr) /*never use directly*/ __attribute__ ((noinline)); +void dtSEAHookArgBlobEnd() /*never use directly*/ __attribute__ ((noinline)); + +void dtSEAHookEndScope(const char* domain, const char* name) __attribute__ ((noinline)); +void dtSEAHookMarker(const char* domain, const char* name, int scope) __attribute__ ((noinline)); +void dtSEAHookCounter(const char* domain, const char* name, double value) __attribute__ ((noinline)); namespace itt_notify { -template -class Task -{ -protected: - __itt_id m_id = __itt_null; - const __itt_domain* m_pDomain; -public: - Task(const __itt_domain* pDomain, __itt_string_handle* pName) - : m_pDomain(pDomain) + class Scope { - m_id = __itt_id_make(const_cast<__itt_domain*>(m_pDomain), reinterpret_cast(pName)); - if (bRegion) + public: + enum Type { - __itt_region_begin(m_pDomain, m_id, __itt_null, pName); + Task, + Region, + Log + }; + + inline Scope(bool task, const char* domain, const char* name) + : m_name(name) + , m_domain(domain) + { +#ifdef KERNEL + OSString * pStr = OSString::withCString(name); + dtSEAHookScope(task ? 0 : 1, pStr->getCStringNoCopy(), pStr->getCStringNoCopy()); + pStr->free(); +#else + dtSEAHookScope(task ? 0 : 1, m_domain, m_name); +#endif } - else + + inline void Arg(const char* name, int val){dtSEAHookArgInt(name, val);} + inline void Arg(const char* name, const char* val){dtSEAHookArgStr(name, val);} + inline void Arg(const char* name, void* val){dtSEAHookArgAddr(name, val);} + inline void ArgBlob(const char* name, const void* val, int size) { - __itt_task_begin(m_pDomain, m_id, __itt_null, pName); + const char* ptr = static_cast(val); +#ifdef KERNEL + OSString * pStr = OSString::withCString(name); + dtSEAHookArgBlobStart(size, pStr->getCStringNoCopy()); + pStr->free(); +#else + dtSEAHookArgBlobStart(size, name); +#endif + for (int i = 0; i < (size / 1024); ++i) + { + dtSEAHookArgBlob1024(ptr); + ptr += 1024; + } + char buff[1024] = {}; + memcpy(buff, ptr, size % 1024); + dtSEAHookArgBlob1024(buff); + dtSEAHookArgBlobEnd(); } - } + ~Scope(){ + dtSEAHookEndScope(m_domain, m_name); + } + protected: + const char* m_name = nullptr; + const char* m_domain = nullptr; + }; - template - typename std::enable_if::value, void>::type AddArg(__itt_string_handle* pName, const T& value) - { - double double_value = value; - __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_double, 1, &double_value); - } + #define ITT_DOMAIN(/*const char* */domain)\ + static const char __sea_domain_name[] = domain - void AddArg(__itt_string_handle* pName, int64_t value) - { - __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_s64, 1, &value); - } + //'group' defines virtual process (null means current process), track defines virtual thread + #define ITT_SCOPE_TRACK(/*const char* */group, /*const char* */ track) - void AddArg(__itt_string_handle* pName, const char* value) - { - __itt_metadata_str_add(m_pDomain, m_id, pName, value, 0); - } + #define ITT_COUNTER(/*const char* */name, /*double */value) dtSEAHookCounter(__sea_domain_name, name, value) - void AddArg(__itt_string_handle* pName, void const* const pValue) + enum MarkerScope { - __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_unknown, 1, const_cast(pValue)); + scope_global, + scope_process, + scope_thread, + scope_task, //means a task that will long until another marker with task scope in this thread occurs + }; + + #define ITT_MARKER(/*const char* */name, /*enum Scope*/scope) dtSEAHookMarker(__sea_domain_name, name, itt_notify::scope) + #define ITT_ARG(/*const char* */name, /*number or string*/ value) __sea_scope__.Arg(name, value) + #define ITT_ARG_BLOB(/*const char* */name, ptr, size) __sea_scope__.ArgBlob(name, ptr, size) + #define ITT_SCOPE(/*bool*/task, /*const char* */name) itt_notify::Scope __sea_scope__(task, __sea_domain_name, name) + #define ITT_SCOPE_TASK(/*const char* */name) ITT_SCOPE(true, name);// XXX ITT_ARG("__file__", __FILE__); ITT_ARG("__line__", __LINE__) + #define ITT_SCOPE_REGION(/*const char* */name) ITT_SCOPE(false, name);//XXX ITT_ARG("__file__", __FILE__); ITT_ARG("__line__", __LINE__) + #define ITT_FUNCTION_TASK() ITT_SCOPE_TASK(__FUNCTION__); //XXX ITT_ARG("__file__", __FILE__); ITT_ARG("__line__", __LINE__) +} + +#else + + #include + #include + #include + #include + #include + + #define INTEL_ITTNOTIFY_API_PRIVATE + #include "ittnotify.h" + + + template + auto is_streamable_impl(int) + -> decltype (T{}, + void(), // Handle evil operator , + std::declval() >> std::declval(), + void(), // Handle evil operator , + std::true_type{}); + + template // fallback, ... has less priority than int + std::false_type is_streamable_impl(...); // fallback, ... has less priority than int + + template + using is_streamable = decltype(is_streamable_impl(0)); + + +#ifdef __OBJC__ + template + struct is_objc_class : std::false_type { }; + + template + struct is_objc_class::value, bool>::type>: std::true_type { }; + + template ::value>::type> + std::ostream& operator<< (std::ostream& stream, T const & t) { + stream << [[t description] UTF8String]; + return stream; } +#endif - ~Task() + namespace itt_notify { + + + template + class Task { - if (bRegion) + protected: + __itt_id m_id = __itt_null; + const __itt_domain* m_pDomain; + public: + Task(const __itt_domain* pDomain, __itt_string_handle* pName) + : m_pDomain(pDomain) { - __itt_region_end(m_pDomain, m_id); + if (!m_pDomain || !m_pDomain->flags) return; + m_id = __itt_id_make(const_cast<__itt_domain*>(m_pDomain), reinterpret_cast(pName)); + if (bRegion) + { + __itt_region_begin(m_pDomain, m_id, __itt_null, pName); + } + else + { + __itt_task_begin(m_pDomain, m_id, __itt_null, pName); + } } - else + + template + typename std::enable_if::value, void>::type AddArg(__itt_string_handle* pName, const T& value) { - __itt_task_end(m_pDomain); + if (!m_pDomain || !m_pDomain->flags) return; + double double_value = value; + __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_double, 1, &double_value); } - } -}; -#ifdef _WIN32 - #define UNICODE_AGNOSTIC(name) name##A -#else - #define UNICODE_AGNOSTIC(name) name -#endif + void AddArg(__itt_string_handle* pName, int64_t value) + { + if (!m_pDomain || !m_pDomain->flags) return; + __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_s64, 1, &value); + } + + void AddArg(__itt_string_handle* pName, const char* value) + { + if (!m_pDomain || !m_pDomain->flags) return; + __itt_metadata_str_add(m_pDomain, m_id, pName, value, 0); + } -#define ITT_DOMAIN(/*const char* */domain)\ - static const __itt_domain* __itt_domain_name = UNICODE_AGNOSTIC(__itt_domain_create)(domain) + void AddArg(__itt_string_handle* pName, void const* const pValue) + { + if (!m_pDomain || !m_pDomain->flags) return; + __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_unknown, 1, const_cast(pValue)); + } -#if defined(_MSC_VER) && _MSC_VER >= 1900 //since VS 2015 magic statics are supported, TODO: check with other compilers - #define ITT_MAGIC_STATIC(static_variable) + template + void AddArg(__itt_string_handle* pName, const T& val, std::enable_if::value>) + { + if (!m_pDomain || !m_pDomain->flags) return; + std::ostringstream os; + os << val; + __itt_metadata_str_add(m_pDomain, m_id, pName, os.str().c_str(), 0); + } + + void AddArg(__itt_string_handle* pName, ...) + { + if (!m_pDomain || !m_pDomain->flags) return; + va_list vl; + va_start(vl, pName); + __itt_metadata_add(m_pDomain, m_id, pName, __itt_metadata_unknown, 1, reinterpret_cast(vl)); + va_end(vl); + } + + ~Task() + { + if (!m_pDomain || !m_pDomain->flags) return; + if (bRegion) + { + __itt_region_end(m_pDomain, m_id); + } + else + { + __itt_task_end(m_pDomain); + } + } + }; + +#ifdef ITT_PROTECT_SCOPE + #define ITT_TOKEN_PASTE(x, y) x ## y + #define ITT_TOKEN_PASTE2(x, y) ITT_TOKEN_PASTE(x, y) + #define ITT_LINE_NAME(name) ITT_TOKEN_PASTE2(name, __LINE__) #else -//the 'while' below is to protect code from crash in multi-threaded environment under compiler without magic statics support - #define ITT_MAGIC_STATIC(static_variable) while(!(static_variable)) std::this_thread::yield(); + #define ITT_LINE_NAME(name) name #endif -#define ITT_SCOPE(region, name)\ - static __itt_string_handle* __itt_scope_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ - ITT_MAGIC_STATIC(__itt_scope_name);\ - itt_notify::Task __itt_scope_item(__itt_domain_name, __itt_scope_name) + #ifdef _WIN32 + #define UNICODE_AGNOSTIC(name) name##A + #else + #define UNICODE_AGNOSTIC(name) name + #endif -#define ITT_SCOPE_TASK(/*const char* */name) ITT_SCOPE(false, name) -#define ITT_SCOPE_REGION(/*const char* */name) ITT_SCOPE(true, name) + #define ITT_DOMAIN(/*const char* */domain)\ + static const __itt_domain* __itt_domain_name = UNICODE_AGNOSTIC(__itt_domain_create)(domain) -#define ITT_FUNCTION_TASK() ITT_SCOPE_TASK(__FUNCTION__); ITT_ARG("__file__", __FILE__); ITT_ARG("__line__", __LINE__) + #if defined(_MSC_VER) && _MSC_VER >= 1900 //since VS 2015 magic statics are supported, TODO: check with other compilers + #define ITT_MAGIC_STATIC(static_variable) + #else + //the 'while' below is to protect code from crash in multi-threaded environment under compiler without magic statics support + #define ITT_MAGIC_STATIC(static_variable) while(!(static_variable)) std::this_thread::yield(); + #endif -#define ITT_ARG(/*const char* */name, /*number or string*/ value) {\ - static __itt_string_handle* __itt_arg_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ - ITT_MAGIC_STATIC(__itt_arg_name);\ - __itt_scope_item.AddArg(__itt_arg_name, value);\ -} + #define ITT_SCOPE(region, name)\ + static __itt_string_handle* ITT_LINE_NAME(__itt_scope_name) = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ + ITT_MAGIC_STATIC(ITT_LINE_NAME(__itt_scope_name));\ + itt_notify::Task ITT_LINE_NAME(__itt_scope_item)(__itt_domain_name, ITT_LINE_NAME(__itt_scope_name)) -enum Scope -{ - scope_global = __itt_scope_global, - scope_process = __itt_scope_track_group, - scope_thread =__itt_scope_track, - scope_task =__itt_scope_task, //means a task that will long until another marker with task scope in this thread occurs -}; - -#define ITT_MARKER(/*const char* */name, /*enum Scope*/scope) {\ - static __itt_string_handle* __itt_marker_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ - ITT_MAGIC_STATIC(__itt_marker_name);\ - __itt_marker(__itt_domain_name, __itt_null, __itt_marker_name, (__itt_scope)itt_notify::scope);\ -} + #define ITT_SCOPE_TASK(/*const char* */name) ITT_SCOPE(false, name) + #define ITT_SCOPE_REGION(/*const char* */name) ITT_SCOPE(true, name) -#define ITT_COUNTER(/*const char* */name, /*double */value) { \ - static __itt_string_handle* __itt_counter_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ - ITT_MAGIC_STATIC(__itt_counter_name);\ - double counter_value = value;\ - __itt_metadata_add(__itt_domain_name, __itt_null, __itt_counter_name, __itt_metadata_double, 1, &counter_value);\ -} + #define ITT_FUNCTION_TASK() ITT_SCOPE_TASK(__FUNCTION__); ITT_ARG("__file__", __FILE__); ITT_ARG("__line__", __LINE__) -class ScopeTrack -{ -public: - ScopeTrack(__itt_track* track) - { - __itt_set_track(track); + #define ITT_ARG(/*const char* */name, /*number or string*/ value) {\ + static __itt_string_handle* __itt_arg_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ + ITT_MAGIC_STATIC(__itt_arg_name);\ + ITT_LINE_NAME(__itt_scope_item).AddArg(__itt_arg_name, value);\ } - ~ScopeTrack() + + enum MarkerScope { - __itt_set_track(nullptr); + scope_global = __itt_scope_global, + scope_process = __itt_scope_track_group, + scope_thread =__itt_scope_track, + scope_task =__itt_scope_task, //means a task that will long until another marker with task scope in this thread occurs + }; + + #define ITT_MARKER(/*const char* */name, /*enum Scope*/scope) {\ + static __itt_string_handle* __itt_marker_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ + ITT_MAGIC_STATIC(__itt_marker_name);\ + __itt_marker(__itt_domain_name, __itt_null, __itt_marker_name, (__itt_scope)itt_notify::scope);\ } -}; -//'group' defines virtual process (null means current process), track defines virtual thread -#define ITT_SCOPE_TRACK(/*const char* */group, /*const char* */ track)\ - static __itt_track* itt_track_name = __itt_track_create(__itt_track_group_create(((group) ? UNICODE_AGNOSTIC(__itt_string_handle_create)(group) : nullptr), __itt_track_group_type_normal), UNICODE_AGNOSTIC(__itt_string_handle_create)(track), __itt_track_type_normal);\ - ITT_MAGIC_STATIC(itt_track_name);\ - itt_notify::ScopeTrack itt_track(itt_track_name); + #define ITT_COUNTER(/*const char* */name, /*double */value) { \ + static __itt_string_handle* __itt_counter_name = UNICODE_AGNOSTIC(__itt_string_handle_create)(name);\ + ITT_MAGIC_STATIC(__itt_counter_name);\ + double counter_value = value;\ + __itt_metadata_add(__itt_domain_name, __itt_null, __itt_counter_name, __itt_metadata_double, 1, &counter_value);\ + } -//TODO: objects + class ScopeTrack + { + public: + ScopeTrack(__itt_track* track) + { + __itt_set_track(track); + } + ~ScopeTrack() + { + __itt_set_track(nullptr); + } + }; + + //'group' defines virtual process (null means current process), track defines virtual thread + #define ITT_SCOPE_TRACK(/*const char* */group, /*const char* */ track)\ + static __itt_track* itt_track_name = __itt_track_create(__itt_track_group_create(((group) ? UNICODE_AGNOSTIC(__itt_string_handle_create)(group) : nullptr), __itt_track_group_type_normal), UNICODE_AGNOSTIC(__itt_string_handle_create)(track), __itt_track_type_normal);\ + ITT_MAGIC_STATIC(itt_track_name);\ + itt_notify::ScopeTrack itt_track(itt_track_name); + + //TODO: objects + + } //namespace itt_notify +#endif -} //namespace itt_notify +#endif //SEA_ITT_NOTIFY_HEADER + + +#ifdef SEA_USE_DTRACE_IMPL //must be included only in one cpp file of module + void dtSEAHookScope(int type, const char* domain, const char* name) {__asm__ __volatile__ ("");} + void dtSEAHookArgInt(const char* name, int value) {__asm__ __volatile__ ("");} + void dtSEAHookArgStr(const char* name, const char* value) {__asm__ __volatile__ ("");} + void dtSEAHookArgAddr(const char* name, const void* value) {__asm__ __volatile__ ("");} + void dtSEAHookEndScope(const char* domain, const char* name) {__asm__ __volatile__ ("");} + void dtSEAHookMarker(const char* domain, const char* name, int scope) {__asm__ __volatile__ ("");} + void dtSEAHookCounter(const char* domain, const char* name, double value) {__asm__ __volatile__ ("");} + void dtSEAHookArgBlobStart(int size, const char* name) /*never use directly*/ {__asm__ __volatile__ ("");} + void dtSEAHookArgBlob1024(const void* ptr) /*never use directly*/ {__asm__ __volatile__ ("");} + void dtSEAHookArgBlobEnd() /*never use directly*/ {__asm__ __volatile__ ("");} + +#endif diff --git a/main.cpp b/main.cpp index 3c6ebdc..eebc2d5 100644 --- a/main.cpp +++ b/main.cpp @@ -1,281 +1,286 @@ -/********************************************************************************************************************************************************************************************************************************************************************************************* -# Intel® Single Event API -# -# This file is provided under the BSD 3-Clause license. -# Copyright (c) 2015, Intel Corporation -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -# Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -**********************************************************************************************************************************************************************************************************************************************************************************************/ -#include "itt_notify.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 - #define setenv _putenv - #include - #undef API_VERSION - #include - #pragma comment(lib, "dbghelp") -#else - #include - #include - - #define setenv putenv - #define _strdup strdup -#endif - -#if (INTPTR_MAX == INT32_MAX) - #define BIT_SUFFIX "32" -#elif INTPTR_MAX == INT64_MAX - #define BIT_SUFFIX "64" -#else - #error "Environment not 32 or 64-bit!" -#endif - -#define INTEL_LIBITTNOTIFY "INTEL_LIBITTNOTIFY" BIT_SUFFIX - -#ifdef __APPLE__ //fat binary is produced, so no bitness is suffixed to dylib name - #define LIB_ITT_NAME "./libIntelSEAPI" -#elif _WIN32 - #define LIB_ITT_NAME "./IntelSEAPI" BIT_SUFFIX -#else - #define LIB_ITT_NAME "./libIntelSEAPI" BIT_SUFFIX -#endif - -#ifdef _WIN32 - #define LIB_ITT LIB_ITT_NAME ".dll" -#elif defined(__APPLE__) - #define LIB_ITT LIB_ITT_NAME ".dylib" -#else - #define LIB_ITT LIB_ITT_NAME ".so" -#endif - -static std::string get_environ_value(const std::string& name) -{ -#ifdef _WIN32 - size_t sz = 0; - char *v = NULL; - _dupenv_s(&v, &sz, name.c_str()); - - std::string ret = v ? v : ""; - free(v); - - return ret; -#else - const char *v = std::getenv(name.c_str()); - return v ? v : ""; -#endif -} - -bool IsVerboseMode() -{ - static bool bVerboseMode = !!get_environ_value("INTEL_SEA_VERBOSE").size(); - return bVerboseMode; -} -#define VerbosePrint(...) {if (IsVerboseMode()) printf(__VA_ARGS__);} - - -int GlobalInit() -{ - std::string val = get_environ_value(INTEL_LIBITTNOTIFY); - if (val.size() && get_environ_value("INTEL_FORCE_SEA").empty()) - { - VerbosePrint("MAIN: %s was already set to %s\n", INTEL_LIBITTNOTIFY, val.c_str()); - } - else - { -#ifndef __ANDROID__ - setenv(_strdup(INTEL_LIBITTNOTIFY "=" LIB_ITT)); - VerbosePrint("MAIN: setting %s = %s\n", INTEL_LIBITTNOTIFY, LIB_ITT); -#endif - } - -#ifdef __ANDROID__ - if (get_environ_value("INTEL_SEA_SAVE_TO").empty()) - setenv(_strdup("INTEL_SEA_SAVE_TO=/data/local/tmp/ISEA")); -#elif defined(__linux__) - if (get_environ_value("INTEL_SEA_SAVE_TO").empty()) - setenv(_strdup("INTEL_SEA_SAVE_TO=/tmp/ISEA")); -#endif - return 1; -} - -int nSetLib = GlobalInit(); - -#if defined(__ANDROID__) -#include -namespace std { //android NDK is missing this functionality - template - std::string to_string(T value) - { - std::ostringstream os; - os << value; - return os.str(); - } -} -#endif - -extern bool g_done; - - -#ifdef _WIN32 - -std::string GetFunctionName(DWORD64 addr, const char* szModulePath) -{ - std::string res; - HANDLE hCurProc = GetCurrentProcess(); - SymSetOptions(SymGetOptions()|SYMOPT_LOAD_LINES|SYMOPT_UNDNAME|SYMOPT_INCLUDE_32BIT_MODULES); - SymInitialize(hCurProc, NULL, TRUE); - uint64_t module = SymLoadModule64(hCurProc, NULL, szModulePath, NULL, 0, 0); - if (!module) return res; - IMAGEHLP_LINE64 line = {sizeof(IMAGEHLP_LINE64)}; - DWORD dwDisplacement = 0; - SymGetLineFromAddr64(hCurProc, module + addr, &dwDisplacement, &line); - if (line.FileName) - { - res += std::string(line.FileName) + "(" + std::to_string(line.LineNumber) + ")\n"; - } - - char buff[sizeof(SYMBOL_INFO) + 1024] = {}; - SYMBOL_INFO * symbol = (SYMBOL_INFO*)buff; - symbol->MaxNameLen = 255; - symbol->SizeOfStruct = sizeof(SYMBOL_INFO); - SymFromAddr(hCurProc, module + addr, nullptr, symbol); - res += symbol->Name; - return res; -} - -BOOL CALLBACK EnumSymbolsCallback(_In_ PSYMBOL_INFO pSymInfo, _In_ ULONG SymbolSize, _In_opt_ PVOID UserContext) -{ - if (!pSymInfo) // || pSymInfo->Tag != 5/*SymTagFunction*/ - return TRUE; - - std::cout << (pSymInfo->Address - pSymInfo->ModBase) << "\t" << pSymInfo->Size << "\t" << pSymInfo->Name; - IMAGEHLP_LINE64 line = { sizeof(IMAGEHLP_LINE64) }; - DWORD dwDisplacement = 0; - SymGetLineFromAddr64(UserContext, pSymInfo->Address, &dwDisplacement, &line); - if (line.FileName) - { - std::cout << "\t" << std::string(line.FileName) + "(" + std::to_string(line.LineNumber) + ")"; - } - - std::cout << std::endl; - - return TRUE; -} - -bool DumpModule(const char* szModulePath) -{ - std::string res; - HANDLE hCurProc = GetCurrentProcess(); - SymSetOptions(SymGetOptions() | SYMOPT_LOAD_LINES | SYMOPT_UNDNAME | SYMOPT_INCLUDE_32BIT_MODULES); - SymInitialize(hCurProc, NULL, TRUE); - uint64_t module = SymLoadModule64(hCurProc, NULL, szModulePath, NULL, 0, 0); - if (!module) return false; - return !!::SymEnumSymbols(hCurProc, module, NULL, EnumSymbolsCallback, hCurProc/*context*/); -} - -int ResolveSymbol(char* request) -{ - char* addr_pos = strrchr(request, ':') + 1; - DWORD64 addr = _atoi64(addr_pos); - if (addr) - { - *(addr_pos - 1) = 0; - std::string res = GetFunctionName(addr, request); - std::cout << res << std::endl; - return res.size() ? 0 : -1; - } - else - { - return DumpModule(request) ? 0 : -1; - } -} - -#endif - -extern __itt_domain* g_domain; - -void ChangePaths() -{ - std::string path = get_environ_value("INTEL_SEA_SAVE_TO"); - if (path.empty()) return; - __itt_string_handle* handle = __itt_string_handle_create("__sea_set_folder"); - int counter = 0; - while (!g_done) - { - std::this_thread::sleep_for(std::chrono::seconds(5)); - __itt_metadata_str_add(g_domain, __itt_null, handle, (!(counter % 2) ? "" : (path + std::to_string(counter)).c_str()), 0); - ++counter; - } -} - -int MeasurePerformance(int work_seconds); -void Main(int work_seconds); - - -#ifdef _WIN32 -int _tmain(int argc, _TCHAR* argv[]) -#else -int main(int argc, char* argv[]) -#endif -{ - int work_seconds = 3; - if (argc > 1) - { -#ifdef _WIN32 - if (strchr(argv[1], ':')) - return ResolveSymbol(argv[1]); -#endif - work_seconds = std::atoi(argv[1]); - } - std::string mode; - if (argc > 2) - mode = argv[2]; - - const char* version2 = __itt_api_version(); - (void)version2; - char path[] = -#ifdef _WIN32 - "c:/temp/trace.json"; -#else - "/tmp/trace.json"; -#endif - - VerbosePrint("Mode: %s\n", mode.c_str()); - - if (std::string::npos != mode.find("perf")) - return MeasurePerformance(work_seconds); - - const char* api_ver = __itt_api_version(); - VerbosePrint("ITT Version: %s\n", api_ver ? api_ver : "Not loaded"); - - //std::thread thrd(ChangePaths); //only for stress testing - Main(work_seconds); - //thrd.join(); - return 0; -} - +/********************************************************************************************************************************************************************************************************************************************************************************************* +# Intel® Single Event API +# +# This file is provided under the BSD 3-Clause license. +# Copyright (c) 2015, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +# Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +**********************************************************************************************************************************************************************************************************************************************************************************************/ +#include "itt_notify.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 + #define setenv _putenv + #include + #undef API_VERSION + #include + #pragma comment(lib, "dbghelp") +#else + #include + #include + + #define setenv putenv + #define _strdup strdup +#endif + +#if (INTPTR_MAX == INT32_MAX) + #define BIT_SUFFIX "32" +#elif INTPTR_MAX == INT64_MAX + #define BIT_SUFFIX "64" +#else + #error "Environment not 32 or 64-bit!" +#endif + +#define INTEL_LIBITTNOTIFY "INTEL_LIBITTNOTIFY" BIT_SUFFIX + +#ifdef __APPLE__ //fat binary is produced, so no bitness is suffixed to dylib name + #define LIB_ITT_NAME "./libIntelSEAPI" +#elif _WIN32 + #define LIB_ITT_NAME "./IntelSEAPI" BIT_SUFFIX +#else + #define LIB_ITT_NAME "./libIntelSEAPI" BIT_SUFFIX +#endif + +#ifdef _WIN32 + #define LIB_ITT LIB_ITT_NAME ".dll" +#elif defined(__APPLE__) + #define LIB_ITT LIB_ITT_NAME ".dylib" +#else + #define LIB_ITT LIB_ITT_NAME ".so" +#endif + +std::string get_environ_value(const std::string& name) +{ +#ifdef _WIN32 + size_t sz = 0; + char *v = NULL; + _dupenv_s(&v, &sz, name.c_str()); + + std::string ret = v ? v : ""; + free(v); + + return ret; +#else + const char *v = std::getenv(name.c_str()); + return v ? v : ""; +#endif +} + +bool IsVerboseMode() +{ +#if defined(_DEBUG) && defined(__ANDROID__) + return true; +#else + static bool bVerboseMode = get_environ_value("INTEL_SEA_VERBOSE").size(); + return bVerboseMode; +#endif +} +#define VerbosePrint(...) {if (IsVerboseMode()) printf(__VA_ARGS__);} + + +int GlobalInit() +{ + std::string val = get_environ_value(INTEL_LIBITTNOTIFY); + if (val.size() && get_environ_value("INTEL_FORCE_SEA").empty()) + { + VerbosePrint("MAIN: %s was already set to %s\n", INTEL_LIBITTNOTIFY, val.c_str()); + } + else + { +#ifndef __ANDROID__ + setenv(_strdup(INTEL_LIBITTNOTIFY "=" LIB_ITT)); + VerbosePrint("MAIN: setting %s = %s\n", INTEL_LIBITTNOTIFY, LIB_ITT); +#endif + } + +#ifdef __ANDROID__ + if (get_environ_value("INTEL_SEA_SAVE_TO").empty()) + setenv(_strdup("INTEL_SEA_SAVE_TO=/data/local/tmp/ISEA")); +#elif defined(__linux__) + if (get_environ_value("INTEL_SEA_SAVE_TO").empty()) + setenv(_strdup("INTEL_SEA_SAVE_TO=/tmp/ISEA")); +#endif + return 1; +} + +int nSetLib = GlobalInit(); + +#if defined(__ANDROID__) +#include +namespace std { //android NDK is missing this functionality + template + std::string to_string(T value) + { + std::ostringstream os; + os << value; + return os.str(); + } +} +#endif + +extern bool g_done; + + +#ifdef _WIN32 + +std::string GetFunctionName(DWORD64 addr, const char* szModulePath) +{ + std::string res; + HANDLE hCurProc = GetCurrentProcess(); + SymSetOptions(SymGetOptions()|SYMOPT_LOAD_LINES|SYMOPT_UNDNAME|SYMOPT_INCLUDE_32BIT_MODULES); + SymInitialize(hCurProc, NULL, TRUE); + uint64_t module = SymLoadModule64(hCurProc, NULL, szModulePath, NULL, 0, 0); + if (!module) return res; + IMAGEHLP_LINE64 line = {sizeof(IMAGEHLP_LINE64)}; + DWORD dwDisplacement = 0; + SymGetLineFromAddr64(hCurProc, module + addr, &dwDisplacement, &line); + if (line.FileName) + { + res += std::string(line.FileName) + "(" + std::to_string(line.LineNumber) + ")\n"; + } + + char buff[sizeof(SYMBOL_INFO) + 1024] = {}; + SYMBOL_INFO * symbol = (SYMBOL_INFO*)buff; + symbol->MaxNameLen = 255; + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + SymFromAddr(hCurProc, module + addr, nullptr, symbol); + res += symbol->Name; + return res; +} + +BOOL CALLBACK EnumSymbolsCallback(_In_ PSYMBOL_INFO pSymInfo, _In_ ULONG SymbolSize, _In_opt_ PVOID UserContext) +{ + if (!pSymInfo) // || pSymInfo->Tag != 5/*SymTagFunction*/ + return TRUE; + + std::cout << (pSymInfo->Address - pSymInfo->ModBase) << "\t" << pSymInfo->Size << "\t" << pSymInfo->Name; + IMAGEHLP_LINE64 line = { sizeof(IMAGEHLP_LINE64) }; + DWORD dwDisplacement = 0; + SymGetLineFromAddr64(UserContext, pSymInfo->Address, &dwDisplacement, &line); + if (line.FileName) + { + std::cout << "\t" << std::string(line.FileName) + "(" + std::to_string(line.LineNumber) + ")"; + } + + std::cout << std::endl; + + return TRUE; +} + +bool DumpModule(const char* szModulePath) +{ + std::string res; + HANDLE hCurProc = GetCurrentProcess(); + SymSetOptions(SymGetOptions() | SYMOPT_LOAD_LINES | SYMOPT_UNDNAME | SYMOPT_INCLUDE_32BIT_MODULES); + SymInitialize(hCurProc, NULL, TRUE); + uint64_t module = SymLoadModule64(hCurProc, NULL, szModulePath, NULL, 0, 0); + if (!module) return false; + return !!::SymEnumSymbols(hCurProc, module, NULL, EnumSymbolsCallback, hCurProc/*context*/); +} + +int ResolveSymbol(char* request) +{ + char* addr_pos = strrchr(request, ':') + 1; + DWORD64 addr = _atoi64(addr_pos); + if (addr) + { + *(addr_pos - 1) = 0; + std::string res = GetFunctionName(addr, request); + std::cout << res << std::endl; + return res.size() ? 0 : -1; + } + else + { + return DumpModule(request) ? 0 : -1; + } +} + +#endif + +extern __itt_domain* g_domain; + +void ChangePaths() +{ + std::string path = get_environ_value("INTEL_SEA_SAVE_TO"); + if (path.empty()) return; + __itt_string_handle* handle = __itt_string_handle_create("__sea_set_folder"); + int counter = 0; + while (!g_done) + { + std::this_thread::sleep_for(std::chrono::seconds(5)); + __itt_metadata_str_add(g_domain, __itt_null, handle, (!(counter % 2) ? "" : (path + std::to_string(counter)).c_str()), 0); + ++counter; + } +} + +int MeasurePerformance(int work_seconds); +void Main(int work_seconds); + + +#ifdef _WIN32 +int _tmain(int argc, _TCHAR* argv[]) +#else +int main(int argc, char* argv[]) +#endif +{ + int work_seconds = 3; + if (argc > 1) + { +#ifdef _WIN32 + if (strchr(argv[1], ':')) + return ResolveSymbol(argv[1]); +#endif + work_seconds = std::atoi(argv[1]); + } + std::string mode; + if (argc > 2) + mode = argv[2]; + + const char* version2 = __itt_api_version(); + (void)version2; + char path[] = +#ifdef _WIN32 + "c:/temp/trace.json"; +#else + "/tmp/trace.json"; +#endif + + VerbosePrint("Mode: %s\n", mode.c_str()); + + if (std::string::npos != mode.find("perf")) + return MeasurePerformance(work_seconds); + + const char* api_ver = __itt_api_version(); + VerbosePrint("ITT Version: %s\n", api_ver ? api_ver : "Not loaded"); + if (!api_ver) + return -1; + //std::thread thrd(ChangePaths); //only for stress testing + Main(work_seconds); + //thrd.join(); + return 0; +} + diff --git a/memory.cpp b/memory.cpp index 6d97313..41b5ff1 100644 --- a/memory.cpp +++ b/memory.cpp @@ -17,6 +17,8 @@ **********************************************************************************************************************************************************************************************************************************************************************************************/ #include "ittnotify.h" +#include "isea_test.h" + #if defined(_DEBUG) && !defined(__ANDROID__) #ifdef UNICODE @@ -25,6 +27,10 @@ __itt_heap_function g_heap = __itt_heap_function_create("CRT", "Memory"); #endif + + +bool bCollectMemory = std::string::npos != get_environ_value("INTEL_SEA_FEATURES").find("mem"); + #ifdef _WIN32 #if _MSC_VER == 1800 //VS2013 #define _CRTBLD //hack, no words @@ -96,7 +102,7 @@ return true; } - bool bInit = InitMemHooks(); + bool bInit = bCollectMemory && InitMemHooks(); #else @@ -211,7 +217,7 @@ return true; } - bool bInit = InitMemHooks(); + bool bInit = bCollectMemory && InitMemHooks(); #elif defined(__linux__) && defined(_DEBUG) diff --git a/runtool/collectors/android.py b/runtool/collectors/android.py index bdec485..1a5d6bf 100644 --- a/runtool/collectors/android.py +++ b/runtool/collectors/android.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import subprocess from sea_runtool import Collector, subst_env_vars @@ -8,8 +9,6 @@ def __init__(self, args): Collector.__init__(self, args) self.adb = self.detect() self.file = None - if self.adb: - self.start() def is_root(self, statics={}): if statics: @@ -33,7 +32,7 @@ def detect(cls): version = parts[parts.index('version') + 1] systraces.append((version, adb)) if systraces: - sorted_by_version = sorted(systraces, key=lambda(ver, _): [int(item) for item in ver.split('.')], reverse=True) + sorted_by_version = sorted(systraces, key=lambda ver__: [int(item) for item in ver__[0].split('.')], reverse=True) return '"%s"' % sorted_by_version[0][1] else: return None @@ -56,6 +55,9 @@ def echo(self, what, where): return out, err def start(self): + if not self.adb: + print("Failed to run without adb...") + return self.file = os.path.join(subst_env_vars(self.args.input), 'atrace-%s.ftrace' % (self.args.cuts[0] if self.args.cuts else '0')) self.echo('0', '/sys/kernel/debug/tracing/tracing_on') self.echo('', '/sys/kernel/debug/tracing/trace') diff --git a/runtool/collectors/ftrace.py b/runtool/collectors/ftrace.py index c37ba2d..6eca428 100644 --- a/runtool/collectors/ftrace.py +++ b/runtool/collectors/ftrace.py @@ -5,6 +5,9 @@ import traceback import subprocess +# http://www.brendangregg.com/perf.html +# sudo perf probe --funcs + sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import sea @@ -118,7 +121,6 @@ def __init__(self, args, remote=False): for event in supported_events: for path in glob.glob('/sys/kernel/debug/tracing/events/*/%s/enable' % event): self.event_list.append(path) - self.start() def echo(self, what, where): self.log("echo %s > %s" % (what, where)) @@ -165,7 +167,8 @@ def start(self): if os.path.exists(self.perf_file): os.remove(self.perf_file) cmd = 'perf record -a -g -o "%s" --pid=%s' % (self.perf_file, self.args.target) - self.perf_proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setpgrp) + self.log(cmd) + self.perf_proc = subprocess.Popen(cmd, shell=True, stdout=self.get_output(), stderr=self.get_output(), preexec_fn=os.setpgrp) def copy_from_target(self, what, where): self.log("copy %s > %s" % (what, where)) @@ -197,6 +200,7 @@ def stop(self, wait=True): shutil.copyfileobj(file_from, file_to) os.remove(file_name) results.append(self.file) + self.execute('chmod -R a+rwX "%s"' % self.args.output) return results diff --git a/runtool/collectors/osx.py b/runtool/collectors/osx.py index 562fddf..bfabdc3 100644 --- a/runtool/collectors/osx.py +++ b/runtool/collectors/osx.py @@ -1,9 +1,41 @@ +from __future__ import print_function +import re import os import sys +import time +from datetime import datetime, timedelta import shutil import subprocess +import threading sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) -from sea_runtool import Collector, get_decoders +import sea +from sea_runtool import Collector, get_decoders, is_domain_enabled, message, get_original_env + +""" + sudo dtrace -l | perl -pe 's/^.*?\S+\s+(\S+?)([0-9]|\s).*/\1/' | sort | uniq > /tmp/dtrace_providers.txt + sudo dtrace -l > /tmp/dtrace_list.txt + dtrace -n 'fbt:::entry { @[probefunc] = count(); }' -c 'ping host' + http://www.brendangregg.com/DTrace/DTrace-cheatsheet.pdf + + https://docs.oracle.com/cd/E19253-01/817-6223/chp-variables-5/index.html + TODO: printf("%s: called from %a\n", probefunc, caller); + objc_runtime$target::: { ustack(); } /*objc_exception_throw*/ + pid$target::objc_msgSend:entry + sudo dtrace -qn 'fbt::*vent13k*:entry/arg3/{printf("%d\n",arg2)}' # keylogger +""" + +DSCRIPT_HEADER = r""" +#pragma D option nolibs +#define GREEDY_ON ++self->greedy_enabled +#define GREEDY_OFF self->greedy_enabled = (self->greedy_enabled > 0) ? (self->greedy_enabled - 1) : self->greedy_enabled + +BEGIN +{ + self->greedy_enabled = 0; +} + + +""" dtrace_context_switch = r""" @@ -25,12 +57,13 @@ curlwpsinfo->pr_lwpid, curlwpsinfo->pr_pri, curpsinfo->pr_fname, args[0]->pr_lwpid, args[0]->pr_pri, args[1]->pr_fname ); - /*{OFF_CPU}*/ } - """ OFF_CPU_STACKS = r""" +sched:::off-cpu +/pid == $target/ +{ printf("%x\tkstack\t%x\t%x:", machtimestamp, pid, tid); stack(); printf("\n%x\tustack\t%x\t%x:", machtimestamp, pid, tid); @@ -40,99 +73,216 @@ jstack(); //TODO: enable better support for jstack-s */ printf("\n"); +} +""" + +dtrace_wakeup = r""" +sched:::wakeup +/curpsinfo->pr_pid == $target || args[1]->pr_pid == $target/ +{ + printf("%x\twkp\t%x\t%x\t%s\t%x\t%s\t%x\t%x\t%x\t%x\n", machtimestamp, + curpsinfo->pr_pid, curlwpsinfo->pr_lwpid, + execname, cpu, + stringof(args[1]->pr_fname), + args[1]->pr_pid, args[0]->pr_lwpid, + args[0]->pr_stype, args[0]->pr_wchan + ); +} + """ + osxaskpass = r"""#!/bin/bash osascript -e 'Tell application "System Events" to display dialog "Password:" default answer "" with hidden answer with title "DTrace requires root priveledges"' -e 'text returned of result' 2>/dev/null """ -pid_dtrace_hooks = r""" +pid_dtrace_hooks = [r""" +pid$target::*dtSEAHookScope*:entry /*{UMD_STACKS}*/ +{ + printf( + "%x\te\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, copyinstr(arg1), copyinstr(arg2) + ); +} +""", r""" +pid$target::*dtSEAHookEndScope*:entry +{ + printf( + "%x\tr\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, copyinstr(arg0), copyinstr(arg1) + ); +} +""", r""" +/* +pid$target::*dtSEAHookArgStr*:entry +{ + printf( + "%x\targ\t%x\t%x\t%s\t%s\n", + machtimestamp, pid, tid, copyinstr(arg0), copyinstr(arg1) + ); +} -pid$target:Metal::entry +pid$target::*dtSEAHookArgInt*:entry { printf( - "%x\te\t%x\t%x\tmtl\t%s\n", machtimestamp, pid, tid, probefunc + "%x\targ\t%x\t%x\t%s\t%d\n", + machtimestamp, pid, tid, copyinstr(arg0), arg1 ); } +*/ + +""" -pid$target:Metal::return +] + +pid_dtrace_hooks += [ # XXX +r""" +objc$target:::entry +/*{CONDITIONS}*/ +{ + printf( + "%x\te\t%x\t%x\tobjc\t%s%s\n", machtimestamp, pid, tid, probemod, probefunc + ); + /*{ARGUMENTS}*/ +} +""", r""" +objc$target:::return +/*{CONDITIONS}*/ { printf( - "%x\tr\t%x\t%x\tmtl\t%s\n", machtimestamp, pid, tid, probefunc + "%x\tr\t%x\t%x\tobjc\t%s%s\n", machtimestamp, pid, tid, probemod, probefunc ); } +""" +] if 0 else [] + + +FOLLOW_CHILD = r""" +//https://www.synack.com/2015/11/17/monitoring-process-creation-via-the-kernel-part-i/ +proc:::exec-success /$target == curpsinfo->pr_ppid/{ + printf("%x\tfollowchild\t%x\t%x\t%s\t%x\t%x\t%s\n", machtimestamp, pid, tid, probename, curpsinfo->pr_ppid, curpsinfo->pr_pid, curpsinfo->pr_psargs); + system("printf \"%d\n\" >> /*{FOLLOW_CHILD}*/", curpsinfo->pr_pid); +} -pid$target:OpenGL:CGLFlushDrawable:entry +proc:::exec /$target == curpsinfo->pr_ppid/{ + printf("%x\tfollowchild\t%x\t%x\t%s\t%x\t%x\t%s\n", machtimestamp, pid, tid, probename, curpsinfo->pr_ppid, curpsinfo->pr_pid, curpsinfo->pr_psargs); + system("printf \"%d\n\" >> /*{FOLLOW_CHILD}*/", curpsinfo->pr_pid); +} + +syscall::exec*:return /$target == curpsinfo->pr_ppid/ { printf( - "%x\te\t%x\t%x\togl\t%s\n", machtimestamp, pid, tid, probefunc + "%x\tfollowchild\t%x\t%x\t%s\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc, probename ); } -pid$target:OpenGL:CGLFlushDrawable:return +syscall::fork:return /$target == curpsinfo->pr_ppid/ { printf( - "%x\tr\t%x\t%x\togl\t%s\n", machtimestamp, pid, tid, probefunc + "%x\tfollowchild\t%x\t%x\t%s\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc, probename ); + system("printf \"%d\n\" >> /*{FOLLOW_CHILD}*/", curpsinfo->pr_pid); } -/* TODO: move under namespace check -pid$target::*dtSEAHookScope*:entry +""" + +BRACKET_FUNC = r""" +pid$target:/*{M:F}*/:entry /*{UMD_STACKS}*/ +/*{CONDITIONS}*/ { printf( - "%x\te\t%x\t%x\t%s:%s\t%s\t%d\n", - machtimestamp, pid, tid, probemod, copyinstr(arg0), copyinstr(arg1), arg2 + "%x\te\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc ); - printf("%x\tustack\t%x\t%x:", machtimestamp, pid, tid); - ustack(); - printf("\n"); + /*{ARGUMENTS}*/ + GREEDY_ON; } +""", r""" +pid$target:/*{M:F}*/:return +/*{CONDITIONS}*/ +{ + GREEDY_OFF; + + printf( + "%x\tr\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc + ); +} +""" -pid$target::*dtSEAHookEndScope*:entry + +def bracket_hook(module='', function=''): + res = [] + for item in BRACKET_FUNC: + res.append(item.replace('/*{M:F}*/', '%s:%s' % (module, function))) + return res + + +def bracket_hook_kmd(module='', function=''): + res = [] + for item in bracket_hook(module, function): + res.append(item.replace('pid$target:', 'fbt:').replace('/*{UMD_STACKS}*/', '/*{KMD_STACKS}*/').replace('/*{CONDITIONS}*/', '')) + return res + + +for mask in ['JavaScriptCore', '*GL*:*gl*', '*GL*:*GL*', 'Metal', '*MTLDriver', '*GLDriver']: # ,'mdtest', 'libigdmd.dylib' + pid_dtrace_hooks += bracket_hook(*mask.split(':')) + + +# TODO: add opencl_api & opencl_cpu providers + +IO_HOOKS = r""" + +fsinfo:::open,fsinfo:::close { printf( - "%x\tr\t%x\t%x\t%s:%s\t%s\n", - machtimestamp, pid, tid, probemod, copyinstr(arg0), copyinstr(arg1) + "%x\tio\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, probename, stringof(args[0]->fi_pathname) ); } -pid$target::*dtSEAHookArgStr*:entry +""" + + +OPEN_CL = r""" +opencl_api$target:::, opencl_cpu$target::: { printf( - "%x\targ\t%x\t%x\t%s\t%s\n", - machtimestamp, pid, tid, copyinstr(arg0), copyinstr(arg1) + "%x\tocl\t%x\t%x\t%s\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc, probename ); } +""" -pid$target::*dtSEAHookArgInt*:entry +# FIXME: extract Interrupt handling and do conditional +fbt_dtrace_hooks = [r""" + +//void kernel_debug_enter(uint32_t coreid, uint32_t debugid, uint64_t timestamp, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t threadid); +fbt::kernel_debug_enter:entry { printf( - "%x\targ\t%x\t%x\t%s\t%d\n", - machtimestamp, pid, tid, copyinstr(arg0), arg1 + "%x\tkd\t%x\t%x\t%s\t%x\t%x\t%x\t%x\t%x\t%x\t%x\t%x\n", machtimestamp, pid, tid, probefunc, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 ); } -*/ -""" +""", r""" -fbt_dtrace_hooks = r""" -/* TODO: move under namespace check -fbt::*dtSEAHookScope*:entry +fbt::*debugid*enabled*:entry { printf( - "%x\te\t%x\t%x\t%s:%s\t%s\t%d\n", - machtimestamp, pid, tid, stringof(probemod), stringof(arg0), stringof(arg1), arg2 + "%x\tkd\t%x\t%x\t%s\t%x\t%x\t%x\t%x\t%x\t%x\n", machtimestamp, pid, tid, probefunc, arg0, arg1, arg2, arg3, arg4, arg5 ); - printf("%x\tkstack\t%x\t%x:", machtimestamp, pid, tid); - stack(); - printf("\n"); } + +""" if not "DISABLED XXX" else '', r""" +fbt::*dtSEAHookScope*:entry /*{KMD_STACKS}*/ +{ + printf( + "%x\te\t%x\t%x\t%s_%s\t%s\t%d\n", + machtimestamp, pid, tid, stringof(probemod), stringof(arg1), stringof(arg2), arg0 + ); +} +""", r""" fbt::*dtSEAHookEndScope*:entry { printf( - "%x\tr\t%x\t%x\t%s:%s\t%s\n", + "%x\tr\t%x\t%x\t%s_%s\t%s\n", machtimestamp, pid, tid, stringof(probemod), stringof(arg0), stringof(arg1) ); } @@ -152,39 +302,165 @@ machtimestamp, pid, tid, stringof(arg0), arg1 ); } + +fbt::*dtSEAHookArgBlobStart*:entry +{ + printf( + "%x\tbs\t%x\t%x\t%x\t%s\t%d\n", + machtimestamp, pid, tid, arg1, stringof(arg1), arg0 + ); + trace(stringof(arg0)); + tracemem(arg0, 10); +} + +fbt::*dtSEAHookArgBlob1024*:entry +{ + printf( + "%x\tblb\t%x\t%x\n", + machtimestamp, pid, tid + ); + tracemem(arg0, 1024); +} + +fbt::*dtSEAHookArgBlobEnd*:entry +{ + printf( + "%x\tbe\t%x\t%x\n", + machtimestamp, pid, tid + ); +} + +""" if not "DISABLED XXX" else '', r""" + +/* + Interrupt handling. + The list of interrupts was obtained by running 'dtrace -l | grep handleInterrupt' */ +fbt:com.apple.driver.AppleAPIC:_ZN28AppleAPICInterruptController15handleInterruptEPvP9IOServicei:entry, +fbt:com.apple.iokit.IOPCIFamily:_ZN8AppleVTD15handleInterruptEP22IOInterruptEventSourcei:entry, +fbt:com.apple.iokit.IOPCIFamily:_ZN32IOPCIMessagedInterruptController15handleInterruptEPvP9IOServicei:entry, +fbt:com.apple.driver.AppleSMBusController:_ZN23AppleSMBusControllerMCP15handleInterruptEP22IOInterruptEventSourcei:entry, +fbt:com.apple.driver.AppleThunderboltNHI:_ZN19AppleThunderboltNHI15handleInterruptEv:entry +{ + printf("%x\tie\t%x\t%x\t%s\t%x\t%s\t%s\n", machtimestamp, + curpsinfo->pr_pid, curlwpsinfo->pr_lwpid, + execname, cpu, probemod, probefunc + ); +} + +fbt:com.apple.driver.AppleAPIC:_ZN28AppleAPICInterruptController15handleInterruptEPvP9IOServicei:return, +fbt:com.apple.iokit.IOPCIFamily:_ZN8AppleVTD15handleInterruptEP22IOInterruptEventSourcei:return, +fbt:com.apple.iokit.IOPCIFamily:_ZN32IOPCIMessagedInterruptController15handleInterruptEPvP9IOServicei:return, +fbt:com.apple.driver.AppleSMBusController:_ZN23AppleSMBusControllerMCP15handleInterruptEP22IOInterruptEventSourcei:return, +fbt:com.apple.driver.AppleThunderboltNHI:_ZN19AppleThunderboltNHI15handleInterruptEv:return +{ + printf("%x\tir\t%x\t%x\t%s\t%x\t%s\t%s\n", machtimestamp, + curpsinfo->pr_pid, curlwpsinfo->pr_lwpid, + execname, cpu, probemod, probefunc + ); +} + +"""] + +HOTSPOTS = bracket_hook() # all modules and functions + +STATS = r""" +pid$target:::entry +{ + self->hotspots[probemod, probefunc] = machtimestamp; +} + +pid$target:::return +/self->hotspots[probemod, probefunc]/ +{ + @hotspots[probemod, probefunc] = avg(machtimestamp - self->hotspots[probemod, probefunc]); + self->hotspots[probemod, probefunc] = 0; +} + +END +{ + printf("\n-=STATS=-\n"); +} """ -HOTSPOTS = r""" +GREEDY_UMD = r""" pid$target:::entry +/self->greedy_enabled/ { printf( "%x\te\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc ); - /*{UMD_STACKS}*/ + /*{ARGUMENTS}*/ } pid$target:::return +/self->greedy_enabled/ { printf( "%x\tr\t%x\t%x\t%s\t%s\n", machtimestamp, pid, tid, probemod, probefunc ); - /*{UMD_STACKS}*/ } + """ -UMD_STACKS = r""" +GREEDY_KMD = GREEDY_UMD.replace('pid$target:', 'fbt:') + + +STACKS = { + +'UMD': r""" +{ printf("%x\tustack\t%x\t%x:", machtimestamp, pid, tid); ustack(); printf("\n"); -""" +} +""", -KMD_STACKS = r""" +'KMD': r""" +{ printf("%x\tkstack\t%x\t%x:", machtimestamp, pid, tid); stack(); printf("\n"); +} """ +} + + +def mach_absolute_time(static={}): + if not static: + import ctypes + libc = ctypes.CDLL('libc.dylib', use_errno=True) + static['mach_absolute_time'] = libc.mach_absolute_time + static['mach_absolute_time'].restype = ctypes.c_uint64 + return static['mach_absolute_time']() + + +class FifoReader(threading.Thread): + def __init__(self, collector, path): + threading.Thread.__init__(self) + self.collector = collector + self.pipe = path + if os.path.exists(self.pipe): + os.remove(self.pipe) + os.mkfifo(self.pipe) + self.file = os.open(self.pipe, os.O_RDWR) + + def run(self): + print('Started reading', self.pipe) + while self.file: + chunks = os.read(self.file, 1024).strip() + for chunk in chunks.split('\n'): + if chunk != 'close': + self.collector.attach(int(chunk)) + print('Read:', chunk) + print('Stopped reading', self.pipe) + + def stop(self): + os.write(self.file, 'close\n') + os.close(self.file) + self.file = None + os.unlink(self.pipe) class DTraceCollector(Collector): @@ -200,22 +476,30 @@ def collect(collector, on): def __init__(self, args): Collector.__init__(self, args) - self.pid = None + self.processes = {} + self.files = [] self.subcollectors = set() + self.attached = set() + decoders = get_decoders() - for decoder_group in decoders.itervalues(): + for decoder_group in decoders.values(): for decoder_class in decoder_group: if any('Subcollector' in str(name) for name in decoder_class.__bases__): self.subcollectors.add(decoder_class) if 'SUDO_ASKPASS' not in os.environ: - os.environ['SUDO_ASKPASS'] = self.create_ask_pass() - if 'DYLD_INSERT_LIBRARIES' in os.environ: - del os.environ['DYLD_INSERT_LIBRARIES'] - self.execute('sudo -A pkill dtrace', env=os.environ) - self.start() + get_original_env()['SUDO_ASKPASS'] = self.create_ask_pass() + assert 'DYLD_INSERT_LIBRARIES' not in os.environ + + self.sudo_execute('pkill dtrace') + self.script = None + if args.follow: + self.fifo_reader = FifoReader(self, os.path.join(self.args.output, 'follow_child.fifo')) + self.prepare() + self.times = [] + self.attach_by_pid = True @staticmethod def create_ask_pass(): @@ -224,7 +508,7 @@ def create_ask_pass(): return path with open(path, 'w') as file: file.write(osxaskpass) - os.chmod(path, 0700) + os.chmod(path, 0o700) return path def gen_gpu_hooks(self, text): # TODO: check /System/Library/Extensions/AppleIntelSKLGraphics.kext/Contents/Info.plist @@ -235,19 +519,19 @@ def gen_gpu_hooks(self, text): # TODO: check /System/Library/Extensions/AppleIn signature = ' '.join(parts[4:-1]) arity = signature.count(',') + 1 name = signature.split('(')[0][1:] - if 'dtHook' in name and parts[-1] == 'entry': + if 'dtHook' in name and parts[-1] == 'entry' and is_domain_enabled('dtHook'): probe = '%s::%s:entry{' % (parts[1], parts[3]) probe += r'printf("%x\t' + name + r'\t%x\t%x' probe += r'\t%x' * arity probe += r'\n", machtimestamp, pid, tid, ' probe += ', '.join(['arg'+str(i) for i in range(0, arity)]) - stack = '' if not self.args.debug else r'printf("%x\tkstack\t%x\t%x:", machtimestamp, pid, tid); stack(); printf("\n%x\tustack\t%x\t%x:", machtimestamp, pid, tid); ustack(); printf("\n");' + stack = '' if not self.args.debug else (STACKS['UMD'] + STACKS['KMD']).replace('{','').replace('}', '') probe += '); %s}\n' % stack probes += probe if not driver: driver = parts[2] DTraceCollector.check_graphics_firmware(driver) - if 'process_token_' in name: + if 'process_token_' in name and is_domain_enabled('process_token'): probe = '%s::%s:%s{' % (parts[1], parts[3], parts[-1]) probe += r'printf("%x\t' + parts[-1][0] + r'\t%x\t%x\tigfx\t' + name.replace('process_token_', '') probe += r'\n", machtimestamp, pid, tid' @@ -264,7 +548,7 @@ def check_graphics_firmware(driver): file_name = '/System/Library/Extensions/%s.kext/Contents/Info.plist' % driver if not os.path.exists(file_name): return - dom = minidom.parse(file_name) + dom = minidom.parse(file_name) # FIXME: import plistlib def find_by_content(el, content): if el.nodeValue == content: @@ -286,72 +570,279 @@ def find_by_content(el, content): else: sibling = sibling.nextSibling if value != '0': - print "Warning: To enable Graphics profiling, set GraphicsFirmwareSelect to 0 in: %s\n\tThen: sudo kextcache -i / & reboot" % file_name + print("Warning: To enable Graphics profiling, set GraphicsFirmwareSelect to 0 in: %s\n\tThen: sudo kextcache -i / & reboot" % file_name) @staticmethod def gen_options(options): return '\n'.join('#pragma D option %s=%s' % (key, str(value)) for key, value in options) + '\n' - def start(self): - # spawn dtrace tracers and exit, all means to stop it must be saved to self members: - # launch command line with dtrace script and remember pid - script = os.path.join(self.args.output, 'script.d') - + def prepare(self): self.files = [os.path.join(self.args.output, 'data-%s.dtrace' % (self.args.cuts[0] if self.args.cuts else '0'))] - if os.path.exists(self.files[0]): - os.remove(self.files[0]) + assert not os.path.exists(self.files[0]) # TODO: remove if not asserts, or return back: was if ... os.remove(self.files[0]) + + dtrace_script = [DSCRIPT_HEADER] + options = [ + ('bufresize', 'auto'), + ] + if self.args.ring: # https://docs.oracle.com/cd/E19253-01/817-6223/chp-buf/index.html + options += [ + ('bufpolicy', 'ring'), + ('bufsize', '64m') # 64 is maximum, system crashes on any bigger, even 65m + ] + else: + options += [ + ('switchrate', '10hz'), # print at 10Hz (instead of 1Hz) - brendangregg + ('bufsize', '4g') + ] - cmd = 'sudo -A dtrace -Z -q -o "%s" -s "%s"' % (self.files[0], script) + dtrace_script.append(self.gen_options(options)) - dtrace_script = [] + if self.args.debug: + dtrace_script.append("#pragma D option debug") + # XXX dtrace_script.append("#pragma D option strsize=512") # 256 is default - if self.args.ring: - dtrace_script.append(self.gen_options([ - ('bufpolicy', 'ring'), - ('bufresize', 'auto'), - ('bufsize', '%dm' % (self.args.ring * 10)) - ])) - - dtrace_script.append(dtrace_context_switch) - dtrace_script.append(fbt_dtrace_hooks) + if is_domain_enabled('context_switches'): + dtrace_script.append(dtrace_context_switch) - (probes, err) = self.execute('sudo -A dtrace -l -m *com.apple.driver.AppleIntel*Graphics*', env=os.environ) - if probes: - dtrace_script.append(self.gen_gpu_hooks(probes)) + if is_domain_enabled('fbt_hooks'): + dtrace_script += fbt_dtrace_hooks - if self.args.target: - dtrace_script.append(pid_dtrace_hooks) - cmd += " -p %s" % self.args.target - if self.args.hotspots: - dtrace_script.append(HOTSPOTS) + if self.args.filesystem: + dtrace_script.append(IO_HOOKS) for subcollector in self.subcollectors: - hooks = subcollector.get_hooks(self.args) + hooks = subcollector.get_hooks(self.args) # support multi pid for subcollectors if hooks: - dtrace_script.append(hooks) + dtrace_script += hooks subcollector.collect(self, True) - dtrace_script = '\n'.join(dtrace_script) + (probes, err) = self.execute('sudo -A dtrace -l -m *com.apple.driver.AppleIntel*Graphics*', log=False) # FIXME: sudo_execute + if probes: + dtrace_script.append(self.gen_gpu_hooks(probes)) + + self.script = dtrace_script + + def sudo_execute(self, cmd): + return self.execute('sudo -E -A ' + cmd) + + def prepare_per_pid(self): + dtrace_script = [] + if self.args.hotspots: + dtrace_script.append(HOTSPOTS) + elif is_domain_enabled('pid_hooks'): + dtrace_script += pid_dtrace_hooks # TODO: add %app_name% hotspots unconditionally + if self.args.stats: + dtrace_script.append(STATS) + + for hook in self.args.hook: + dtrace_script += bracket_hook(*hook.split(':')) + + if is_domain_enabled('instrument_target'): + if self.args.victim: + mod_name = os.path.basename(self.args.victim) + elif self.args.target: + (out, err) = DTraceCollector.execute('ps -p %d -o args' % self.args.target, log=False) + if not err: + lines = out.strip().split('\n') + if len(lines) > 1: + executable = lines[1].split()[0] + mod_name = executable.split('/')[-1] + + print('Auto-instrumented module:', mod_name) + dtrace_script += bracket_hook(mod_name.replace(' ', '*')) + + if is_domain_enabled('opencl'): + dtrace_script.append(OPEN_CL) + + if self.args.greedy: + dtrace_script.append(GREEDY_UMD) + # dtrace_script.append(GREEDY_KMD) # FIXME: hangs the system so far + + return dtrace_script + + def patch_per_pid(self, pids, items): + result = [] + for item in items: + if '$target:' in item: + for pid in pids: + result.append(item.replace('$target:', '%s:' % str(pid))) + else: + result.append(item) + + if self.args.follow: + fc = FOLLOW_CHILD.replace('/*{FOLLOW_CHILD}*/', self.fifo_reader.pipe) + for pid in pids: + result.append(fc.replace('$target', str(pid))) + return result + + def get_cmd(self, out, script, pids=[]): + # -C Run the C preprocessor + # -Z Permit probe descriptions that match zero probes + # -w Permit destructive actions in D programs + cmd = 'sudo -E -A dtrace -C -Z -w -o "%s" -s "%s"' % (out, script) # FIXME: sudo_execute + if self.args.verbose != 'info': + cmd += ' -q' # Set quiet mode + else: + # -S Show D compiler intermediate code + # -v Print an interface stability report + # -V Report the highest D programming interface version + # -e Exit after compiling + # -l List probes instead of enabling them + cmd += ' ' + + for pid in pids: + cmd += " -p %s" % pid + return cmd + + def launch_victim(self, victim, env): + proc = self.run_dtrace(victim=victim, env=env) + if not proc: + return None - if self.args.stacks: - dtrace_script = dtrace_script.replace('/*{OFF_CPU}*/', OFF_CPU_STACKS) - dtrace_script = dtrace_script.replace('/*{KMD_STACKS}*/', KMD_STACKS) - dtrace_script = dtrace_script.replace('/*{UMD_STACKS}*/', UMD_STACKS) + class PopenWrapper: + def __init__(self, parent, victim): + self.parent = parent + cmd = 'pgrep -n "%s"' % os.path.basename(victim[0]) + while True: + data, err = parent.execute(cmd) + if data: + self.pid = int(data) + break + time.sleep(1) + + def send_signal(self, sig): + self.parent.sudo_execute('kill -%d %d' % (sig, self.pid)) + + def wait(self, sec=10): + proc['proc'].wait() + """ XXX + for x in range(0, sec): + proc['proc'].poll() + time.sleep(1) + """ + return proc['proc'].returncode + + def communicate(self): + self.wait() + return None, None + + return PopenWrapper(self, victim) + + def run_dtrace(self, attach_by_name=False, victim=None, env=None): + self.attach_by_pid = False + # spawn dtrace tracers and exit, all means to stop it must be saved to self members: + # launch command line with dtrace script and remember pid + script = os.path.join(self.args.output, 'script.d') + cmd = self.get_cmd(self.files[0], script) + dtrace_script = self.script[:] + + hooks = [] + for hook in self.args.hook_kmd: + hooks += bracket_hook_kmd(*hook.split(':')) + + dtrace_script += hooks + # The target is known only when start is called, so doing part of preparation here + if attach_by_name: + dtrace_script += self.prepare_per_pid() + cmd += " -W %s" % os.path.basename(self.args.victim) + elif victim: + dtrace_script += self.prepare_per_pid() + cmd += ' -c "%s"' % ' '.join(victim) + else: + assert not any('$target' in item for item in dtrace_script) + if self.args.follow: + self.fifo_reader.start() + pids = self.args.target if isinstance(self.args.target, list) else [self.args.target] + for pid in pids: + cmd += " -p %s" % pid + print("Attaching PIDs:", pids) + items = self.prepare_per_pid() + dtrace_script += self.patch_per_pid(pids, items) + for pid in pids: + if self.args.stacks: + dtrace_script.append(OFF_CPU_STACKS.replace('$target', str(pid))) + if is_domain_enabled('wakeups'): + dtrace_script.append(dtrace_wakeup.replace('$target', str(pid))) + for hook in hooks: + dtrace_script.append(hook.replace('/*{CONDITIONS}*/', '/pid == %s/' % str(pid))) + + if self.args.stacks: + dtrace_script = self.patch_stacks(pids, dtrace_script) + + if self.args.args: + res = [] + tmpl = r'printf("%x\targs\t%x\t%x'\ + + (r'\t%d' * self.args.args)\ + + r'\n", machtimestamp, pid, tid, '\ + + r', '.join(['arg%d' % i for i in range(self.args.args)])\ + + r');' + for item in dtrace_script: + res.append(item.replace('/*{ARGUMENTS}*/', tmpl)) + dtrace_script = res + + # remove duplicates from the list: + dtrace_script = [item for n, item in enumerate(dtrace_script) if item not in dtrace_script[:n]] + dtrace_script = '\n'.join(dtrace_script) with open(script, 'w') as file: file.write(dtrace_script) - self.collect_system_info() - proc = subprocess.Popen(cmd, shell=True, stdin=None, stdout=self.output, stderr=self.output, env=os.environ) - self.pid = proc.pid + return self.run_parallel(cmd, env) + + def start(self): # FIXME: see man dtrace -W option for proper attach + self.times.append(datetime.now()) + if self.attach_by_pid: + self.run_dtrace() + + def run_parallel(self, cmd, env=None): self.log(cmd) - self.log("pid: %d" % proc.pid) + proc = self.processes.setdefault(cmd, {}) + proc['proc'] = subprocess.Popen(cmd, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env = env or os.environ) + proc['pid'] = proc['proc'].pid + self.log("%s -> pid: %d" % (cmd, proc['proc'].pid)) + return proc + + @staticmethod + def patch_stacks(pids, items): + reg_exp = re.compile(r"""(.*)\/\*\{(.*)_STACKS\}\*\/.*""", re.IGNORECASE | re.DOTALL) + result = [] + for item in items: + result.append(item) + if '_STACKS}*/' in item: + lines = item.strip().split('\n') + assert lines[-1].strip().endswith('}') + res = reg_exp.search(lines[0]) + what, where = res.groups() + condition = '/$target == pid/' if pids else '' + code = '\n%s %s %s' % (what, condition, STACKS[where]) + if pids: + for pid in pids: + result.append(code.replace('$target', str(pid))) + else: + result.append(code) + return result + + def attach(self, pid): + if pid in self.attached: + return + pids = [pid] + list(self.get_pid_children(pid)) + self.attached |= set(pids) + items = self.prepare_per_pid() + dtrace_script = [DSCRIPT_HEADER] + [item for item in self.script if '$target' in item] + dtrace_script += self.patch_per_pid(pids, items) + script = os.path.join(self.args.output, 'script_%d.d' % pid) + dtrace_script = '\n'.join(dtrace_script) + self.files.append(os.path.join(self.args.output, 'data-%d-%s.dtrace' % (pid, self.args.cuts[0] if self.args.cuts else '0'))) + cmd = self.get_cmd(self.files[-1], script, pids) + with open(script, 'w') as file: + file.write(dtrace_script) + self.run_parallel(cmd) @staticmethod def get_pid_children(parent): (out, err) = DTraceCollector.execute('ps -o pid,ppid -ax', log=False) if err: - print err + print(err) return for line in out.split('\n'): if not line: @@ -363,25 +854,107 @@ def get_pid_children(parent): if str(parent) == ppid: yield int(pid) + @staticmethod + def locate(what, statics={}): + try: + if not statics: + res = subprocess.check_output(['locate', '-S']).decode("utf-8") + statics['locate'] = 'WARNING' not in res + if not statics['locate']: + print(res) + if statics['locate']: + return subprocess.check_output(['locate', what]).decode("utf-8").split('\n') + except Exception: + pass + return [] + + def collect_codes(self): + items = self.locate("*.codes") + files = [item.strip() for item in items if not item.startswith('/Volumes')] + + items = self.locate("kdebug.h") + files += [item.strip() for item in items if not item.startswith('/Volumes')] + + filtered = {} + for file in files: + if not file or not os.path.exists(file): + continue + name = os.path.basename(file) + size = os.path.getsize(file) + if size and (name not in filtered or os.path.getsize(filtered[name]) < size): + filtered[name] = file + for file in filtered.values(): + shutil.copy(file, self.args.output) + + items = self.locate("IntelGPUSignposts.plist") + plists = [] + for line in items: + line = line.strip() + if line: + + plists.append((os.path.getmtime(line), line)) # finding newest + if plists: + plist = sorted(plists)[-1][1] + shutil.copy(plist, self.args.output) + def collect_system_info(self): with open(os.path.join(self.args.output, 'sysinfo.txt'), 'w') as file: - (probes, err) = self.execute('sysctl -a', env=os.environ, stdout=file) + (probes, err) = self.execute('sysctl -a', stdout=file) def stop(self, wait=True): - self.log("pid: %s" % str(self.pid)) + if self.args.follow: + print('Stopping: Follow child...') + self.fifo_reader.stop() + + for name, data in self.processes.items(): + print('Stopping:', name) + pids = [data['pid']] + list(self.get_pid_children(data['pid'])) + for pid in pids: + self.sudo_execute("kill -2 %d" % pid) + for pid in pids: + try: + os.waitpid(pid, 0) + except: + pass + + if not data['proc']: + continue + out, err = data['proc'].communicate() + message(None, "\n\n -= Target %s output =- {\n" % name) + if out: + self.log("Trace %s out:\n%s" % (name, out.decode())) + message(None, out.strip()) + message(None, "-" * 50) + if err: + self.log("Trace %s err:\n%s" % (name, err.decode()), True) + message(None, err.strip()) + message(None, "}\n\n") + + if data['proc'].returncode != 0: + message('error', '%s(%d) has exited with error code %d check logs for details' % (name, data['pid'], data['proc'].returncode)) + for subcollector in self.subcollectors: + print('Stopping:', subcollector) subcollector.collect(self, False) - if not self.pid: - return [] - dtrace_pids = [self.pid] + list(self.get_pid_children(self.pid)) - for pid in dtrace_pids: # FIXME: check if it has parent pid as well. Looks as we kill only children. - self.execute("sudo -A kill -2 %d" % pid, env=os.environ) - for pid in dtrace_pids: - try: - os.waitpid(pid, 0) - except: - pass - return self.files + + self.times.append(datetime.now()) + + sys_log = os.path.join(self.args.output, 'sys_log.json') + with open(sys_log, 'w') as file: + cmd = 'log show --source --style json --debug --signpost' # --last 1m --start, --end 'YYYY-MM-DD HH:MM:SS' + cmd += self.times[1].strftime(" --end '%Y-%m-%d %H:%M:%S'") + if self.args.ring: + cmd += (self.times[1] - timedelta(seconds=self.args.ring)).strftime(" --start '%Y-%m-%d %H:%M:%S'") + else: + cmd += self.times[0].strftime(" --start '%Y-%m-%d %H:%M:%S'") + self.execute(cmd, stdout=file) # FIXME: get time of collection or ring size + + self.collect_system_info() + self.collect_codes() + + res = self.files + [sys_log] + + return res @classmethod def available(cls): @@ -389,7 +962,7 @@ def available(cls): return False (out, err) = cls.execute('csrutil status') if 'disabled' not in out: - print 'Please do: "csrutil disable" from Recovery OS terminal to be able using dtrace...' + print('Please do: "csrutil disable" from Recovery OS terminal to be able using dtrace...') return False return True @@ -400,5 +973,7 @@ def available(cls): 'collector': DTraceCollector }] - +if __name__ == "__main__": + print(mach_absolute_time()) + DTraceCollector.check_graphics_firmware('com.apple.driver.AppleIntelSKLGraphics') diff --git a/runtool/collectors/win.py b/runtool/collectors/win.py index 2e522b6..aa4087b 100644 --- a/runtool/collectors/win.py +++ b/runtool/collectors/win.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import sys import time @@ -16,6 +17,11 @@ def relog_etl(frm, to): sea.ITT('win').relog(frm, to) +def async_exec(cmd, title=None, env=None): + cmd = 'start "%s" /MIN /LOW %s' % (title if title else cmd, cmd) + subprocess.Popen(cmd, shell=True, stdin=None, stdout=None, stderr=None, creationflags=0x00000008, env=env) # DETACHED_PROCESS + + class WPRCollector(Collector): def __init__(self, args): Collector.__init__(self, args) @@ -25,8 +31,6 @@ def __init__(self, args): self.file = os.path.join(args.output, "wpa-%s.etl" % (self.args.cuts[0] if self.args.cuts else '0')) else: self.file = os.path.join(args.output, "wpa.etl") - if self.wpr: - self.start() @classmethod def detect(cls, statics={}): @@ -37,6 +41,7 @@ def detect(cls, statics={}): for wpr in wprs: proc = subprocess.Popen('"%s" /?' % wpr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = proc.communicate() + out = out.decode() if err: return None for line in out.split('\n'): @@ -48,7 +53,7 @@ def detect(cls, statics={}): break if not res: return None - statics['res'] = sorted(res, key=lambda(_, ver): [int(item) for item in ver.split('.')], reverse=True)[0][0] + statics['res'] = sorted(res, key=lambda __ver: [int(item) for item in __ver[1].split('.')], reverse=True)[0][0] return statics['res'] @staticmethod @@ -67,6 +72,9 @@ def get_options(): yield parts[0], parts[0] in ['DiskIO', 'FileIO', 'GPU', 'GeneralProfile', 'Handle', 'Heap', 'Network', 'Power', 'Video', 'VirtualAllocation'] def start(self): + if not self.wpr: + print("Failed to start without WPR...") + return if self.is_recording(): self.cancel() profile = os.path.normpath(os.path.join(self.args.bindir, '..', 'ETW', 'IntelSEAPI.wprp')) @@ -107,13 +115,13 @@ def stop(self, wait=True): time.sleep(1) return [self.file] else: - sea.prepare_environ(self.args) - self.stop_wpr(self.wpr, self.file, self.args.output) + env = sea.prepare_environ(self.args) + self.stop_wpr(self.wpr, self.file, self.args.output, env) return [self.file] @classmethod - def stop_wpr(cls, wpr, file, output): - (out, err) = cls.execute('"%s" -stop "%s"' % (wpr, file)) + def stop_wpr(cls, wpr, file, output, env=None): + (out, err) = cls.execute('"%s" -stop "%s"' % (wpr, file), env=env) if err: return [] assert(file in out) @@ -171,7 +179,7 @@ def stop(self, wait=True, complete=True): (out, err) = self.execute('logman stop GPA_GPUVIEW -ets') if err and complete: - print err + print(err) environ = os.environ.copy() environ['TLOG'] = 'NORMAL' @@ -193,7 +201,7 @@ def merge(cls, gpuview, file, started, wait, args=None): cmd = '"%s" -merge Merged.etl IntelSEAPI.etl "%s"' % (xperf, os.path.basename(file)) (out, err) = Collector.execute(cmd, cwd=started) if err and (os.path.basename(file) not in err): - print err + print(err) relog_etl(os.path.join(started, os.path.basename(file)), file) shutil.rmtree(started) else: @@ -254,6 +262,10 @@ def start(self): if is_domain_enabled('OculusVR'): file.write('"{553787FC-D3D7-4F5E-ACB2-1597C7209B3C}"\n') count += 1 + if is_domain_enabled('Intel_Graphics_D3D10'): + file.write('"{AD367E62-97EF-4B20-8235-E8AB49DB0C23}"\n') + count += 1 + if count: cmd = 'logman start GPA_SEA -ct perf -bs 1024 -nb 120 480' cmd += ' -pf "%s" -o "%s" %s -ets' % (logman_pf, self.files[0], (('-max %d -f bincirc' % (self.args.ring * 15)) if self.args.ring else '')) @@ -332,6 +344,11 @@ def start(self): return self else: del self.files[-1] + + self.files.append('%s/etw_profilers.logman' % self.args.output) + cmd = 'cmd /c logman query providers ^> "%s"' % self.files[-1] + async_exec(cmd, 'Collecting ETW providers') + return self def stop(self, wait=True): # TODO: stop without waits @@ -349,6 +366,7 @@ def stop(self, wait=True): # TODO: stop without waits proc = subprocess.Popen('xperf -stop GPA_SEA', shell=True) if wait: proc.wait() + return self.files diff --git a/runtool/decoders/Adreno.py b/runtool/decoders/Adreno.py index 7b34af8..d1ed2fa 100644 --- a/runtool/decoders/Adreno.py +++ b/runtool/decoders/Adreno.py @@ -1,3 +1,4 @@ +from __future__ import print_function TRACK_INDEX, TRACK_NAME = -1, 'GPU' @@ -32,7 +33,7 @@ def gpu_queue(self, ctx, inflight, timestamp): if not delta: return thread = self.gpu.thread(ctx, 'GPU %d' % ctx) - for i in xrange(abs(delta)): + for i in range(abs(delta)): if delta > 0: task = thread.task(str(self.task_counter), 'Adreno', overlapped=True).begin(timestamp + len(self.gpu_tasks) * 1000, self.task_counter) self.gpu_tasks.append(task) @@ -51,7 +52,7 @@ def cpu_queue(self, pid, tid, ctx, queued, timestamp): state['queued'] = queued if not delta: return - for i in xrange(abs(delta)): + for i in range(abs(delta)): if delta > 0: self.cpu_tasks.append( self.callbacks.process(pid).thread(tid) @@ -92,11 +93,11 @@ def handle_record(self, proc, pid, tid, cpu, flags, timestamp, name, args): elif name == 'adreno_cmdbatch_submitted': args = self.parse_args(args) self.gpu_queue(int(args['ctx']), int(args['inflight']), timestamp) - # print 'submitted', args + # print('submitted', args) elif name == 'adreno_cmdbatch_retired': args = self.parse_args(args) self.gpu_queue(int(args['ctx']), int(args['inflight']), timestamp) - # print 'retired', args + # print('retired', args) elif name == 'kgsl_waittimestamp_entry': # ctx here is important, but absent in _exit, and adreno_drawctxt_wait does the same """ thread = self.callbacks.process(pid).thread(tid) @@ -173,7 +174,7 @@ def handle_record(self, proc, pid, tid, cpu, flags, timestamp, name, args): elif name in ['kgsl_mem_timestamp_queue', 'kgsl_mem_timestamp_free']: pass else: - print name + print(name) def finalize(self): pass diff --git a/runtool/decoders/MSNT_SystemTrace.py b/runtool/decoders/MSNT_SystemTrace.py index 1781195..5a8ff42 100644 --- a/runtool/decoders/MSNT_SystemTrace.py +++ b/runtool/decoders/MSNT_SystemTrace.py @@ -1,26 +1,32 @@ +from __future__ import print_function +import os import sys import socket +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) +from python_compat import basestring + def resolve_host(ip): try: return socket.gethostbyaddr(ip)[0] except: return None + def convert_numbers(obj): if isinstance(obj, dict): - for k, v in obj.iteritems(): + for k, v in obj.items(): obj[k] = convert_numbers(v) elif isinstance(obj, list): new = [convert_numbers(v) for v in obj] del obj[:] obj.extend(new) - elif hasattr(obj, '__iter__'): - for v in obj: - convert_numbers(v) elif isinstance(obj, basestring): if obj.isdigit(): return int(obj) + elif hasattr(obj, '__iter__'): + for v in obj: + convert_numbers(v) return obj @@ -60,8 +66,8 @@ def handle_record(self, system, data, info): elif info['EventName'] == 'SystemConfig': if info['Opcode'] in ['Video', 'NIC', 'PhyDisk', 'LogDisk', 'CPU', 'Platform']: self.callbacks.add_metadata(info['Opcode'], convert_numbers(data)) - elif sys.gettrace(): - print 'EventName:', info['EventName'], 'Opcode:', info['Opcode'] + else: + self.callbacks.set_thread_name(int(data['ProcessId']), int(data['ThreadId']), data['ThreadName']) def on_receive(self, size, now, pid, source, target): receiver = self.tcpip.setdefault(source, {'packets': [], 'started': None}) @@ -104,7 +110,7 @@ def end_receiver(receiver): receiver['started'] = None def finalize(self): - for target, receiver in self.tcpip.iteritems(): + for target, receiver in self.tcpip.items(): self.end_receiver(receiver) packets = receiver['packets'] if packets: diff --git a/runtool/decoders/PVR.py b/runtool/decoders/PVR.py index f33a3ab..633f7ef 100644 --- a/runtool/decoders/PVR.py +++ b/runtool/decoders/PVR.py @@ -1,3 +1,6 @@ +import os +import sys +sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'importers'))) from etw import GPUQueue TRACK_INDEX, TRACK_NAME = -1, 'GPU' diff --git a/runtool/exporters/BestTraceFormat.py b/runtool/exporters/BestTraceFormat.py index 9871cce..97fc943 100644 --- a/runtool/exporters/BestTraceFormat.py +++ b/runtool/exporters/BestTraceFormat.py @@ -12,10 +12,10 @@ def __init__(self, args, tree): """Open the .btf file and write its header.""" TaskCombiner.__init__(self, args, tree) self.file = open(self.get_targets()[-1], "w+b") - self.file.write('#version 2.1.3\n') - self.file.write('#creator GDP-SEA\n') - self.file.write('#creationDate 2014-02-19T11:39:20Z\n') - self.file.write('#timeScale ns\n') + self._write('#version 2.1.3\n') + self._write('#creator GDP-SEA\n') + self._write('#creationDate 2014-02-19T11:39:20Z\n') + self._write('#timeScale ns\n') def get_targets(self): return [self.args.output + ".btf"] @@ -28,8 +28,12 @@ def complete_task(self, type, b, e): """ #