From f29e1de344faf00a0ca85c7cae389e1cefb21261 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 29 Apr 2019 14:44:58 -0400 Subject: [PATCH 01/75] crash.types.task: improve task flag handling This commit adds knowledge of the task flags for newer releases. In Linux v3.14, several elements were removed from task_state_array. In Linux v4.4, TASK_PARKED was renumbered to be in task_state_array. This commit handles the right things and will complain if the flags change again. Signed-off-by: Jeff Mahoney --- crash/commands/ps.py | 17 ++---- crash/types/task.py | 127 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 107 insertions(+), 37 deletions(-) diff --git a/crash/commands/ps.py b/crash/commands/ps.py index 78996cb649a..36815346499 100755 --- a/crash/commands/ps.py +++ b/crash/commands/ps.py @@ -437,7 +437,7 @@ def task_state_string(self, task): pass buf = '??' - if hasattr(TF, 'TASK_DEAD'): + if TF.has_flag('TASK_DEAD'): try: buf = self.task_states[state & ~TF.TASK_DEAD] except KeyError: @@ -522,17 +522,15 @@ def setup_task_states(self): TF.TASK_RUNNING : "RU", TF.TASK_INTERRUPTIBLE : "IN", TF.TASK_UNINTERRUPTIBLE : "UN", - TF.TASK_ZOMBIE : "ZO", + TF.EXIT_ZOMBIE : "ZO", TF.TASK_STOPPED : "ST", } - if hasattr(TF, 'TASK_EXCLUSIVE'): - self.task_states[TF.TASK_EXCLUSIVE] = "EX" - if hasattr(TF, 'TASK_SWAPPING'): + if TF.has_flag('TASK_SWAPPING'): self.task_states[TF.TASK_SWAPPING] = "SW" - if hasattr(TF, 'TASK_DEAD'): + if TF.has_flag('TASK_DEAD'): self.task_states[TF.TASK_DEAD] = "DE" - if hasattr(TF, 'TASK_TRACING_STOPPED'): + if TF.has_flag('TASK_TRACING_STOPPED'): self.task_states[TF.TASK_TRACING_STOPPED] = "TR" def execute(self, argv): @@ -540,10 +538,7 @@ def execute(self, argv): sort_by_last_run = lambda x: -x.info.last_run() if not hasattr(self, 'task_states'): - try: - self.setup_task_states() - except AttributeError: - raise CommandLineError("The task subsystem is not available.") + self.setup_task_states() sort_by = sort_by_pid if argv.l: diff --git a/crash/types/task.py b/crash/types/task.py index 5e930f2c5e7..e3b2a701a30 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -13,16 +13,44 @@ def get_value(symname): if sym[0]: return sym[0].value() +# This is pretty painful. These are all #defines so none of them end +# up with symbols in the kernel. The best approximation we have is +# task_state_array which doesn't include all of them. All we can do +# is make some assumptions based on the changes upstream. This will +# be fragile. class TaskStateFlags(CrashBaseClass): __types__ = [ 'char *', 'struct task_struct' ] __symvals__ = [ 'task_state_array' ] - __symbol_callbacks__ = [ ('task_state_array', 'task_state_flags_callback') ] - __delayed_values__ = [ 'TASK_RUNNING', 'TASK_INTERRUPTIBLE', - 'TASK_UNINTERRUPTIBLE', 'TASK_ZOMBIE', - 'TASK_STOPPED', 'TASK_SWAPPING', 'TASK_EXCLUSIVE' ] + __symbol_callbacks__ = [ ('task_state_array', '_task_state_flags_callback') ] + + TASK_RUNNING = 0 + + TASK_FLAG_UNINITIALIZED = -1 + + TASK_INTERRUPTIBLE: int=TASK_FLAG_UNINITIALIZED + TASK_UNINTERRUPTIBLE: int=TASK_FLAG_UNINITIALIZED + TASK_STOPPED: int=TASK_FLAG_UNINITIALIZED + EXIT_ZOMBIE: int=TASK_FLAG_UNINITIALIZED + TASK_DEAD: int=TASK_FLAG_UNINITIALIZED + EXIT_DEAD: int=TASK_FLAG_UNINITIALIZED + TASK_SWAPPING: int=TASK_FLAG_UNINITIALIZED + TASK_TRACING_STOPPED: int=TASK_FLAG_UNINITIALIZED + TASK_WAKEKILL: int=TASK_FLAG_UNINITIALIZED + TASK_WAKING: int=TASK_FLAG_UNINITIALIZED + TASK_PARKED: int=TASK_FLAG_UNINITIALIZED + __TASK_IDLE: int=TASK_FLAG_UNINITIALIZED + + TASK_NOLOAD: int=TASK_FLAG_UNINITIALIZED + TASK_NEW: int=TASK_FLAG_UNINITIALIZED + TASK_IDLE: int=TASK_FLAG_UNINITIALIZED + + @classmethod + def has_flag(cls, flagname): + v = getattr(cls, flagname) + return v != cls.TASK_FLAG_UNINITIALIZED @classmethod - def task_state_flags_callback(cls, symbol): + def _task_state_flags_callback(cls, symbol): count = array_size(cls.task_state_array) bit = 0 @@ -33,45 +61,92 @@ def task_state_flags_callback(cls, symbol): '(sleeping)' : 'TASK_INTERRUPTIBLE', '(disk sleep)' : 'TASK_UNINTERRUPTIBLE', '(stopped)' : 'TASK_STOPPED', - '(zombie)' : 'TASK_ZOMBIE', - #'(dead)' : 'TASK_DEAD', + '(zombie)' : 'EXIT_ZOMBIE', + 'x (dead)' : 'TASK_DEAD', + 'X (dead)' : 'EXIT_DEAD', '(swapping)' : 'TASK_SWAPPING', - #'(tracing stop)' : 'TASK_TRACING_STOPPED', + '(tracing stop)' : 'TASK_TRACING_STOPPED', '(wakekill)' : 'TASK_WAKEKILL', '(waking)' : 'TASK_WAKING', + '(parked)' : 'TASK_PARKED', + '(idle)' : '__TASK_IDLE', } for key in state_strings: if key in state: - try: - dv = get_delayed_lookup(cls, state_strings[key]) - dv.callback(bit) - except KeyError: - setattr(cls, state_strings[key], bit) - if '(dead)' in state: - cls.TASK_DEAD = bit - if '(tracing stop)' in state: - cls.TASK_TRACING_STOPPED = bit + setattr(cls, state_strings[key], bit) + if bit == 0: bit = 1 else: bit <<= 1 - cls.check_state_bits() + + # Linux 4.14 re-introduced TASK_PARKED into task_state_array + # which renumbered some bits + if not cls.has_flag('TASK_PARKED') and cls.has_flag('TASK_DEAD'): + newbits = cls.TASK_PARKED << 1 + cls.TASK_DEAD = newbits + cls.TASK_WAKEKILL = newbits << 1 + cls.TASK_WAKING = newbits << 2 + cls.TASK_NOLOAD = newbits << 3 + cls.TASK_NEW = newbits << 4 + + assert(cls.TASK_PARKED == 0x0040) + assert(cls.TASK_DEAD == 0x0080) + assert(cls.TASK_WAKEKILL == 0x0100) + assert(cls.TASK_WAKING == 0x0200) + + # Linux 3.14 removed several elements from task_state_array + # so we'll have to make some assumptions. + # TASK_NOLOAD wasn't introduced until 4.2 and wasn't added + # to task_state_array until v4.14. There's no way to + # detect whether the use of the flag is valid for a particular + # kernel release. + elif not cls.has_flag('TASK_DEAD'): + if cls.EXIT_ZOMBIE > cls.EXIT_DEAD: + newbits = cls.EXIT_ZOMBIE << 1 + else: + newbits = cls.EXIT_DEAD << 1 + cls.TASK_DEAD = newbits + cls.TASK_WAKEKILL = newbits << 1 + cls.TASK_WAKING = newbits << 2 + cls.TASK_PARKED = newbits << 3 + cls.TASK_NOLOAD = newbits << 4 + cls.TASK_NEW = newbits << 5 + + assert(cls.TASK_DEAD == 0x0040) + assert(cls.TASK_WAKEKILL == 0x0080) + assert(cls.TASK_WAKING == 0x0100) + assert(cls.TASK_PARKED == 0x0200) + else: + assert(cls.TASK_DEAD == 64) + assert(cls.TASK_WAKEKILL == 128) + assert(cls.TASK_WAKING == 256) + assert(cls.TASK_PARKED == 512) + + if cls.has_flag('TASK_NOLOAD'): + assert(cls.TASK_NOLOAD == 1024) + cls.TASK_IDLE = cls.TASK_NOLOAD | cls.TASK_UNINTERRUPTIBLE + assert(cls.TASK_IDLE == 1026) + if cls.has_flag('TASK_NEW'): + assert(cls.TASK_NEW == 2048) + + cls._check_state_bits() @classmethod - def check_state_bits(cls): + def _check_state_bits(cls): required = [ 'TASK_RUNNING', 'TASK_INTERRUPTIBLE', 'TASK_UNINTERRUPTIBLE', - 'TASK_ZOMBIE', + 'EXIT_ZOMBIE', 'TASK_STOPPED', ] missing = [] for bit in required: - if not hasattr(cls, bit): + if not cls.has_flag(bit): missing.append(bit) if len(missing): @@ -156,9 +231,9 @@ def get_thread_info(self): def get_last_cpu(self): try: - return self.task_struct['cpu'] + return int(self.task_struct['cpu']) except gdb.error as e: - return self.thread_info['cpu'] + return int(self.thread_info['cpu']) def task_state(self): state = int(self.task_struct['state']) @@ -171,10 +246,10 @@ def maybe_dead(self): known = TF.TASK_INTERRUPTIBLE known |= TF.TASK_UNINTERRUPTIBLE - known |= TF.TASK_ZOMBIE + known |= TF.EXIT_ZOMBIE known |= TF.TASK_STOPPED - if hasattr(TF, 'TASK_SWAPPING'): + if TF.has_flag('TASK_SWAPPING'): known |= TF.TASK_SWAPPING return (state & known) == 0 @@ -185,7 +260,7 @@ def is_exiting(self): return self.task_flags() & PF_EXITING def is_zombie(self): - return self.task_state() & TF.TASK_ZOMBIE + return self.task_state() & TF.EXIT_ZOMBIE def update_mem_usage(self): if self.mem_valid: From 86422cbbcd459338418d50eb459bc7d5a4f46dc0 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 29 Apr 2019 14:45:17 -0400 Subject: [PATCH 02/75] crash.commands.ps: add support for TASK_IDLE Kernel 4.2 introduced TASK_NOLOAD, which when combined with TASK_UNINTERRUPTIBLE, produced TASK_IDLE. This mask is used for kernel threads, so without support for the flags, `ps' shows ?? for kernel threads. Signed-off-by: Jeff Mahoney --- crash/commands/ps.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crash/commands/ps.py b/crash/commands/ps.py index 36815346499..4b4bb919482 100755 --- a/crash/commands/ps.py +++ b/crash/commands/ps.py @@ -436,19 +436,21 @@ def task_state_string(self, task): except AttributeError: pass - buf = '??' - if TF.has_flag('TASK_DEAD'): - try: - buf = self.task_states[state & ~TF.TASK_DEAD] - except KeyError: - pass + buf = None - if state & TF.TASK_DEAD and task.maybe_dead(): - buf = self.task_states[TF.TASK_DEAD] + for bits in sorted(self.task_states.keys(), reverse=True): + if (state & bits) == bits: + buf = self.task_states[bits] + break + if state & TF.TASK_DEAD and task.maybe_dead(): + buf = self.task_states[TF.TASK_DEAD] if buf is not None and exclusive: buf += "EX" + if buf is None: + print(f"Unknown state {state} found") + return buf @classmethod @@ -532,6 +534,8 @@ def setup_task_states(self): self.task_states[TF.TASK_DEAD] = "DE" if TF.has_flag('TASK_TRACING_STOPPED'): self.task_states[TF.TASK_TRACING_STOPPED] = "TR" + if TF.has_flag('TASK_IDLE'): + self.task_states[TF.TASK_IDLE] = "ID" def execute(self, argv): sort_by_pid = lambda x: x.info.task_struct['pid'] From 68f663025da3106266132e170ca888eeba975d7b Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 19 Sep 2018 10:47:38 +0200 Subject: [PATCH 03/75] crash.commands.help: sort commands in help output It's more user-friendly to be able to locate whether a command is present alphabetically. Signed-off-by: Jeff Mahoney --- crash/commands/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crash/commands/help.py b/crash/commands/help.py index 43fa3c5e97c..8d0b4d13d57 100644 --- a/crash/commands/help.py +++ b/crash/commands/help.py @@ -28,7 +28,7 @@ def __init__(self): def execute(self, argv): if not argv.args: print("Available commands:") - for cmd in self.commands: + for cmd in sorted(self.commands): text = self.commands[cmd].__doc__ if text: summary = text.split('\n')[0].strip() From 4fb4c3ad82b85a522a3490febb1dad378fff207a Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 29 Apr 2019 14:40:13 -0400 Subject: [PATCH 04/75] crash.util: add struct_has_member The use of anonymous structures and unions means that things like: struct foo { struct { int x; }; }; if 'x' in cls.foo_type: # will evaluate false when foo.x works fine in C code. In order to make these less painful for subsystem modules, we add a struct_has_member helper that does the right thing to resolve the member. Signed-off-by: Jeff Mahoney --- crash/util.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crash/util.py b/crash/util.py index 0c5905fdd3f..12608f1096b 100644 --- a/crash/util.py +++ b/crash/util.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Union + import gdb from crash.infra import CrashBaseClass, export from crash.exceptions import MissingTypeError, MissingSymbolError +TypeSpecifier = Union [ gdb.Type, gdb.Value, str, gdb.Symbol ] + class OffsetOfError(Exception): """Generic Exception for offsetof errors""" def __init__(self, message): @@ -96,6 +100,37 @@ def container_of(self, val, gdbtype, member): offset = offsetof(gdbtype, member) return (val.cast(charp) - offset).cast(gdbtype.pointer()).dereference() + @export + @staticmethod + def struct_has_member(gdbtype: TypeSpecifier, name: str) -> bool: + """ + Returns whether a structure has a given member name. + + A typical method of determining whether a structure has a member is just + to check the fields list. That generally works but falls apart when + the structure contains an anonymous union or substructure since + it will push the members one level deeper in the namespace. + + This routine provides a simple interface that covers those details. + + Args: + val (gdb.Type, gdb.Value, str, gdb.Symbol): The object for which + to resolve the type to search for the member + name (str): The name of the member to query + + Returns: + bool: Whether the member is present in the specified type + + Raises: + TypeError: An invalid argument has been provided. + + """ + try: + x = TypesUtilClass.offsetof(gdbtype, name) + return True + except InvalidComponentError: + return False + @export @staticmethod def get_symbol_value(symname, block=None, domain=None): From 7fe8aa76c4092728217f15c7035dc32c76b065d0 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 29 Apr 2019 14:43:50 -0400 Subject: [PATCH 05/75] crash.types.task: handle anonymous sub-structure in mm_struct Kernel v4.19 moved most of mm_struct into an anonymous sub-structure. Even though C code can access members directly, the gdb type infrastructure reflects the actual type layout. This means that things like "if 'rss_stat' in cls.mm_struct_type" will return false even if the member is present. To cope with this, use struct_has_member instead, which does the right things when detecting whether a struct member is present. Signed-off-by: Jeff Mahoney --- crash/types/task.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/crash/types/task.py b/crash/types/task.py index e3b2a701a30..9a87111d6a8 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -2,7 +2,7 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.util import array_size +from crash.util import array_size, struct_has_member from crash.infra import CrashBaseClass from crash.infra.lookup import DelayedValue, ClassProperty, get_delayed_lookup @@ -212,7 +212,7 @@ def init_task_types(cls, task): cls.task_struct_type = task.type fields = cls.task_struct_type.fields() cls.task_state_has_exit_state = 'exit_state' in fields - cls.mm_struct_fields = gdb.lookup_type('struct mm_struct').keys() + cls.mm_struct_type = gdb.lookup_type('struct mm_struct') cls.pick_get_rss() cls.pick_last_run() cls.init_mm = get_value('init_mm') @@ -322,12 +322,11 @@ def get_rss_stat_field(self): def get_anon_file_rss_fields(self): mm = self.task_struct['mm'] rss = 0 - for name in ['_anon_rss', '_file_rss']: - if name in mm_struct_fields: - if mm[name].type == self.atomic_long_type: - rss += int(mm[name]['counter']) - else: - rss += int(mm[name]) + for name in cls.anon_file_rss_fields: + if mm[name].type == self.atomic_long_type: + rss += int(mm[name]['counter']) + else: + rss += int(mm[name]) return rss # The Pythonic way to do this is by generating the LinuxTask class @@ -335,20 +334,28 @@ def get_anon_file_rss_fields(self): # select the proper function and assign it to the class. @classmethod def pick_get_rss(cls): - if 'rss' in cls.mm_struct_fields: + if struct_has_member(cls.mm_struct_type, 'rss'): cls.get_rss = cls.get_rss_field - elif '_rss' in cls.mm_struct_fields: + elif struct_has_member(cls.mm_struct_type, '_rss'): cls.get_rss = cls.get__rss_field - elif 'rss_stat' in cls.mm_struct_fields: + elif struct_has_member(cls.mm_struct_type, 'rss_stat'): cls.MM_FILEPAGES = get_value('MM_FILEPAGES') cls.MM_ANONPAGES = get_value('MM_ANONPAGES') cls.get_rss = cls.get_rss_stat_field - elif '_anon_rss' in cls.mm_struct_fields or \ - '_file_rss' in cls.mm_struct_fields: + else: + cls.anon_file_rss_fields = [] + + if struct_has_member(cls.mm_struct_type, '_file_rss'): + cls.anon_file_rss_fields.append('_file_rss') + + if struct_has_member(cls.mm_struct_type, '_anon_rss'): + cls.anon_file_rss_fields.append('_anon_rss') + cls.atomic_long_type = gdb.lookup_type('atomic_long_t') cls.get_rss = cls.get_anon_file_rss_fields - else: - raise RuntimeError("No method to retrieve RSS from task found.") + + if len(cls.anon_file_rss_fields): + raise RuntimeError("No method to retrieve RSS from task found.") def last_run__last_run(self): return int(self.task_struct['last_run']) From b8de0ab6528437dc5012e0760728a13317c82a47 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 30 Apr 2019 21:26:20 -0400 Subject: [PATCH 06/75] crash.util: add uuid decoding With upcoming file system subsystem modules, we'll want a common way to handle UUID decoding. XFS uses uuid_t while btrfs uses an array of u8. This introduces helpers into crash.util: - decode_uuid -- decodes the byte array - decode_uuid_t -- decodes the uuid_t Signed-off-by: Jeff Mahoney --- crash/util.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/crash/util.py b/crash/util.py index 12608f1096b..1f240aeae00 100644 --- a/crash/util.py +++ b/crash/util.py @@ -4,6 +4,8 @@ from typing import Union import gdb +import uuid + from crash.infra import CrashBaseClass, export from crash.exceptions import MissingTypeError, MissingSymbolError @@ -70,7 +72,7 @@ def __init__(self, member, gdbtype): self.type = gdbtype class TypesUtilClass(CrashBaseClass): - __types__ = [ 'char *' ] + __types__ = [ 'char *', 'uuid_t' ] @export def container_of(self, val, gdbtype, member): @@ -411,3 +413,67 @@ def array_for_each(value): size = array_size(value) for i in range(array_size(value)): yield value[i] + + @export + @classmethod + def decode_uuid(cls, value: gdb.Value) -> uuid.UUID: + """ + Decode an array of bytes that describes a UUID into a Python-style + UUID object + + Args: + value (gdb.Value): The UUID to decode + + Returns: + uuid.UUID: The UUID object that describes the value + + Raises: + TypeError: value is not gdb.Value or does not describe a 16-byte array. + + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if (value.type.code != gdb.TYPE_CODE_ARRAY or + value[0].type.sizeof != 1 or + value.type.sizeof != 16): + raise TypeError("value must describe an array of 16 bytes") + + u = 0 + for i in range(0, 16): + u <<= 8 + u += int(value[i]) + + return uuid.UUID(int=u) + + @export + @classmethod + def decode_uuid_t(cls, value: gdb.Value) -> uuid.UUID: + """ + Decode a Linux kernel uuid_t into a Python-style UUID object + + Args: + value (gdb.Value): The uuid_t to be decoded + + Returns: + uuid.UUID: The UUID object that describes the value + + Raises: + TypeError: value is not gdb.Value + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if value.type != self.uuid_t_type: + if (value.type.code == gdb.TYPE_CODE_PTR and + value.type.target() == self.uuid_t_type): + value = value.dereference() + else: + raise TypeError("value must describe a uuid_t") + + if 'b' in cls.uuid_t_type: + member = 'b' + else: + member = '__u_bits' + + return cls.decode_uuid(value[member]) From e75bc230832aeff0c55a665cb0ed7d21d9d9e4df Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 26 Apr 2019 21:34:48 +0200 Subject: [PATCH 07/75] crash.util: add decode_flags helper The decode_flags helper takes a gdb.Value representing an integer and a dictionary of int -> str that maps the powers of 2 to flag names and produces a human-readable string describing the flags. If no name is found FLAG_$number is used instead. Signed-off-by: Jeff Mahoney --- crash/util.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crash/util.py b/crash/util.py index 1f240aeae00..bfa1548502d 100644 --- a/crash/util.py +++ b/crash/util.py @@ -6,6 +6,8 @@ import gdb import uuid +from typing import Dict + from crash.infra import CrashBaseClass, export from crash.exceptions import MissingTypeError, MissingSymbolError @@ -414,6 +416,47 @@ def array_for_each(value): for i in range(array_size(value)): yield value[i] + @export + @staticmethod + def decode_flags(value: gdb.Value, names: Dict[int, str], + separator: str="|") -> str: + """ + Present a bitfield of individual flags in a human-readable format. + + Args: + value (gdb.Value): + The value containing the flags to be decoded. + names (dict of int->str): + A dictionary containing mappings for each bit number to + a human-readable name. Any flags found that do not have + a matching value in the dict will be displayed as FLAG_. + separator (str, defaults to "|"): + The string to use as a separator between each flag name in the + output. + + Returns: + str: A human-readable string displaying the flag values. + + Raises: + TypeError: value is not gdb.Value or names is not dict. + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if not isinstance(names, dict): + raise TypeError("names must be a dictionary of int -> str") + + flags_val = int(value) + flags = [] + for n in range(0, value.type.sizeof << 3): + if flags_val & (1 << n): + try: + flags.append(names[1 << n]) + except KeyError: + flags.append("FLAG_{}".format(n)) + + return separator.join(flags) + @export @classmethod def decode_uuid(cls, value: gdb.Value) -> uuid.UUID: From 81cb9717907420dd8ca1f39b2ea84cd68abaf4d2 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 19 Sep 2018 10:48:13 +0200 Subject: [PATCH 08/75] crash.types.{,k}list: use the type from the symbol when available Internally, gdb treats the type loaded from a typed symbol and a type symbol differently and wants to do the full type comparison dance. If we use the typed symbol directly, we can use a pointer comparison. Signed-off-by: Jeff Mahoney --- crash/types/klist.py | 4 ++++ crash/types/list.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/crash/types/klist.py b/crash/types/klist.py index 88e9b0a7aad..7ccb49fa7b3 100644 --- a/crash/types/klist.py +++ b/crash/types/klist.py @@ -20,6 +20,8 @@ def klist_for_each(self, klist): elif klist.type != self.klist_type: raise TypeError("klist must be gdb.Value representing 'struct klist' or 'struct klist *' not {}" .format(klist.type)) + if klist.type is not self.klist_type: + self.klist_type = klist.type for node in list_for_each_entry(klist['k_list'], self.klist_node_type, 'n_node'): @@ -32,4 +34,6 @@ def klist_for_each_entry(self, klist, gdbtype, member): for node in klist_for_each(klist): if node.type != self.klist_node_type: raise TypeError("Type {} found. Expected {}.".format(node.type), self.klist_node_type.pointer()) + if node.type is not self.klist_node_type: + self.klist_node_type = node.type yield container_of(node, gdbtype, member) diff --git a/crash/types/list.py b/crash/types/list.py index 380a367c8d9..68c3a33672f 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -31,6 +31,8 @@ def list_for_each(self, list_head, include_head=False, reverse=False, elif list_head.type != self.list_head_type: raise TypeError("Must be struct list_head not {}" .format(str(list_head.type))) + if list_head.type is not self.list_head_type: + self.list_head_type = list_head.type fast = None if int(list_head.address) == 0: raise CorruptListError("list_head is NULL pointer.") From 736fdf787ea74d3c6f6adae9f7ac9bf21c1c42be Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 26 Apr 2019 21:24:20 +0200 Subject: [PATCH 09/75] crash.types.list: add list_empty The list_empty method returns a boolean indicating whether a list_head describes an empty list. Signed-off-by: Jeff Mahoney --- crash/types/list.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crash/types/list.py b/crash/types/list.py index 68c3a33672f..ccfa0ccb6ed 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -116,3 +116,12 @@ def list_for_each_entry(self, list_head, gdbtype, member, include_head=False, re raise TypeError("Type {} found. Expected struct list_head *." .format(str(node.type))) yield container_of(node, gdbtype, member) + + @export + def list_empty(self, list_head): + addr = int(list_head.address) + if list_head.type.code == gdb.TYPE_CODE_PTR: + addr = int(list_head) + + return addr == int(list_head['next']) + From df92870a25056df166a8ca9c85875e339e6d8e80 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 10/75] crash.types.list: fix cycle tests The cycle tests aren't passing exact_cycles=True and will loop forever. Signed-off-by: Jeff Mahoney --- crash/types/list.py | 7 +++++-- tests/test_list.py | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crash/types/list.py b/crash/types/list.py index ccfa0ccb6ed..53df2c5cfd1 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -110,8 +110,11 @@ def list_for_each(self, list_head, include_head=False, reverse=False, raise pending_exception @export - def list_for_each_entry(self, list_head, gdbtype, member, include_head=False, reverse=False): - for node in list_for_each(list_head, include_head=include_head, reverse=reverse): + def list_for_each_entry(self, list_head, gdbtype, member, + include_head=False, reverse=False, + exact_cycles=False): + for node in list_for_each(list_head, include_head=include_head, + reverse=reverse, exact_cycles=exact_cycles): if node.type != self.list_head_type.pointer(): raise TypeError("Type {} found. Expected struct list_head *." .format(str(node.type))) diff --git a/tests/test_list.py b/tests/test_list.py index a5efe9081ec..bea22c4a267 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -68,7 +68,7 @@ def test_cycle_list(self): expected_count = short_list.type.sizeof // short_list[0].type.sizeof count = 0 with self.assertRaises(ListCycleError): - for node in list_for_each(normal_list): + for node in list_for_each(normal_list, exact_cycles=True): count += 1 def test_corrupt_list(self): @@ -77,7 +77,7 @@ def test_corrupt_list(self): expected_count = short_list.type.sizeof // short_list[0].type.sizeof count = 0 with self.assertRaises(CorruptListError): - for node in list_for_each(normal_list): + for node in list_for_each(normal_list, exact_cycles=True): count += 1 def test_normal_container_list_with_string(self): @@ -110,7 +110,7 @@ def test_cycle_container_list_with_string(self): count = 0 with self.assertRaises(ListCycleError): for node in list_for_each_entry(cycle_list, 'struct container', - 'list'): + 'list', exact_cycles=True): count += 1 def test_cycle_container_list_with_type(self): @@ -122,7 +122,7 @@ def test_cycle_container_list_with_type(self): count = 0 with self.assertRaises(ListCycleError): for node in list_for_each_entry(cycle_list, struct_container, - 'list'): + 'list', exact_cycles=True): count += 1 def test_bad_container_list_with_string(self): From 87aec8167af0a3f558c6f7094547b715a4df168b Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 11/75] tests: clear file during test teardown The tests that load files (or targets) need to tear them down so subsequent tests don't get tripped up by them. Signed-off-by: Jeff Mahoney --- tests/test_infra_lookup.py | 3 +++ tests/test_list.py | 3 +++ tests/test_objfile_callbacks.py | 3 +++ tests/test_percpu.py | 1 + tests/test_syscache.py | 3 +++ tests/test_syscmd.py | 3 +++ tests/test_util.py | 3 +++ 7 files changed, 19 insertions(+) diff --git a/tests/test_infra_lookup.py b/tests/test_infra_lookup.py index acb7098fb24..9c3f50e86d4 100644 --- a/tests/test_infra_lookup.py +++ b/tests/test_infra_lookup.py @@ -204,6 +204,9 @@ class TestMinimalSymbolCallback(unittest.TestCase): def setUp(self): gdb.execute("file") + def tearDown(self): + gdb.execute("file") + def load_file(self): gdb.execute("file tests/test-util") diff --git a/tests/test_list.py b/tests/test_list.py index bea22c4a267..d78bba53ee1 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -15,6 +15,9 @@ def setUp(self): gdb.execute("file tests/test-list") self.list_head = gdb.lookup_type("struct list_head") + def tearDown(self): + gdb.execute("file") + def test_none_list(self): count = 0 with self.assertRaises(TypeError): diff --git a/tests/test_objfile_callbacks.py b/tests/test_objfile_callbacks.py index 7b0591d8c3b..74aba9105fc 100644 --- a/tests/test_objfile_callbacks.py +++ b/tests/test_objfile_callbacks.py @@ -11,6 +11,9 @@ class TestCallback(unittest.TestCase): def setUp(self): gdb.execute("file") + def tearDown(self): + gdb.execute("file") + def load_file(self): gdb.execute("file tests/test-util") diff --git a/tests/test_percpu.py b/tests/test_percpu.py index 773c7969449..a3f25c80f3b 100644 --- a/tests/test_percpu.py +++ b/tests/test_percpu.py @@ -31,6 +31,7 @@ def setUp(self): def tearDown(self): try: gdb.execute("detach", to_string=True) + gdb.execute("file") except gdb.error: print() pass diff --git a/tests/test_syscache.py b/tests/test_syscache.py index 2b69b93440e..2f719090f67 100644 --- a/tests/test_syscache.py +++ b/tests/test_syscache.py @@ -23,6 +23,9 @@ def setUp(self): gdb.execute("file tests/test-syscache") self.cycle_namespace() + def tearDown(self): + gdb.execute("file") + def cycle_namespace(self): import crash.cache.syscache reload(crash.cache.syscache) diff --git a/tests/test_syscmd.py b/tests/test_syscmd.py index d78c5472725..7387d18600c 100644 --- a/tests/test_syscmd.py +++ b/tests/test_syscmd.py @@ -15,6 +15,9 @@ def setUp(self): gdb.execute("file tests/test-syscache", to_string=True) self.cmd = SysCommand("pysys") + def tearDown(self): + gdb.execute("file") + def test_sys(self): old_stdout = sys.stdout sys.stdout = StringIO() diff --git a/tests/test_util.py b/tests/test_util.py index 9abbeb97295..e3cf78aa56d 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -21,6 +21,9 @@ def setUp(self): self.ulongsize = self.ulong.sizeof self.test_struct = gdb.lookup_type("struct test") + def tearDown(self): + gdb.execute("file") + def test_invalid_python_type(self): with self.assertRaises(InvalidArgumentError): offset = offsetof(self, 'dontcare') From ff7ef81ec8a9bb0aaf9fd3c7294857a0e86d048a Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 12/75] crash.kernel: convert setup to use DelayedSymvals Now that we have DelayedAttributes everywhere, the setup code can be converted to use it. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index 0da18160c78..1bb87cabdce 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -19,7 +19,8 @@ class CrashKernel(CrashBaseClass): __types__ = [ 'struct module' ] - __symvals__ = [ 'modules' ] + __symvals__ = [ 'modules', 'init_task' ] + __symbols__ = [ 'runqueues'] def __init__(self, vmlinux_filename, searchpath=None): self.findmap = {} @@ -225,11 +226,9 @@ def load_debuginfo(self, objfile, name=None, verbose=False): def setup_tasks(self): gdb.execute('set print thread-events 0') - init_task = gdb.lookup_global_symbol('init_task') - task_list = init_task.value()['tasks'] - runqueues = gdb.lookup_global_symbol('runqueues') + task_list = self.init_task['tasks'] - rqs = get_percpu_var(runqueues) + rqs = get_percpu_var(self.runqueues) rqscurrs = {int(x["curr"]) : k for (k, x) in rqs.items()} self.pid_to_task_struct = {} @@ -239,9 +238,12 @@ def setup_tasks(self): task_count = 0 tasks = [] - for taskg in list_for_each_entry(task_list, init_task.type, 'tasks', include_head=True): + for taskg in list_for_each_entry(task_list, self.init_task.type, + 'tasks', include_head=True): tasks.append(taskg) - for task in list_for_each_entry(taskg['thread_group'], init_task.type, 'thread_group'): + for task in list_for_each_entry(taskg['thread_group'], + self.init_task.type, + 'thread_group'): tasks.append(task) for task in tasks: From a132e7617b26c1ee002925b249bbf7dc5356a071 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 13/75] crash.arch: add baseline ppc64 support This commit adds baseline ppc64 support. It should be enough to populate the thread list but this is an old commit that needs refreshing. Signed-off-by: Jeff Mahoney --- crash/arch/ppc64.py | 30 ++++++++++++++++++++++++++++++ crash/kdump/target.py | 1 + crash/kernel.py | 3 ++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 crash/arch/ppc64.py diff --git a/crash/arch/ppc64.py b/crash/arch/ppc64.py new file mode 100644 index 00000000000..3ab8eb7e159 --- /dev/null +++ b/crash/arch/ppc64.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb + +from crash.arch import CrashArchitecture, register, KernelFrameFilter + +class Powerpc64Architecture(CrashArchitecture): + ident = "powerpc:common64" + aliases = ["ppc64", "elf64-powerpc"] + + def __init__(self): + super(Powerpc64Architecture, self).__init__() + self.ulong_type = gdb.lookup_type('unsigned long') + thread_info_type = gdb.lookup_type('struct thread_info') + self.thread_info_p_type = thread_info_type.pointer() + + # Stop stack traces with addresses below this + self.filter = KernelFrameFilter(0xffff000000000000) + + def setup_thread_info(self, thread): + task = thread.info.task_struct + thread_info = task['stack'].cast(self.thread_info_p_type) + thread.info.set_thread_info(thread_info) + + @classmethod + def get_stack_pointer(cls, thread_struct): + return thread_struct['ksp'] + +register(Powerpc64Architecture) diff --git a/crash/kdump/target.py b/crash/kdump/target.py index d76e8001b47..948ca2b6a19 100644 --- a/crash/kdump/target.py +++ b/crash/kdump/target.py @@ -8,6 +8,7 @@ import addrxlat import crash.arch import crash.arch.x86_64 +import crash.arch.ppc64 class SymbolCallback(object): "addrxlat symbolic callback" diff --git a/crash/kernel.py b/crash/kernel.py index 1bb87cabdce..a8acc3d1457 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -39,7 +39,8 @@ def set_gdb_arch(self): elf_to_gdb = { ('EM_X86_64', 'ELFCLASS64') : 'i386:x86-64', ('EM_386', 'ELFCLASS32') : 'i386', - ('EM_S390', 'ELFCLASS64') : 's390:64-bit' + ('EM_S390', 'ELFCLASS64') : 's390:64-bit', + ('EM_PPC64', 'ELFCLASS64') : 'powerpc:common64' } try: From bbe3d6f3971c5005968c4a14dd89dc15495b5987 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 14/75] crash.types.bitmap: add find first/last/next helpers This commit adds some typical helpers for bitmaps: - find_first_set_bit - find_next_set_bit - find_last_set_bit - find_first_zero_bit - find_next_zero_bit Signed-off-by: Jeff Mahoney --- crash/types/bitmap.py | 279 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 275 insertions(+), 4 deletions(-) diff --git a/crash/types/bitmap.py b/crash/types/bitmap.py index b7f343c3933..aad3a970d20 100644 --- a/crash/types/bitmap.py +++ b/crash/types/bitmap.py @@ -1,8 +1,12 @@ #!/usr/bin/python3 # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable + import gdb +from math import log + from crash.infra import CrashBaseClass, export class TypesBitmapClass(CrashBaseClass): @@ -12,17 +16,45 @@ class TypesBitmapClass(CrashBaseClass): bits_per_ulong = None @classmethod - def setup_ulong(cls, gdbtype): + def _check_bitmap_type(cls, bitmap: gdb.Value) -> None: + if ((bitmap.type.code != gdb.TYPE_CODE_ARRAY or + bitmap[0].type.code != cls.unsigned_long_type.code or + bitmap[0].type.sizeof != cls.unsigned_long_type.sizeof) and + (bitmap.type.code != gdb.TYPE_CODE_PTR or + bitmap.type.target().code != cls.unsigned_long_type.code or + bitmap.type.target().sizeof != cls.unsigned_long_type.sizeof)): + raise TypeError("bitmaps are expected to be arrays of unsigned long not `{}'" + .format(bitmap.type)) + + @classmethod + def setup_ulong(cls, gdbtype: gdb.Type) -> None: cls.bits_per_ulong = gdbtype.sizeof * 8 @export @classmethod - def for_each_set_bit(cls, bitmap): + def for_each_set_bit(cls, bitmap: gdb.Value, + size_in_bytes: int=None) -> Iterable[int]: + """ + Yield each set bit in a bitmap + + Args: + bitmap (gdb.Value: + The bitmap to iterate + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Yields: + int: The position of a bit that is set + """ + cls._check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof # FIXME: callback not workie? cls.bits_per_ulong = cls.unsigned_long_type.sizeof * 8 - size = bitmap.type.sizeof * 8 + size = size_in_bytes * 8 idx = 0 bit = 0 while size > 0: @@ -39,4 +71,243 @@ def for_each_set_bit(cls, bitmap): size -= cls.bits_per_ulong idx += 1 - + + @classmethod + def _find_first_set_bit(cls, val: gdb.Value) -> int: + r = 1 + + if val == 0: + return 0 + + if (val & 0xffffffff) == 0: + val >>= 32 + r += 32 + + if (val & 0xffff) == 0: + val >>= 16 + r += 16 + + if (val & 0xff) == 0: + val >>= 8 + r += 8 + + if (val & 0xf) == 0: + val >>= 4 + r += 4 + + if (val & 0x3) == 0: + val >>= 2 + r += 2 + + if (val & 0x1) == 0: + val >>= 1 + r += 1 + + return r + + @export + @classmethod + def find_next_zero_bit(cls, bitmap: gdb.Value, start: int, + size_in_bytes: int=None) -> int: + """ + Return the next unset bit in the bitmap starting at position `start', + inclusive. + + Args: + bitmap (gdb.Value: + The bitmap to test + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the first bit that is unset or 0 if all are set + """ + cls._check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + elements = size_in_bytes // cls.unsigned_long_type.sizeof + + if start > size_in_bytes << 3: + raise IndexError("Element {} is out of range ({} elements)" + .format(start, elements)) + + element = start // (cls.unsigned_long_type.sizeof << 3) + offset = start % (cls.unsigned_long_type.sizeof << 3) + + for n in range(element, elements): + item = ~bitmap[n] + if item == 0: + continue + + if offset > 0: + item &= ~((1 << offset) - 1) + + v = cls._find_first_set_bit(item) + if v > 0: + ret = n * (cls.unsigned_long_type.sizeof << 3) + v + assert(ret >= start) + return ret + + offset = 0 + + return 0 + + @export + @classmethod + def find_first_zero_bit(cls, bitmap: gdb.Value, + size_in_bytes: int=None) -> int: + """ + Return the first unset bit in the bitmap + + Args: + bitmap (gdb.Value: + The bitmap to scan + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + + Returns: + int: The position of the first bit that is unset + """ + return cls.find_next_zero_bit(bitmap, 0, size_in_bytes) + + @export + @classmethod + def find_next_set_bit(cls, bitmap: gdb.Value, start: int, + size_in_bytes: int=None) -> int: + """ + Return the next set bit in the bitmap starting at position `start', + inclusive. + + Args: + bitmap (gdb.Value: + The bitmap to scan + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the next bit that is set, or 0 if all are + unset + """ + cls._check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + elements = size_in_bytes // cls.unsigned_long_type.sizeof + + if start > size_in_bytes << 3: + raise IndexError("Element {} is out of range ({} elements)" + .format(start, elements)) + + element = start // (cls.unsigned_long_type.sizeof << 3) + offset = start % (cls.unsigned_long_type.sizeof << 3) + + for n in range(element, elements): + if bitmap[n] == 0: + continue + + item = bitmap[n] + if offset > 0: + item &= ~((1 << offset) - 1) + + v = cls._find_first_set_bit(item) + if v > 0: + ret = n * (cls.unsigned_long_type.sizeof << 3) + v + assert(ret >= start) + return ret + + offset = 0 + + return 0 + + @export + @classmethod + def find_first_set_bit(cls, bitmap: gdb.Value, + size_in_bytes: int=None) -> int: + """ + Return the first set bit in the bitmap + + Args: + bitmap (gdb.Value: + The bitmap to scan + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the first bit that is set, or 0 if all are + unset + """ + return cls.find_next_set_bit(bitmap, 0, size_in_bytes) + + @classmethod + def _find_last_set_bit(cls, val: gdb.Value) -> int: + r = cls.unsigned_long_type.sizeof << 3 + + if val == 0: + return 0 + + if (val & 0xffffffff00000000) == 0: + val <<= 32 + r -= 32 + + if (val & 0xffff000000000000) == 0: + val <<= 16 + r -= 16 + + if (val & 0xff00000000000000) == 0: + val <<= 8 + r -= 8 + + if (val & 0xf000000000000000) == 0: + val <<= 4 + r -= 4 + + if (val & 0xc000000000000000) == 0: + val <<= 2 + r -= 2 + + if (val & 0x8000000000000000) == 0: + val <<= 1 + r -= 1 + + return r + + @export + @classmethod + def find_last_set_bit(cls, bitmap: gdb.Value, + size_in_bytes: int=None) -> int: + """ + Return the last set bit in the bitmap + + Args: + bitmap (gdb.Value: + The bitmap to scan + + Returns: + int: The position of the last bit that is set, or 0 if all are unset + """ + cls._check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + elements = size_in_bytes // cls.unsigned_long_type.sizeof + + for n in range(elements - 1, -1, -1): + if bitmap[n] == 0: + continue + + v = cls._find_last_set_bit(bitmap[n]) + if v > 0: + return n * (cls.unsigned_long_type.sizeof << 3) + v + + return 0 From 527e04427a95e617c80231ef5427b3de1fed8554 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 15/75] crash: use new gdb.Target to create standalone target When I rebased crash-python-gdb to an 8.3 prerelease, I found that targets have been converted to C++. That necessitated a rewrite of much of the target code, and I cleaned up some rough edges. With the new target, we load the vmcore using a simple 'target kdumpfile /path/to/vmcore' command that can be used entirely outside of the crash semantic code. This means we can debug the target more easily and use it in the testing code without having to parse everything for every test. This commit converts crash to use the new target but doesn't exploit it for testing yet. Signed-off-by: Jeff Mahoney --- crash.sh | 22 +++++-- crash/kdump/__init__.py | 2 - crash/kernel.py | 109 ++++++------------------------- crash/session.py | 20 ++---- crash/types/node.py | 2 +- crash/types/page.py | 2 +- kdump/__init__.py | 0 {crash/kdump => kdump}/target.py | 88 +++++++++++++------------ tests/test_target.py | 24 +++++-- 9 files changed, 107 insertions(+), 162 deletions(-) delete mode 100644 crash/kdump/__init__.py create mode 100644 kdump/__init__.py rename {crash/kdump => kdump}/target.py (56%) diff --git a/crash.sh b/crash.sh index cd47cca23eb..77535b82151 100755 --- a/crash.sh +++ b/crash.sh @@ -118,6 +118,15 @@ set prompt py-crash> set height 0 set print pretty on +file "$KERNEL" + +python +from kdump.target import Target +target = Target(debug=False) +end + +target kdumpfile $VMCORE + python import sys import traceback @@ -129,8 +138,8 @@ except RuntimeError as e: sys.exit(1) path = "$SEARCHDIRS".split(' ') try: - x = crash.session.Session("$KERNEL", "$VMCORE", "$ZKERNEL", path) - print("The 'pyhelp' command will list the command extensions.") + x = crash.session.Session(path) + print("The 'pyhelp' command will list the command extensions.") except gdb.error as e: print("crash-python: {}, exiting".format(str(e)), file=sys.stderr) traceback.print_exc() @@ -140,16 +149,19 @@ except RuntimeError as e: file=sys.stderr) traceback.print_exc() sys.exit(1) + +target.unregister() +del target EOF # This is how we debug gdb problems when running crash if [ "$DEBUGMODE" = "gdb" ]; then - RUN="run -nh -q -x $GDBINIT" + RUN="run -nx -q -x $GDBINIT" echo $RUN > /tmp/gdbinit - gdb $GDB -nh -q -x /tmp/gdbinit + gdb $GDB -nx -q -x /tmp/gdbinit elif [ "$DEBUGMODE" = "valgrind" ]; then valgrind --keep-stacktraces=alloc-and-free $GDB -nh -q -x $GDBINIT else - $GDB -nh -q -x $GDBINIT + $GDB -nx -q -x $GDBINIT fi diff --git a/crash/kdump/__init__.py b/crash/kdump/__init__.py deleted file mode 100644 index 9e72c13b9b3..00000000000 --- a/crash/kdump/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: diff --git a/crash/kernel.py b/crash/kernel.py index a8acc3d1457..8e4257f5688 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -4,15 +4,15 @@ import gdb import sys import os.path +import crash.arch +import crash.arch.x86_64 +import crash.arch.ppc64 from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry from crash.types.percpu import get_percpu_var from crash.types.list import list_for_each_entry import crash.cache.tasks from crash.types.task import LinuxTask -import crash.kdump -import crash.kdump.target -from kdumpfile import kdumpfile from elftools.elf.elffile import ELFFile LINUX_KERNEL_PID = 1 @@ -22,98 +22,29 @@ class CrashKernel(CrashBaseClass): __symvals__ = [ 'modules', 'init_task' ] __symbols__ = [ 'runqueues'] - def __init__(self, vmlinux_filename, searchpath=None): + def __init__(self, searchpath=None): self.findmap = {} - self.vmlinux_filename = vmlinux_filename self.searchpath = searchpath - f = open(self.vmlinux_filename, 'rb') - self.elffile = ELFFile(f) - - self.set_gdb_arch() - - def set_gdb_arch(self): - mach = self.elffile['e_machine'] - e_class = self.elffile['e_ident']['EI_CLASS'] - - elf_to_gdb = { - ('EM_X86_64', 'ELFCLASS64') : 'i386:x86-64', - ('EM_386', 'ELFCLASS32') : 'i386', - ('EM_S390', 'ELFCLASS64') : 's390:64-bit', - ('EM_PPC64', 'ELFCLASS64') : 'powerpc:common64' - } - - try: - gdbarch = elf_to_gdb[(mach, e_class)] - except KeyError as e: - raise RuntimeError("no mapping for {}:{} to gdb architecture found.".format(mach, e_class)) - gdb.execute("set arch {}".format(gdbarch), to_string=True) - - def open_kernel(self): - if self.base_offset is None: - raise RuntimeError("Base offset is unconfigured.") + sym = gdb.lookup_symbol('vsnprintf', None)[0] + if sym is None: + raise RuntimeError("Missing vsnprintf indicates that there is no kernel image loaded.") - self.load_sections() - - try: - list_type = gdb.lookup_type('struct list_head') - except gdb.error as e: - self.load_debuginfo(gdb.objfiles()[0], None) - try: - list_type = gdb.lookup_type('struct list_head') - except gdb.error as e: - raise RuntimeError("Couldn't locate debuginfo for {}" - .format(self.vmlinux_filename)) + f = open(gdb.objfiles()[0].filename, 'rb') + self.elffile = ELFFile(f) - self.target.setup_arch() + archname = sym.symtab.objfile.architecture.name() + archclass = crash.arch.get_architecture(archname) + self.arch = archclass() - def get_sections(self): - sections = {} + self.target = gdb.current_target() + self.vmcore = self.target.kdump - text = self.elffile.get_section_by_name('.text') + self.target.fetch_registers = self.fetch_registers - for section in self.elffile.iter_sections(): - if (section['sh_addr'] < text['sh_addr'] and - section.name != '.data..percpu'): - continue - sections[section.name] = section['sh_addr'] - - return sections - - def load_sections(self): - sections = self.get_sections() - - line = "" - - # .data..percpu shouldn't have relocation applied but it does. - # Perhaps it's due to the address being 0 and it being handled - # as unspecified in the parameter list. -# for section, addr in sections.items(): -# if addr == 0: -# line += " -s {} {:#x}".format(section, addr) - - # The gdb internals are subtle WRT how symbols are mapped. - # Minimal symbols are mapped using the offset for the section - # that contains them. That means that using providing an address - # for .text here gives a base address with no offset and minimal - # symbols in .text (like __switch_to_asm) will not have the correct - # addresses after relocation. - cmd = "add-symbol-file {} -o {:#x} {} ".format(self.vmlinux_filename, - self.base_offset, line) - gdb.execute(cmd, to_string=True) - - def attach_vmcore(self, vmcore_filename, debug=False): - self.vmcore_filename = vmcore_filename - self.vmcore = kdumpfile(vmcore_filename) - self.target = crash.kdump.target.Target(self.vmcore, debug) - - self.base_offset = 0 - try: - KERNELOFFSET = "linux.vmcoreinfo.lines.KERNELOFFSET" - attr = self.vmcore.attr.get(KERNELOFFSET, "0") - self.base_offset = int(attr, base=16) - except Exception as e: - print(e) + def fetch_registers(self, register): + thread = gdb.selected_thread() + return self.arch.fetch_register(thread, register.regnum) def for_each_module(self): for module in list_for_each_entry(self.modules, self.module_type, @@ -264,9 +195,9 @@ def setup_tasks(self): continue thread.name = task['comm'].string() - self.target.arch.setup_thread_info(thread) + self.arch.setup_thread_info(thread) ltask.attach_thread(thread) - ltask.set_get_stack_pointer(self.target.arch.get_stack_pointer) + ltask.set_get_stack_pointer(self.arch.get_stack_pointer) crash.cache.tasks.cache_task(ltask) diff --git a/crash/session.py b/crash/session.py index 581a5ce6394..3296887575c 100644 --- a/crash/session.py +++ b/crash/session.py @@ -17,10 +17,6 @@ class Session(object): commands and subsystems. Args: - kernel_exec (str, optional): The path to the kernel executable - vmcore (str, optional): The path to the vmcore - kernelpath (str, optional): The path the kernel name to use - when reporting errors searchpath (list of str, optional): Paths to directory trees to search for kernel modules and debuginfo debug (bool, optional, default=False): Whether to enable verbose @@ -28,25 +24,17 @@ class Session(object): """ - def __init__(self, kernel_exec=None, vmcore=None, kernelpath=None, - searchpath=None, debug=False): - self.vmcore_filename = vmcore - + def __init__(self, searchpath=None, debug=False): print("crash-python initializing...") if searchpath is None: searchpath = [] - if kernel_exec: - self.kernel = crash.kernel.CrashKernel(kernel_exec, searchpath) - self.kernel.attach_vmcore(vmcore, debug) - self.kernel.open_kernel() + self.kernel = crash.kernel.CrashKernel(searchpath) autoload_submodules('crash.cache') autoload_submodules('crash.subsystem') autoload_submodules('crash.commands') - if kernel_exec: - self.kernel.setup_tasks() - self.kernel.load_modules(searchpath) - + self.kernel.setup_tasks() + self.kernel.load_modules(searchpath) diff --git a/crash/types/node.py b/crash/types/node.py index 2e59f07f2cf..1e460d9cbb0 100644 --- a/crash/types/node.py +++ b/crash/types/node.py @@ -14,7 +14,7 @@ class TypesNodeUtilsClass(CrashBaseClass): @export def numa_node_id(self, cpu): - if gdb.current_target().arch.ident == "powerpc:common64": + if gdb.current_target().arch.name() == "powerpc:common64": return int(self.numa_cpu_lookup_table[cpu]) else: return int(get_percpu_var(self.numa_node, cpu)) diff --git a/crash/types/page.py b/crash/types/page.py index 7f8d3a0321a..54487aa2bd7 100644 --- a/crash/types/page.py +++ b/crash/types/page.py @@ -52,7 +52,7 @@ class Page(CrashBaseClass): def setup_page_type(cls, gdbtype): # TODO: should check config, but that failed to work on ppc64, hardcode # 64k for now - if gdb.current_target().arch.ident == "powerpc:common64": + if gdb.current_target().arch.name() == "powerpc:common64": cls.PAGE_SHIFT = 16 # also a config cls.directmap_base = 0xc000000000000000 diff --git a/kdump/__init__.py b/kdump/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/crash/kdump/target.py b/kdump/target.py similarity index 56% rename from crash/kdump/target.py rename to kdump/target.py index 948ca2b6a19..d374707e8e7 100644 --- a/crash/kdump/target.py +++ b/kdump/target.py @@ -6,9 +6,6 @@ from kdumpfile import kdumpfile, KDUMP_KVADDR from kdumpfile.exceptions import * import addrxlat -import crash.arch -import crash.arch.x86_64 -import crash.arch.ppc64 class SymbolCallback(object): "addrxlat symbolic callback" @@ -31,38 +28,50 @@ def __call__(self, symtype, *args): raise addrxlat.NoDataError() class Target(gdb.Target): - def __init__(self, vmcore, debug=False): - if not isinstance(vmcore, kdumpfile): - raise TypeError("vmcore must be of type kdumpfile") - self.arch = None + def __init__(self, debug=False): + super().__init__() self.debug = debug - self.kdump = vmcore + self.shortname = "kdumpfile" + self.longname = "Use a Linux kernel kdump file as a target" + + self.register() + + def open(self, filename, from_tty): + + if len(gdb.objfiles()) == 0: + raise gdb.GdbError("kdumpfile target requires kernel to be already loaded for symbol resolution") + try: + self.kdump = kdumpfile(file=filename) + except Exception as e: + raise gdb.GdbError("Failed to open `{}': {}" + .format(filename, str(e))) + + self.kdump.attr['addrxlat.ostype'] = 'linux' ctx = self.kdump.get_addrxlat_ctx() ctx.cb_sym = SymbolCallback(ctx) - self.kdump.attr['addrxlat.ostype'] = 'linux' - # So far we've read from the kernel image, now that we've setup - # the architecture, we're ready to plumb into the target - # infrastructure. - super().__init__() + KERNELOFFSET = "linux.vmcoreinfo.lines.KERNELOFFSET" + try: + attr = self.kdump.attr.get(KERNELOFFSET, "0") + self.base_offset = int(attr, base=16) + except Exception as e: + self.base_offset = 0 + + vmlinux = gdb.objfiles()[0].filename - def setup_arch(self): - archname = self.kdump.attr.arch.name - archclass = crash.arch.get_architecture(archname) - if not archclass: - raise NotImplementedError("Architecture {} is not supported yet." - .format(archname)) + # Load the kernel at the relocated address + gdb.execute("add-symbol-file {} -o {:#x} -s .data..percpu 0" + .format(vmlinux, self.base_offset)) - # Doesn't matter what symbol as long as it's everywhere - # Use vsnprintf since 'printk' can be dropped with CONFIG_PRINTK=n - sym = gdb.lookup_symbol('vsnprintf', None)[0] - if sym is None: - raise RuntimeError("Missing vsnprintf indicates there is no kernel image loaded.") - if sym.symtab.objfile.architecture.name() != archclass.ident: - raise TypeError("Dump file is for `{}' but provided kernel is for `{}'" - .format(archname, archclass.ident)) + # Clear out the old symbol cache + gdb.execute("file {}".format(vmlinux)) - self.arch = archclass() + def close(self): + try: + self.unregister() + except: + pass + del self.kdump @classmethod def report_error(cls, addr, length, error): @@ -70,7 +79,7 @@ def report_error(cls, addr, length, error): .format(length, addr, str(error)), file=sys.stderr) - def to_xfer_partial(self, obj, annex, readbuf, writebuf, offset, ln): + def xfer_partial(self, obj, annex, readbuf, writebuf, offset, ln): ret = -1 if obj == self.TARGET_OBJECT_MEMORY: try: @@ -93,28 +102,21 @@ def to_xfer_partial(self, obj, annex, readbuf, writebuf, offset, ln): raise IOError("Unknown obj type") return ret - @staticmethod - def to_thread_alive(ptid): + def thread_alive(self, ptid): return True - @staticmethod - def to_pid_to_str(ptid): + def pid_to_str(self, ptid): return "pid {:d}".format(ptid[1]) - def to_fetch_registers(self, register): - thread = gdb.selected_thread() - self.arch.fetch_register(thread, register.regnum) - return True + def fetch_registers(self, register): + return False - @staticmethod - def to_prepare_to_store(thread): + def prepare_to_store(self, thread): pass # We don't need to store anything; The regcache is already written. - @staticmethod - def to_store_registers(thread): + def store_registers(self, thread): pass - @staticmethod - def to_has_execution(ptid): + def has_execution(self, ptid): return False diff --git a/tests/test_target.py b/tests/test_target.py index 29877ed2bb3..27df6590904 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -4,17 +4,31 @@ import unittest import gdb import os.path -from crash.kdump.target import Target +from kdump.target import Target class TestUtil(unittest.TestCase): def setUp(self): + gdb.execute("file") self.do_real_tests = os.path.exists("tests/vmcore") + def tearDown(self): + try: + x = gdb.current_target() + del x + except: + pass + gdb.execute('target exec') + def test_bad_file(self): - with self.assertRaises(TypeError): - x = Target("/does/not/exist") + x = Target() + with self.assertRaises(gdb.error): + gdb.execute('target kdumpfile /does/not/exist') + x.unregister() def test_real_open_with_no_kernel(self): if self.do_real_tests: - with self.assertRaises(RuntimeError): - x = Target("tests/vmcore") + x = Target() + with self.assertRaises(gdb.error): + gdb.execute('target kdumpfile tests/vmcore') + x.unregister() + From afdd5b261b159ef1ecdda603343cb1b2f2d61704 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Tue, 7 May 2019 11:24:18 -0400 Subject: [PATCH 16/75] crash.types.module: create module for modules This module contains for_each_module and a new for_each_module_section. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 21 ++++------------- crash/types/module.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 crash/types/module.py diff --git a/crash/kernel.py b/crash/kernel.py index 8e4257f5688..2281d7235be 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -11,6 +11,7 @@ from crash.types.list import list_for_each_entry from crash.types.percpu import get_percpu_var from crash.types.list import list_for_each_entry +from crash.types.module import for_each_module, for_each_module_section import crash.cache.tasks from crash.types.task import LinuxTask from elftools.elf.elffile import ELFFile @@ -18,8 +19,7 @@ LINUX_KERNEL_PID = 1 class CrashKernel(CrashBaseClass): - __types__ = [ 'struct module' ] - __symvals__ = [ 'modules', 'init_task' ] + __symvals__ = [ 'init_task' ] __symbols__ = [ 'runqueues'] def __init__(self, searchpath=None): @@ -46,21 +46,10 @@ def fetch_registers(self, register): thread = gdb.selected_thread() return self.arch.fetch_register(thread, register.regnum) - def for_each_module(self): - for module in list_for_each_entry(self.modules, self.module_type, - 'list'): - yield module - def get_module_sections(self, module): - attrs = module['sect_attrs'] out = [] - for sec in range(0, attrs['nsections']): - attr = attrs['attrs'][sec] - name = attr['name'].string() - if name == '.text': - continue - out.append("-s {} {:#x}".format(name, int(attr['address']))) - + for (name, addr) in for_each_module_section(module): + out.append("-s {} {:#x}".format(name, addr)) return " ".join(out) def load_modules(self, verbose=False): @@ -68,7 +57,7 @@ def load_modules(self, verbose=False): sys.stdout.flush() failed = 0 loaded = 0 - for module in self.for_each_module(): + for module in for_each_module(): modname = "{}".format(module['name'].string()) modfname = "{}.ko".format(modname) found = False diff --git a/crash/types/module.py b/crash/types/module.py new file mode 100644 index 00000000000..3a17b245345 --- /dev/null +++ b/crash/types/module.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +from typing import Iterable, Tuple + +import gdb +from crash.infra import CrashBaseClass, export +from crash.types.list import list_for_each_entry + +class Module(CrashBaseClass): + __symvals__ = [ 'modules'] + __types__ = [ 'struct module' ] + + @classmethod + @export + def for_each_module(cls) -> Iterable[gdb.Value]: + """ + Iterate over each module in the modules list + + Yields: + gdb.Value(): The next module on the list + + """ + for module in list_for_each_entry(cls.modules, cls.module_type, + 'list'): + yield module + + @classmethod + @export + def for_each_module_section(cls, module: gdb.Value) \ + -> Iterable[Tuple[str, int]]: + """ + Iterate over each ELF section in a loaded module + + This routine iterates over the 'sect_attrs' member of the + 'struct module' already in memory. For ELF sections from the + module at rest, use pyelftools on the module file. + + Args: + module (gdb.Value): The struct module to iterate + + Yields: + (str, int): A 2-tuple containing the name and address + of the section + """ + attrs = module['sect_attrs'] + + for sec in range(0, attrs['nsections']): + attr = attrs['attrs'][sec] + name = attr['name'].string() + if name == '.text': + continue + + yield (name, int(attr['address'])) From 96be84342be20c4355ae6129d838d8ec18593450 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 30 Apr 2019 22:54:14 -0400 Subject: [PATCH 17/75] kdump.target: don't specify offset for .data..percpu The "right" thing to do is for .data..percpu to be loaded at offset 0. Unfortunately, that only works when debuginfo is embedded in the binary. When separate debuginfo is used, section offsets can't be specified and gdb interprets an offset of 0 to mean "immediately after the preceding section." In order to make the rest of the percpu code sane, we'll let gdb make the same assumption when embedded debuginfo is used. Signed-off-by: Jeff Mahoney --- crash/types/percpu.py | 21 --------------------- kdump/target.py | 11 +++++++++-- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index c1034d23223..7c446f45977 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -18,14 +18,6 @@ class TypesPerCPUClass(CrashBaseClass): dynamic_offset_cache = None - # TODO: put this somewhere else - arch? - @classmethod - def setup_kaslr_offset(cls): - offset = int(gdb.lookup_minimal_symbol("_text").value().address) - offset -= int(gdb.lookup_minimal_symbol("phys_startup_64").value().address) - offset -= 0xffffffff80000000 - cls.kaslr_offset = offset - @classmethod def setup_per_cpu_size(cls, symbol): try: @@ -36,9 +28,6 @@ def setup_per_cpu_size(cls, symbol): @classmethod def setup_nr_cpus(cls, ignored): cls.nr_cpus = array_size(cls.__per_cpu_offset) - # piggyback on this as it seems those minsymbols at the time of - # their callback yield offset of 0 - cls.setup_kaslr_offset() @classmethod def __add_to_offset_cache(cls, base, start, end): @@ -53,9 +42,6 @@ def __setup_dynamic_offset_cache(cls): for slot in range(cls.pcpu_nr_slots): for chunk in list_for_each_entry(cls.pcpu_slot[slot], cls.pcpu_chunk_type, 'list'): chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) - # __per_cpu_start is adjusted by KASLR, but dynamic offsets are - # not, so we have to subtract the offset - chunk_base += int(cls.__per_cpu_start) - cls.kaslr_offset off = 0 start = None @@ -149,13 +135,6 @@ def get_percpu_var_nocheck(self, var, cpu=None, is_symbol=False): addr += var.cast(self.char_p_type) addr -= self.__per_cpu_start - # if we got var from symbol, it means KASLR relocation was applied to - # the offset, it was applied also to __per_cpu_start, which cancels out - # If var wasn't a symbol, we have to undo the adjustion to - # __per_cpu_start, otherwise we get a bogus address - if not is_symbol: - addr += self.kaslr_offset - vartype = var.type return addr.cast(vartype).dereference() diff --git a/kdump/target.py b/kdump/target.py index d374707e8e7..fb4e11cf121 100644 --- a/kdump/target.py +++ b/kdump/target.py @@ -59,9 +59,16 @@ def open(self, filename, from_tty): vmlinux = gdb.objfiles()[0].filename + # Load the kernel at the relocated address - gdb.execute("add-symbol-file {} -o {:#x} -s .data..percpu 0" - .format(vmlinux, self.base_offset)) + # Unfortunately, the percpu section has an offset of 0 and + # ends up getting placed at the offset base. This is easy + # enough to handle in the percpu code. + result = gdb.execute("add-symbol-file {} -o {:#x}" + .format(vmlinux, self.base_offset), + to_string=True) + if self.debug: + print(result) # Clear out the old symbol cache gdb.execute("file {}".format(vmlinux)) From fa71477fa55c0cd5b199e77d5f15249dc275d616 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 6 May 2019 14:22:33 -0400 Subject: [PATCH 18/75] crash.types.percpu: handle missing pcpu_nr_slots Kernels prior to 2.6.30 didn't have dynamic percpu ranges. The test cases have also not been extended to cover the dynamic ranges. This commit catches DelayedAttributeError so the test cases can pass. Signed-off-by: Jeff Mahoney --- crash/types/percpu.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index 7c446f45977..7ffb90902f9 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -103,16 +103,20 @@ def __is_percpu_var(self, var): return int(v) < self.per_cpu_size def __is_percpu_var_dynamic(self, var): - if self.dynamic_offset_cache is None: - self.__setup_dynamic_offset_cache() + try: + if self.dynamic_offset_cache is None: + self.__setup_dynamic_offset_cache() - var = int(var) - # TODO: we could sort the list... - for (start, end) in self.dynamic_offset_cache: - if var >= start and var < end: - return True + var = int(var) + # TODO: we could sort the list... + for (start, end) in self.dynamic_offset_cache: + if var >= start and var < end: + return True - return False + return False + except DelayedAttributeError: + # This can happen with the testcases or in kernels prior to 2.6.30 + pass @export def is_percpu_var(self, var): From c6ec4bcaa028cc2f0dc2a9b2edf63d6c6c56d09d Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 08:36:23 -0400 Subject: [PATCH 19/75] crash.types.percpu: get_percpu_var, raise exception with passed val If the value passed to get_percpu_var isn't a percpu, we raise an exception -- but the exception contained the processed value instead of the passed one. That can be misleading when debugging. Also, handle the val=None case that can occur as we try to treat the value as a pointer. Signed-off-by: Jeff Mahoney --- crash/types/percpu.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index 7ffb90902f9..60f1a8ab4f0 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -144,6 +144,7 @@ def get_percpu_var_nocheck(self, var, cpu=None, is_symbol=False): @export def get_percpu_var(self, var, cpu=None): + orig_var = var # Percpus can be: # - actual objects, where we'll need to use the address. # - pointers to objects, where we'll need to use the target @@ -159,6 +160,6 @@ def get_percpu_var(self, var, cpu=None): var = var.address if not self.is_percpu_var(var): var = var.address - if not self.is_percpu_var(var): - raise TypeError("Argument {} does not correspond to a percpu pointer.".format(var)) - return self.get_percpu_var_nocheck(var, cpu, is_symbol) + if var is None or not self.is_percpu_var(var): + raise TypeError("Argument {} does not correspond to a percpu pointer.".format(orig_var)) + return self.get_percpu_var_nocheck(var, cpu, is_symbol, nr_cpus) From 7502b34e3b3239225598eeb1d699adda530412d9 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Tue, 7 May 2019 11:55:00 -0400 Subject: [PATCH 20/75] crash.types.cpu: fix online cpu mask and add possible cpu mask --- crash/types/cpu.py | 67 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/crash/types/cpu.py b/crash/types/cpu.py index a5c63f26d7b..29fc3d666bd 100644 --- a/crash/types/cpu.py +++ b/crash/types/cpu.py @@ -3,24 +3,77 @@ import gdb from crash.infra import CrashBaseClass, export -from crash.util import container_of, find_member_variant, get_symbol_value from crash.types.bitmap import for_each_set_bit +from crash.exceptions import DelayedAttributeError + +from typing import List, Iterable # this wraps no particular type, rather it's a placeholder for # functions to iterate over online cpu's etc. class TypesCPUClass(CrashBaseClass): + __symbol_callbacks__ = [ ('cpu_online_mask', '_setup_online_mask'), + ('__cpu_online_mask', '_setup_online_mask'), + ('cpu_possible_mask', '_setup_possible_mask'), + ('__cpu_possible_mask', '_setup_possible_mask') ] - __symbol_callbacks__ = [ ('cpu_online_mask', 'setup_cpus_mask') ] - - cpus_online = None + cpus_online: List[int] = list() + cpus_possible: List[int] = list() @classmethod - def setup_cpus_mask(cls, cpu_mask): - bits = cpu_mask.value()["bits"] + def _setup_online_mask(cls, symbol: gdb.Symbol) -> None: + cls.cpu_online_mask = symbol.value() + bits = cls.cpu_online_mask["bits"] cls.cpus_online = list(for_each_set_bit(bits)) @export - def for_each_online_cpu(self): + def for_each_online_cpu(self) -> Iterable[int]: + """ + Yield CPU numbers of all online CPUs + + Yields: + int: Number of a possible CPU location + """ for cpu in self.cpus_online: yield cpu + @export + def highest_online_cpu_nr(self) -> None: + """ + Return The highest online CPU number + + Returns: + int: The highest online CPU number + """ + if not TypesCPUClass.cpus_online : + raise DelayedAttributeError(self.__class__.__name__, 'cpus_online') + return self.cpus_online[-1] + + @classmethod + def _setup_possible_mask(cls, cpu_mask: gdb.Symbol) -> None: + cls.cpu_possible_mask = cpu_mask.value() + bits = cls.cpu_possible_mask["bits"] + cls.cpus_possible = list(for_each_set_bit(bits)) + + @export + def for_each_possible_cpu(self) -> Iterable[int]: + """ + Yield CPU numbers of all possible CPUs + + Yields: + int: Number of a possible CPU location + """ + for cpu in self.cpus_possible: + yield cpu + + @export + def highest_possible_cpu_nr(self) -> int: + """ + Return The highest possible CPU number + + Returns: + int: The highest possible CPU number + """ + if not self.cpus_possible: + raise DelayedAttributeError(self.__class__.__name__, + 'cpus_possible') + return self.cpus_possible[-1] From 36a58b8c019157ef8cd903820dbac84fa9865916 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 21/75] crash.types.percpu: better percpu handling Newer kernels do percpu differently than the SLE11 kernel we wrote the percpu code to use. It fails on newer kernels, causing crash-python to abort while setting up tasks. This commit adds support for reading the cpu_possible_mask to gather percpu variables. NOTE: Before this lands in master, we need to ensure it works with sparsely numbered CPUs. I'm not convinced it does yet. Signed-off-by: Jeff Mahoney --- crash/types/percpu.py | 399 +++++++++++++++++++++++++++++++----------- 1 file changed, 295 insertions(+), 104 deletions(-) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index 60f1a8ab4f0..251bb4303fa 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -1,165 +1,356 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Dict, Union, List, Tuple + import gdb from crash.infra import CrashBaseClass, export -from crash.util import array_size +from crash.util import array_size, struct_has_member from crash.types.list import list_for_each_entry from crash.exceptions import DelayedAttributeError +from crash.types.bitmap import find_first_set_bit, find_last_set_bit +from crash.types.bitmap import find_next_set_bit, find_next_zero_bit +from crash.types.page import Page +from crash.types.cpu import highest_possible_cpu_nr + +class PerCPUError(TypeError): + fmt = "{} does not correspond to a percpu pointer." + def __init__(self, var): + super().__init__(self.fmt.format(var)) + +SymbolOrValue = Union[gdb.Value, gdb.Symbol] +PerCPUReturn = Union[gdb.Value, Dict[int, gdb.Value]] class TypesPerCPUClass(CrashBaseClass): - __types__ = [ 'char *', 'struct pcpu_chunk' ] + __types__ = [ 'void *', 'char *', 'struct pcpu_chunk', + 'struct percpu_counter' ] __symvals__ = [ '__per_cpu_offset', 'pcpu_base_addr', 'pcpu_slot', - 'pcpu_nr_slots' ] + 'pcpu_nr_slots', 'pcpu_group_offsets' ] __minsymvals__ = ['__per_cpu_start', '__per_cpu_end' ] - __minsymbol_callbacks__ = [ ('__per_cpu_start', 'setup_per_cpu_size'), - ('__per_cpu_end', 'setup_per_cpu_size') ] - __symbol_callbacks__ = [ ('__per_cpu_offset', 'setup_nr_cpus') ] + __minsymbol_callbacks__ = [ ('__per_cpu_start', '_setup_per_cpu_size'), + ('__per_cpu_end', '_setup_per_cpu_size') ] + __symbol_callbacks__ = [ ('__per_cpu_offset', '_setup_nr_cpus') ] - dynamic_offset_cache = None + dynamic_offset_cache: List[Tuple[int, int]] = list() + static_ranges: Dict[int, int] = dict() + last_cpu = -1 + nr_cpus = 0 @classmethod - def setup_per_cpu_size(cls, symbol): + def _setup_per_cpu_size(cls, symbol: gdb.Symbol) -> None: + try: + size = cls.__per_cpu_end - cls.__per_cpu_start + except DelayedAttributeError: + pass + + cls.static_ranges[0] = size + if cls.__per_cpu_start != 0: + cls.static_ranges[cls.__per_cpu_start] = size + try: - cls.per_cpu_size = cls.__per_cpu_end - cls.__per_cpu_start + # This is only an optimization so we don't return NR_CPUS values + # when there are far fewer CPUs on the system. + cls.last_cpu = highest_possible_cpu_nr() except DelayedAttributeError: pass @classmethod - def setup_nr_cpus(cls, ignored): + def _setup_nr_cpus(cls, ignored: gdb.Symbol) -> None: cls.nr_cpus = array_size(cls.__per_cpu_offset) + if cls.last_cpu == -1: + cls.last_cpu = cls.nr_cpus + @classmethod - def __add_to_offset_cache(cls, base, start, end): + def _add_to_offset_cache(cls, base: int, start: int, end: int) -> None: cls.dynamic_offset_cache.append((base + start, base + end)) @classmethod - def __setup_dynamic_offset_cache(cls): - # TODO: interval tree would be more efficient, but this adds no 3rd - # party module dependency... - cls.dynamic_offset_cache = list() + def dump_ranges(cls) -> None: + """ + Dump all percpu ranges to stdout + """ + for (start, size) in cls.static_ranges.items(): + print(f"static start={start:#x}, size={size:#x}") + if cls.dynamic_offset_cache: + for (start, end) in cls.dynamic_offset_cache: + print(f"dynamic start={start:#x}, end={end:#x}") + + @classmethod + def _setup_dynamic_offset_cache_area_map(cls, chunk: gdb.Value) -> None: used_is_negative = None - for slot in range(cls.pcpu_nr_slots): - for chunk in list_for_each_entry(cls.pcpu_slot[slot], cls.pcpu_chunk_type, 'list'): - chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) - - off = 0 - start = None - _map = chunk['map'] - map_used = int(chunk['map_used']) - - # Prior to 3.14 commit 723ad1d90b56 ("percpu: store offsets - # instead of lengths in ->map[]"), negative values in map - # meant the area is used, and the absolute value is area size. - # After the commit, the value is area offset for unused, and - # offset | 1 for used (all offsets have to be even). The value - # at index 'map_used' is a 'sentry' which is the total size | - # 1. There is no easy indication of whether kernel includes - # the commit, unless we want to rely on version numbers and - # risk breakage in case of backport to older version. Instead - # employ a heuristic which scans the first chunk, and if no - # negative value is found, assume the kernel includes the - # commit. - if used_is_negative is None: - used_is_negative = False - for i in range(map_used): - val = int(_map[i]) - if val < 0: - used_is_negative = True - break - - if used_is_negative: - for i in range(map_used): - val = int(_map[i]) - if val < 0: - if start is None: - start = off - else: - if start is not None: - cls.__add_to_offset_cache(chunk_base, start, off) - start = None - off += abs(val) + chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) + + off = 0 + start = None + _map = chunk['map'] + map_used = int(chunk['map_used']) + + # Prior to 3.14 commit 723ad1d90b56 ("percpu: store offsets + # instead of lengths in ->map[]"), negative values in map + # meant the area is used, and the absolute value is area size. + # After the commit, the value is area offset for unused, and + # offset | 1 for used (all offsets have to be even). The value + # at index 'map_used' is a 'sentry' which is the total size | + # 1. There is no easy indication of whether kernel includes + # the commit, unless we want to rely on version numbers and + # risk breakage in case of backport to older version. Instead + # employ a heuristic which scans the first chunk, and if no + # negative value is found, assume the kernel includes the + # commit. + if used_is_negative is None: + used_is_negative = False + for i in range(map_used): + val = int(_map[i]) + if val < 0: + used_is_negative = True + break + + if used_is_negative: + for i in range(map_used): + val = int(_map[i]) + if val < 0: + if start is None: + start = off + else: if start is not None: - cls.__add_to_offset_cache(chunk_base, start, off) + cls._add_to_offset_cache(chunk_base, start, off) + start = None + off += abs(val) + if start is not None: + cls._add_to_offset_cache(chunk_base, start, off) + else: + for i in range(map_used): + off = int(_map[i]) + if off & 1 == 1: + off -= 1 + if start is None: + start = off else: - for i in range(map_used): - off = int(_map[i]) - if off & 1 == 1: - off -= 1 - if start is None: - start = off - else: - if start is not None: - cls.__add_to_offset_cache(chunk_base, start, off) - start = None if start is not None: - off = int(_map[map_used]) - 1 - cls.__add_to_offset_cache(chunk_base, start, off) + cls._add_to_offset_cache(chunk_base, start, off) + start = None + if start is not None: + off = int(_map[map_used]) - 1 + cls._add_to_offset_cache(chunk_base, start, off) + + + @classmethod + def _setup_dynamic_offset_cache_bitmap(cls, chunk: gdb.Value) -> None: + group_offset = int(cls.pcpu_group_offsets[0]) + size_in_bytes = int(chunk['nr_pages']) * Page.PAGE_SIZE + size_in_bits = size_in_bytes << 3 + start = -1 + end = 0 + + chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) + cls._add_to_offset_cache(chunk_base, 0, size_in_bytes) - def __is_percpu_var(self, var): - if int(var) < self.__per_cpu_start: - return False - v = var.cast(self.char_p_type) - self.__per_cpu_start - return int(v) < self.per_cpu_size + @classmethod + def _setup_dynamic_offset_cache(cls) -> None: + # TODO: interval tree would be more efficient, but this adds no 3rd + # party module dependency... + use_area_map = struct_has_member(cls.pcpu_chunk_type, 'map') + for slot in range(cls.pcpu_nr_slots): + for chunk in list_for_each_entry(cls.pcpu_slot[slot], cls.pcpu_chunk_type, 'list'): + if use_area_map: + cls._setup_dynamic_offset_cache_area_map(chunk) + else: + cls._setup_dynamic_offset_cache_bitmap(chunk) - def __is_percpu_var_dynamic(self, var): + def _is_percpu_var_dynamic(self, var: int) -> bool: try: - if self.dynamic_offset_cache is None: - self.__setup_dynamic_offset_cache() + if not self.dynamic_offset_cache: + self._setup_dynamic_offset_cache() - var = int(var) # TODO: we could sort the list... for (start, end) in self.dynamic_offset_cache: if var >= start and var < end: return True - - return False except DelayedAttributeError: # This can happen with the testcases or in kernels prior to 2.6.30 pass + return False + + # The resolved percpu address + def _is_static_percpu_address(self, addr: int) -> bool: + for start in self.static_ranges: + size = self.static_ranges[start] + for cpu in range(0, self.last_cpu): + offset = int(__per_cpu_offset[cpu]) + start + if addr >= offset and addr < offset + size: + return True + return False + + # The percpu virtual address + def is_static_percpu_var(self, addr: int) -> bool: + """ + Returns whether the provided address is within the bounds of + the percpu static ranges + + Args: + addr (int): The address to query + + Returns: + bool: whether this address belongs to a static range + """ + for start in self.static_ranges: + for cpu in range(0, self.last_cpu): + size = self.static_ranges[start] + if addr >= start and addr < start + size: + return True + return False + + # The percpu range should start at offset 0 but gdb relocation + # treats 0 as a special value indicating it should just be after + # the previous section. It's possible to override this while + # loading debuginfo but not when debuginfo is embedded. + def _relocated_offset(self, var): + addr=int(var) + start = self.__per_cpu_start + size = self.static_ranges[start] + if addr >= start and addr < start + size: + return addr - start + return addr + @export - def is_percpu_var(self, var): + def is_percpu_var(self, var: SymbolOrValue) -> bool: + """ + Returns whether the provided value or symbol falls within + any of the percpu ranges + + Args: + var: (gdb.Value or gdb.Symbol): The value to query + + Returns: + bool: whether the value belongs to any percpu range + """ if isinstance(var, gdb.Symbol): var = var.value().address - if self.__is_percpu_var(var): + + var = int(var) + if self.is_static_percpu_var(var): return True - if self.__is_percpu_var_dynamic(var): + if self._is_percpu_var_dynamic(var): return True return False - def get_percpu_var_nocheck(self, var, cpu=None, is_symbol=False): + def get_percpu_var_nocheck(self, var: SymbolOrValue, cpu: int=None, + nr_cpus: int=None) -> PerCPUReturn: + """ + Retrieve a per-cpu variable for one or all CPUs without performing + range checks + + Per-cpus come in a few forms: + - "Array" of objects + - "Array" of pointers to objects + - Pointers to either of those + + If we want to get the typing right, we need to recognize each one + and figure out what type to pass back. We do want to dereference + pointer to a percpu but we don't want to dereference a percpu + pointer. + + Args: + var (gdb.Symbol, gdb.MinSymbol, gdb.Value): + The value to use to resolve the percpu location + cpu (int, optional, default=None): The cpu for which to return + the per-cpu value. A value of None will return a dictionary + of [cpu, value] for all CPUs. + nr_cpus(int, optional, default=None): + + Returns: + gdb.Value: If cpu is specified, the value corresponding to + the specified CPU. + dict(int, gdb.Value): If cpu is not specified, the values + corresponding to every CPU in a dictionary indexed by CPU + number. + + Raises: + TypeError: var is not gdb.Symbol or gdb.Value + ValueError: cpu is less than 0 + ValueError: nr_cpus is less-or-equal to 0 + """ + if nr_cpus is None: + nr_cpus = self.last_cpu + if nr_cpus < 0: + raise ValueError("nr_cpus must be > 0") if cpu is None: vals = {} - for cpu in range(0, self.nr_cpus): - vals[cpu] = self.get_percpu_var_nocheck(var, cpu, is_symbol) + for cpu in range(0, nr_cpus): + vals[cpu] = self.get_percpu_var_nocheck(var, cpu, nr_cpus) return vals + elif cpu < 0: + raise ValueError("cpu must be >= 0") addr = self.__per_cpu_offset[cpu] - addr += var.cast(self.char_p_type) - addr -= self.__per_cpu_start + if addr > 0: + addr += self._relocated_offset(var) - vartype = var.type - return addr.cast(vartype).dereference() + val = gdb.Value(addr).cast(var.type) + if var.type != self.void_p_type: + val = val.dereference() + return val @export - def get_percpu_var(self, var, cpu=None): + def get_percpu_var(self, var: SymbolOrValue, cpu: int=None, + nr_cpus: int=None) -> PerCPUReturn: + """ + Retrieve a per-cpu variable for one or all CPUs + + Per-cpus come in a few forms: + - "Array" of objects + - "Array" of pointers to objects + - Pointers to either of those + + If we want to get the typing right, we need to recognize each one + and figure out what type to pass back. We do want to dereference + pointer to a percpu but we don't want to dereference a percpu + pointer. + + Args: + var (gdb.Symbol, gdb.MinSymbol, gdb.Value): + The value to use to resolve the percpu location + cpu (int, optional, default=None): The cpu for which to return + the per-cpu value. A value of None will return a dictionary + of [cpu, value] for all CPUs. + nr_cpus(int, optional, default=None): + + Returns: + gdb.Value: If cpu is specified, the value corresponding to + the specified CPU. + dict(int, gdb.Value): If cpu is not specified, the values + corresponding to every CPU in a dictionary indexed by CPU + number. + + Raises: + TypeError: var is not gdb.Symbol or gdb.Value + PerCPUError: var does not fall into any percpu range + ValueError: cpu is less than 0 + ValueError: nr_cpus is less-or-equal to 0 + """ orig_var = var - # Percpus can be: - # - actual objects, where we'll need to use the address. - # - pointers to objects, where we'll need to use the target - # - a pointer to a percpu object, where we'll need to use the - # address of the target - is_symbol = False if isinstance(var, gdb.Symbol) or isinstance(var, gdb.MinSymbol): var = var.value() - is_symbol = True if not isinstance(var, gdb.Value): raise TypeError("Argument must be gdb.Symbol or gdb.Value") - if var.type.code != gdb.TYPE_CODE_PTR: - var = var.address - if not self.is_percpu_var(var): - var = var.address - if var is None or not self.is_percpu_var(var): - raise TypeError("Argument {} does not correspond to a percpu pointer.".format(orig_var)) - return self.get_percpu_var_nocheck(var, cpu, is_symbol, nr_cpus) + + if var.type.code == gdb.TYPE_CODE_PTR: + # The percpu contains pointers + if var.address is not None and self.is_percpu_var(var.address): + var = var.address + # Pointer to a percpu + elif self.is_percpu_var(var): + if var.type != self.void_p_type: + var = var.dereference().address + assert(self.is_percpu_var(var)) + else: + raise PerCPUError(orig_var) + # object is a percpu + elif self.is_percpu_var(var.address): + var = var.address + else: + raise PerCPUError(orig_var) + + return self.get_percpu_var_nocheck(var, cpu, nr_cpus) From 118bbfe83fa9867aae5bc011a99e6c792f6dd6b8 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 21 May 2019 12:59:08 -0400 Subject: [PATCH 22/75] crash.types.percpu: separate the all-cpus and single-cpu cases --- crash/kernel.py | 4 +- crash/types/percpu.py | 162 +++++++++++++++++++----------------------- tests/test_percpu.py | 20 +++--- 3 files changed, 87 insertions(+), 99 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index 2281d7235be..53715b30c49 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -9,7 +9,7 @@ import crash.arch.ppc64 from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry -from crash.types.percpu import get_percpu_var +from crash.types.percpu import get_percpu_vars from crash.types.list import list_for_each_entry from crash.types.module import for_each_module, for_each_module_section import crash.cache.tasks @@ -149,7 +149,7 @@ def setup_tasks(self): task_list = self.init_task['tasks'] - rqs = get_percpu_var(self.runqueues) + rqs = get_percpu_vars(self.runqueues) rqscurrs = {int(x["curr"]) : k for (k, x) in rqs.items()} self.pid_to_task_struct = {} diff --git a/crash/types/percpu.py b/crash/types/percpu.py index 251bb4303fa..eb21d857569 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -19,9 +19,20 @@ def __init__(self, var): super().__init__(self.fmt.format(var)) SymbolOrValue = Union[gdb.Value, gdb.Symbol] -PerCPUReturn = Union[gdb.Value, Dict[int, gdb.Value]] class TypesPerCPUClass(CrashBaseClass): + """ + Per-cpus come in a few forms: + - "Array" of objects + - "Array" of pointers to objects + - Pointers to either of those + + If we want to get the typing right, we need to recognize each one + and figure out what type to pass back. We do want to dereference + pointer to a percpu but we don't want to dereference a percpu + pointer. + """ + __types__ = [ 'void *', 'char *', 'struct pcpu_chunk', 'struct percpu_counter' ] __symvals__ = [ '__per_cpu_offset', 'pcpu_base_addr', 'pcpu_slot', @@ -236,52 +247,34 @@ def is_percpu_var(self, var: SymbolOrValue) -> bool: return True return False - def get_percpu_var_nocheck(self, var: SymbolOrValue, cpu: int=None, - nr_cpus: int=None) -> PerCPUReturn: - """ - Retrieve a per-cpu variable for one or all CPUs without performing - range checks - - Per-cpus come in a few forms: - - "Array" of objects - - "Array" of pointers to objects - - Pointers to either of those - - If we want to get the typing right, we need to recognize each one - and figure out what type to pass back. We do want to dereference - pointer to a percpu but we don't want to dereference a percpu - pointer. + def _resolve_percpu_var(self, var): + orig_var = var + if isinstance(var, gdb.Symbol) or isinstance(var, gdb.MinSymbol): + var = var.value() + if not isinstance(var, gdb.Value): + raise TypeError("Argument must be gdb.Symbol or gdb.Value") - Args: - var (gdb.Symbol, gdb.MinSymbol, gdb.Value): - The value to use to resolve the percpu location - cpu (int, optional, default=None): The cpu for which to return - the per-cpu value. A value of None will return a dictionary - of [cpu, value] for all CPUs. - nr_cpus(int, optional, default=None): + if var.type.code == gdb.TYPE_CODE_PTR: + # The percpu contains pointers + if var.address is not None and self.is_percpu_var(var.address): + var = var.address + # Pointer to a percpu + elif self.is_percpu_var(var): + if var.type != types.void_p_type: + var = var.dereference().address + assert(self.is_percpu_var(var)) + else: + raise PerCPUError(orig_var) + # object is a percpu + elif self.is_percpu_var(var.address): + var = var.address + else: + raise PerCPUError(orig_var) - Returns: - gdb.Value: If cpu is specified, the value corresponding to - the specified CPU. - dict(int, gdb.Value): If cpu is not specified, the values - corresponding to every CPU in a dictionary indexed by CPU - number. + return var - Raises: - TypeError: var is not gdb.Symbol or gdb.Value - ValueError: cpu is less than 0 - ValueError: nr_cpus is less-or-equal to 0 - """ - if nr_cpus is None: - nr_cpus = self.last_cpu - if nr_cpus < 0: - raise ValueError("nr_cpus must be > 0") - if cpu is None: - vals = {} - for cpu in range(0, nr_cpus): - vals[cpu] = self.get_percpu_var_nocheck(var, cpu, nr_cpus) - return vals - elif cpu < 0: + def _get_percpu_var(self, var: SymbolOrValue, cpu: int) -> gdb.Value: + if cpu < 0: raise ValueError("cpu must be >= 0") addr = self.__per_cpu_offset[cpu] @@ -294,63 +287,58 @@ def get_percpu_var_nocheck(self, var: SymbolOrValue, cpu: int=None, return val @export - def get_percpu_var(self, var: SymbolOrValue, cpu: int=None, - nr_cpus: int=None) -> PerCPUReturn: + def get_percpu_var(self, var: SymbolOrValue, cpu: int) -> gdb.Value: """ - Retrieve a per-cpu variable for one or all CPUs - - Per-cpus come in a few forms: - - "Array" of objects - - "Array" of pointers to objects - - Pointers to either of those - - If we want to get the typing right, we need to recognize each one - and figure out what type to pass back. We do want to dereference - pointer to a percpu but we don't want to dereference a percpu - pointer. + Retrieve a per-cpu variable for a single CPU Args: var (gdb.Symbol, gdb.MinSymbol, gdb.Value): The value to use to resolve the percpu location - cpu (int, optional, default=None): The cpu for which to return - the per-cpu value. A value of None will return a dictionary - of [cpu, value] for all CPUs. - nr_cpus(int, optional, default=None): + cpu (int): The cpu for which to return the per-cpu value. Returns: gdb.Value: If cpu is specified, the value corresponding to the specified CPU. - dict(int, gdb.Value): If cpu is not specified, the values - corresponding to every CPU in a dictionary indexed by CPU - number. Raises: TypeError: var is not gdb.Symbol or gdb.Value PerCPUError: var does not fall into any percpu range ValueError: cpu is less than 0 - ValueError: nr_cpus is less-or-equal to 0 """ - orig_var = var - if isinstance(var, gdb.Symbol) or isinstance(var, gdb.MinSymbol): - var = var.value() - if not isinstance(var, gdb.Value): - raise TypeError("Argument must be gdb.Symbol or gdb.Value") + var = self._resolve_percpu_var(var) + return self._get_percpu_var(var, cpu) - if var.type.code == gdb.TYPE_CODE_PTR: - # The percpu contains pointers - if var.address is not None and self.is_percpu_var(var.address): - var = var.address - # Pointer to a percpu - elif self.is_percpu_var(var): - if var.type != self.void_p_type: - var = var.dereference().address - assert(self.is_percpu_var(var)) - else: - raise PerCPUError(orig_var) - # object is a percpu - elif self.is_percpu_var(var.address): - var = var.address - else: - raise PerCPUError(orig_var) + @export + def get_percpu_vars(self, var: SymbolOrValue, + nr_cpus: int=None) -> Dict[int, gdb.Value]: + """ + Retrieve a per-cpu variable for all CPUs + + Args: + var (gdb.Symbol, gdb.MinSymbol, gdb.Value): + The value to use to resolve the percpu location + nr_cpus(int, optional, default=None): The number of CPUs to + return results for. None (or unspecified) will use + the highest possible CPU count. + + Returns: + dict(int, gdb.Value): The values corresponding to every CPU + in a dictionary indexed by CPU number. + + Raises: + TypeError: var is not gdb.Symbol or gdb.Value + PerCPUError: var does not fall into any percpu range + ValueError: nr_cpus is <= 0 + """ + if nr_cpus is None: + nr_cpus = self.last_cpu + + if nr_cpus <= 0: + raise ValueError("nr_cpus must be > 0") + + vals = dict() - return self.get_percpu_var_nocheck(var, cpu, nr_cpus) + var = self._resolve_percpu_var(var) + for cpu in range(0, nr_cpus): + vals[cpu] = self._get_percpu_var(var, cpu) + return vals diff --git a/tests/test_percpu.py b/tests/test_percpu.py index a3f25c80f3b..0be6f2ed1c8 100644 --- a/tests/test_percpu.py +++ b/tests/test_percpu.py @@ -5,7 +5,7 @@ import gdb import crash -import crash.types.percpu +import crash.types.percpu as percpu class TestPerCPU(unittest.TestCase): def setUp(self): @@ -40,28 +40,28 @@ def tearDown(self): def test_struct_test(self): var = gdb.lookup_symbol('struct_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val['x'] == cpu) self.assertTrue(val.type == self.test_struct) def test_ulong_test(self): var = gdb.lookup_symbol('ulong_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val == cpu) self.assertTrue(val.type == self.ulong_type) def test_ulong_ptr_test(self): var = gdb.lookup_symbol('ptr_to_ulong_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val.type == self.ulong_type.pointer()) self.assertTrue(val.dereference() == cpu) def test_voidp_test(self): var = gdb.lookup_symbol('voidp_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val is not None) self.assertTrue(val.type == self.voidp) self.assertTrue(int(val) == 0xdeadbeef) @@ -69,7 +69,7 @@ def test_voidp_test(self): def test_struct_test_ptr(self): var = gdb.lookup_symbol('ptr_to_struct_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val['x'] == cpu) self.assertTrue(val.type == self.test_struct.pointer()) @@ -77,14 +77,14 @@ def test_struct_test_ptr(self): def test_percpu_ptr_sym(self): var = gdb.lookup_symbol('percpu_test', None)[0] self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val.type == self.test_struct) # This is a pointer to an unbound percpu var def test_percpu_ptr_val(self): var = gdb.lookup_symbol('percpu_test', None)[0].value() self.assertTrue(var is not None) - for cpu, val in list(crash.types.percpu.get_percpu_var(var).items()): + for cpu, val in list(percpu.get_percpu_vars(var).items()): self.assertTrue(val.type == self.test_struct) # This is a saved pointer to an bound percpu var (e.g. normal ptr) @@ -92,7 +92,7 @@ def test_non_percpu_sym(self): var = gdb.lookup_symbol('non_percpu_test', None)[0] self.assertTrue(var is not None) with self.assertRaises(TypeError): - x = crash.types.percpu.get_percpu_var(var, 0) + x = percpu.get_percpu_var(var, 0) self.assertTrue(var.value()['x'] == 0) # This is a pointer to an bound percpu var (e.g. normal ptr) @@ -100,5 +100,5 @@ def test_non_percpu_ptr(self): var = gdb.lookup_symbol('non_percpu_test', None)[0].value() self.assertTrue(var is not None) with self.assertRaises(TypeError): - x = crash.types.percpu.get_percpu_var(var, 0) + x = percpu.get_percpu_var(var, 0) self.assertTrue(var['x'] == 0) From 7b38ed424a20b9002b97c76b0e1a3e41da994442 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 30 Apr 2019 17:35:31 -0400 Subject: [PATCH 23/75] crash.types.percpu: add percpu_counter_sum This commit adds support for calculating the contents of percpu counters. Signed-off-by: Jeff Mahoney --- crash/types/percpu.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index eb21d857569..5bca27ce79a 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -342,3 +342,31 @@ def get_percpu_vars(self, var: SymbolOrValue, for cpu in range(0, nr_cpus): vals[cpu] = self._get_percpu_var(var, cpu) return vals + + @export + def percpu_counter_sum(self, var: SymbolOrValue) -> int: + """ + Returns the sum of a percpu counter + + Args: + var (gdb.Value or gdb.Symbol): The percpu counter to sum + + Returns: + int: the sum of all components of the percpu counter + """ + if isinstance(var, gdb.Symbol): + var = var.value() + + if not (var.type == self.percpu_counter_type or + (var.type.code == gdb.TYPE_CODE_PTR and + var.type.target() == self.percpu_counter_type)): + raise TypeError("var must be gdb.Symbol or gdb.Value describing `{}' not `{}'" + .format(self.percpu_counter_type, var.type)) + + total = int(var['count']) + + v = get_percpu_vars(var['counters']) + for cpu in v: + total += int(v[cpu]) + + return total From a515eefc8abd7e3f45352ea201a852b5291e2f36 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 24/75] crash: auto-select crashing task and print backtrace on startup The most common task using a kernel debugger is to examine a crashed task. This autoselects and prints the backtrace on startup if such a task exists. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 10 ++++++++++ crash/session.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/crash/kernel.py b/crash/kernel.py index 53715b30c49..b81d66e8402 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -15,6 +15,7 @@ import crash.cache.tasks from crash.types.task import LinuxTask from elftools.elf.elffile import ELFFile +from crash.util import get_symbol_value LINUX_KERNEL_PID = 1 @@ -41,6 +42,7 @@ def __init__(self, searchpath=None): self.vmcore = self.target.kdump self.target.fetch_registers = self.fetch_registers + self.crashing_thread = None def fetch_registers(self, register): thread = gdb.selected_thread() @@ -167,6 +169,11 @@ def setup_tasks(self): 'thread_group'): tasks.append(task) + try: + crashing_cpu = int(get_symbol_value('crashing_cpu')) + except Exception as e: + crashing_cpu = None + for task in tasks: cpu = None regs = None @@ -177,12 +184,15 @@ def setup_tasks(self): ltask = LinuxTask(task, active, cpu, regs) ptid = (LINUX_KERNEL_PID, task['pid'], 0) + try: thread = gdb.selected_inferior().new_thread(ptid, ltask) except gdb.error as e: print("Failed to setup task @{:#x}".format(int(task.address))) continue thread.name = task['comm'].string() + if active and crashing_cpu is not None and cpu == crashing_cpu: + self.crashing_thread = thread self.arch.setup_thread_info(thread) ltask.attach_thread(thread) diff --git a/crash/session.py b/crash/session.py index 3296887575c..2959aa1d3cd 100644 --- a/crash/session.py +++ b/crash/session.py @@ -38,3 +38,19 @@ def __init__(self, searchpath=None, debug=False): self.kernel.setup_tasks() self.kernel.load_modules(searchpath) + if self.kernel.crashing_thread: + try: + result = gdb.execute("thread {}" + .format(self.kernel.crashing_thread.num), + to_string=True) + if debug: + print(result) + except gdb.error as e: + print("Error while switching to crashed thread: {}" + .format(str(e))) + print("Further debugging may not be possible.") + return + + print("Backtrace from crashing task (PID {:d}):" + .format(self.kernel.crashing_thread.ptid[1])) + gdb.execute("where") From f6feb546fdb8f1b5b2cb1b272d5a5a7fd2a2ce78 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 08:48:35 -0400 Subject: [PATCH 25/75] crash.sh: use $TMPDIR for gdbinit in --gdb mode We currently drop the gdbinit in /tmp, which is not correct. Signed-off-by: Jeff Mahoney --- crash.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crash.sh b/crash.sh index 77535b82151..d51bd17876c 100755 --- a/crash.sh +++ b/crash.sh @@ -158,8 +158,8 @@ EOF if [ "$DEBUGMODE" = "gdb" ]; then RUN="run -nx -q -x $GDBINIT" - echo $RUN > /tmp/gdbinit - gdb $GDB -nx -q -x /tmp/gdbinit + echo $RUN > $TMPDIR/gdbinit-debug + gdb $GDB -nx -q -x $TMPDIR/gdbinit-debug elif [ "$DEBUGMODE" = "valgrind" ]; then valgrind --keep-stacktraces=alloc-and-free $GDB -nh -q -x $GDBINIT else From fe1c40ebb16c3771da3335fee81959ea3cf85a84 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 09:02:37 -0400 Subject: [PATCH 26/75] crash.kernel: use new KernelError exception to report errors during startup crash.kernel currently raises RuntimeError when we should be raising a more specific exception. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 5 ++++- crash/session.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index b81d66e8402..84bcf0a36f8 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -17,6 +17,9 @@ from elftools.elf.elffile import ELFFile from crash.util import get_symbol_value +class CrashKernelError(RuntimeError): + pass + LINUX_KERNEL_PID = 1 class CrashKernel(CrashBaseClass): @@ -29,7 +32,7 @@ def __init__(self, searchpath=None): sym = gdb.lookup_symbol('vsnprintf', None)[0] if sym is None: - raise RuntimeError("Missing vsnprintf indicates that there is no kernel image loaded.") + raise CrashKernelError("Missing vsnprintf indicates that there is no kernel image loaded.") f = open(gdb.objfiles()[0].filename, 'rb') self.elffile = ELFFile(f) diff --git a/crash/session.py b/crash/session.py index 2959aa1d3cd..cbd2ab560dc 100644 --- a/crash/session.py +++ b/crash/session.py @@ -6,6 +6,7 @@ from crash.infra import autoload_submodules import crash.kernel +from crash.kernel import CrashKernelError from kdumpfile import kdumpfile class Session(object): @@ -35,8 +36,13 @@ def __init__(self, searchpath=None, debug=False): autoload_submodules('crash.subsystem') autoload_submodules('crash.commands') - self.kernel.setup_tasks() - self.kernel.load_modules(searchpath) + try: + self.kernel.setup_tasks() + self.kernel.load_modules(searchpath) + except CrashKernelError as e: + print(str(e)) + print("Further debugging may not be possible.") + return if self.kernel.crashing_thread: try: From 18707a4c9a0aecf5e16ed50878387b882febfa0e Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 10:00:14 -0400 Subject: [PATCH 27/75] crash.kernel: add debug option to load_modules When debugging module loading, it's useful to see the full path to the module. crash.kernel.CrashKernel.load_module now accepts a debug option to make that a bit more verbose. The 'verbose' mode still prints the module name and the normal mode prints dots. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index 84bcf0a36f8..827cd40c9d1 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -57,7 +57,7 @@ def get_module_sections(self, module): out.append("-s {} {:#x}".format(name, addr)) return " ".join(out) - def load_modules(self, verbose=False): + def load_modules(self, verbose=False, debug=False): print("Loading modules...", end='') sys.stdout.flush() failed = 0 @@ -78,8 +78,14 @@ def load_modules(self, verbose=False): else: addr = int(module['core_layout']['base']) - if verbose: + if debug: + print("Loading {} at {:#x}".format(modpath, addr)) + elif verbose: print("Loading {} at {:#x}".format(modname, addr)) + else: + print(".", end='') + sys.stdout.flush() + sections = self.get_module_sections(module) gdb.execute("add-symbol-file {} {:#x} {}" .format(modpath, addr, sections), From 823e505024c3464e4002ee6239f1479e0ce5571d Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 09:32:17 -0400 Subject: [PATCH 28/75] crash.kernel: use objfile.has_symbols() to detect debuginfo Looking up a symbol is a hacky way of discovering whether we have debuginfo. Now that gdb-python has objfile.has_symbols(), we can use that instead. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index 827cd40c9d1..98c94980b01 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -29,15 +29,17 @@ class CrashKernel(CrashBaseClass): def __init__(self, searchpath=None): self.findmap = {} self.searchpath = searchpath + obj = gdb.objfiles()[0] + kernel = os.path.basename(obj.filename) - sym = gdb.lookup_symbol('vsnprintf', None)[0] - if sym is None: - raise CrashKernelError("Missing vsnprintf indicates that there is no kernel image loaded.") + if not obj.has_symbols(): + raise CrashKernelError("Couldn't locate debuginfo for {}" + .format(kernel)) f = open(gdb.objfiles()[0].filename, 'rb') self.elffile = ELFFile(f) - archname = sym.symtab.objfile.architecture.name() + archname = obj.architecture.name() archclass = crash.arch.get_architecture(archname) self.arch = archclass() @@ -90,10 +92,12 @@ def load_modules(self, verbose=False, debug=False): gdb.execute("add-symbol-file {} {:#x} {}" .format(modpath, addr, sections), to_string=True) - sal = gdb.find_pc_line(addr) - if sal.symtab is None: - objfile = gdb.lookup_objfile(modpath) + + objfile = gdb.lookup_objfile(modpath) + if not objfile.has_symbols(): self.load_debuginfo(objfile, modpath) + elif debug: + print(" + has debug symbols") # We really should check the version, but GDB doesn't export # a way to lookup sections. From 58b7c35cbc62fde294e9758131a77f030fc7520c Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 15:01:33 -0400 Subject: [PATCH 29/75] crash.util: promote to be its own sub-package With the new delayed lookup code landing in crash.util, it makes sense to split it into its own sub-package. Signed-off-by: Jeff Mahoney --- crash/{util.py => util/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crash/{util.py => util/__init__.py} (100%) diff --git a/crash/util.py b/crash/util/__init__.py similarity index 100% rename from crash/util.py rename to crash/util/__init__.py From 41754292003b29b02e0da642bf127e95fd177839 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 09:46:45 -0400 Subject: [PATCH 30/75] crash: add static percpu regions for modules Static percpu variables added by modules aren't addressable right now. This commit adds the static ranges to the percpu mappings. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 5 +++++ crash/types/percpu.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/crash/kernel.py b/crash/kernel.py index 98c94980b01..ca93ed766d1 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -89,6 +89,11 @@ def load_modules(self, verbose=False, debug=False): sys.stdout.flush() sections = self.get_module_sections(module) + + percpu = int(module['percpu']) + if percpu > 0: + sections += " -s .data..percpu {:#x}".format(percpu) + gdb.execute("add-symbol-file {} {:#x} {}" .format(modpath, addr, sections), to_string=True) diff --git a/crash/types/percpu.py b/crash/types/percpu.py index 5bca27ce79a..bbc65c96dec 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -7,6 +7,7 @@ from crash.infra import CrashBaseClass, export from crash.util import array_size, struct_has_member from crash.types.list import list_for_each_entry +from crash.types.module import for_each_module from crash.exceptions import DelayedAttributeError from crash.types.bitmap import find_first_set_bit, find_last_set_bit from crash.types.bitmap import find_next_set_bit, find_next_zero_bit @@ -44,6 +45,7 @@ class TypesPerCPUClass(CrashBaseClass): dynamic_offset_cache: List[Tuple[int, int]] = list() static_ranges: Dict[int, int] = dict() + module_ranges: Dict[int, int] = dict() last_cpu = -1 nr_cpus = 0 @@ -72,6 +74,16 @@ def _setup_nr_cpus(cls, ignored: gdb.Symbol) -> None: if cls.last_cpu == -1: cls.last_cpu = cls.nr_cpus + @classmethod + def _setup_module_ranges(cls, modules: gdb.Symbol) -> None: + for module in for_each_module(): + start = int(module['percpu']) + if start == 0: + continue + + size = int(module['percpu_size']) + cls.module_ranges[start] = size + @classmethod def _add_to_offset_cache(cls, base: int, start: int, end: int) -> None: cls.dynamic_offset_cache.append((base + start, base + end)) @@ -83,6 +95,8 @@ def dump_ranges(cls) -> None: """ for (start, size) in cls.static_ranges.items(): print(f"static start={start:#x}, size={size:#x}") + for (start, size) in cls.module_ranges.items(): + print(f"module start={start:#x}, size={size:#x}") if cls.dynamic_offset_cache: for (start, end) in cls.dynamic_offset_cache: print(f"dynamic start={start:#x}, end={end:#x}") @@ -225,6 +239,24 @@ def _relocated_offset(self, var): return addr - start return addr + def is_module_percpu_var(self, addr: int) -> bool: + """ + Returns whether the provided value or symbol falls within + any of the percpu ranges for modules + + Args: + addr (int): The address to query + + Returns: + bool: whether this address belongs to a module range + """ + for start in self.module_ranges: + for cpu in range(0, self.last_cpu): + size = self.module_ranges[start] + if addr >= start and addr < start + size: + return True + return False + @export def is_percpu_var(self, var: SymbolOrValue) -> bool: """ @@ -243,6 +275,8 @@ def is_percpu_var(self, var: SymbolOrValue) -> bool: var = int(var) if self.is_static_percpu_var(var): return True + if self.is_module_percpu_var(var): + return True if self._is_percpu_var_dynamic(var): return True return False From 40352e285b6c5432fc804f0d8b50a96d172fd4ac Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 09:54:47 -0400 Subject: [PATCH 31/75] crash.kernel: handle failure to add module symbols Rather than dump a gdb.error back to the UI, catch and raise a Crash-specific exception that gives some context. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index ca93ed766d1..b32eba5f691 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -94,9 +94,15 @@ def load_modules(self, verbose=False, debug=False): if percpu > 0: sections += " -s .data..percpu {:#x}".format(percpu) - gdb.execute("add-symbol-file {} {:#x} {}" - .format(modpath, addr, sections), - to_string=True) + try: + result = gdb.execute("add-symbol-file {} {:#x} {}" + .format(modpath, addr, sections), + to_string=True) + except gdb.error as e: + raise CrashKernelError("Error while loading module `{}': {}" + .format(modname, str(e))) + if debug: + print(result) objfile = gdb.lookup_objfile(modpath) if not objfile.has_symbols(): From 9a86bc6c55accb9babc39b105556cb27fe0fab0b Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 10:12:24 -0400 Subject: [PATCH 32/75] crash.kernel: improve and document debuginfo and module loading In an ideal world, debuginfo packages would be installed under /usr/lib/debug and gdb would locate debuginfo automatically. Practically, this isn't the case. Recent SUSE kernel debuginfo packages don't create the symlinks for modules for automatic loading. Even if they did, the more common use case is to maintain a cache of debuginfo files outside of /usr/lib/debug. This commit improves the handling of the loading of modules and debuginfo. We have some sane defaults, documented in crash.kernel and add some command line options to override those. Signed-off-by: Jeff Mahoney --- crash.sh | 134 +++++++++++--- crash/kernel.py | 441 +++++++++++++++++++++++++++++++++++++++++------ crash/session.py | 26 ++- pycrash | 1 + pycrash.asciidoc | 83 +++++++-- 5 files changed, 584 insertions(+), 101 deletions(-) create mode 120000 pycrash diff --git a/crash.sh b/crash.sh index d51bd17876c..408760c7a26 100755 --- a/crash.sh +++ b/crash.sh @@ -3,38 +3,98 @@ usage() { cat <&2 -usage: $(basename $0) [-d|--search-dir ] +usage: $(basename $0) [options] -Debugging options: ---gdb Run the embedded gdb underneath a separate gdb instance. - This is useful for debugging issues in gdb that are seen - while running crash-python. ---valgrind Run the embedded gdb underneath valgrind. - This is useful for debugging memory leaks in gdb patches. ---nofiles Start up without loading any object files. - This is useful for testing delayed lookup error handling. +Options: +-r | --root + Use the specified directory as the root for all file searches. When + using properly configured .build-id symbolic links, this is the + best method to use as the debuginfo will be loaded automatically via + gdb without searching for filenames. + +-m | --modules + Use the specified directory to search for modules + +-d | --modules-debuginfo + Use the specified directory to search for module debuginfo + +-D | --vmlinux-debuginfo + Use the specified directory to search for vmlinux debuginfo + +-b | --build-dir + Use the specified directory as the root for all file searches. This + directory should be the root of a built kernel source tree. This is + shorthand for "-r -m . -d . -D ." and will override preceding + options. +Debugging options: +--debug + Enable noisy output for debugging the debugger +-v | --verbose + Enable verbose output for debugging the debugger +--gdb + Run the embedded gdb underneath a separate gdb instance. This is useful + for debugging issues in gdb that are seen while running crash-python. +--valgrind + Run the embedded gdb underneath valgrind. This is useful + for debugging memory leaks in gdb patches. END exit 1 } -TEMP=$(getopt -o 'd:h' --long 'search-dir:,gdb,valgrind,nofiles,help' -n "$(basename $0)" -- "$@") +TEMP=$(getopt -o 'vr:d:m:D:b:h' --long 'verbose,root:,modules-debuginfo:,modules:,vmlinux-debuginfo:,build-dir:,debug,gdb,valgrind,help' -n "$(basename $0)" -- "$@") if [ $? -ne 0 ]; then - echo "Terminating." >&2 - exit 1 + usage fi eval set -- "$TEMP" unset TEMP +VERBOSE=False +DEBUG=False + while true; do case "$1" in - '-d'|'--search-dir') - SEARCHDIRS="$SEARCHDIRS $2" + '-r'|'--root') + SEARCH_DIRS="$SEARCH_DIRS $2" + shift 2 + continue + ;; + '-m'|'--modules') + MODULES="$MODULES $2" + shift 2 + continue + ;; + '-d'|'--modules-debuginfo') + MODULES_DEBUGINFO="$MODULES_DEBUGINFO $2" + shift 2 + continue + ;; + '-D'|'--vmlinux-debuginfo') + VMLINUX_DEBUGINFO="$VMLINUX_DEBUGINFO $2" + shift 2 + continue + ;; + '-b'|'--build-dir') + SEARCH_DIRS="$2" + VMLINUX_DEBUGINFO="." + MODULES="." + MODULES_DEBUGINFO="." shift 2 continue + ;; + '-v'|'--verbose') + VERBOSE="True" + shift + continue + ;; + '--debug') + DEBUG="True" + shift + continue ;; + '--gdb') DEBUGMODE=gdb shift @@ -45,11 +105,6 @@ while true; do shift continue ;; - '--nofiles') - NOFILES=yes - shift - continue - ;; '-h'|'--help') usage ;; '--') @@ -63,7 +118,7 @@ while true; do esac done -if [ "$#" -ne 2 -a -z "$NOFILES" ]; then +if [ "$#" -ne 2 ]; then usage fi @@ -111,7 +166,15 @@ else fi VMCORE=$2 +for path in $SEARCH_DIRS; do + if test -n "$DFD"; then + DFD="$DFD:$path" + else + DFD="$path" + fi +done cat << EOF >> $GDBINIT +set debug-file-directory $DFD:/usr/lib/debug set build-id-verbose 0 set python print-stack full set prompt py-crash> @@ -132,13 +195,40 @@ import sys import traceback try: import crash.session + from crash.kernel import CrashKernel except RuntimeError as e: print("crash-python: {}, exiting".format(str(e)), file=sys.stderr) traceback.print_exc() sys.exit(1) -path = "$SEARCHDIRS".split(' ') + +roots = None +module_path = None +module_debuginfo_path = None +vmlinux_debuginfo = None +verbose=$VERBOSE +debug=$DEBUG + +s = "$SEARCH_DIRS" +if len(s) > 0: + roots = s.split(" ") + +s = "$VMLINUX_DEBUGINFO" +if len(s) > 0: + vmlinux_debuginfo = s.split(" ") + +s = "$MODULES" +if len(s) > 0: + module_path = s.split(" ") + +s = "$MODULES_DEBUGINFO" +if len(s) > 0: + module_debuginfo_path = s.split(" ") + try: - x = crash.session.Session(path) + kernel = CrashKernel(roots, vmlinux_debuginfo, module_path, + module_debuginfo_path, verbose, debug) + + x = crash.session.Session(kernel, verbose=verbose, debug=debug) print("The 'pyhelp' command will list the command extensions.") except gdb.error as e: print("crash-python: {}, exiting".format(str(e)), file=sys.stderr) diff --git a/crash/kernel.py b/crash/kernel.py index b32eba5f691..40066ff74ff 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -3,6 +3,8 @@ import gdb import sys +import re +import fnmatch import os.path import crash.arch import crash.arch.x86_64 @@ -13,24 +15,257 @@ from crash.types.list import list_for_each_entry from crash.types.module import for_each_module, for_each_module_section import crash.cache.tasks +import crash.cache.syscache from crash.types.task import LinuxTask from elftools.elf.elffile import ELFFile from crash.util import get_symbol_value +from typing import Pattern, Union, List, Dict, Any + class CrashKernelError(RuntimeError): pass +class NoMatchingFileError(FileNotFoundError): + pass + LINUX_KERNEL_PID = 1 +PathSpecifier = Union[List[str], str] + class CrashKernel(CrashBaseClass): + __types__ = [ 'char *' ] __symvals__ = [ 'init_task' ] __symbols__ = [ 'runqueues'] - def __init__(self, searchpath=None): - self.findmap = {} - self.searchpath = searchpath + + def __init__(self, roots: PathSpecifier=None, + vmlinux_debuginfo: PathSpecifier=None, + module_path: PathSpecifier=None, + module_debuginfo_path: PathSpecifier=None, + verbose: bool=False, debug: bool=False): + """ + Initialize a basic kernel semantic debugging session. + + This means that we load the following: + - Kernel image symbol table (and debuginfo, if not integrated) + relocated to the base offset used by kASLR + - Kernel modules that were loaded on the the crashed system (again, + with debuginfo if not integrated) + - Percpu ranges used by kernel module + - Architecture-specific details + - Linux tasks populated into the GDB thread table + + If kernel module files and debuginfo cannot be located, backtraces + may be incomplete if the addresses used by the modules are crossed. + Percpu ranges will be properly loaded regardless. + + For arguments that accept paths to specify a base directory to be + used, the entire directory structure will be read and cached to + speed up subsequent searches. Still, reading large directory trees + is a time consuming operation and being exact as possible will + improve startup time. + + Args: + root (str or list of str, None for defaults): The roots of trees + to search for debuginfo files. When specified, all roots + will be searched using the following arguments (including + the absolute paths in the defaults if unspecified). + + Defaults to: / + + vmlinux_debuginfo (str or list of str, None for defaults): The + location of the separate debuginfo file corresponding + to the kernel being debugged. + + Defaults to: + - .debug + - ./vmlinux-.debug + - /usr/lib/debug/.build-id/xx/.debug + - /usr/lib/debug/.debug + - /usr/lib/debug/boot/.debug + - /usr/lib/debug/boot/vmlinux- + + module_path (string, None for defaults): The base directory to + be used to search for kernel modules (e.g. module.ko) to be + used to load symbols for the kernel being debugged. + + Defaults to: + - ./modules + - /lib/modules/ + + module_debuginfo_path (string, None for defaults): The base + directory to search for debuginfo matching the kernel + modules already loaded. + + Defaults to: + - ./modules.debug + - /usr/lib/debug/.build-id/xx/.debug + - /usr/lib/debug/lib/modules/ + Raises: + CrashKernelError: If the kernel debuginfo cannot be loaded. + TypeError: If any of the arguments are not None, str, + or list of str + + """ + self.findmap: Dict[str, Dict[Any, Any]] = dict() + self.modules_order: Dict[str, Dict[str, str]] = dict() obj = gdb.objfiles()[0] kernel = os.path.basename(obj.filename) + debugroot = "/usr/lib/debug" + + version = self.extract_version() + + if roots is None: + self.roots = [ "/" ] + elif (isinstance(roots, list) and len(roots) > 0 and + isinstance(roots[0], str)): + x = None + for root in roots: + if os.path.exists(root): + if x is None: + x = [ root ] + else: + x.append(root) + else: + print("root {} does not exist".format(root)) + + if x is None: + x = [ "/" ] + self.roots = x + elif (isinstance(roots, str)): + x = None + if os.path.exists(roots): + if x is None: + x = [ roots ] + else: + x.append(roots) + if x is None: + x = [ "/" ] + self.roots = x + else: + raise TypeError("roots must be None, str, or list of str") + + if verbose: + print("roots={}".format(self.roots)) + + if vmlinux_debuginfo is None: + x = [] + defaults = [ + "{}.debug".format(kernel), + "vmlinux-{}.debug".format(version), + "{}/{}.debug".format(debugroot, kernel), + "{}/boot/{}.debug".format(debugroot, + os.path.basename(kernel)), + "{}/boot/vmlinux-{}.debug".format(debugroot, version), + ] + for root in self.roots: + for mpath in defaults: + path = "{}/{}".format(root, mpath) + if os.path.exists(path): + if x is None: + x = [path] + else: + x.append(path) + + self.vmlinux_debuginfo = x + + elif (isinstance(vmlinux_debuginfo, list) and + len(vmlinux_debuginfo) > 0 and + isinstance(vmlinux_debuginfo[0], str)): + self.vmlinux_debuginfo = vmlinux_debuginfo + elif isinstance(vmlinux_debuginfo, str): + self.vmlinux_debuginfo = [ vmlinux_debuginfo ] + else: + raise TypeError("vmlinux_debuginfo must be None, str, or list of str") + + if verbose: + print("vmlinux_debuginfo={}".format(self.vmlinux_debuginfo)) + + if module_path is None: + x = [] + + path = "modules" + if os.path.exists(path): + x.append(path) + + for root in self.roots: + path = "{}/lib/modules/{}".format(root, version) + if os.path.exists(path): + x.append(path) + + self.module_path = x + elif (isinstance(module_path, list) and + isinstance(module_path[0], str)): + x = [] + + for root in self.roots: + for mpath in module_path: + path = "{}/{}".format(root, mpath) + if os.path.exists(path): + x.append(path) + + self.module_path = x + elif isinstance(module_path, str): + x = [] + + if os.path.exists(module_path): + x.append(module_path) + + self.module_path = x + else: + raise TypeError("module_path must be None, str, or list of str") + + if verbose: + print("module_path={}".format(self.module_path)) + + if module_debuginfo_path is None: + x = [] + + path = "modules.debug" + if os.path.exists(path): + x.append(path) + + for root in self.roots: + path = "{}/{}/lib/modules/{}".format(root, debugroot, version) + if os.path.exists(path): + x.append(path) + self.module_debuginfo_path = x + elif (isinstance(module_debuginfo_path, list) and + isinstance(module_debuginfo_path[0], str)): + x = [] + + for root in self.roots: + for mpath in module_debuginfo_path: + path = "{}/{}".format(root, mpath) + if os.path.exists(path): + x.append(path) + + self.module_debuginfo_path = x + elif isinstance(module_debuginfo_path, str): + x = [] + + for root in self.roots: + path = "{}/{}".format(root, module_debuginfo_path) + if os.path.exists(path): + x.append(path) + + self.module_debuginfo_path = x + else: + raise TypeError("module_debuginfo_path must be None, str, or list of str") + + if verbose: + print("module_debuginfo_path={}".format(self.module_debuginfo_path)) + + # We need separate debuginfo. Let's go find it. + if not obj.has_symbols(): + print("Loading debug symbols for vmlinux") + for path in [self.build_id_path(obj)] + self.vmlinux_debuginfo: + try: + obj.add_separate_debug_file(path) + if obj.has_symbols(): + break + except gdb.error as e: + pass if not obj.has_symbols(): raise CrashKernelError("Couldn't locate debuginfo for {}" @@ -49,28 +284,50 @@ def __init__(self, searchpath=None): self.target.fetch_registers = self.fetch_registers self.crashing_thread = None - def fetch_registers(self, register): + # When working without a symbol table, we still need to be able + # to resolve version information. + def get_minsymbol_as_string(self, name: str) -> str: + sym = gdb.lookup_minimal_symbol(name).value() + + return sym.address.cast(self.char_p_type).string() + + def extract_version(self) -> str: + try: + uts = get_symbol_value('init_uts_ns') + return uts['name']['release'].string() + except (AttributeError, NameError): + pass + + banner = self.get_minsymbol_as_string('linux_banner') + + return banner.split(' ')[2] + + def fetch_registers(self, register: gdb.Register) -> None: thread = gdb.selected_thread() - return self.arch.fetch_register(thread, register.regnum) + self.arch.fetch_register(thread, register.regnum) - def get_module_sections(self, module): + def get_module_sections(self, module: gdb.Value) -> str: out = [] for (name, addr) in for_each_module_section(module): out.append("-s {} {:#x}".format(name, addr)) return " ".join(out) - def load_modules(self, verbose=False, debug=False): - print("Loading modules...", end='') - sys.stdout.flush() + def load_modules(self, verbose: bool=False, debug: bool=False) -> None: + version = crash.cache.syscache.utsname.release + print("Loading modules for {}".format(version), end='') + if verbose: + print(":", flush=True) failed = 0 loaded = 0 for module in for_each_module(): modname = "{}".format(module['name'].string()) modfname = "{}.ko".format(modname) found = False - for path in self.searchpath: - modpath = self.find_module_file(modfname, path) - if not modpath: + for path in self.module_path: + + try: + modpath = self.find_module_file(modfname, path) + except NoMatchingFileError: continue found = True @@ -106,7 +363,7 @@ def load_modules(self, verbose=False, debug=False): objfile = gdb.lookup_objfile(modpath) if not objfile.has_symbols(): - self.load_debuginfo(objfile, modpath) + self.load_module_debuginfo(objfile, modpath, verbose) elif debug: print(" + has debug symbols") @@ -120,6 +377,8 @@ def load_modules(self, verbose=False, debug=False): print("Couldn't find module file for {}".format(modname)) failed += 1 else: + if not objfile.has_symbols(): + print("Couldn't find debuginfo for {}".format(modname)) loaded += 1 if (loaded + failed) % 10 == 10: print(".", end='') @@ -134,43 +393,131 @@ def load_modules(self, verbose=False, debug=False): del self.findmap self.findmap = {} - def find_module_file(self, name, path): - if not path in self.findmap: - self.findmap[path] = {} + @staticmethod + def normalize_modname(mod: str) -> str: + return mod.replace('-', '_') + + def cache_modules_order(self, path: str) -> None: + self.modules_order[path] = dict() + order = os.path.join(path, "modules.order") + try: + f = open(order) + for line in f.readlines(): + modpath = line.rstrip() + modname = self.normalize_modname(os.path.basename(modpath)) + if modname[:7] == "kernel/": + modname = modname[7:] + modpath = os.path.join(path, modpath) + if os.path.exists(modpath): + self.modules_order[path][modname] = modpath + f.close() + except OSError: + pass + + def get_module_path_from_modules_order(self, path: str, name: str) -> str: + if not path in self.modules_order: + self.cache_modules_order(path) - for root, dirs, files in os.walk(path): - for filename in files: - nname = filename.replace('-', '_') - self.findmap[path][nname] = os.path.join(root, filename) try: - nname = name.replace('-', '_') - return self.findmap[path][nname] + return self.modules_order[path][name] except KeyError: - return None - - def load_debuginfo(self, objfile, name=None, verbose=False): - if name is None: - name = objfile.filename - if ".gz" in name: - name = name.replace(".gz", "") - filename = "{}.debug".format(os.path.basename(name)) - filepath = None - - # Check current directory first - if os.path.exists(filename): - filepath = filename - else: - for path in self.searchpath: - filepath = self.find_module_file(filename, path) - if filepath: - break + raise NoMatchingFileError(name) - if filepath: - objfile.add_separate_debug_file(filepath) + def cache_file_tree(self, path, regex: Pattern[str]=None) -> None: + if not path in self.findmap: + self.findmap[path] = { + 'filters' : [], + 'files' : {}, + } + + # If we've walked this path with no filters, we have everything + # already. + if self.findmap[path]['filters'] is None: + return + + if regex is None: + self.findmap[path]['filters'] = None else: - print("Could not locate debuginfo for {}".format(name)) + pattern = regex.pattern + if pattern in self.findmap[path]['filters']: + return + self.findmap[path]['filters'].append(pattern) + + for root, dirs, files in os.walk(path): + for filename in files: + modname = self.normalize_modname(filename) + + if regex and regex.match(modname) is None: + continue - def setup_tasks(self): + modpath = os.path.join(root, filename) + self.findmap[path]['files'][modname] = modpath + + def get_file_path_from_tree_search(self, path: str, name: str, + regex: Pattern[str]=None) -> str: + self.cache_file_tree(path, regex) + + try: + modname = self.normalize_modname(name) + return self.findmap[path]['files'][modname] + except KeyError: + raise NoMatchingFileError(name) + + def find_module_file(self, name: str, path: str) -> str: + try: + return self.get_module_path_from_modules_order(path, name) + except NoMatchingFileError: + pass + + regex = re.compile(fnmatch.translate("*.ko")) + return self.get_file_path_from_tree_search(path, name, regex) + + def find_module_debuginfo_file(self, name: str, path: str) -> str: + regex = re.compile(fnmatch.translate("*.ko.debug")) + return self.get_file_path_from_tree_search(path, name, regex) + + @staticmethod + def build_id_path(objfile: gdb.Objfile) -> str: + build_id = objfile.build_id + return ".build_id/{}/{}.debug".format(build_id[0:2], build_id[2:]) + + def try_load_debuginfo(self, objfile: gdb.Objfile, + path: str, verbose: bool=False) -> bool: + try: + if verbose: + print(" + Loading debuginfo: {}".format(path)) + objfile.add_separate_debug_file(path) + if objfile.has_symbols(): + return True + except gdb.error as e: + print(e) + + return False + + def load_module_debuginfo(self, objfile: gdb.Objfile, + modpath: str=None, verbose: bool=False) -> None: + if modpath is None: + modpath = objfile.filename + if ".gz" in modpath: + modpath = modpath.replace(".gz", "") + filename = "{}.debug".format(os.path.basename(modpath)) + + build_id_path = self.build_id_path(objfile) + + for path in self.module_debuginfo_path: + filepath = "{}/{}".format(path, build_id_path) + if self.try_load_debuginfo(objfile, filepath, verbose): + break + + try: + filepath = self.find_module_debuginfo_file(filename, path) + except NoMatchingFileError: + continue + + if self.try_load_debuginfo(objfile, filepath, verbose): + break + + def setup_tasks(self) -> None: gdb.execute('set print thread-events 0') task_list = self.init_task['tasks'] @@ -178,8 +525,6 @@ def setup_tasks(self): rqs = get_percpu_vars(self.runqueues) rqscurrs = {int(x["curr"]) : k for (k, x) in rqs.items()} - self.pid_to_task_struct = {} - print("Loading tasks...", end='') sys.stdout.flush() @@ -196,7 +541,7 @@ def setup_tasks(self): try: crashing_cpu = int(get_symbol_value('crashing_cpu')) except Exception as e: - crashing_cpu = None + crashing_cpu = -1 for task in tasks: cpu = None @@ -215,7 +560,7 @@ def setup_tasks(self): print("Failed to setup task @{:#x}".format(int(task.address))) continue thread.name = task['comm'].string() - if active and crashing_cpu is not None and cpu == crashing_cpu: + if active and cpu == crashing_cpu: self.crashing_thread = thread self.arch.setup_thread_info(thread) diff --git a/crash/session.py b/crash/session.py index cbd2ab560dc..91298b13d35 100644 --- a/crash/session.py +++ b/crash/session.py @@ -5,32 +5,26 @@ import sys from crash.infra import autoload_submodules -import crash.kernel -from crash.kernel import CrashKernelError -from kdumpfile import kdumpfile +from crash.kernel import CrashKernel, CrashKernelError class Session(object): """ crash.Session is the main driver component for crash-python - The Session class loads the kernel, kernel modules, debuginfo, - and vmcore and auto loads any sub modules for autoinitializing - commands and subsystems. + The Session class loads the kernel modules, sets up tasks, and auto loads + any sub modules for autoinitializing commands and subsystems. Args: - searchpath (list of str, optional): Paths to directory trees to - search for kernel modules and debuginfo + kernel (CrashKernel): The kernel to debug during this session + verbose (bool, optional, default=False): Whether to enable verbose + output debug (bool, optional, default=False): Whether to enable verbose debugging output """ - - - def __init__(self, searchpath=None, debug=False): + def __init__(self, kernel: CrashKernel, verbose: bool=False, + debug: bool=False) -> None: print("crash-python initializing...") - if searchpath is None: - searchpath = [] - - self.kernel = crash.kernel.CrashKernel(searchpath) + self.kernel = kernel autoload_submodules('crash.cache') autoload_submodules('crash.subsystem') @@ -38,7 +32,7 @@ def __init__(self, searchpath=None, debug=False): try: self.kernel.setup_tasks() - self.kernel.load_modules(searchpath) + self.kernel.load_modules(verbose=verbose, debug=debug) except CrashKernelError as e: print(str(e)) print("Further debugging may not be possible.") diff --git a/pycrash b/pycrash new file mode 120000 index 00000000000..0c3d27cfed4 --- /dev/null +++ b/pycrash @@ -0,0 +1 @@ +crash.sh \ No newline at end of file diff --git a/pycrash.asciidoc b/pycrash.asciidoc index ed216071e55..c4d1bced9f5 100644 --- a/pycrash.asciidoc +++ b/pycrash.asciidoc @@ -7,34 +7,87 @@ pycrash - a Linux kernel crash dump debugger written in Python SYNOPSIS -------- -*pycrash* [options] +*pycrash* [options] DESCRIPTION ----------- The *pycrash* utility is a Linux kernel crash debugger written in Python. It improves upon the original crash tool by adding support for symbolic -backtraces and in that it is easily extensible by the user. +backtraces and in that it is easily extensible by the user using a rich +python interface that offers semantic helpers for various subsystems. In order to operate properly, full debuginfo is required for the kernel -image and all modules in use. +image and all modules in use. Without options specifying other paths, +the following defaults are used for locating the debuginfo and modules: + +Kernel debuginfo: + +* .debug +* ./vmlinux-.debug +* /usr/lib/debug/.build-id//.debug +* /usr/lib/debug/.debug +* /usr/lib/debug/boot/vmlinux-.debug +* /usr/lib/debug/boot/vmlinux- + +Module path: + +* ./modules +* /lib/modules/ + +Module debuginfo path: + +* ./modules.debug +* /usr/lib/debug/.build-id/xx/.debug +* /usr/lib/debug/lib/modules/ + +The build-id and kernel-version fields are detected within the kernel +and modules and cannot be overridden. + OPTIONS ------- -*-d|--search-dir *:: -Specify a directory to search recursively for modules or debuginfo for -the kernel or modules. -+ -This option may be specified multiple times. + +Each of the following options may be specified multiple times. + +*-r | --root *:: + Use the specified directory as the root for all file searches. When + using properly configured .build-id symbolic links, this is the + best method to use as the debuginfo will be loaded automatically via + gdb without searching for filenames. If this is the only option + specified, the defaults documented above will be used relative to + each root. + +*-m | --modules *:: + Use the specified directory to search for modules + +*-d | --modules-debuginfo *:: + Use the specified directory to search for module debuginfo + +*-D | --vmlinux-debuginfo *:: + Use the specified directory to search for vmlinux debuginfo + +*-b | --build-dir *:: + Use the specified directory as the root for all file searches. This + directory should be the root of a built kernel source tree. This is + shorthand for *-r -m . -d . -D .* and will override preceding + options. + +DEBUGGING OPTIONS: +------------------ + +*-v | --verbose*:: + Enable verbose output for debugging the debugger + +*--debug*:: + Enable even noisier output for debugging the debugger *--gdb*:: -Start the gdb instance used with crash-python within gdb. -+ -This option is primarily intended for debugging gdb issues. + Run the embedded gdb underneath a separate gdb instance. This is useful + for debugging issues in gdb that are seen while running crash-python. *--valgrind*:: -Start the gdb instance used with crash-python within valgrind. -+ -This option is primarily intended for debugging gdb issues. + Run the embedded gdb underneath valgrind. This is useful + for debugging memory leaks in gdb patches. EXIT STATUS ----------- @@ -49,4 +102,4 @@ Please refer to the GitHub repository at https://github.com/jeffmahoney/crash-py SEE ALSO -------- `gdb`(1) -`libdkumpfil` +`libdkumpfile` From 95b3d0d56dc3e55783d20228500e9bbce0684919 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Thu, 25 Apr 2019 20:52:45 -0400 Subject: [PATCH 33/75] crash.kernel: check module version on load Since the beginning of this project, we've assumed that the modules we load match the kernel being debugged. We don't do anything to verify that assumption. This commit uses the version and vermagic to validate module contents, if available. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index 40066ff74ff..d43bf3cae08 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -28,6 +28,29 @@ class CrashKernelError(RuntimeError): class NoMatchingFileError(FileNotFoundError): pass +class ModinfoMismatchError(ValueError): + def __init__(self, attribute, path, value, expected_value): + self.path = path + self.value = value + self.expected_value = expected_value + self.attribute = attribute + + def __str__(self): + return "module {} has mismatched {} (got `{}' expected `{}')".format( + self.path, self.attribute, self.value, self.expected_value) + +class ModVersionMismatchError(ModinfoMismatchError): + def __init__(self, path, module_value, expected_value): + super(ModVersionMismatchError, self).__init__('vermagic', + path, module_value, + expected_value) + +class ModSourceVersionMismatchError(ModinfoMismatchError): + def __init__(self, path, module_value, expected_value): + super(ModSourceVersionMismatchError, self).__init__('srcversion', + path, module_value, + expected_value) + LINUX_KERNEL_PID = 1 PathSpecifier = Union[List[str], str] @@ -271,8 +294,7 @@ def __init__(self, roots: PathSpecifier=None, raise CrashKernelError("Couldn't locate debuginfo for {}" .format(kernel)) - f = open(gdb.objfiles()[0].filename, 'rb') - self.elffile = ELFFile(f) + self.vermagic = self.extract_vermagic() archname = obj.architecture.name() archclass = crash.arch.get_architecture(archname) @@ -302,6 +324,38 @@ def extract_version(self) -> str: return banner.split(' ')[2] + def extract_vermagic(self) -> str: + try: + magic = get_symbol_value('vermagic') + return magic.string() + except (AttributeError, NameError): + pass + + return self.get_minsymbol_as_string('vermagic') + + def extract_modinfo_from_module(self, modpath: str) -> Dict[str, str]: + f = open(modpath, 'rb') + + d = None + try: + elf = ELFFile(f) + modinfo = elf.get_section_by_name('.modinfo') + + d = {} + for line in modinfo.data().split(b'\x00'): + val = line.decode('utf-8') + if val: + eq = val.index('=') + d[val[0:eq]] = val[eq + 1:] + except Exception as e: + print(e) + del d + d = dict() + + del elf + f.close() + return d + def fetch_registers(self, register: gdb.Register) -> None: thread = gdb.selected_thread() self.arch.fetch_register(thread, register.regnum) @@ -312,6 +366,28 @@ def get_module_sections(self, module: gdb.Value) -> str: out.append("-s {} {:#x}".format(name, addr)) return " ".join(out) + def check_module_version(self, modpath: str, module: gdb.Value) -> None: + modinfo = self.extract_modinfo_from_module(modpath) + + vermagic = None + if 'vermagic' in modinfo: + vermagic = modinfo['vermagic'] + + if vermagic != self.vermagic: + raise ModVersionMismatchError(modpath, vermagic, self.vermagic) + + mi_srcversion = None + if 'srcversion' in modinfo: + mi_srcversion = modinfo['srcversion'] + + mod_srcversion = None + if 'srcversion' in module.type: + mod_srcversion = module['srcversion'].string() + + if mi_srcversion != mod_srcversion: + raise ModSourceVersionMismatchError(modpath, mi_srcversion, + mod_srcversion) + def load_modules(self, verbose: bool=False, debug: bool=False) -> None: version = crash.cache.syscache.utsname.release print("Loading modules for {}".format(version), end='') @@ -330,6 +406,13 @@ def load_modules(self, verbose: bool=False, debug: bool=False) -> None: except NoMatchingFileError: continue + try: + self.check_module_version(modpath, module) + except ModinfoMismatchError as e: + if verbose: + print(str(e)) + continue + found = True if 'module_core' in module.type: @@ -367,8 +450,6 @@ def load_modules(self, verbose: bool=False, debug: bool=False) -> None: elif debug: print(" + has debug symbols") - # We really should check the version, but GDB doesn't export - # a way to lookup sections. break if not found: From 917c3a9e562eda46bd3800878f0a3b2d251a1e18 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Thu, 2 May 2019 14:12:47 -0400 Subject: [PATCH 34/75] crash.commands.task: allow 'task' command with no argument to show current task Since we use the 'task' command to select tasks instead of the gdb 'thread' command directly, the 'task' command should report the current task when no argument is given. Signed-off-by: Jeff Mahoney --- crash/commands/task.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crash/commands/task.py b/crash/commands/task.py index f50553dc7ad..6aac802ca47 100644 --- a/crash/commands/task.py +++ b/crash/commands/task.py @@ -25,14 +25,18 @@ def __init__(self, name): parser = ArgumentParser(prog=name) - parser.add_argument('pid', type=int, nargs=1) + parser.add_argument('pid', type=int, nargs=argparse.REMAINDER) parser.format_usage = lambda: "thread \n" Command.__init__(self, name, parser) def execute(self, args): try: - thread = crash.cache.tasks.get_task(args.pid[0]).thread + if args.pid: + thread = crash.cache.tasks.get_task(args.pid[0]).thread + else: + thread = gdb.selected_thread() + gdb.execute("thread {}".format(thread.num)) except KeyError: print("No such task with pid {}".format(args.pid[0])) From a18e2b7fdae2b01045d0618e3f331d6f372e38e0 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 17 May 2019 16:18:12 -0400 Subject: [PATCH 35/75] crash.infra.callbacks: explicitly connect to callbacks Having the callback objects connect to the gdb callback infrastructure made initialization complicated in some cases. By making it explicit, we can ensure the callback object is fully initialized before it receives its first callback. Signed-off-by: Jeff Mahoney --- crash/infra/callback.py | 25 ++++++++++++++++++------- crash/infra/lookup.py | 15 ++++++++++++--- tests/test_objfile_callbacks.py | 2 ++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/crash/infra/callback.py b/crash/infra/callback.py index 8307d16e630..08a0accaf8b 100644 --- a/crash/infra/callback.py +++ b/crash/infra/callback.py @@ -24,27 +24,38 @@ class ObjfileEventCallback(object): methods. """ def __init__(self): - self.completed = True - completed = False + self.completed = False + self.connected = False self.setup_symbol_cache_flush_callback() + def connect_callback(self): + if self.completed: + raise CallbackCompleted(self) + + if self.connected: + return + + self.connected = True + # We don't want to do lookups immediately if we don't have # an objfile. It'll fail for any custom types but it can # also return builtin types that are eventually changed. if len(gdb.objfiles()) > 0: result = self.check_ready() if not (result is None or result is False): - completed = self.callback(result) + self.completed = self.callback(result) - if completed is False: - self.completed = False + if self.completed is False: gdb.events.new_objfile.connect(self._new_objfile_callback) + return self.completed + def complete(self): if not self.completed: gdb.events.new_objfile.disconnect(self._new_objfile_callback) self.completed = True + self.connected = False else: raise CallbackCompleted(self) @@ -81,7 +92,7 @@ def check_ready(self): A return value other than None or False will be passed to the callback. """ - return True + raise NotImplementedError("check_ready must be implemented by derived class.") def callback(self, result): """ @@ -93,4 +104,4 @@ def callback(self, result): Args: result: The result to pass to the callback """ - pass + raise NotImplementedError("callback must be implemented by derived class.") diff --git a/crash/infra/lookup.py b/crash/infra/lookup.py index c350586e17b..56bb0f79393 100644 --- a/crash/infra/lookup.py +++ b/crash/infra/lookup.py @@ -20,10 +20,13 @@ def __init__(self, name, callback, symbol_file=None): symbol is discovered symbol_file (str, optional, default=None): Name of symbol file """ + super().__init__() + self.name = name self.symbol_file = symbol_file self.callback = callback - super().__init__() + + self.connect_callback() def check_ready(self): return gdb.lookup_minimal_symbol(self.name, self.symbol_file, None) @@ -46,10 +49,13 @@ def __init__(self, name, callback, domain=gdb.SYMBOL_VAR_DOMAIN): domain (gdb.Symbol constant, i.e. SYMBOL_*_DOMAIN): The domain to search for the symbol """ + super().__init__() + self.name = name self.domain = domain self.callback = callback - super().__init__() + + self.connect_callback() def check_ready(self): return gdb.lookup_symbol(self.name, None, self.domain)[0] @@ -78,10 +84,13 @@ class TypeCallback(ObjfileEventCallback): objfile and returns the gdb.Type associated with it. """ def __init__(self, name, callback, block=None): + super().__init__() + self.name = name self.block = block self.callback = callback - super().__init__() + + self.connect_callback() def check_ready(self): try: diff --git a/tests/test_objfile_callbacks.py b/tests/test_objfile_callbacks.py index 74aba9105fc..ae1906e3dc8 100644 --- a/tests/test_objfile_callbacks.py +++ b/tests/test_objfile_callbacks.py @@ -24,6 +24,8 @@ def __init__(self): self.checked = False super(test_class, self).__init__() + self.connect_callback() + def check_ready(self): self.checked = True return safe_get_symbol_value('main') From e0fa23ee9062f3c7900b7538b1aa463904485802 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 14:09:03 -0400 Subject: [PATCH 36/75] crash.infra.lookup: move type name resolution into TypeCallback In preparation to eliminate CrashBaseClass, we need to pull type name resolution out of the DelayedLookups class that will be eventually removed. This commit moves it into TypeCallback where it can also be used by DelayedType. Signed-off-by: Jeff Mahoney --- crash/exceptions.py | 7 ++-- crash/infra/lookup.py | 67 +++++++++++++++++++------------------- crash/types/cpu.py | 5 ++- tests/test_infra_lookup.py | 14 ++++---- 4 files changed, 46 insertions(+), 47 deletions(-) diff --git a/crash/exceptions.py b/crash/exceptions.py index 593bb63f8bb..e7ddd27a359 100644 --- a/crash/exceptions.py +++ b/crash/exceptions.py @@ -18,6 +18,7 @@ class DelayedAttributeError(AttributeError): The attribute has been declared but the symbol to fill it has not yet been located. """ - def __init__(self, owner, name): - msg = "{} has delayed attribute {} but it has not been completed." - super().__init__(msg.format(owner, name)) + def __init__(self, name): + msg = "Delayed attribute {} has not been completed." + self.name = name + super().__init__(msg.format(name)) diff --git a/crash/infra/lookup.py b/crash/infra/lookup.py index 56bb0f79393..f155f818c3d 100644 --- a/crash/infra/lookup.py +++ b/crash/infra/lookup.py @@ -86,12 +86,35 @@ class TypeCallback(ObjfileEventCallback): def __init__(self, name, callback, block=None): super().__init__() - self.name = name + (self.name, self.attrname, self.pointer) = self.resolve_type(name) + self.block = block self.callback = callback self.connect_callback() + @staticmethod + def resolve_type(name): + pointer = False + name = name.strip() + if name[-1] == '*': + pointer = True + name = name[:-1].strip() + + attrname = name + if name.startswith('struct '): + attrname = name[7:].strip() + + if pointer: + attrname += '_p_type' + else: + attrname += '_type' + + name = name + attrname = attrname.replace(' ', '_') + + return (name, attrname, pointer) + def check_ready(self): try: return gdb.lookup_type(self.name, self.block) @@ -111,9 +134,9 @@ def __init__(self, name): self.name = name self.value = None - def get(self, owner): + def get(self): if self.value is None: - raise DelayedAttributeError(owner, self.name) + raise DelayedAttributeError(self.name) return self.value def callback(self, value): @@ -153,15 +176,13 @@ class DelayedType(DelayedValue): """ A DelayedValue for types. """ - def __init__(self, name, pointer=False): + def __init__(self, name): """ Args: - name (str): The name of the type. Must not be a pointer type. - pointer (bool, optional, default=False): Whether the requested - type should be returned as a pointer to that type. + name (str): The name of the type. """ + (name, attrname, self.pointer) = TypeCallback.resolve_type(name) super().__init__(name) - self.pointer = pointer self.cb = TypeCallback(name, self.callback) def __str__(self): @@ -201,7 +222,7 @@ def __init__(self, get): self.get = get def __get__(self, instance, owner): - return self.get(owner) + return self.get() class DelayedLookups(object): """ @@ -210,26 +231,6 @@ class DelayedLookups(object): special names. These are documented in the _CrashBaseMeta documentation. """ - @classmethod - def _resolve_type(cls, name): - pointer = False - name = name.strip() - if name[-1] == '*': - pointer = True - name = name[:-1].strip() - - attrname = name - if name.startswith('struct '): - attrname = name[7:].strip() - - if pointer: - attrname += '_p_type' - else: - attrname += '_type' - - attrname = attrname.replace(' ', '_') - return (name, attrname, pointer) - @classmethod def name_check(cls, dct, name, attrname): try: @@ -260,9 +261,8 @@ def setup_delayed_lookups_for_class(cls, clsname, dct): if not isinstance(dct['__types__'], list): raise TypeError('__types__ attribute must be a list of strings') for typ in dct['__types__']: - (lookupname, attrname, pointer) = cls._resolve_type(typ) - cls.add_lookup(clsname, dct, lookupname, - DelayedType(lookupname, pointer), attrname) + t = DelayedType(typ) + cls.add_lookup(clsname, dct, t.name, t, t.attrname) del dct['__types__'] if '__symbols__' in dct: if not isinstance(dct['__symbols__'], list): @@ -304,9 +304,8 @@ def setup_named_callbacks(this_cls, cls, dct): callbacks = [] if '__type_callbacks__' in dct: for (typ, callback) in dct['__type_callbacks__']: - (lookupname, attrname, pointer) = this_cls._resolve_type(typ) cb = getattr(cls, callback) - callbacks.append(TypeCallback(lookupname, cb)) + callbacks.append(TypeCallback(typ, cb)) del dct['__type_callbacks__'] if '__symbol_callbacks__' in dct: diff --git a/crash/types/cpu.py b/crash/types/cpu.py index 29fc3d666bd..1a83dee38e2 100644 --- a/crash/types/cpu.py +++ b/crash/types/cpu.py @@ -45,7 +45,7 @@ def highest_online_cpu_nr(self) -> None: int: The highest online CPU number """ if not TypesCPUClass.cpus_online : - raise DelayedAttributeError(self.__class__.__name__, 'cpus_online') + raise DelayedAttributeError('cpus_online') return self.cpus_online[-1] @classmethod @@ -74,6 +74,5 @@ def highest_possible_cpu_nr(self) -> int: int: The highest possible CPU number """ if not self.cpus_possible: - raise DelayedAttributeError(self.__class__.__name__, - 'cpus_possible') + raise DelayedAttributeError('cpus_possible') return self.cpus_possible[-1] diff --git a/tests/test_infra_lookup.py b/tests/test_infra_lookup.py index 9c3f50e86d4..a1acb52ecb5 100644 --- a/tests/test_infra_lookup.py +++ b/tests/test_infra_lookup.py @@ -16,7 +16,7 @@ class TestDelayedLookupSetup(unittest.TestCase): def test_resolve_struct_normal(self): spec = 'struct test' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'struct test') self.assertTrue(attrname == 'test_type') self.assertFalse(pointer) @@ -24,7 +24,7 @@ def test_resolve_struct_normal(self): def test_resolve_struct_normal_pointer(self): spec = 'struct test *' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'struct test') self.assertTrue(attrname == 'test_p_type') self.assertTrue(pointer) @@ -32,7 +32,7 @@ def test_resolve_struct_normal_pointer(self): def test_resolve_struct_leading_whitespace(self): spec = ' struct test' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'struct test') self.assertTrue(attrname == 'test_type') self.assertFalse(pointer) @@ -40,7 +40,7 @@ def test_resolve_struct_leading_whitespace(self): def test_resolve_struct_trailing_whitespace(self): spec = 'struct test ' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'struct test') self.assertTrue(attrname == 'test_type') self.assertFalse(pointer) @@ -48,7 +48,7 @@ def test_resolve_struct_trailing_whitespace(self): def test_resolve_struct_middle_whitespace(self): spec = 'struct test' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'struct test') self.assertTrue(attrname == 'test_type') self.assertFalse(pointer) @@ -56,7 +56,7 @@ def test_resolve_struct_middle_whitespace(self): def test_resolve_char(self): spec = 'char' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'char') self.assertTrue(attrname == 'char_type') self.assertFalse(pointer) @@ -64,7 +64,7 @@ def test_resolve_char(self): def test_resolve_char_pointer(self): spec = 'char *' - (name, attrname, pointer) = DelayedLookups._resolve_type(spec) + (name, attrname, pointer) = TypeCallback.resolve_type(spec) self.assertTrue(name == 'char') self.assertTrue(attrname == 'char_p_type') self.assertTrue(pointer) From 5dee123e8735f9b95c7d6a8b40a53ea1d8a8012b Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 15:03:37 -0400 Subject: [PATCH 37/75] crash.util.symbols: introduce simplified delayed lookups The CrashBaseClass method of resolving types and symbols automatically is overcomplicated. We have classes in modules that have no use for them other than to access the type and symbol resolution. We then have to export helpers to the module namespace for convenience. It's not obvious to most people why any of this stuff is the way it is, which makes contributing a challenge. This commit introduces a new crash.utils.symbol module that offers a simplified way to do it. We still have the automatic importing into a namespace, but the namespaces are not embedded in a class. Instead, we have a DelayedCollection object that is extended for particular uses. For example, we can convert for_each_super_block to the following: from crash.util.symbols import Types, Symvals types = Types(['struct super_block', 'struct module *']) symvals = Symvals(['super_blocks']) def for_each_super_block(): for sb in for_each_list_entry(symvals.super_blocks, types.super_block_type, s_list) yield sb Unfortunately, moving the attribute creation outside of the metaclass means that it's subject to Python name mangling. This means that symbols prefixed with two or more underscores cannot be accessed via the attribute "dot" interface. The DelayedCollection class supports attribute access via the dict "[name]" interface as well as via a get(name) accessor. Signed-off-by: Jeff Mahoney --- crash/infra/lookup.py | 40 ++++-- crash/util/symbols.py | 108 +++++++++++++++++ tests/test_util_symbols.py | 241 +++++++++++++++++++++++++++++++++++++ 3 files changed, 377 insertions(+), 12 deletions(-) create mode 100644 crash/util/symbols.py create mode 100644 tests/test_util_symbols.py diff --git a/crash/infra/lookup.py b/crash/infra/lookup.py index f155f818c3d..4b086f12548 100644 --- a/crash/infra/lookup.py +++ b/crash/infra/lookup.py @@ -7,7 +7,20 @@ from crash.infra.callback import ObjfileEventCallback from crash.exceptions import DelayedAttributeError -class MinimalSymbolCallback(ObjfileEventCallback): +class NamedCallback(ObjfileEventCallback): + """ + A base class for Callbacks with names + """ + def __init__(self, name, attrname=None): + super().__init__() + + self.name = name + self.attrname = self.name + + if attrname is not None: + self.attrname = attrname + +class MinimalSymbolCallback(NamedCallback): """ A callback that executes when the named minimal symbol is discovered in the objfile and returns the gdb.MinimalSymbol. @@ -20,9 +33,8 @@ def __init__(self, name, callback, symbol_file=None): symbol is discovered symbol_file (str, optional, default=None): Name of symbol file """ - super().__init__() + super().__init__(name) - self.name = name self.symbol_file = symbol_file self.callback = callback @@ -35,7 +47,7 @@ def __str__(self): .format(self.__class__.__name__, self.name, self.symbol_file, self.callback)) -class SymbolCallback(ObjfileEventCallback): +class SymbolCallback(NamedCallback): """ A callback that executes when the named symbol is discovered in the objfile and returns the gdb.Symbol. @@ -49,9 +61,8 @@ def __init__(self, name, callback, domain=gdb.SYMBOL_VAR_DOMAIN): domain (gdb.Symbol constant, i.e. SYMBOL_*_DOMAIN): The domain to search for the symbol """ - super().__init__() + super().__init__(name) - self.name = name self.domain = domain self.callback = callback @@ -78,15 +89,15 @@ def check_ready(self): pass return None -class TypeCallback(ObjfileEventCallback): +class TypeCallback(NamedCallback): """ A callback that executes when the named type is discovered in the objfile and returns the gdb.Type associated with it. """ def __init__(self, name, callback, block=None): - super().__init__() + (name, attrname, self.pointer) = self.resolve_type(name) - (self.name, self.attrname, self.pointer) = self.resolve_type(name) + super().__init__(name, attrname) self.block = block self.callback = callback @@ -130,8 +141,11 @@ class DelayedValue(object): A generic class for making class attributes available that describe to-be-loaded symbols, minimal symbols, and types. """ - def __init__(self, name): + def __init__(self, name, attrname=None): self.name = name + self.attrname = attrname + if self.attrname is None: + self.attrname = name self.value = None def get(self): @@ -155,6 +169,7 @@ def __init__(self, name): """ super().__init__(name) self.cb = MinimalSymbolCallback(name, self.callback) + def __str__(self): return "{} attached with {}".format(self.__class__, str(self.cb)) @@ -169,6 +184,7 @@ def __init__(self, name): """ super().__init__(name) self.cb = SymbolCallback(name, self.callback) + def __str__(self): return "{} attached with {}".format(self.__class__, str(self.cb)) @@ -182,11 +198,11 @@ def __init__(self, name): name (str): The name of the type. """ (name, attrname, self.pointer) = TypeCallback.resolve_type(name) - super().__init__(name) + super().__init__(name, attrname) self.cb = TypeCallback(name, self.callback) def __str__(self): - return "{} attached with {}".format(self.__class__, str(self.cb)) + return "{} attached with {}".format(self.__class__, str(self.callback)) def callback(self, value): if self.pointer: diff --git a/crash/util/symbols.py b/crash/util/symbols.py new file mode 100644 index 00000000000..c7f2a767a57 --- /dev/null +++ b/crash/util/symbols.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +from typing import Type, List, Tuple, Callable, Union, Dict + +import gdb + +from crash.infra.lookup import DelayedType, DelayedSymbol, DelayedSymval +from crash.infra.lookup import DelayedValue, DelayedMinimalSymbol +from crash.infra.lookup import DelayedMinimalSymval +from crash.infra.lookup import NamedCallback, TypeCallback +from crash.infra.lookup import SymbolCallback, MinimalSymbolCallback +from crash.exceptions import DelayedAttributeError + +CollectedValue = Union[gdb.Type, gdb.Value, gdb.Symbol, gdb.MinSymbol] + +class DelayedCollection(object): + def __init__(self, cls: Type[DelayedValue], names: Union[List[str,], str]): + self.attrs: Dict[str, DelayedValue] = {} + + if isinstance(names, str): + names = [ names ] + + for name in names: + t = cls(name) + self.attrs[t.attrname] = t + + def get(self, name): + if name not in self.attrs: + raise NameError(f"'{self.__class__}' object has no '{name}'") + + if self.attrs[name].value is not None: + setattr(self, name, self.attrs[name].value) + return self.attrs[name].value + + raise DelayedAttributeError(name) + + def override(self, name: str, value: CollectedValue): + if not name in self.attrs: + raise RuntimeError(f"{name} is not part of this collection") + + self.attrs[name].value = value + + def __getitem__(self, name): + try: + return self.get(name) + except NameError as e: + raise KeyError(str(e)) + + def __getattr__(self, name): + try: + return self.get(name) + except NameError as e: + raise AttributeError(str(e)) + +class Types(DelayedCollection): + def __init__(self, names): + super(Types, self).__init__(DelayedType, names) + + def override(self, name, value): + (ignore, name, pointer) = TypeCallback.resolve_type(name) + + super().override(name, value) + +class Symbols(DelayedCollection): + def __init__(self, names): + super(Symbols, self).__init__(DelayedSymbol, names) + +class Symvals(DelayedCollection): + def __init__(self, names): + super(Symvals, self).__init__(DelayedSymval, names) + +class MinimalSymbols(DelayedCollection): + def __init__(self, names): + super(MinimalSymbols, self).__init__(DelayedMinimalSymbol, names) + +class MinimalSymvals(DelayedCollection): + def __init__(self, names): + super(MinimalSymvals, self).__init__(DelayedMinimalSymval, names) + +class DelayedValues(DelayedCollection): + def __init__(self, names): + super(DelayedValues, self).__init__(DelayedDelayedValue, names) + +CallbackSpecifier = Tuple[str, Callable] +CallbackSpecifiers = Union[List[CallbackSpecifier], CallbackSpecifier] + +class CallbackCollection(object): + def __init__(self, cls: Type[NamedCallback], cbs: CallbackSpecifiers): + if isinstance(cbs, tuple): + cbs = [ cbs ] + + for cb in cbs: + t = cls(cb[0], cb[1]) + setattr(self, t.attrname, t) + +class TypeCallbacks(CallbackCollection): + def __init__(self, cbs): + super().__init__(TypeCallback, cbs) + +class SymbolCallbacks(CallbackCollection): + def __init__(self, cbs): + super().__init__(SymbolCallback, cbs) + +class MinimalSymbolCallbacks(CallbackCollection): + def __init__(self, cbs): + super().__init__(MinimalSymbolCallback, cbs) + diff --git a/tests/test_util_symbols.py b/tests/test_util_symbols.py new file mode 100644 index 00000000000..3bb43e9f372 --- /dev/null +++ b/tests/test_util_symbols.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import unittest +import gdb + +from crash.exceptions import DelayedAttributeError + +from crash.util.symbols import MinimalSymbols, Symbols, Symvals, Types +from crash.util.symbols import TypeCallbacks, SymbolCallbacks +from crash.util.symbols import MinimalSymbolCallbacks + +class TestDelayedContainers(unittest.TestCase): + def setUp(self): + gdb.execute("file") + + def load_file(self): + gdb.execute("file tests/test-util") + + def msymbol_test(self): + class Test(object): + msymbols = MinimalSymbols([ 'test_struct' ]) + return Test + + def test_bad_msymbol_name(self): + test = self.msymbol_test() + x = test.msymbols + with self.assertRaises(AttributeError): + y = x.bad_symbol_name + + def test_msymbol_unavailable_at_start(self): + test = self.msymbol_test() + x = test().msymbols + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + + def test_msymbol_available_on_load(self): + test = self.msymbol_test() + x = test().msymbols + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + self.load_file() + self.assertTrue(isinstance(x.test_struct, gdb.MinSymbol)) + + def test_msymbol_available_at_start(self): + test = self.msymbol_test() + x = test().msymbols + self.load_file() + + self.assertTrue(isinstance(x.test_struct, gdb.MinSymbol)) + + def symbol_test(self): + class Test(object): + symbols = Symbols([ 'test_struct' ]) + return Test + + def test_bad_symbol_name(self): + test = self.symbol_test() + x = test.symbols + with self.assertRaises(AttributeError): + y = x.bad_symbol_name + + def test_symbol_unavailable_at_start(self): + test = self.symbol_test() + x = test().symbols + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + + def test_symbol_available_on_load(self): + test = self.symbol_test() + x = test().symbols + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + self.load_file() + self.assertTrue(isinstance(x.test_struct, gdb.Symbol)) + + def test_symbol_available_at_start(self): + test = self.symbol_test() + self.load_file() + + x = test().symbols + self.assertTrue(isinstance(x.test_struct, gdb.Symbol)) + + def symval_test(self): + class Test(object): + symvals = Symvals( [ 'test_struct' ] ) + return Test + + def test_bad_symval_name(self): + test = self.symval_test() + x = test.symvals + with self.assertRaises(AttributeError): + y = x.bad_symval_name + + def test_symval_unavailable_at_start(self): + test = self.symval_test() + x = test().symvals + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + + def test_symval_available_on_load(self): + test = self.symval_test() + x = test().symvals + with self.assertRaises(DelayedAttributeError): + y = x.test_struct + self.load_file() + self.assertTrue(isinstance(x.test_struct, gdb.Value)) + + def test_symval_available_at_start(self): + test = self.symval_test() + self.load_file() + + x = test().symvals + self.assertTrue(isinstance(x.test_struct, gdb.Value)) + + def type_test(self): + class Test(object): + types = Types( [ 'struct test' ] ) + return Test + + def test_bad_type_name(self): + test = self.type_test() + x = test.types + with self.assertRaises(AttributeError): + y = x.bad_type_name + + def test_type_unavailable_at_start(self): + test = self.type_test() + x = test().types + with self.assertRaises(DelayedAttributeError): + y = x.test_type + + def test_type_available_on_load(self): + test = self.type_test() + x = test().types + with self.assertRaises(DelayedAttributeError): + y = x.test_type + self.load_file() + y = x.test_type + self.assertTrue(isinstance(y, gdb.Type)) + + def test_type_available_at_start(self): + test = self.type_test() + self.load_file() + + x = test().types + y = x.test_type + self.assertTrue(isinstance(y, gdb.Type)) + + def ptype_test(self): + class Test(object): + types = Types( [ 'struct test *' ]) + return Test + + def test_bad_ptype_name(self): + test = self.ptype_test() + x = test.types + with self.assertRaises(AttributeError): + y = x.bad_ptype_name + + def test_p_type_unavailable_at_start(self): + test = self.ptype_test() + x = test().types + with self.assertRaises(DelayedAttributeError): + y = x.test_p_type + + def test_p_type_available_on_load(self): + test = self.ptype_test() + x = test().types + with self.assertRaises(DelayedAttributeError): + y = x.test_p_type + self.load_file() + y = x.test_p_type + self.assertTrue(isinstance(y, gdb.Type)) + + def test_p_type_available_at_start(self): + test = self.ptype_test() + self.load_file() + + x = test().types + y = x.test_p_type + self.assertTrue(isinstance(y, gdb.Type)) + + def type_callback_test(self): + class Test(object): + class nested(object): + ulong_valid = False + + @classmethod + def check_ulong(cls, gdbtype): + cls.ulong_valid = True + + type_cbs = TypeCallbacks( [ ('unsigned long', + nested.check_ulong) ] ) + return Test + + def test_type_callback_nofile(self): + test = self.type_callback_test() + x = test().nested + self.assertFalse(x.ulong_valid) + with self.assertRaises(AttributeError): + y = x.unsigned_long_type + + def test_type_callback(self): + test = self.type_callback_test() + x = test().nested + self.load_file() + self.assertTrue(x.ulong_valid) + with self.assertRaises(AttributeError): + y = x.unsigned_long_type + + def type_callback_test_multi(self): + class Test(object): + class nested(object): + types = Types( [ 'unsigned long' ] ) + + ulong_valid = False + + @classmethod + def check_ulong(cls, gdbtype): + cls.ulong_valid = True + + type_cbs = TypeCallbacks( [ ('unsigned long', + nested.check_ulong) ] ) + + return Test + + def test_type_callback_nofile_multi(self): + test = self.type_callback_test_multi() + x = test().nested + self.assertFalse(x.ulong_valid) + with self.assertRaises(DelayedAttributeError): + y = x.types.unsigned_long_type + + def test_type_callback_multi(self): + test = self.type_callback_test_multi() + x = test().nested + self.load_file() + self.assertTrue(x.ulong_valid) + y = x.types.unsigned_long_type + self.assertTrue(isinstance(y, gdb.Type)) + self.assertTrue(y.sizeof > 4) From cc56e256dde2164449985184915c76b57de8d19a Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 26 Apr 2019 21:36:51 +0200 Subject: [PATCH 38/75] crash.subsystem.filesystem: add super_flags super_flags returns a human-readable string describing the flags in super_block->s_flags. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 81 +++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index 410cdd02ea2..fbba96ec0e0 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -2,12 +2,73 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.util import container_of +from crash.util import container_of, decode_flags from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry from crash.subsystem.storage import block_device_name from crash.subsystem.storage import Storage as block +MS_RDONLY = 1 +MS_NOSUID = 2 +MS_NODEV = 4 +MS_NOEXEC = 8 +MS_SYNCHRONOUS = 16 +MS_REMOUNT = 32 +MS_MANDLOCK = 64 +MS_DIRSYNC = 128 +MS_NOATIME = 1024 +MS_NODIRATIME = 2048 +MS_BIND = 4096 +MS_MOVE = 8192 +MS_REC = 16384 +MS_VERBOSE = 32768 +MS_SILENT = 32768 +MS_POSIXACL = (1<<16) +MS_UNBINDABLE = (1<<17) +MS_PRIVATE = (1<<18) +MS_SLAVE = (1<<19) +MS_SHARED = (1<<20) +MS_RELATIME = (1<<21) +MS_KERNMOUNT = (1<<22) +MS_I_VERSION = (1<<23) +MS_STRICTATIME = (1<<24) +MS_LAZYTIME = (1<<25) +MS_NOSEC = (1<<28) +MS_BORN = (1<<29) +MS_ACTIVE = (1<<30) +MS_NOUSER = (1<<31) + +SB_FLAGS = { + MS_RDONLY : "MS_RDONLY", + MS_NOSUID : "MS_NOSUID", + MS_NODEV : "MS_NODEV", + MS_NOEXEC : "MS_NOEXEC", + MS_SYNCHRONOUS : "MS_SYNCHRONOUS", + MS_REMOUNT : "MS_REMOUNT", + MS_MANDLOCK : "MS_MANDLOCK", + MS_DIRSYNC : "MS_DIRSYNC", + MS_NOATIME : "MS_NOATIME", + MS_NODIRATIME : "MS_NODIRATIME", + MS_BIND : "MS_BIND", + MS_MOVE : "MS_MOVE", + MS_REC : "MS_REC", + MS_SILENT : "MS_SILENT", + MS_POSIXACL : "MS_POSIXACL", + MS_UNBINDABLE : "MS_UNBINDABLE", + MS_PRIVATE : "MS_PRIVATE", + MS_SLAVE : "MS_SLAVE", + MS_SHARED : "MS_SHARED", + MS_RELATIME : "MS_RELATIME", + MS_KERNMOUNT : "MS_KERNMOUNT", + MS_I_VERSION : "MS_I_VERSION", + MS_STRICTATIME : "MS_STRICTATIME", + MS_LAZYTIME : "MS_LAZYTIME", + MS_NOSEC : "MS_NOSEC", + MS_BORN : "MS_BORN", + MS_ACTIVE : "MS_ACTIVE", + MS_NOUSER : "MS_NOUSER", +} + class FileSystem(CrashBaseClass): __types__ = [ 'struct dio *', 'struct buffer_head *' ] @@ -37,7 +98,7 @@ def _register_end_bio_bh_io_sync(cls, sym): @export @staticmethod - def super_fstype(sb): + def super_fstype(sb: gdb.Value) -> str: """ Returns the file system type's name for a given superblock. @@ -50,6 +111,22 @@ def super_fstype(sb): """ return sb['s_type']['name'].string() + @export + @staticmethod + def super_flags(sb: gdb.Value) -> str: + """ + Returns the flags associated with the given superblock. + + Args: + sb (gdb.Value): The struct super_block for + which to return the flags. + + Returns: + str: The flags field in human-readable form. + + """ + return decode_flags(sb['s_flags'], SB_FLAGS) + @export @classmethod def register_buffer_head_decoder(cls, sym, decoder): From 5f69e54dec4853899538a68b59d537c8636dc06f Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 18 Sep 2018 05:28:41 -0400 Subject: [PATCH 39/75] crash.subsystem.filesystem: add for_each_super_block, get_super_block Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 54 +++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index fbba96ec0e0..1ba7151c5cd 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable, Union + import gdb -from crash.util import container_of, decode_flags +from crash.util import container_of, get_typed_pointer, decode_flags from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry from crash.subsystem.storage import block_device_name from crash.subsystem.storage import Storage as block +AddressSpecifier = Union[int, str, gdb.Value] + MS_RDONLY = 1 MS_NOSUID = 2 MS_NODEV = 4 @@ -71,7 +75,9 @@ class FileSystem(CrashBaseClass): __types__ = [ 'struct dio *', - 'struct buffer_head *' ] + 'struct buffer_head *', + 'struct super_block' ] + __symvals__ = [ 'super_blocks' ] __symbol_callbacks__ = [ ('dio_bio_end_io', '_register_dio_bio_end'), ('dio_bio_end_aio', '_register_dio_bio_end'), @@ -320,4 +326,48 @@ def decode_end_buffer_write_sync(cls, bh): } return chain + @export + @classmethod + def for_each_super_block(cls) -> Iterable[gdb.Value]: + """ + Iterate over the list of super blocks and yield each one. + + Args: + None + + Yields: + gdb.Value + """ + for sb in list_for_each_entry(cls.super_blocks, cls.super_block_type, + 's_list'): + yield sb + + @export + @classmethod + def get_super_block(cls, desc: AddressSpecifier, + force: bool=False) -> gdb.Value: + """ + Given an address description return a gdb.Value that contains + a struct super_block at that address. + + Args: + desc (gdb.Value, str, or int): The address for which to provide + a casted pointer + force (bool): Skip testing whether the value is available. + + Returns: + gdb.Value: The super_block at the requested + location + + Raises: + gdb.NotAvailableError: The target value was not available. + """ + sb = get_typed_pointer(desc, cls.super_block_type).dereference() + if not force: + try: + x = int(sb['s_dev']) + except gdb.NotAvailableError: + raise gdb.NotAvailableError(f"no superblock available at `{desc}'") + return sb + inst = FileSystem() From 7d720160761db8fe6de92af7c21999d56122a958 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 19 Sep 2018 10:49:40 +0200 Subject: [PATCH 40/75] crash.subsystem.filesystem.mount: use type from symbol when available Internally, gdb treats the type loaded from a typed symbol and a type symbol differently and wants to do the full type comparison dance. If we use the typed symbol directly, we can use a pointer comparison. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/mount.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crash/subsystem/filesystem/mount.py b/crash/subsystem/filesystem/mount.py index 9422dc0bc36..b9578985f6f 100644 --- a/crash/subsystem/filesystem/mount.py +++ b/crash/subsystem/filesystem/mount.py @@ -62,6 +62,11 @@ def for_each_mount_nsproxy(self, task): def real_mount(cls, vfsmnt): if (vfsmnt.type == cls.mount_type or vfsmnt.type == cls.mount_type.pointer()): + t = vfsmnt.type + if t.code == gdb.TYPE_CODE_PTR: + t = t.target() + if t is not cls.mount_type: + cls.mount_type = t return vfsmnt return container_of(vfsmnt, cls.mount_type, 'mnt') From b4facc03f5f05049335d70e77fc549e3d8fc4865 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 21:07:17 -0400 Subject: [PATCH 41/75] crash.subsystem.filesystem.mount: use crash.util.decode_flags Now that we have crash.util.decode_flags, we can use that instead of open-coding it in mount_flags. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/mount.py | 63 +++++++++++------------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/crash/subsystem/filesystem/mount.py b/crash/subsystem/filesystem/mount.py index b9578985f6f..2407dceb1dd 100644 --- a/crash/subsystem/filesystem/mount.py +++ b/crash/subsystem/filesystem/mount.py @@ -6,7 +6,7 @@ from crash.infra import CrashBaseClass, export from crash.subsystem.filesystem import super_fstype from crash.types.list import list_for_each_entry -from crash.util import container_of +from crash.util import container_of, decode_flags, struct_has_member MNT_NOSUID = 0x01 MNT_NODEV = 0x02 @@ -20,6 +20,25 @@ MNT_SHARED = 0x1000 MNT_UNBINDABLE = 0x2000 +MNT_FLAGS = { + MNT_NOSUID : "MNT_NOSUID", + MNT_NODEV : "MNT_NODEV", + MNT_NOEXEC : "MNT_NOEXEC", + MNT_NOATIME : "MNT_NOATIME", + MNT_NODIRATIME : "MNT_NODIRATIME", + MNT_RELATIME : "MNT_RELATIME", + MNT_READONLY : "MNT_READONLY", +} + +MNT_FLAGS_HIDDEN = { + MNT_SHRINKABLE : "[MNT_SHRINKABLE]", + MNT_WRITE_HOLD : "[MNT_WRITE_HOLD]", + MNT_SHARED : "[MNT_SHARED]", + MNT_UNBINDABLE : "[MNT_UNBINDABLE]", +} +MNT_FLAGS_HIDDEN.update(MNT_FLAGS) + + class Mount(CrashBaseClass): __types__ = [ 'struct mount', 'struct vfsmount' ] __symvals__ = [ 'init_task' ] @@ -73,45 +92,11 @@ def real_mount(cls, vfsmnt): @export @classmethod def mount_flags(cls, mnt, show_hidden=False): - flags = int(mnt['mnt_flags']) - - if flags & MNT_READONLY: - flagstr = "ro" - else: - flagstr = "rw" - - if flags & MNT_NOSUID: - flagstr += ",nosuid" - - if flags & MNT_NODEV: - flagstr += ",nodev" - - if flags & MNT_NOEXEC: - flagstr += ",noexec" - - if flags & MNT_NOATIME: - flagstr += ",noatime" - - if flags & MNT_NODIRATIME: - flagstr += ",nodiratime" - - if flags & MNT_RELATIME: - flagstr += ",relatime" - + if struct_has_member(mnt, 'mnt'): + mnt = mnt['mnt'] if show_hidden: - if flags & MNT_SHRINKABLE: - flagstr += ",[MNT_SHRINKABLE]" - - if flags & MNT_WRITE_HOLD: - flagstr += ",[MNT_WRITE_HOLD]" - - if flags & MNT_SHARED: - flagstr += ",[MNT_SHARED]" - - if flags & MNT_UNBINDABLE: - flagstr += ",[MNT_UNBINDABLE]" - - return flagstr + return decode_flags(mnt['mnt_flags'], MNT_FLAGS_HIDDEN, ",") + return decode_flags(mnt['mnt_flags'], MNT_FLAGS, ",") @export @staticmethod From b2817c94fb5218e0bbba0ec1085513b4e6de1a00 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 30 Apr 2019 22:34:45 -0400 Subject: [PATCH 42/75] crash.subsystem.filesystem: include sector number when describing dio bio Knowing a particular bio is dio isn't entirely helpful when there are multiples of them. This includes the sector number as well. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index 1ba7151c5cd..de92f6f0d65 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -193,8 +193,9 @@ def decode_dio_bio(cls, bio): offset = dio['block_in_file'] << dio['blkbits'] chain = { - 'description' : "{:x} bio: Direct I/O for {} inode {} on {}".format( - int(bio), fstype, dio['inode']['i_ino'], dev), + 'description' : "{:x} bio: Direct I/O for {} inode {}, sector {} on {}".format( + int(bio), fstype, dio['inode']['i_ino'], + bio['bi_sector'], dev), 'bio' : bio, 'dio' : dio, 'fstype' : fstype, From b7085be43b7e6183347c02edea8d3f2cdf0e18dd Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 15:15:09 -0400 Subject: [PATCH 43/75] crash.subsystem.{filesystem,storage}: refactor bio decoders The current bio decoders are essentially open coded everywhere. This commit formalizes the interface to make it easier to extend. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 218 ----------------------- crash/subsystem/filesystem/decoders.py | 138 ++++++++++++++ crash/subsystem/filesystem/ext3.py | 80 ++++----- crash/subsystem/storage/__init__.py | 89 +-------- crash/subsystem/storage/decoders.py | 159 +++++++++++++++++ crash/subsystem/storage/device_mapper.py | 197 ++++++++++---------- 6 files changed, 427 insertions(+), 454 deletions(-) create mode 100644 crash/subsystem/filesystem/decoders.py create mode 100644 crash/subsystem/storage/decoders.py diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index de92f6f0d65..32b74dec738 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -78,30 +78,6 @@ class FileSystem(CrashBaseClass): 'struct buffer_head *', 'struct super_block' ] __symvals__ = [ 'super_blocks' ] - __symbol_callbacks__ = [ - ('dio_bio_end_io', '_register_dio_bio_end'), - ('dio_bio_end_aio', '_register_dio_bio_end'), - ('mpage_end_io', '_register_mpage_end_io'), - ('end_bio_bh_io_sync', '_register_end_bio_bh_io_sync') ] - - buffer_head_decoders = {} - - @classmethod - def _register_dio_bio(cls, symval): - block.register_bio_decoder(cls.dio_bio_end, cls.decode_dio_bio) - - @classmethod - def _register_dio_bio_end(cls, sym): - block.register_bio_decoder(sym, cls.decode_dio_bio) - - @classmethod - def _register_mpage_end_io(cls, sym): - block.register_bio_decoder(sym, cls.decode_mpage) - - @classmethod - def _register_end_bio_bh_io_sync(cls, sym): - block.register_bio_decoder(sym, cls.decode_bio_buffer_head) - @export @staticmethod def super_fstype(sb: gdb.Value) -> str: @@ -133,200 +109,6 @@ def super_flags(sb: gdb.Value) -> str: """ return decode_flags(sb['s_flags'], SB_FLAGS) - @export - @classmethod - def register_buffer_head_decoder(cls, sym, decoder): - """ - Registers a buffer_head decoder with the filesystem subsystem. - - A buffer_head decoder is a method thats acepts a buffer_head, - potentially interprets the private members of the buffer_head, - and returns a dictionary. The only mandatory member of the - dictionary is 'description' which contains a human-readable - description of the purpose of this buffer_head. - - If the buffer_head is part of a stack, the 'next' item should contain - the next object in the stack. It does not necessarily need to be - a buffer_head. It does need to have a 'decoder' item declared - that will accept the given object. The decoder does not need to - be registered unless it will be a top-level decoder. - - Other items can be added as-needed to allow informed callers - to obtain direct informatiom. - - Args: - sym (gdb.Value): - The kernel function used as buffer_head->b_h_end_io callback - """ - - cls.buffer_head_decoders[sym] = decoder - - @classmethod - def decode_dio_bio(cls, bio): - """ - Decodes a bio used for direct i/o. - - This method decodes a bio generated by the direct-io component of - the file system subsystem. The bio can either have been submitted - directly or asynchronously. - - Args: - bio(gdb.Value): The struct bio to be decoded, generated - by the direct i/o component - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of the bio - - bio (gdb.Value): The struct bio being decoded - - dio (gdb.Value): The direct i/o component of - the bio - - fstype (str): The name of the file system which submitted - this bio - - inode (gdb.Value): The struct inode, if any, - that owns the file associated with this bio - - offset (int): The offset within the file, in bytes - - devname (str): The device name associated with this bio - """ - dio = bio['bi_private'].cast(cls.dio_p_type) - fstype = cls.super_fstype(dio['inode']['i_sb']) - dev = block_device_name(dio['inode']['i_sb']['s_bdev']) - offset = dio['block_in_file'] << dio['blkbits'] - - chain = { - 'description' : "{:x} bio: Direct I/O for {} inode {}, sector {} on {}".format( - int(bio), fstype, dio['inode']['i_ino'], - bio['bi_sector'], dev), - 'bio' : bio, - 'dio' : dio, - 'fstype' : fstype, - 'inode' : dio['inode'], - 'offset' : offset, - 'devname' : dev, - } - return chain - - @classmethod - def decode_mpage(cls, bio): - """ - Decodes a bio used for multipage i/o. - - This method decodes a bio generated by the mpage component of - the file system subsystem. - - Args: - bio(gdb.Value): The struct bio to be decoded, generated - by the mpage component - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of the bio - - bio (gdb.Value): The struct bio being decoded - - fstype (str): The name of the file system which submitted - this bio - - inode (gdb.Value): The struct inode, if any, - that owns the file associated with this bio - """ - inode = bio['bi_io_vec'][0]['bv_page']['mapping']['host'] - fstype = cls.super_fstype(inode['i_sb']) - chain = { - 'description' : - "{:x} bio: Multipage I/O: inode {}, type {}, dev {}".format( - int(bio), inode['i_ino'], fstype, - block_device_name(bio['bi_bdev'])), - 'bio' : bio, - 'fstype' : fstype, - 'inode' : inode, - } - return chain - - @classmethod - def decode_bio_buffer_head(cls, bio): - """ - Decodes a bio used to perform i/o for buffer_heads - - This method decodes a bio generated by buffer head submission. - - Args: - bio(gdb.Value): The struct bio to be decoded, generated - by buffer head submission - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of the bio - - bio (gdb.Value): The struct bio being decoded - - next (gdb.Value): The buffer_head that - initiated this bio. - - decoder (gdb.Value): - A decoder for the buffer head - """ - bh = bio['bi_private'].cast(cls.buffer_head_p_type) - chain = { - 'description' : - "{:x} bio: Bio representation of buffer head".format(int(bio)), - 'bio' : bio, - 'next' : bh, - 'decoder' : cls.decode_buffer_head, - } - - return chain - - @classmethod - def decode_buffer_head(cls, bh): - """ - Decodes a struct buffer_head - - This method decodes a struct buffer_head, using an - implementation-specific decoder, if available - - Args: - bio(gdb.Value): The struct buffer_head to be - decoded. - - Returns: - dict: Minimally contains the following items. - - description (str): Human-readable description of the bio - - bh (gdb.Value): The struct buffer_head - Additional items may be available based on the - implmentation-specific decoder. - """ - endio = bh['b_end_io'] - try: - return cls.buffer_head_decoders[endio](bh) - except KeyError: - pass - desc = "{:x} buffer_head: for dev {}, block {}, size {} (undecoded)".format( - int(bh), block_device_name(bh['b_bdev']), - bh['b_blocknr'], bh['b_size']) - chain = { - 'description' : desc, - 'bh' : bh, - } - return chain - - @classmethod - def decode_end_buffer_write_sync(cls, bh): - """ - Decodes a struct buffer_head submitted by file systems for routine - synchronous writeback. - - Args: - bio(gdb.Value): The struct buffer_head to be - decoded. - - Returns: - dict: Minimally contains the following items. - - description (str): Human-readable description of the bio - - bh (gdb.Value): The struct buffer_head - """ - desc = ("{:x} buffer_head: for dev {}, block {}, size {} (unassociated)" - .format(block_device_name(bh['b_bdev']), - bh['b_blocknr'], bh['b_size'])) - chain = { - 'description' : desc, - 'bh' : bh, - } - return chain - @export @classmethod def for_each_super_block(cls) -> Iterable[gdb.Value]: diff --git a/crash/subsystem/filesystem/decoders.py b/crash/subsystem/filesystem/decoders.py new file mode 100644 index 00000000000..badd71189e2 --- /dev/null +++ b/crash/subsystem/filesystem/decoders.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +from crash.util.symbols import Types +from crash.subsystem.storage import block_device_name +from crash.subsystem.storage.decoders import Decoder, decode_bh +from crash.subsystem.filesystem import super_fstype + +class DIOBioDecoder(Decoder): + """ + Decodes a bio used for direct i/o. + + This method decodes a bio generated by the direct-io component of + the file system subsystem. The bio can either have been submitted + directly or asynchronously. + + Args: + bio(gdb.Value): The struct bio to be decoded, generated + by the direct i/o component + """ + + types = Types([ 'struct dio *' ]) + __endio__ = [ 'dio_bio_end_io', 'dio_bio_end_io' ] + description = "{:x} bio: Direct I/O for {} inode {}, sector {} on {}" + + def __init__(self, bio): + super().__init__() + self.bio = bio + + def interpret(self): + self.dio = self.bio['bi_private'].cast(self.types.dio_p_type) + self.fstype = super_fstype(self.dio['inode']['i_sb']) + self.dev = block_device_name(self.dio['inode']['i_sb']['s_bdev']) + self.offset = self.dio['block_in_file'] << self.dio['blkbits'] + + def __str__(self): + return self.description.format(int(self.bio), self.fstype, + self.dio['inode']['i_ino'], + self.bio['bi_sector'], self.dev) + + def __next__(self): + return None + +DIOBioDecoder.register() + +class DecodeMPage(Decoder): + """ + Decodes a bio used for multipage i/o. + + This method decodes a bio generated by the mpage component of + the file system subsystem. + + Args: + bio(gdb.Value): The struct bio to be decoded, generated + by the mpage component + + Returns: + dict: Contains the following items: + - description (str): Human-readable description of the bio + - bio (gdb.Value): The struct bio being decoded + - fstype (str): The name of the file system which submitted + this bio + - inode (gdb.Value): The struct inode, if any, + that owns the file associated with this bio + """ + + __endio__ = 'mpage_end_io' + + description = "{:x} bio: Multipage I/O: inode {}, type {}, dev {}" + + def __init__(self, bio): + super().__init__() + + self.bio = bio + + def interpret(self): + self.inode = bio['bi_io_vec'][0]['bv_page']['mapping']['host'] + self.fstype = super_fstype(inode['i_sb']) + + def __str__(self): + return self.description.format(int(self.bio), self.inode['i_ino'], + self.fstype, + block_device_name(bio['bi_bdev'])) + +DecodeMPage.register() + +class DecodeBioBH(Decoder): + """ + Decodes a bio used to perform i/o for buffer_heads + + This method decodes a bio generated by buffer head submission. + + Args: + bio(gdb.Value): The struct bio to be decoded, generated + by buffer head submission + + """ + types = Types([ 'struct buffer_head *' ]) + __endio__ = 'end_bio_bh_io_sync' + description = "{:x} bio: Bio representation of buffer head" + + def __init__(self, bio): + super().__init__() + self.bio = bio + + def interpret(self): + self.bh = self.bio['bi_private'].cast(self.types.buffer_head_p_type) + + def __str__(self): + return self.description.format(int(bio)) + + def __next__(self): + return decode_bh(self.bh) + +DecodeBioBH.register() + +class DecodeSyncWBBH(Decoder): + """ + Decodes a struct buffer_head submitted by file systems for routine + synchronous writeback. + + Args: + bio(gdb.Value): The struct buffer_head to be + decoded. + """ + __endio__ = 'end_buffer_write_sync' + description = "{:x} buffer_head: for dev {}, block {}, size {} (unassociated)" + + def __init__(self, bh): + super().__init__() + self.bh = bh + + def __str__(self): + self.description.format(block_device_name(bh['b_bdev']), + self.bh['b_blocknr'], self.bh['b_size']) + +DecodeSyncWBBH.register() diff --git a/crash/subsystem/filesystem/ext3.py b/crash/subsystem/filesystem/ext3.py index 71f45e77613..320804d6228 100644 --- a/crash/subsystem/filesystem/ext3.py +++ b/crash/subsystem/filesystem/ext3.py @@ -5,53 +5,33 @@ from crash.infra import CrashBaseClass from crash.util import get_symbol_value -from crash.subsystem.filesystem import register_buffer_head_decoder - -class Ext3(CrashBaseClass): - __symbol_callbacks__ = [ - ('journal_end_buffer_io_sync', '_register_journal_buffer_io_sync') ] - - @classmethod - def _register_journal_buffer_io_sync(cls, sym): - # ext3/ext4 and jbd/jbd2 share names but not implementations - b = gdb.block_for_pc(int(sym.value().address)) - sym = get_symbol_value('journal_end_buffer_io_sync', b) - - register_buffer_head_decoder(sym, cls.decode_journal_buffer_io_sync) - - @classmethod - def decode_journal_buffer_io_sync(cls, bh): - """ - Decodes an ext3 journal buffer - - This method decodes a struct buffer_head with and end_io callback - of journal_end_buffer_io_sync. - - Args: - bh (gdb.Value): The struct buffer_head to - decode - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of - the buffer head - - bh (gdb.Value): The buffer head being - decoded - - fstype (str): The name of the file system type being decoded - - devname (str): The name of the device the file system uses - - offset (int): The offset, in bytes, of the block described - - length (int): The length of the block described - """ - - fstype = "journal on ext3" - devname = block_device_name(bh['b_bdev']) - chain = { - 'bh' : bh, - 'description' : "{:x} buffer_head: {} journal block (jbd) on {}".format(int(bh), fstype, devname), - 'fstype' : fstype, - 'devname' : devname, - 'offset' : int(bh['b_blocknr']) * int(bh['b_size']), - 'length' : int(bh['b_size']) - } - - return chain +from crash.subsystem.storage.decoders import Decoder + +class Ext3Decoder(Decoder): + """ + Decodes an ext3 journal buffer + + This decodes a struct buffer_head with an end_io callback + of journal_end_buffer_io_sync. + + Args: + bh (gdb.Value): The struct buffer_head to decode + """ + + __endio__ = 'journal_end_buffer_io_sync' + description = "{:x} buffer_head: {} journal block (jbd) on {}" + + def __init__(self, bh): + super().__init__() + self.bh = bh + + def interpret(self): + self.fstype = "journal on ext3" + self.devname = block_device_name(self.bh['b_bdev']) + self.offset = int(self.bh['b_blocknr']) * int(self.bh['b_size']) + self.length = int(self.bh['b_size']) + + def __str__(self): + return self.description(int(self.bh), fstype, devname) + +Ext3Decoder.register() diff --git a/crash/subsystem/storage/__init__.py b/crash/subsystem/storage/__init__.py index d210011edb4..5228e1d290c 100644 --- a/crash/subsystem/storage/__init__.py +++ b/crash/subsystem/storage/__init__.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable + import gdb from crash.util import container_of from crash.infra import CrashBaseClass, export from crash.types.classdev import for_each_class_device +from . import decoders import crash.exceptions class Storage(CrashBaseClass): @@ -23,8 +26,6 @@ class Storage(CrashBaseClass): ( 'part_type', '_check_types' ) ] __type_callbacks__ = [ ('struct device_type', '_check_types' ) ] - bio_decoders = {} - @classmethod def _check_types(cls, result): try: @@ -43,45 +44,7 @@ def _check_types(cls, result): @export @classmethod - def register_bio_decoder(cls, sym, decoder): - """ - Registers a bio decoder with the storage subsystem. - - A bio decoder is a method that accepts a bio, potentially - interprets the private members of the bio, and returns - a dictionary. The only mandatory member of the dictionary - is 'description' which contains a human-readable description - of the purpose of this bio. - - If the bio is part of a stack, the 'next' item should contain - the next object in the stack. It does not necessarily need - to be a bio. It does need to have a 'decoder' item declared - that will accept the given object. The decoder does not - need to be registered unless it will be a top-level decoder. - - Other items can be added as-needed to allow informed callers - to obtain direct information. - - Args: - sym (gdb.Symbol or gdb.Value): - The Symbol or Value describing a kernel function used as - a bio->b_end_io callback - decoder (method): A Python method that accepts a - gdb.Value(struct bio) - - Raises: - TypeError: sym is not a gdb.Symbol or gdb.Value - """ - - if isinstance(sym, gdb.Symbol): - sym = sym.value().address - elif not isinstance(sym, gdb.Value): - raise TypeError("register_bio_decoder expects gdb.Symbol or gdb.Value") - cls.bio_decoders[int(sym)] = decoder - - @export - @classmethod - def for_each_bio_in_stack(cls, bio): + def for_each_bio_in_stack(cls, bio: gdb.Value) -> Iterable[decoders.Decoder]: """ Iterates and decodes each bio involved in a stacked storage environment @@ -90,7 +53,7 @@ def for_each_bio_in_stack(cls, bio): processed by each level's decoder. The stack will be interrupted if an encountered object doesn't have a decoder specified. - See register_bio_decoder for more detail. + See crash.subsystem.storage.decoder.register_decoder for more detail. Args: bio (gdb.Value): The initial struct bio to start @@ -102,44 +65,10 @@ def for_each_bio_in_stack(cls, bio): Additional items may be available based on the implmentation-specific decoder. """ - first = cls.bio_decoders[int(bio['bi_end_io'])](bio) - if first: - yield first - while 'decoder' in first: - first = first['decoder'](first['next']) - yield first - - @export - @classmethod - def decode_bio(cls, bio): - """ - Decodes a single bio, if possible - - This method will return a dictionary describing a single bio - after decoding it using a registered decoder, if available. - - If no decoder is registered, a generic description will be - returned in the dictionary's 'description' field. - - Args: - bio (gdb.Value): The bio to decode - - Returns: - dict: Contains, minimally, the following item. - - description (str): A human-readable description of the bio. - Additional items may be available based on the - implmentation-specific decoder. - """ - - try: - return cls.bio_decoders[int(bio['bi_end_io'])](bio) - except KeyError: - chain = { - 'description' : "{:x} bio: undecoded bio on {} ({})".format( - int(bio), block_device_name(bio['bi_bdev']), - bio['bi_end_io']), - } - return chain + decoder = decoders.decode_bio(bio) + while decoder is not None: + yield decoder + decoder = next(decoder) @export def dev_to_gendisk(self, dev): diff --git a/crash/subsystem/storage/decoders.py b/crash/subsystem/storage/decoders.py new file mode 100644 index 00000000000..d31ea298074 --- /dev/null +++ b/crash/subsystem/storage/decoders.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +from typing import Union, List +from crash.infra import CrashBaseClass +from crash.infra.lookup import SymbolCallback + +EndIOSpecifier = Union[int, str, List[str], gdb.Value, gdb.Symbol, None] + +decoders = {} + +class Decoder(CrashBaseClass): + + __endio__: EndIOSpecifier = None + + def __init__(self): + self.interpreted = False + + def interpret(self) -> None: + pass + + def __getattr__(self, name): + if self.interpreted: + raise AttributeError(f"No such attribute `{name}'") + + self.interpret() + self.interpreted = True + return getattr(self, name) + + @classmethod + def register(cls): + register_decoder(cls.__endio__, cls) + + def __str__(self) -> str: + pass + + def __next__(self): + return None + + +class DecodeBufferHead(Decoder): + """ + Decodes a struct buffer_head + + This method decodes a generic struct buffer_head, when no + implementation-specific decoder is available + + Args: + bio(gdb.Value): The struct buffer_head to be + decoded. + """ + + description = "{:x} buffer_head: for dev {}, block {}, size {} (undecoded)" + + def __init__(self, bh: gdb.Value): + super().__init__() + self.bh = bh + + def interpret(self): + pass + + def __str__(self): + return self.description.format(int(self.bh), + block_device_name(self.bh['b_bdev']), + self.bh['b_blocknr'], self.bh['b_size']) + +def register_decoder(endio: EndIOSpecifier, decoder: Decoder) -> None: + """ + Registers a bio/buffer_head decoder with the storage subsystem. + + A decoder is a class that accepts a bio, buffer_head, or other object, + potentially interprets the private members of the object, and + returns a Decoder object that describes it. + + The only mandatory part of a Decoder is the __str__ method to + print the description. + + If the bio is part of a stack, the __next__ method will contain + the next Decoder object in the stack. It does not necessarily need + to be a bio. The Decoder does not need to be registered unless it + will be a top-level decoder. + + Other attributes can be added as-needed to allow informed callers + to obtain direct information. + + Args: + endio (str, list of str, gdb.Symbol, gdb.Value, or int): The function + used as an endio callback. + + The str or list of str arguments are used to register a callback + such that the Decoder is registered when the symbol is available. + + The gdb.Symbol, gdb.Value, and int versions are to be used + once the symbol is available for resolution. + + If in doubt, use the names instead of the symbols objects. + + decoder (Decoder): The decoder class used to handle this object. + + """ + debug = False + if isinstance(endio, str): + if debug: + print(f"Registering {endio} as callback") + x = SymbolCallback(endio, lambda a: register_decoder(a, decoder)) + return + elif isinstance(endio, list) and isinstance(endio[0], str): + for sym in endio: + if debug: + print(f"Registering {sym} as callback") + x = SymbolCallback(sym, lambda a: register_decoder(a, decoder)) + return + + if isinstance(endio, gdb.Symbol): + endio = endio.value() + + if isinstance(endio, gdb.Value): + endio = int(endio.address) + + if debug: + print(f"Registering {endio:#x} for real") + + decoders[endio] = decoder + +class GenericBioDecoder(Decoder): + description = "{:x} bio: undecoded bio on {} ({})" + def __init__(self, bio): + super().__init__() + self.bio = bio + + def __str__(self): + return self.description.format(int(self.bio), + block_device_name(self.bio['bi_bdev']), + bio['bi_end_io']) + +def decode_bio(bio: gdb.Value) -> Decoder: + """ + Decodes a single bio, if possible + + This method will return a Decoder object describing a single bio + after decoding it using a registered decoder, if available. + + If no decoder is registered, a generic description will be used. + + Args: + bio (gdb.Value): The bio to decode + """ + + try: + return decoders[int(bio['bi_end_io'])](bio) + except KeyError: + return GenericBioDecoder(bio) + +def decode_bh(bh: gdb.Value) -> Decoder: + try: + return decoders[int(bh['b_endio'])](bh) + except KeyError: + return DecodeBufferHead(bh) diff --git a/crash/subsystem/storage/device_mapper.py b/crash/subsystem/storage/device_mapper.py index 0030bf4cd7c..74d5a1807b4 100644 --- a/crash/subsystem/storage/device_mapper.py +++ b/crash/subsystem/storage/device_mapper.py @@ -4,76 +4,46 @@ import gdb from crash.infra import CrashBaseClass -from crash.subsystem.storage import Storage as block from crash.subsystem.storage import block_device_name +from crash.subsystem.storage.decoders import Decoder, decode_bio -class DeviceMapper(CrashBaseClass): - __types__ = [ 'struct dm_rq_clone_bio_info *', - 'struct dm_target_io *' ] - __symbol_callbacks__ = [ - ('end_clone_bio', '_register_end_clone_bio'), - ('clone_endio', '_register_clone_endio') ] +class ClonedBioReqDecoder(Decoder): + """ + Decodes a request-based device mapper cloned bio - @classmethod - def _register_end_clone_bio(cls, sym): - if 'clone' in cls.dm_rq_clone_bio_info_p_type.target(): - getter = cls._get_clone_bio_rq_info_3_7 - else: - getter = cls._get_clone_bio_rq_info_old - cls._get_clone_bio_rq_info = getter - block.register_bio_decoder(sym, cls.decode_clone_bio_rq) + This decodes a cloned bio generated by request-based device mapper targets. - @classmethod - def _register_clone_endio(cls, sym): - if 'clone' in cls.dm_target_io_p_type.target(): - getter = cls._get_clone_bio_tio_3_15 - else: - getter = cls._get_clone_bio_tio_old - cls._get_clone_bio_tio = getter - block.register_bio_decoder(sym, cls.decode_clone_bio) + Args: + bio(gdb.Value): A struct bio generated by a + request-based device mapper target - @classmethod - def decode_clone_bio_rq(cls, bio): - """ - Decodes a request-based device mapper cloned bio - - This method decodes a cloned bio generated by request-based - device mapper targets. - - Args: - bio(gdb.Value): A struct bio generated by a - request-based device mapper target - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of the bio - - bio (gdb.Value): The provided bio - - tio (gdb.Value(): The struct - dm_target_io for this bio - - next (gdb.Value): The original bio that was - the source of this one - - decoder (method(gdb.Value)): The decoder for - the original bio - """ - - info = cls._get_clone_bio_rq_info(bio) - - # We can pull the related bios together here if required - # b = bio['bi_next'] - # while int(b) != 0: - # b = b['bi_next'] - - chain = { - 'bio' : bio, - 'tio' : info['tio'], - 'next' : info['orig'], - 'description' : - '{:x} bio: Request-based Device Mapper on {}'.format( - int(bio), block_device_name(bio['bi_bdev'])), - 'decoder' : block.decode_bio, - } - - return chain + """ + __types__ = [ 'struct dm_rq_clone_bio_info *' ] + __endio__ = 'end_clone_bio' + description = '{:x} bio: Request-based Device Mapper on {}' + + _get_clone_bio_rq_info = None + + def __init__(self, bio): + super().__init__() + self.bio = bio + if cls._get_clone_bio_rq_info is None: + if 'clone' in cls.dm_rq_clone_bio_info_p_type.target(): + getter = cls._get_clone_bio_rq_info_3_7 + else: + getter = cls._get_clone_bio_rq_info_old + cls._get_clone_bio_rq_info = getter + + def interpret(self): + self.info = cls._get_clone_bio_rq_info(bio) + self.tio = self.info['tio'] + + def __str__(self): + self.description.format(int(self.bio), + block_device_name(self.bio['bi_bdev'])) + + def __next__(self): + return decode_bio(self.info['orig']) @classmethod def _get_clone_bio_rq_info_old(cls, bio): @@ -83,47 +53,60 @@ def _get_clone_bio_rq_info_old(cls, bio): def _get_clone_bio_rq_info_3_7(cls, bio): return container_of(bio, cls.dm_rq_clone_bio_info_p_type, 'clone') - @classmethod - def decode_clone_bio(cls, bio): - """ - Decodes a bio-based device mapper cloned bio - - This method decodes a cloned bio generated by request-based - device mapper targets. - - Args: - bio(gdb.Value): A struct bio generated by a - bio-based device mapper target - - Returns: - dict: Contains the following items: - - description (str): Human-readable description of the bio - - bio (gdb.Value): The provided bio - - tio (gdb.Value): The struct - dm_target_tio for this bio - - next (gdb.Value): The original bio that was - the source of this one - - decoder (method(gdb.Value)): The decoder for the - original bio - """ - tio = cls._get_clone_bio_tio(bio) - - next_bio = tio['io']['bio'] - - chain = { - 'description' : "{:x} bio: device mapper clone: {}[{}] -> {}[{}]".format( - int(bio), - block_device_name(bio['bi_bdev']), - int(bio['bi_sector']), - block_device_name(next_bio['bi_bdev']), - int(next_bio['bi_sector'])), - 'bio' : bio, - 'tio' : tio, - 'next' : next_bio, - 'decoder' : block.decode_bio, - } - - return chain +ClonedBioReqDecoder.register() + +class ClonedBioDecoder(Decoder): + """ + Decodes a bio-based device mapper cloned bio + + This method decodes a cloned bio generated by request-based + device mapper targets. + + Args: + bio(gdb.Value): A struct bio generated by a + bio-based device mapper target + + Returns: + dict: Contains the following items: + - description (str): Human-readable description of the bio + - bio (gdb.Value): The provided bio + - tio (gdb.Value): The struct + dm_target_tio for this bio + - next (gdb.Value): The original bio that was + the source of this one + - decoder (method(gdb.Value)): The decoder for the + original bio + """ + __types__ = [ 'struct dm_target_io *' ] + _get_clone_bio_tio = None + __endio__ = 'clone_endio' + description = "{:x} bio: device mapper clone: {}[{}] -> {}[{}]" + + def __init__(self, bio): + super().__init__() + self.bio = bio + + if _get_clone_bio_tio is None: + if 'clone' in cls.dm_target_io_p_type.target(): + getter = cls._get_clone_bio_tio_3_15 + else: + getter = cls._get_clone_bio_tio_old + cls._get_clone_bio_tio = getter + + def interpret(self): + self.tio = cls._get_clone_bio_tio(bio) + self.next_bio = tio['io']['bio'] + + def __str__(self): + return self.description.format( + int(self.bio), + block_device_name(bself.io['bi_bdev']), + int(bself.io['bi_sector']), + block_device_name(self.next_bio['bi_bdev']), + int(self.next_bio['bi_sector'])) + + def __next__(self): + return decode_bio(self.next_bio) @classmethod def _get_clone_bio_tio_old(cls, bio): @@ -133,3 +116,5 @@ def _get_clone_bio_tio_old(cls, bio): def _get_clone_bio_tio_3_15(cls, bio): return container_of(bio['bi_private'], cls.dm_clone_bio_info_p_type, 'clone') + +ClonedBioDecoder.register() From c27620be539e72d89ce24f262e2dea2a4a507113 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 6 May 2019 16:26:29 -0400 Subject: [PATCH 44/75] crash.types.vmstat: remove dead code The versions of the static methods that build the num name lists are redundant since the callbacks perform that work. Also, the static methods are using a version of the API that doesn't exist anymore. Signed-off-by: Jeff Mahoney --- crash/types/vmstat.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/crash/types/vmstat.py b/crash/types/vmstat.py index 139c3237a26..6d2cc77fd29 100644 --- a/crash/types/vmstat.py +++ b/crash/types/vmstat.py @@ -10,12 +10,13 @@ class VmStat(CrashBaseClass): __types__ = ['enum zone_stat_item', 'enum vm_event_item'] + __symbols__ = [ 'vm_event_states' ] __type_callbacks__ = [ ('enum zone_stat_item', 'check_enum_type'), ('enum vm_event_item', 'check_enum_type') ] nr_stat_items = None nr_event_items = None - + vm_stat_names = None vm_event_names = None @@ -40,34 +41,26 @@ def __populate_names(cls, enum_type, items_name): for field in enum_type.fields(): if field.enumval < nr_items: - names[field.enumval] = field.name - + names[field.enumval] = field.name + return (nr_items, names) @staticmethod def get_stat_names(): - if VmStat.vm_stat_names is None: - VmStat.vm_stat_names = VmStat.__populate_names( - VmStat.nr_stat_items, "enum zone_stat_item") return VmStat.vm_stat_names @staticmethod def get_event_names(): - if VmStat.vm_event_names is None: - VmStat.vm_event_names = VmStat.__populate_names( - VmStat.nr_event_items, "enum vm_event_item") return VmStat.vm_event_names - @staticmethod + @classmethod def get_events(): - states_sym = gdb.lookup_global_symbol("vm_event_states") nr = VmStat.nr_event_items events = [0] * nr for cpu in for_each_online_cpu(): - states = get_percpu_var(states_sym, cpu) + states = get_percpu_var(cls.vm_event_states, cpu) for item in range(0, nr): events[item] += int(states["event"][item]) return events - From fcb22c423780a8a898db63ddfe69e6184e76909a Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 3 May 2019 15:17:13 -0400 Subject: [PATCH 45/75] crash: remove CrashBaseClass This commit removes CrashBaseClass from the project entirely. This simplifies the code immensely and means contributors don't need to understand and/or potentially debug the black magic behind the namespace, implicit singleton, exporting, and type/symbol loading in order to get started. The pattern now is to use crash.types.symbols to declare a Types, Symbols, Symvals, etc object at the top of the module that contains any needed types or symbols. These are still declared as an array of names and the naming structure (e.g. struct_name_type) is the same as with CrashBaseClass. They just appear in separate collections under separate names. Most functions will be at the module level and will reference these objects directly. If a state object is required, a class is still used but it can inherit from `object' instead. If callbacks are used on class attributes (instead of instance attributes) they must be declared outside of the class. This allows us to eliminate many classes entirely and reduce others to simple state holder classes. In some cases, like with percpu, it makes sense to just use module instance and marshall the calls from the module level to the instance explicitly. Signed-off-by: Jeff Mahoney --- crash/addrxlat.py | 19 +- crash/cache/__init__.py | 4 +- crash/cache/syscache.py | 51 +- crash/commands/__init__.py | 4 +- crash/commands/dmesg.py | 37 +- crash/infra/__init__.py | 98 --- crash/infra/lookup.py | 109 --- crash/kernel.py | 21 +- crash/subsystem/filesystem/__init__.py | 154 ++--- crash/subsystem/filesystem/btrfs.py | 54 +- crash/subsystem/filesystem/ext3.py | 2 - crash/subsystem/filesystem/mount.py | 229 +++---- crash/subsystem/storage/__init__.py | 562 ++++++++------- crash/subsystem/storage/blocksq.py | 82 ++- crash/subsystem/storage/decoders.py | 3 +- crash/subsystem/storage/device_mapper.py | 18 +- crash/types/bitmap.py | 540 +++++++-------- crash/types/classdev.py | 17 +- crash/types/cpu.py | 108 +-- crash/types/klist.py | 51 +- crash/types/list.py | 217 +++--- crash/types/module.py | 85 ++- crash/types/node.py | 74 +- crash/types/page.py | 88 ++- crash/types/percpu.py | 191 ++++-- crash/types/slab.py | 154 ++--- crash/types/task.py | 44 +- crash/types/vmstat.py | 44 +- crash/types/zone.py | 34 +- crash/util/__init__.py | 825 +++++++++++------------ tests/test_infra.py | 87 --- tests/test_infra_lookup.py | 362 +--------- tests/test_syscache.py | 2 +- 33 files changed, 1814 insertions(+), 2556 deletions(-) delete mode 100644 tests/test_infra.py diff --git a/crash/addrxlat.py b/crash/addrxlat.py index 5d00e75db27..03e24658f91 100644 --- a/crash/addrxlat.py +++ b/crash/addrxlat.py @@ -3,7 +3,6 @@ import gdb import addrxlat -from crash.infra import CrashBaseClass, export from crash.cache.syscache import utsname from crash.util import offsetof @@ -39,7 +38,7 @@ def cb_read32(self, faddr): def cb_read64(self, faddr): return int(gdb.Value(faddr.addr).cast(self.uint64_ptr).dereference()) -class CrashAddressTranslation(CrashBaseClass): +class CrashAddressTranslation(object): def __init__(self): try: target = gdb.current_target() @@ -62,14 +61,12 @@ def __init__(self): self.is_non_auto = True break - @export - def addrxlat_context(self): - return self.context +__impl = CrashAddressTranslation() +def addrxlat_context(): + return __impl.context - @export - def addrxlat_system(self): - return self.system +def addrxlat_system(): + return __impl.system - @export - def addrxlat_is_non_auto(self): - return self.is_non_auto +def addrxlat_is_non_auto(): + return __impl.is_non_auto diff --git a/crash/cache/__init__.py b/crash/cache/__init__.py index 502299d3ebf..2afb4a5aa6e 100644 --- a/crash/cache/__init__.py +++ b/crash/cache/__init__.py @@ -7,9 +7,9 @@ import glob import importlib -from crash.infra import CrashBaseClass, autoload_submodules +from crash.infra import autoload_submodules -class CrashCache(CrashBaseClass): +class CrashCache(object): def refresh(self): pass diff --git a/crash/cache/syscache.py b/crash/cache/syscache.py index ff875a1d5e1..b5d9a666115 100644 --- a/crash/cache/syscache.py +++ b/crash/cache/syscache.py @@ -11,14 +11,14 @@ from crash.exceptions import DelayedAttributeError from crash.cache import CrashCache from crash.util import array_size -from crash.infra import export -from crash.infra.lookup import get_delayed_lookup +from crash.util.symbols import Types, Symvals, SymbolCallbacks +from crash.infra.lookup import DelayedValue class CrashUtsnameCache(CrashCache): - __symvals__ = [ 'init_uts_ns' ] + symvals = Symvals([ 'init_uts_ns' ]) def load_utsname(self): - self.utsname = self.init_uts_ns['name'] + self.utsname = self.symvals.init_uts_ns['name'] return self.utsname def init_utsname_cache(self): @@ -43,8 +43,8 @@ def __getattr__(self, name): return getattr(self.__class__, name) class CrashConfigCache(CrashCache): - __types__ = [ 'char *' ] - __symvals__ = [ 'kernel_config_data' ] + types = Types([ 'char *' ]) + symvals = Symvals([ 'kernel_config_data' ]) def __getattr__(self, name): if name == 'config_buffer': @@ -70,8 +70,8 @@ def decompress_config_buffer(self): MAGIC_END = 'IKCFG_ED' # Must cast it to char * to do the pointer arithmetic correctly - data_addr = self.kernel_config_data.address.cast(self.char_p_type) - data_len = self.kernel_config_data.type.sizeof + data_addr = self.symvals.kernel_config_data.address.cast(self.types.char_p_type) + data_len = self.symvals.kernel_config_data.type.sizeof buf_len = len(MAGIC_START) buf = self.read_buf_str(data_addr, buf_len) @@ -119,14 +119,18 @@ def __getitem__(self, name): return None class CrashKernelCache(CrashCache): - __symvals__ = [ 'avenrun' ] - __symbol_callbacks__ = [ - ( 'jiffies', 'setup_jiffies' ), - ( 'jiffies_64', 'setup_jiffies' ) ] - __delayed_values__ = [ 'jiffies' ] + symvals = Symvals([ 'avenrun' ]) jiffies_ready = False adjust_jiffies = False + + jiffies_dv = DelayedValue('jiffies') + + @property + def jiffies(self): + v = self.jiffies_dv.get() + return v + def __init__(self, config): CrashCache.__init__(self) self.config = config @@ -157,8 +161,8 @@ def format_loadavg(metrics): def get_loadavg_values(self): metrics = [] - for index in range(0, array_size(self.avenrun)): - metrics.append(self.calculate_loadavg(self.avenrun[index])) + for index in range(0, array_size(self.symvals.avenrun)): + metrics.append(self.calculate_loadavg(self.symvals.avenrun[index])) return metrics @@ -170,6 +174,11 @@ def get_loadavg(self): except DelayedAttributeError: return "Unknown" + @classmethod + def set_jiffies(cls, value): + cls.jiffies_dv.value = None + cls.jiffies_dv.callback(value) + @classmethod def setup_jiffies(cls, symbol): if cls.jiffies_ready: @@ -187,7 +196,7 @@ def setup_jiffies(cls, symbol): jiffies = int(gdb.lookup_global_symbol('jiffies').value()) cls.adjust_jiffies = False - delayed = get_delayed_lookup(cls, 'jiffies').callback(jiffies) + cls.set_jiffies(jiffies) def adjusted_jiffies(self): if self.adjust_jiffies: @@ -199,10 +208,14 @@ def get_uptime(self): self.uptime = timedelta(seconds=self.adjusted_jiffies() // self.hz) return self.uptime - @export - def jiffies_to_msec(self, jiffies): - return 1000 // self.hz * jiffies +symbol_cbs = SymbolCallbacks( [( 'jiffies', + CrashKernelCache.setup_jiffies ), + ( 'jiffies_64', + CrashKernelCache.setup_jiffies ) ]) utsname = CrashUtsnameCache() config = CrashConfigCache() kernel = CrashKernelCache(config) + +def jiffies_to_msec(jiffies): + return 1000 // kernel.hz * jiffies diff --git a/crash/commands/__init__.py b/crash/commands/__init__.py index 7b1772196c0..fac1b28a82b 100644 --- a/crash/commands/__init__.py +++ b/crash/commands/__init__.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: -from crash.infra import CrashBaseClass - import gdb import os @@ -20,7 +18,7 @@ class ArgumentParser(argparse.ArgumentParser): def error(self, message): raise CommandLineError(message) -class Command(CrashBaseClass, gdb.Command): +class Command(gdb.Command): commands = {} def __init__(self, name, parser=None): self.name = "py" + name diff --git a/crash/commands/dmesg.py b/crash/commands/dmesg.py index 19d38398d80..18a286cf96a 100644 --- a/crash/commands/dmesg.py +++ b/crash/commands/dmesg.py @@ -8,6 +8,11 @@ from crash.commands import Command, ArgumentParser from crash.exceptions import DelayedAttributeError +from crash.util.symbols import Types, Symvals + +types = Types([ 'struct printk_log *' , 'char *' ]) +symvals = Symvals([ 'log_buf', 'log_buf_len', 'log_first_idx', 'log_next_idx', + 'clear_seq', 'log_first_seq', 'log_next_seq' ]) class LogTypeException(Exception): pass @@ -148,10 +153,6 @@ def __init__(self, name): parser.format_usage = lambda: 'log [-tdm]\n' Command.__init__(self, name, parser) - __types__ = [ 'struct printk_log *' , 'char *' ] - __symvals__ = [ 'log_buf', 'log_buf_len', 'log_first_idx', 'log_next_idx', - 'clear_seq', 'log_first_seq', 'log_next_seq' ] - @classmethod def filter_unstructured_log(cls, log, args): lines = log.split('\n') @@ -168,11 +169,11 @@ def filter_unstructured_log(cls, log, args): return '\n'.join(lines) def log_from_idx(self, logbuf, idx, dict_needed=False): - msg = (logbuf + idx).cast(self.printk_log_p_type) + msg = (logbuf + idx).cast(types.printk_log_p_type) try: - textval = (msg.cast(self.char_p_type) + - self.printk_log_p_type.target().sizeof) + textval = (msg.cast(types.char_p_type) + + types.printk_log_p_type.target().sizeof) text = textval.string(length=int(msg['text_len'])) except UnicodeDecodeError as e: print(e) @@ -197,8 +198,8 @@ def log_from_idx(self, logbuf, idx, dict_needed=False): if dict_needed: dict_len = int(msg['dict_len']) - d = (msg.cast(self.char_p_type) + - self.printk_log_p_type.target().sizeof + textlen) + d = (msg.cast(types.char_p_type) + + types.printk_log_p_type.target().sizeof + textlen) s = '' for i in range(0, dict_len): @@ -214,19 +215,19 @@ def log_from_idx(self, logbuf, idx, dict_needed=False): def get_log_msgs(self, dict_needed=False): try: - idx = self.log_first_idx + idx = symvals.log_first_idx except DelayedAttributeError as e: raise LogTypeException('not structured log') - if self.clear_seq < self.log_first_seq: - self.clear_seq = self.log_first_seq + if symvals.clear_seq < symvals.log_first_seq: + symvals.clear_seq = symvals.log_first_seq - seq = self.clear_seq - idx = self.log_first_idx + seq = symvals.clear_seq + idx = symvals.log_first_idx - while seq < self.log_next_seq: - msg = self.log_from_idx(self.log_buf, idx, dict_needed) + while seq < symvals.log_next_seq: + msg = self.log_from_idx(symvals.log_buf, idx, dict_needed) seq += 1 idx = msg['next'] yield msg @@ -250,11 +251,11 @@ def handle_structured_log(self, args): print('{}'.format(d.encode('string_escape'))) def handle_logbuf(self, args): - if self.log_buf_len and self.log_buf: + if symvals.log_buf_len and symvals.log_buf: if args.d: raise LogInvalidOption("Unstructured logs don't offer key/value pair support") - print(self.filter_unstructured_log(self.log_buf.string('utf-8', 'replace'), args)) + print(self.filter_unstructured_log(symvals.log_buf.string('utf-8', 'replace'), args)) def execute(self, args): try: diff --git a/crash/infra/__init__.py b/crash/infra/__init__.py index 7a92c877039..5d53aa0888b 100644 --- a/crash/infra/__init__.py +++ b/crash/infra/__init__.py @@ -4,106 +4,8 @@ import sys import glob import os.path -import inspect import importlib -from crash.infra.lookup import DelayedLookups - -class export_wrapper(object): - def __init__(self, mod, cls, func): - self.cls = cls - self.func = func - - if not hasattr(mod, '_export_wrapper_singleton_dict'): - mod._export_wrapper_singleton_dict = {} - self.singleton_dict = mod._export_wrapper_singleton_dict - - def __call__(self, *args, **kwargs): - try: - obj = self.singleton_dict[self.cls] - except KeyError: - obj = self.cls() - self.singleton_dict[self.cls] = obj - - if isinstance(self.func, classmethod): - return self.func.__func__(self.cls, *args, **kwargs) - elif isinstance(self.func, staticmethod): - return self.func.__func__(*args, **kwargs) - else: - return self.func(obj, *args, **kwargs) - -def register_singleton(mod, obj): - if not hasattr(mod, '_export_wrapper_singleton_dict'): - raise RuntimeError("Class {} has no exported members." - .format(obj.__class__.__name__)) - - mod._export_wrapper_singleton_dict[obj.__class__] = obj - -def export(func): - """This marks the function for export to the module namespace. - The class must inherit from CrashBaseClass.""" - if isinstance(func, staticmethod) or isinstance(func, classmethod): - func.__func__.__export_to_module__ = True - else: - func.__export_to_module__ = True - return func - -class _CrashBaseMeta(type): - """ - This metaclass handles both exporting methods to the module namespace - and handling asynchronous loading of types and symbols. To enable it, - all you need to do is define your class as follows: - - class Foo(CrashBaseClass): - ... - - There are several special class variables that are interpreted during - class (not instance) creation. - - The following create properties in the class that initially - raise MissingSymbolError but contain the requested information when - made available. The properties for types will be the name of the type, - with 'struct ' removed and _type appended. E.g. 'struct test' becomes - test_type. If it's a pointer type, _p is appended after the type name, - e.g. 'struct test *' becomes test_p_type. The properties for the symbols - are named with the symbol name. If there is a naming collision, - NameError is raised. - __types__ -- A list consisting of type names. Pointer are handled in - Pointer are handled in a manner similarly to how - they are handled in C code. e.g. 'char *'. - __symbols__ -- A list of symbol names - __minsymbols__ -- A list of minimal symbols - __symvals__ -- A list of symbol names that will return the value - associated with the symbol instead of the symbol itself. - - The following set up callbacks when the requested type or symbol value - is available. These each accept a list of 2-tuples, (specifier, callback). - The callback is passed the type or symbol requested. - __type_callbacks__ - __symbol_callbacks__ - """ - def __new__(cls, name, parents, dct): - DelayedLookups.setup_delayed_lookups_for_class(name, dct) - return type.__new__(cls, name, parents, dct) - - def __init__(cls, name, parents, dct): - super(_CrashBaseMeta, cls).__init__(name, parents, dct) - cls.setup_exports_for_class(cls, dct) - DelayedLookups.setup_named_callbacks(cls, dct) - - @staticmethod - def setup_exports_for_class(cls, dct): - mod = sys.modules[dct['__module__']] - for name, decl in dct.items(): - if (hasattr(decl, '__export_to_module__') or - ((isinstance(decl, classmethod) or - isinstance(decl, staticmethod)) and - hasattr(decl.__func__, "__export_to_module__"))): - setattr(mod, name, export_wrapper(mod, cls, decl)) - -class CrashBaseClass(metaclass=_CrashBaseMeta): - pass - def autoload_submodules(caller, callback=None): mods = [] try: diff --git a/crash/infra/lookup.py b/crash/infra/lookup.py index 4b086f12548..fd3b582fdd9 100644 --- a/crash/infra/lookup.py +++ b/crash/infra/lookup.py @@ -232,112 +232,3 @@ def callback(self, value): def __str__(self): return "{} attached with {}".format(self.__class__, str(self.cb)) - -class ClassProperty(object): - def __init__(self, get): - self.get = get - - def __get__(self, instance, owner): - return self.get() - -class DelayedLookups(object): - """ - A class for handling dynamic creation of class attributes that - contain delayed values. The attributes are specified using - special names. These are documented in the _CrashBaseMeta - documentation. - """ - @classmethod - def name_check(cls, dct, name, attrname): - try: - collision = dct['__delayed_lookups__'][attrname] - except KeyError: - return - - raise NameError("DelayedLookup name collision: `{}' and `{}' -> `{}'" - .format(name, collision.name, attrname)) - - @classmethod - def add_lookup(cls, clsname, dct, name, attr, attrname=None): - if attrname is None: - attrname = name - cls.name_check(dct, name, attrname) - dct['__delayed_lookups__'][attrname] = attr - if attrname.startswith('__'): - attrname = '_{}{}'.format(clsname, attrname) - dct[attrname] = ClassProperty(attr.get) - - @classmethod - def setup_delayed_lookups_for_class(cls, clsname, dct): - if '__delayed_lookups__' in dct: - raise NameError("Name `delayed_lookups' is reserved when using DelayedLookups") - dct['__delayed_lookups__'] = {} - - if '__types__' in dct: - if not isinstance(dct['__types__'], list): - raise TypeError('__types__ attribute must be a list of strings') - for typ in dct['__types__']: - t = DelayedType(typ) - cls.add_lookup(clsname, dct, t.name, t, t.attrname) - del dct['__types__'] - if '__symbols__' in dct: - if not isinstance(dct['__symbols__'], list): - raise TypeError('__symbols__ attribute must be a list of strings') - for symname in dct['__symbols__']: - cls.add_lookup(clsname, dct, symname, DelayedSymbol(symname)) - del dct['__symbols__'] - if '__minsymbols__' in dct: - if not isinstance(dct['__minsymbols__'], list): - raise TypeError('__minsymbols_ attribute must be a list of strings') - for symname in dct['__minsymbols__']: - cls.add_lookup(clsname, dct, symname, - DelayedMinimalSymbol(symname)) - del dct['__minsymbols__'] - if '__symvals__' in dct: - if not isinstance(dct['__symvals__'], list): - raise TypeError('__symvals__ attribute must be a list of strings') - for symname in dct['__symvals__']: - cls.add_lookup(clsname, dct, symname, DelayedSymval(symname)) - del dct['__symvals__'] - - if '__minsymvals__' in dct: - if not isinstance(dct['__minsymvals__'], list): - raise TypeError('__minsymvals__ attribute must be a list of strings') - for symname in dct['__minsymvals__']: - cls.add_lookup(clsname, dct, symname, - DelayedMinimalSymval(symname)) - del dct['__minsymvals__'] - - if '__delayed_values__' in dct: - if not isinstance(dct['__delayed_values__'], list): - raise TypeError('__delayed_values__ attribute must be a list of strings') - for propname in dct['__delayed_values__']: - cls.add_lookup(clsname, dct, propname, DelayedValue(propname)) - del dct['__delayed_values__'] - - @classmethod - def setup_named_callbacks(this_cls, cls, dct): - callbacks = [] - if '__type_callbacks__' in dct: - for (typ, callback) in dct['__type_callbacks__']: - cb = getattr(cls, callback) - callbacks.append(TypeCallback(typ, cb)) - del dct['__type_callbacks__'] - - if '__symbol_callbacks__' in dct: - for (sym, callback) in dct['__symbol_callbacks__']: - cb = getattr(cls, callback) - callbacks.append(SymbolCallback(sym, cb)) - del dct['__symbol_callbacks__'] - if '__minsymbol_callbacks__' in dct: - for (sym, callback) in dct['__minsymbol_callbacks__']: - cb = getattr(cls, callback) - callbacks.append(MinimalSymbolCallback(sym, cb)) - del dct['__minsymbol_callbacks__'] - if callbacks: - dct['__delayed_lookups__']['__callbacks__'] = callbacks - -def get_delayed_lookup(cls, name): - return cls.__delayed_lookups__[name] - - diff --git a/crash/kernel.py b/crash/kernel.py index d43bf3cae08..c8f410400d0 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -9,7 +9,6 @@ import crash.arch import crash.arch.x86_64 import crash.arch.ppc64 -from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry from crash.types.percpu import get_percpu_vars from crash.types.list import list_for_each_entry @@ -19,6 +18,7 @@ from crash.types.task import LinuxTask from elftools.elf.elffile import ELFFile from crash.util import get_symbol_value +from crash.util.symbols import Types, Symvals, Symbols from typing import Pattern, Union, List, Dict, Any @@ -55,11 +55,10 @@ def __init__(self, path, module_value, expected_value): PathSpecifier = Union[List[str], str] -class CrashKernel(CrashBaseClass): - __types__ = [ 'char *' ] - __symvals__ = [ 'init_task' ] - __symbols__ = [ 'runqueues'] - +class CrashKernel(object): + types = Types([ 'char *' ]) + symvals = Symvals([ 'init_task' ]) + symbols = Symbols([ 'runqueues']) def __init__(self, roots: PathSpecifier=None, vmlinux_debuginfo: PathSpecifier=None, @@ -311,7 +310,7 @@ def __init__(self, roots: PathSpecifier=None, def get_minsymbol_as_string(self, name: str) -> str: sym = gdb.lookup_minimal_symbol(name).value() - return sym.address.cast(self.char_p_type).string() + return sym.address.cast(self.types.char_p_type).string() def extract_version(self) -> str: try: @@ -601,9 +600,9 @@ def load_module_debuginfo(self, objfile: gdb.Objfile, def setup_tasks(self) -> None: gdb.execute('set print thread-events 0') - task_list = self.init_task['tasks'] + task_list = self.symvals.init_task['tasks'] - rqs = get_percpu_vars(self.runqueues) + rqs = get_percpu_vars(self.symbols.runqueues) rqscurrs = {int(x["curr"]) : k for (k, x) in rqs.items()} print("Loading tasks...", end='') @@ -611,11 +610,11 @@ def setup_tasks(self) -> None: task_count = 0 tasks = [] - for taskg in list_for_each_entry(task_list, self.init_task.type, + for taskg in list_for_each_entry(task_list, self.symvals.init_task.type, 'tasks', include_head=True): tasks.append(taskg) for task in list_for_each_entry(taskg['thread_group'], - self.init_task.type, + self.symvals.init_task.type, 'thread_group'): tasks.append(task) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index 32b74dec738..ee2f2dbf8bb 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -5,10 +5,13 @@ import gdb from crash.util import container_of, get_typed_pointer, decode_flags -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Types, Symvals +from crash.infra.lookup import DelayedSymval, DelayedType from crash.types.list import list_for_each_entry from crash.subsystem.storage import block_device_name -from crash.subsystem.storage import Storage as block + +types = Types('struct super_block') +symvals = Symvals('super_blocks') AddressSpecifier = Union[int, str, gdb.Value] @@ -73,84 +76,69 @@ MS_NOUSER : "MS_NOUSER", } -class FileSystem(CrashBaseClass): - __types__ = [ 'struct dio *', - 'struct buffer_head *', - 'struct super_block' ] - __symvals__ = [ 'super_blocks' ] - @export - @staticmethod - def super_fstype(sb: gdb.Value) -> str: - """ - Returns the file system type's name for a given superblock. - - Args: - sb (gdb.Value): The struct super_block for - which to return the file system type's name - - Returns: - str: The file system type's name - """ - return sb['s_type']['name'].string() - - @export - @staticmethod - def super_flags(sb: gdb.Value) -> str: - """ - Returns the flags associated with the given superblock. - - Args: - sb (gdb.Value): The struct super_block for - which to return the flags. - - Returns: - str: The flags field in human-readable form. - - """ - return decode_flags(sb['s_flags'], SB_FLAGS) - - @export - @classmethod - def for_each_super_block(cls) -> Iterable[gdb.Value]: - """ - Iterate over the list of super blocks and yield each one. - - Args: - None - - Yields: - gdb.Value - """ - for sb in list_for_each_entry(cls.super_blocks, cls.super_block_type, - 's_list'): - yield sb - - @export - @classmethod - def get_super_block(cls, desc: AddressSpecifier, - force: bool=False) -> gdb.Value: - """ - Given an address description return a gdb.Value that contains - a struct super_block at that address. - - Args: - desc (gdb.Value, str, or int): The address for which to provide - a casted pointer - force (bool): Skip testing whether the value is available. - - Returns: - gdb.Value: The super_block at the requested - location - - Raises: - gdb.NotAvailableError: The target value was not available. - """ - sb = get_typed_pointer(desc, cls.super_block_type).dereference() - if not force: - try: - x = int(sb['s_dev']) - except gdb.NotAvailableError: - raise gdb.NotAvailableError(f"no superblock available at `{desc}'") - return sb - -inst = FileSystem() +def super_fstype(sb: gdb.Value) -> str: + """ + Returns the file system type's name for a given superblock. + + Args: + sb (gdb.Value): The struct super_block for + which to return the file system type's name + + Returns: + str: The file system type's name + """ + return sb['s_type']['name'].string() + +def super_flags(sb: gdb.Value) -> str: + """ + Returns the flags associated with the given superblock. + + Args: + sb (gdb.Value): The struct super_block for + which to return the flags. + + Returns: + str: The flags field in human-readable form. + + """ + return decode_flags(sb['s_flags'], SB_FLAGS) + +def for_each_super_block() -> Iterable[gdb.Value]: + """ + Iterate over the list of super blocks and yield each one. + + Args: + None + + Yields: + gdb.Value + """ + for sb in list_for_each_entry(symvals.super_blocks, + types.super_block_type, 's_list'): + yield sb + +def get_super_block(desc: AddressSpecifier, force: bool=False) -> gdb.Value: + """ + Given an address description return a gdb.Value that contains + a struct super_block at that address. + + Args: + desc (gdb.Value, str, or int): The address for which to provide + a casted pointer + force (bool): Skip testing whether the value is available. + + Returns: + gdb.Value: The super_block at the requested + location + + Raises: + gdb.NotAvailableError: The target value was not available. + """ + sb = get_typed_pointer(desc, types.super_block_type).dereference() + if not force: + try: + x = int(sb['s_dev']) + except gdb.NotAvailableError: + raise gdb.NotAvailableError(f"no superblock available at `{desc}'") + + return sb diff --git a/crash/subsystem/filesystem/btrfs.py b/crash/subsystem/filesystem/btrfs.py index 1515f00862b..d42790dee39 100644 --- a/crash/subsystem/filesystem/btrfs.py +++ b/crash/subsystem/filesystem/btrfs.py @@ -3,40 +3,38 @@ import gdb -from crash.infra import CrashBaseClass +from crash.util.symbols import Types -class BtrfsFileSystem(CrashBaseClass): - __types__ = [ 'struct btrfs_inode', 'struct btrfs_fs_info *' ] +types = Types([ 'struct btrfs_inode', 'struct btrfs_fs_info *', + 'struct btrfs_fs_info' ]) - @classmethod - def btrfs_inode(cls, vfs_inode): - """ - Converts a VFS inode to a btrfs inode +def btrfs_inode(vfs_inode): + """ + Converts a VFS inode to a btrfs inode - This method converts a struct inode to a struct btrfs_inode. + This method converts a struct inode to a struct btrfs_inode. - Args: - vfs_inode (gdb.Value): The struct inode to convert - to a struct btrfs_inode + Args: + vfs_inode (gdb.Value): The struct inode to convert + to a struct btrfs_inode - Returns: - gdb.Value: The converted struct btrfs_inode - """ - return container_of(vfs_inode, cls.btrfs_inode_type, 'vfs_inode') + Returns: + gdb.Value: The converted struct btrfs_inode + """ + return container_of(vfs_inode, types.btrfs_inode_type, 'vfs_inode') - @classmethod - def btrfs_sb_info(cls, super_block): - """ - Converts a VFS superblock to a btrfs fs_info +def btrfs_fs_info(super_block): + """ + Converts a VFS superblock to a btrfs fs_info - This method converts a struct super_block to a struct btrfs_fs_info + This method converts a struct super_block to a struct btrfs_fs_info - Args: - super_block (gdb.Value): The struct super_block - to convert to a struct btrfs_fs_info. + Args: + super_block (gdb.Value): The struct super_block + to convert to a struct btrfs_fs_info. - Returns: - gdb.Value: The converted struct - btrfs_fs_info - """ - return super_block['s_fs_info'].cast(cls.btrfs_fs_info_p_type) + Returns: + gdb.Value: The converted struct + btrfs_fs_info + """ + return super_block['s_fs_info'].cast(types.btrfs_fs_info_p_type) diff --git a/crash/subsystem/filesystem/ext3.py b/crash/subsystem/filesystem/ext3.py index 320804d6228..cdad6cf5a90 100644 --- a/crash/subsystem/filesystem/ext3.py +++ b/crash/subsystem/filesystem/ext3.py @@ -3,8 +3,6 @@ import gdb -from crash.infra import CrashBaseClass -from crash.util import get_symbol_value from crash.subsystem.storage.decoders import Decoder class Ext3Decoder(Decoder): diff --git a/crash/subsystem/filesystem/mount.py b/crash/subsystem/filesystem/mount.py index 2407dceb1dd..44ebda4d080 100644 --- a/crash/subsystem/filesystem/mount.py +++ b/crash/subsystem/filesystem/mount.py @@ -3,10 +3,10 @@ import gdb -from crash.infra import CrashBaseClass, export from crash.subsystem.filesystem import super_fstype from crash.types.list import list_for_each_entry from crash.util import container_of, decode_flags, struct_has_member +from crash.util.symbols import Types, Symvals, TypeCallbacks, SymbolCallbacks MNT_NOSUID = 0x01 MNT_NODEV = 0x02 @@ -38,139 +38,124 @@ } MNT_FLAGS_HIDDEN.update(MNT_FLAGS) +types = Types([ 'struct mount', 'struct vfsmount' ]) +symvals = Symvals([ 'init_task' ]) -class Mount(CrashBaseClass): - __types__ = [ 'struct mount', 'struct vfsmount' ] - __symvals__ = [ 'init_task' ] - __type_callbacks__ = [ ('struct vfsmount', 'check_mount_type' ) ] - __symbol_callbacks__ = [ ('init_task', 'check_task_interface' ) ] - +class Mount(object): @classmethod def for_each_mount_impl(cls, task): raise NotImplementedError("Mount.for_each_mount is unhandled on this kernel version.") @classmethod - def check_mount_type(cls, gdbtype): - try: - cls.mount_type = gdb.lookup_type('struct mount') - except gdb.error: - # Older kernels didn't separate mount from vfsmount - cls.mount_type = cls.vfsmount_type + def for_each_mount_nsproxy(cls, task): + return list_for_each_entry(task['nsproxy']['mnt_ns']['list'], + types.mount_type, 'mnt_list') @classmethod def check_task_interface(cls, symval): try: - nsproxy = cls.init_task['nsproxy'] + nsproxy = symvals.init_task['nsproxy'] cls.for_each_mount_impl = cls.for_each_mount_nsproxy except KeyError: print("check_task_interface called but no init_task?") pass - @export - def for_each_mount(self, task=None): - if task is None: - task = self.init_task - return self.for_each_mount_impl(task) - - def for_each_mount_nsproxy(self, task): - return list_for_each_entry(task['nsproxy']['mnt_ns']['list'], - self.mount_type, 'mnt_list') - - @export - @classmethod - def real_mount(cls, vfsmnt): - if (vfsmnt.type == cls.mount_type or - vfsmnt.type == cls.mount_type.pointer()): - t = vfsmnt.type - if t.code == gdb.TYPE_CODE_PTR: - t = t.target() - if t is not cls.mount_type: - cls.mount_type = t - return vfsmnt - return container_of(vfsmnt, cls.mount_type, 'mnt') - - @export - @classmethod - def mount_flags(cls, mnt, show_hidden=False): - if struct_has_member(mnt, 'mnt'): - mnt = mnt['mnt'] - if show_hidden: - return decode_flags(mnt['mnt_flags'], MNT_FLAGS_HIDDEN, ",") - return decode_flags(mnt['mnt_flags'], MNT_FLAGS, ",") - - @export - @staticmethod - def mount_super(mnt): - try: - sb = mnt['mnt']['mnt_sb'] - except gdb.error: - sb = mnt['mnt_sb'] - return sb - - @export - @staticmethod - def mount_root(mnt): - try: - mnt = mnt['mnt'] - except gdb.error: - pass - - return mnt['mnt_root'] - - @export - @classmethod - def mount_fstype(cls, mnt): - return super_fstype(cls.mount_super(mnt)) - - @export - @classmethod - def mount_device(cls, mnt): - devname = mnt['mnt_devname'].string() - if devname is None: - devname = "none" - return devname - - @export - @classmethod - def d_path(cls, mnt, dentry, root=None): - if root is None: - root = cls.init_task['fs']['root'] - - if dentry.type.code != gdb.TYPE_CODE_PTR: - dentry = dentry.address - - if mnt.type.code != gdb.TYPE_CODE_PTR: - mnt = mnt.address - - mount = cls.real_mount(mnt) - if mount.type.code != gdb.TYPE_CODE_PTR: - mount = mount.address - - try: - mnt = mnt['mnt'].address - except gdb.error: - pass - - name = "" - - # Gone are the days where finding the root was as simple as - # dentry == dentry->d_parent - while dentry != root['dentry'] or mnt != root['mnt']: - if dentry == mnt['mnt_root'] or dentry == dentry['d_parent']: - if dentry != mnt['mnt_root']: - return None - if mount != mount['mnt_parent']: - dentry = mount['mnt_mountpoint'] - mount = mount['mnt_parent'] - try: - mnt = mount['mnt'].address - except gdb.error: - mnt = mount - continue - break - - name = "/" + dentry['d_name']['name'].string() + name - dentry = dentry['d_parent'] - if not name: - name = '/' - return name +def check_mount_type(gdbtype): + try: + types.mount_type = gdb.lookup_type('struct mount') + except gdb.error: + # Older kernels didn't separate mount from vfsmount + types.mount_type = types.vfsmount_type + +def for_each_mount(task=None): + if task is None: + task = symvals.init_task + return Mount.for_each_mount_impl(task) + +def real_mount(vfsmnt): + if (vfsmnt.type == types.mount_type or + vfsmnt.type == types.mount_type.pointer()): + t = vfsmnt.type + if t.code == gdb.TYPE_CODE_PTR: + t = t.target() + if t is not types.mount_type: + types.mount_type = t + return vfsmnt + return container_of(vfsmnt, types.mount_type, 'mnt') + +def mount_flags(mnt, show_hidden=False): + if struct_has_member(mnt, 'mnt'): + mnt = mnt['mnt'] + if show_hidden: + return decode_flags(mnt['mnt_flags'], MNT_FLAGS_HIDDEN, ",") + return decode_flags(mnt['mnt_flags'], MNT_FLAGS, ",") + +def mount_super(mnt): + try: + sb = mnt['mnt']['mnt_sb'] + except gdb.error: + sb = mnt['mnt_sb'] + return sb + +def mount_root(mnt): + try: + mnt = mnt['mnt'] + except gdb.error: + pass + + return mnt['mnt_root'] + +def mount_fstype(mnt): + return super_fstype(mount_super(mnt)) + +def mount_device(mnt): + devname = mnt['mnt_devname'].string() + if devname is None: + devname = "none" + return devname + +def d_path(mnt, dentry, root=None): + if root is None: + root = symvals.init_task['fs']['root'] + + if dentry.type.code != gdb.TYPE_CODE_PTR: + dentry = dentry.address + + if mnt.type.code != gdb.TYPE_CODE_PTR: + mnt = mnt.address + + mount = real_mount(mnt) + if mount.type.code != gdb.TYPE_CODE_PTR: + mount = mount.address + + try: + mnt = mnt['mnt'].address + except gdb.error: + pass + + name = "" + + # Gone are the days where finding the root was as simple as + # dentry == dentry->d_parent + while dentry != root['dentry'] or mnt != root['mnt']: + if dentry == mnt['mnt_root'] or dentry == dentry['d_parent']: + if dentry != mnt['mnt_root']: + return None + if mount != mount['mnt_parent']: + dentry = mount['mnt_mountpoint'] + mount = mount['mnt_parent'] + try: + mnt = mount['mnt'].address + except gdb.error: + mnt = mount + continue + break + + name = "/" + dentry['d_name']['name'].string() + name + dentry = dentry['d_parent'] + if not name: + name = '/' + return name + +type_cbs = TypeCallbacks([ ('struct vfsmount', check_mount_type ) ]) +symbols_cbs = SymbolCallbacks([ ('init_task', Mount.check_task_interface ) ]) diff --git a/crash/subsystem/storage/__init__.py b/crash/subsystem/storage/__init__.py index 5228e1d290c..37cf821d8d9 100644 --- a/crash/subsystem/storage/__init__.py +++ b/crash/subsystem/storage/__init__.py @@ -4,302 +4,280 @@ from typing import Iterable import gdb +from gdb.types import get_basic_type from crash.util import container_of -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Types, Symvals, SymbolCallbacks, TypeCallbacks from crash.types.classdev import for_each_class_device from . import decoders import crash.exceptions -class Storage(CrashBaseClass): - __types__ = [ 'struct gendisk', - 'struct hd_struct', - 'struct device', - 'struct device_type', - 'struct bdev_inode' ] - __symvals__ = [ 'block_class', - 'blockdev_superblock', - 'disk_type', - 'part_type' ] - __symbol_callbacks = [ - ( 'disk_type', '_check_types' ), - ( 'part_type', '_check_types' ) ] - __type_callbacks__ = [ ('struct device_type', '_check_types' ) ] - - @classmethod - def _check_types(cls, result): - try: - if cls.part_type.type.unqualified() != cls.device_type_type: - raise TypeError("part_type expected to be {} not {}" - .format(cls.device_type_type, - cls.part_type.type)) - - if cls.disk_type.type.unqualified() != cls.device_type_type: - raise TypeError("disk_type expected to be {} not {}" - .format(cls.device_type_type, - cls.disk_type.type)) - cls.types_checked = True - except crash.exceptions.DelayedAttributeError: - pass - - @export - @classmethod - def for_each_bio_in_stack(cls, bio: gdb.Value) -> Iterable[decoders.Decoder]: - """ - Iterates and decodes each bio involved in a stacked storage environment - - This method will return a dictionary describing each object - in the storage stack, starting with the provided bio, as - processed by each level's decoder. The stack will be interrupted - if an encountered object doesn't have a decoder specified. - - See crash.subsystem.storage.decoder.register_decoder for more detail. - - Args: - bio (gdb.Value): The initial struct bio to start - decoding - - Yields: - dict : Contains, minimally, the following item. - - description (str): A human-readable description of the bio. - Additional items may be available based on the - implmentation-specific decoder. - """ - decoder = decoders.decode_bio(bio) - while decoder is not None: - yield decoder - decoder = next(decoder) - - @export - def dev_to_gendisk(self, dev): - """ - Converts a struct device that is embedded in a struct gendisk - back to the struct gendisk. - - Args: - dev (gdb.Value) : A struct device contained within - a struct gendisk. No checking is performed. Results - if other structures are provided are undefined. - - Returns: - gdb.Value : The converted struct hd_struct - """ - return container_of(dev, self.gendisk_type, 'part0.__dev') - - @export - def dev_to_part(self, dev): - """ - Converts a struct device that is embedded in a struct hd_struct - back to the struct hd_struct. - - Args: - dev (gdb.Value): A struct device embedded within a - struct hd_struct. No checking is performed. Results if other - structures are provided are undefined. - - Returns: - gdb.Value(struct hd_struct): The converted struct hd_struct - - """ - return container_of(dev, self.hd_struct_type, '__dev') - - @export - def gendisk_to_dev(self, gendisk): - """ - Converts a struct gendisk that embeds a struct device to - the struct device. - - Args: - dev (gdb.Value): A struct gendisk that embeds - a struct device. No checking is performed. Results - if other structures are provided are undefined. - - Returns: - gdb.Value: The converted struct device - """ - - return gendisk['part0']['__dev'].address - - @export - def part_to_dev(self, part): - """ - Converts a struct hd_struct that embeds a struct device to - the struct device. - - Args: - dev (gdb.Value): A struct hd_struct that embeds - a struct device. No checking is performed. Results if - other structures are provided are undefined. - - Returns: - gdb.Value: The converted struct device - """ - return part['__dev'].address - - @export - def for_each_block_device(self, subtype=None): - """ - Iterates over each block device registered with the block class. - - This method iterates over the block_class klist and yields every - member found. The members are either struct gendisk or - struct hd_struct, depending on whether it describes an entire - disk or a partition, respectively. - - The members can be filtered by providing a subtype, which - corresponds to a the the type field of the struct device. - - Args: - subtype (gdb.Value, optional): The struct - device_type that will be used to match and filter. Typically - 'disk_type' or 'device_type' - - Yields: - gdb.Value - A struct gendisk - or struct hd_struct that meets the filter criteria. - - Raises: - RuntimeError: An unknown device type was encountered during - iteration. - """ - - if subtype: - if subtype.type.unqualified() == self.device_type_type: - subtype = subtype.address - elif subtype.type.unqualified() != self.device_type_type.pointer(): - raise TypeError("subtype must be {} not {}" - .format(self.device_type_type.pointer(), - subtype.type.unqualified())) - for dev in for_each_class_device(self.block_class, subtype): - if dev['type'] == self.disk_type.address: - yield self.dev_to_gendisk(dev) - elif dev['type'] == self.part_type.address: - yield self.dev_to_part(dev) - else: - raise RuntimeError("Encountered unexpected device type {}" - .format(dev['type'])) - - @export - def for_each_disk(self): - """ - Iterates over each block device registered with the block class - that corresponds to an entire disk. - - This is an alias for for_each_block_device(disk_type) - """ - - return self.for_each_block_device(self.disk_type) - - @export - def gendisk_name(self, gendisk): - """ - Returns the name of the provided block device. - - This method evaluates the block device and returns the name, - including partition number, if applicable. - - Args: - gendisk(gdb.Value): - A struct gendisk or struct hd_struct for which to return - the name - - Returns: - str: the name of the block device - - Raises: - TypeError: gdb.Value does not describe a struct gendisk or - struct hd_struct - """ - if gendisk.type.code == gdb.TYPE_CODE_PTR: - gendisk = gendisk.dereference() - - if gendisk.type.unqualified() == self.gendisk_type: - return gendisk['disk_name'].string() - elif gendisk.type.unqualified() == self.hd_struct_type: - parent = self.dev_to_gendisk(self.part_to_dev(gendisk)['parent']) - return "{}{:d}".format(self.gendisk_name(parent), - int(gendisk['partno'])) +types = Types([ 'struct gendisk', 'struct hd_struct', 'struct device', + 'struct device_type', 'struct bdev_inode' ]) +symvals = Symvals([ 'block_class', 'blockdev_superblock', 'disk_type', + 'part_type' ]) + +def for_each_bio_in_stack(bio: gdb.Value) -> Iterable[decoders.Decoder]: + """ + Iterates and decodes each bio involved in a stacked storage environment + + This method will yield a Decoder object describing each level + in the storage stack, starting with the provided bio, as + processed by each level's decoder. The stack will be interrupted + if an encountered object doesn't have a decoder specified. + + See crash.subsystem.storage.decoders for more detail. + + Args: + bio (gdb.Value): The initial struct bio to start + decoding + + Yields: + Decoder + """ + decoder = decoders.decode_bio(bio) + while decoder is not None: + yield decoder + decoder = next(decoder) + +def dev_to_gendisk(dev): + """ + Converts a struct device that is embedded in a struct gendisk + back to the struct gendisk. + + Args: + dev (gdb.Value) : A struct device contained within + a struct gendisk. No checking is performed. Results + if other structures are provided are undefined. + + Returns: + gdb.Value : The converted struct hd_struct + """ + return container_of(dev, types.gendisk_type, 'part0.__dev') + +def dev_to_part(dev): + """ + Converts a struct device that is embedded in a struct hd_struct + back to the struct hd_struct. + + Args: + dev (gdb.Value): A struct device embedded within a + struct hd_struct. No checking is performed. Results if other + structures are provided are undefined. + + Returns: + gdb.Value: The converted struct hd_struct + + """ + return container_of(dev, types.hd_struct_type, '__dev') + +def gendisk_to_dev(gendisk): + """ + Converts a struct gendisk that embeds a struct device to + the struct device. + + Args: + dev (gdb.Value): A struct gendisk that embeds + a struct device. No checking is performed. Results + if other structures are provided are undefined. + + Returns: + gdb.Value: The converted struct device + """ + + return gendisk['part0']['__dev'].address + +def part_to_dev(part): + """ + Converts a struct hd_struct that embeds a struct device to + the struct device. + + Args: + dev (gdb.Value): A struct hd_struct that embeds + a struct device. No checking is performed. Results if + other structures are provided are undefined. + + Returns: + gdb.Value: The converted struct device + """ + return part['__dev'].address + + +def for_each_block_device(subtype: gdb.Value=None) -> Iterable[gdb.Value]: + """ + Iterates over each block device registered with the block class. + + This method iterates over the block_class klist and yields every + member found. The members are either struct gendisk or + struct hd_struct, depending on whether it describes an entire + disk or a partition, respectively. + + The members can be filtered by providing a subtype, which + corresponds to a the the type field of the struct device. + + Args: + subtype (gdb.Value, optional): The struct + device_type that will be used to match and filter. Typically + 'disk_type' or 'device_type' + + Yields: + gdb.Value or + gdb.Value: + A struct gendisk or struct hd_struct that meets + the filter criteria. + + Raises: + RuntimeError: An unknown device type was encountered during + iteration. + """ + + if subtype: + if get_basic_type(subtype.type) == types.device_type_type: + subtype = subtype.address + elif get_basic_type(subtype.type) != types.device_type_type.pointer(): + raise TypeError("subtype must be {} not {}" + .format(types.device_type_type.pointer(), + subtype.type.unqualified())) + for dev in for_each_class_device(symvals.block_class, subtype): + if dev['type'] == symvals.disk_type.address: + yield dev_to_gendisk(dev) + elif dev['type'] == symvals.part_type.address: + yield dev_to_part(dev) else: - raise TypeError("expected {} or {}, not {}" - .format(self.gendisk_type, self.hd_struct_type, - gendisk.type.unqualified())) - - @export - def block_device_name(self, bdev): - """ - Returns the name of the provided block device. - - This method evaluates the block device and returns the name, - including partition number, if applicable. - - Args: - bdev(gdb.Value): A struct block_device for - which to return the name - - Returns: - str: the name of the block device - """ - return self.gendisk_name(bdev['bd_disk']) - - @export - def is_bdev_inode(self, inode): - """ - Tests whether the provided struct inode describes a block device - - This method evaluates the inode and returns a True or False, - depending on whether the inode describes a block device. - - Args: - bdev(gdb.Value): The struct inode to test whether - it describes a block device. - - Returns: - bool: True if the inode describes a block device, False otherwise. - """ - return inode['i_sb'] == self.blockdev_superblock - - @export - def inode_to_block_device(self, inode): - """ - Returns the block device associated with this inode. - - If the inode describes a block device, return that block device. - Otherwise, raise TypeError. - - Args: - inode(gdb.Value): The struct inode for which to - return the associated block device - - Returns: - gdb.Value: The struct block_device associated - with the provided struct inode - - Raises: - TypeError: inode does not describe a block device - """ - if inode['i_sb'] != self.blockdev_superblock: - raise TypeError("inode does not correspond to block device") - return container_of(inode, self.bdev_inode_type, 'vfs_inode')['bdev'] - - @export - def inode_on_bdev(self, inode): - """ - Returns the block device associated with this inode. - - If the inode describes a block device, return that block device. - Otherwise, return the block device, if any, associated - with the inode's super block. - - Args: - inode(gdb.Value): The struct inode for which to - return the associated block device - - Returns: - gdb.Value: The struct block_device associated - with the provided struct inode - """ - if self.is_bdev_inode(inode): - return self.inode_to_block_device(inode) - else: - return inode['i_sb']['s_bdev'] -inst = Storage() + raise RuntimeError("Encountered unexpected device type {}" + .format(dev['type'])) + +def for_each_disk(): + """ + Iterates over each block device registered with the block class + that corresponds to an entire disk. + + This is an alias for for_each_block_device(disk_type) + """ + + return for_each_block_device(symvals.disk_type) + +def gendisk_name(gendisk): + """ + Returns the name of the provided block device. + + This method evaluates the block device and returns the name, + including partition number, if applicable. + + Args: + gendisk(gdb.Value): + A struct gendisk or struct hd_struct for which to return + the name + + Returns: + str: the name of the block device + + Raises: + TypeError: gdb.Value does not describe a struct gendisk or + struct hd_struct + """ + if gendisk.type.code == gdb.TYPE_CODE_PTR: + gendisk = gendisk.dereference() + + if get_basic_type(gendisk.type) == types.gendisk_type: + return gendisk['disk_name'].string() + elif get_basic_type(gendisk.type) == types.hd_struct_type: + parent = dev_to_gendisk(part_to_dev(gendisk)['parent']) + return "{}{:d}".format(gendisk_name(parent), int(gendisk['partno'])) + else: + raise TypeError("expected {} or {}, not {}" + .format(types.gendisk_type, types.hd_struct_type, + gendisk.type.unqualified())) + +def block_device_name(bdev): + """ + Returns the name of the provided block device. + + This method evaluates the block device and returns the name, + including partition number, if applicable. + + Args: + bdev(gdb.Value): A struct block_device for + which to return the name + + Returns: + str: the name of the block device + """ + return gendisk_name(bdev['bd_disk']) + +def is_bdev_inode(inode): + """ + Tests whether the provided struct inode describes a block device + + This method evaluates the inode and returns a True or False, + depending on whether the inode describes a block device. + + Args: + bdev(gdb.Value): The struct inode to test whether + it describes a block device. + + Returns: + bool: True if the inode describes a block device, False otherwise. + """ + return inode['i_sb'] == symvals.blockdev_superblock + +def inode_to_block_device(inode): + """ + Returns the block device associated with this inode. + + If the inode describes a block device, return that block device. + Otherwise, raise TypeError. + + Args: + inode(gdb.Value): The struct inode for which to + return the associated block device + + Returns: + gdb.Value: The struct block_device associated + with the provided struct inode + + Raises: + TypeError: inode does not describe a block device + """ + if inode['i_sb'] != symvals.blockdev_superblock: + raise TypeError("inode does not correspond to block device") + return container_of(inode, types.bdev_inode_type, 'vfs_inode')['bdev'] + +def inode_on_bdev(inode): + """ + Returns the block device associated with this inode. + + If the inode describes a block device, return that block device. + Otherwise, return the block device, if any, associated + with the inode's super block. + + Args: + inode(gdb.Value): The struct inode for which to + return the associated block device + + Returns: + gdb.Value: The struct block_device associated + with the provided struct inode + """ + if is_bdev_inode(inode): + return inode_to_block_device(inode) + else: + return inode['i_sb']['s_bdev'] + +def _check_types(result): + try: + if symvals.part_type.type.unqualified() != types.device_type_type: + raise TypeError("part_type expected to be {} not {}" + .format(symvals.device_type_type, + types.part_type.type)) + + if symvals.disk_type.type.unqualified() != types.device_type_type: + raise TypeError("disk_type expected to be {} not {}" + .format(symvals.device_type_type, + types.disk_type.type)) + except crash.exceptions.DelayedAttributeError: + pass + +symbol_cbs = SymbolCallbacks([ ( 'disk_type', _check_types ), + ( 'part_type', _check_types )] ) +type_cbs = TypeCallbacks([ ('struct device_type', _check_types ) ]) diff --git a/crash/subsystem/storage/blocksq.py b/crash/subsystem/storage/blocksq.py index b2b10d5a723..53533c0c7b0 100644 --- a/crash/subsystem/storage/blocksq.py +++ b/crash/subsystem/storage/blocksq.py @@ -3,52 +3,48 @@ import gdb -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Types from crash.types.list import list_for_each_entry from crash.cache.syscache import kernel class NoQueueError(RuntimeError): pass -class SingleQueueBlock(CrashBaseClass): - __types__ = [ 'struct request' ] - - @export - def for_each_request_in_queue(self, queue): - """ - Iterates over each struct request in request_queue - - This method iterates over the request_queue's queuelist and - returns a request for each member. - - Args: - queue(gdb.Value): The struct request_queue - used to iterate - - Yields: - gdb.Value: Each struct request contained within - the request_queue's queuelist - """ - if int(queue) == 0: - raise NoQueueError("Queue is NULL") - return list_for_each_entry(queue['queue_head'], self.request_type, - 'queuelist') - - @export - @classmethod - def request_age_ms(cls, request): - """ - Returns the age of the request in milliseconds - - This method returns the difference between the current time - (jiffies) and the request's start_time, in milliseconds. - - Args: - request(gdb.Value): The struct request used - to determine age - - Returns: - int: Difference between the request's start_time and - current jiffies in milliseconds. - """ - return kernel.jiffies_to_msec(kernel.jiffies - request['start_time']) +types = Types([ 'struct request' ]) + +def for_each_request_in_queue(queue): + """ + Iterates over each struct request in request_queue + + This method iterates over the request_queue's queuelist and + returns a request for each member. + + Args: + queue(gdb.Value): The struct request_queue + used to iterate + + Yields: + gdb.Value: Each struct request contained within + the request_queue's queuelist + """ + if int(queue) == 0: + raise NoQueueError("Queue is NULL") + return list_for_each_entry(queue['queue_head'], types.request_type, + 'queuelist') + +def request_age_ms(request): + """ + Returns the age of the request in milliseconds + + This method returns the difference between the current time + (jiffies) and the request's start_time, in milliseconds. + + Args: + request(gdb.Value): The struct request used + to determine age + + Returns: + int: Difference between the request's start_time and + current jiffies in milliseconds. + """ + return kernel.jiffies_to_msec(kernel.jiffies - request['start_time']) diff --git a/crash/subsystem/storage/decoders.py b/crash/subsystem/storage/decoders.py index d31ea298074..df3484e5222 100644 --- a/crash/subsystem/storage/decoders.py +++ b/crash/subsystem/storage/decoders.py @@ -3,14 +3,13 @@ import gdb from typing import Union, List -from crash.infra import CrashBaseClass from crash.infra.lookup import SymbolCallback EndIOSpecifier = Union[int, str, List[str], gdb.Value, gdb.Symbol, None] decoders = {} -class Decoder(CrashBaseClass): +class Decoder(object): __endio__: EndIOSpecifier = None diff --git a/crash/subsystem/storage/device_mapper.py b/crash/subsystem/storage/device_mapper.py index 74d5a1807b4..37a2a1a45c8 100644 --- a/crash/subsystem/storage/device_mapper.py +++ b/crash/subsystem/storage/device_mapper.py @@ -3,7 +3,7 @@ import gdb -from crash.infra import CrashBaseClass +from crash.util.symbols import Types from crash.subsystem.storage import block_device_name from crash.subsystem.storage.decoders import Decoder, decode_bio @@ -18,7 +18,7 @@ class ClonedBioReqDecoder(Decoder): request-based device mapper target """ - __types__ = [ 'struct dm_rq_clone_bio_info *' ] + types = Types([ 'struct dm_rq_clone_bio_info *' ]) __endio__ = 'end_clone_bio' description = '{:x} bio: Request-based Device Mapper on {}' @@ -28,7 +28,7 @@ def __init__(self, bio): super().__init__() self.bio = bio if cls._get_clone_bio_rq_info is None: - if 'clone' in cls.dm_rq_clone_bio_info_p_type.target(): + if 'clone' in cls.types.dm_rq_clone_bio_info_p_type.target(): getter = cls._get_clone_bio_rq_info_3_7 else: getter = cls._get_clone_bio_rq_info_old @@ -47,11 +47,11 @@ def __next__(self): @classmethod def _get_clone_bio_rq_info_old(cls, bio): - return bio['bi_private'].cast(cls.dm_rq_clone_bio_info_p_type) + return bio['bi_private'].cast(cls.types.dm_rq_clone_bio_info_p_type) @classmethod def _get_clone_bio_rq_info_3_7(cls, bio): - return container_of(bio, cls.dm_rq_clone_bio_info_p_type, 'clone') + return container_of(bio, cls.types.dm_rq_clone_bio_info_p_type, 'clone') ClonedBioReqDecoder.register() @@ -77,7 +77,7 @@ class ClonedBioDecoder(Decoder): - decoder (method(gdb.Value)): The decoder for the original bio """ - __types__ = [ 'struct dm_target_io *' ] + types = Types([ 'struct dm_target_io *' ]) _get_clone_bio_tio = None __endio__ = 'clone_endio' description = "{:x} bio: device mapper clone: {}[{}] -> {}[{}]" @@ -87,7 +87,7 @@ def __init__(self, bio): self.bio = bio if _get_clone_bio_tio is None: - if 'clone' in cls.dm_target_io_p_type.target(): + if 'clone' in cls.types.dm_target_io_p_type.target(): getter = cls._get_clone_bio_tio_3_15 else: getter = cls._get_clone_bio_tio_old @@ -110,11 +110,11 @@ def __next__(self): @classmethod def _get_clone_bio_tio_old(cls, bio): - return bio['bi_private'].cast(cls.dm_target_io_p_type) + return bio['bi_private'].cast(cls.types.dm_target_io_p_type) @classmethod def _get_clone_bio_tio_3_15(cls, bio): return container_of(bio['bi_private'], - cls.dm_clone_bio_info_p_type, 'clone') + cls.types.dm_clone_bio_info_p_type, 'clone') ClonedBioDecoder.register() diff --git a/crash/types/bitmap.py b/crash/types/bitmap.py index aad3a970d20..be84a585a5c 100644 --- a/crash/types/bitmap.py +++ b/crash/types/bitmap.py @@ -4,310 +4,280 @@ from typing import Iterable import gdb - from math import log -from crash.infra import CrashBaseClass, export - -class TypesBitmapClass(CrashBaseClass): - __types__ = [ 'unsigned long' ] - __type_callbacks__ = [ ('unsigned long', 'setup_ulong') ] - - bits_per_ulong = None - - @classmethod - def _check_bitmap_type(cls, bitmap: gdb.Value) -> None: - if ((bitmap.type.code != gdb.TYPE_CODE_ARRAY or - bitmap[0].type.code != cls.unsigned_long_type.code or - bitmap[0].type.sizeof != cls.unsigned_long_type.sizeof) and - (bitmap.type.code != gdb.TYPE_CODE_PTR or - bitmap.type.target().code != cls.unsigned_long_type.code or - bitmap.type.target().sizeof != cls.unsigned_long_type.sizeof)): - raise TypeError("bitmaps are expected to be arrays of unsigned long not `{}'" - .format(bitmap.type)) - - @classmethod - def setup_ulong(cls, gdbtype: gdb.Type) -> None: - cls.bits_per_ulong = gdbtype.sizeof * 8 - - @export - @classmethod - def for_each_set_bit(cls, bitmap: gdb.Value, - size_in_bytes: int=None) -> Iterable[int]: - """ - Yield each set bit in a bitmap - - Args: - bitmap (gdb.Value: - The bitmap to iterate - size_in_bytes (int): The size of the bitmap if the type is - unsigned long *. - - Yields: - int: The position of a bit that is set - """ - cls._check_bitmap_type(bitmap) - - if size_in_bytes is None: - size_in_bytes = bitmap.type.sizeof - - # FIXME: callback not workie? - cls.bits_per_ulong = cls.unsigned_long_type.sizeof * 8 - - size = size_in_bytes * 8 - idx = 0 - bit = 0 - while size > 0: - ulong = bitmap[idx] - - if ulong != 0: - for off in range(min(size, cls.bits_per_ulong)): - if ulong & 1 != 0: - yield bit - bit += 1 - ulong >>= 1 - else: - bit += cls.bits_per_ulong - - size -= cls.bits_per_ulong - idx += 1 - - @classmethod - def _find_first_set_bit(cls, val: gdb.Value) -> int: - r = 1 - - if val == 0: - return 0 - - if (val & 0xffffffff) == 0: - val >>= 32 - r += 32 - - if (val & 0xffff) == 0: - val >>= 16 - r += 16 - - if (val & 0xff) == 0: - val >>= 8 - r += 8 - - if (val & 0xf) == 0: - val >>= 4 - r += 4 - - if (val & 0x3) == 0: - val >>= 2 - r += 2 - - if (val & 0x1) == 0: - val >>= 1 - r += 1 - - return r - - @export - @classmethod - def find_next_zero_bit(cls, bitmap: gdb.Value, start: int, - size_in_bytes: int=None) -> int: - """ - Return the next unset bit in the bitmap starting at position `start', - inclusive. - - Args: - bitmap (gdb.Value: - The bitmap to test - start (int): The bit number to use as a starting position. If - the bit at this position is unset, it will be the first - bit number yielded. - size_in_bytes (int): The size of the bitmap if the type is - unsigned long *. - - Returns: - int: The position of the first bit that is unset or 0 if all are set - """ - cls._check_bitmap_type(bitmap) - - if size_in_bytes is None: - size_in_bytes = bitmap.type.sizeof - - elements = size_in_bytes // cls.unsigned_long_type.sizeof - - if start > size_in_bytes << 3: - raise IndexError("Element {} is out of range ({} elements)" - .format(start, elements)) - - element = start // (cls.unsigned_long_type.sizeof << 3) - offset = start % (cls.unsigned_long_type.sizeof << 3) - - for n in range(element, elements): - item = ~bitmap[n] - if item == 0: - continue - - if offset > 0: - item &= ~((1 << offset) - 1) - - v = cls._find_first_set_bit(item) - if v > 0: - ret = n * (cls.unsigned_long_type.sizeof << 3) + v - assert(ret >= start) - return ret - - offset = 0 - - return 0 - - @export - @classmethod - def find_first_zero_bit(cls, bitmap: gdb.Value, - size_in_bytes: int=None) -> int: - """ - Return the first unset bit in the bitmap - - Args: - bitmap (gdb.Value: - The bitmap to scan - start (int): The bit number to use as a starting position. If - the bit at this position is unset, it will be the first - bit number yielded. - - Returns: - int: The position of the first bit that is unset - """ - return cls.find_next_zero_bit(bitmap, 0, size_in_bytes) - - @export - @classmethod - def find_next_set_bit(cls, bitmap: gdb.Value, start: int, - size_in_bytes: int=None) -> int: - """ - Return the next set bit in the bitmap starting at position `start', - inclusive. - - Args: - bitmap (gdb.Value: - The bitmap to scan - start (int): The bit number to use as a starting position. If - the bit at this position is unset, it will be the first - bit number yielded. - size_in_bytes (int): The size of the bitmap if the type is - unsigned long *. - - Returns: - int: The position of the next bit that is set, or 0 if all are - unset - """ - cls._check_bitmap_type(bitmap) - - if size_in_bytes is None: - size_in_bytes = bitmap.type.sizeof - - elements = size_in_bytes // cls.unsigned_long_type.sizeof - - if start > size_in_bytes << 3: - raise IndexError("Element {} is out of range ({} elements)" - .format(start, elements)) - - element = start // (cls.unsigned_long_type.sizeof << 3) - offset = start % (cls.unsigned_long_type.sizeof << 3) - - for n in range(element, elements): - if bitmap[n] == 0: - continue - - item = bitmap[n] - if offset > 0: - item &= ~((1 << offset) - 1) - - v = cls._find_first_set_bit(item) - if v > 0: - ret = n * (cls.unsigned_long_type.sizeof << 3) + v - assert(ret >= start) - return ret - - offset = 0 - +from crash.util.symbols import Types + +types = Types('unsigned long') + +def _check_bitmap_type(bitmap: gdb.Value) -> None: + if ((bitmap.type.code != gdb.TYPE_CODE_ARRAY or + bitmap[0].type.code != types.unsigned_long_type.code or + bitmap[0].type.sizeof != types.unsigned_long_type.sizeof) and + (bitmap.type.code != gdb.TYPE_CODE_PTR or + bitmap.type.target().code != types.unsigned_long_type.code or + bitmap.type.target().sizeof != types.unsigned_long_type.sizeof)): + raise TypeError("bitmaps are expected to be arrays of unsigned long not `{}'" + .format(bitmap.type)) + +def for_each_set_bit(bitmap: gdb.Value, + size_in_bytes: int=None) -> Iterable[int]: + """ + Yield each set bit in a bitmap + + Args: + bitmap (gdb.Value: + The bitmap to iterate + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Yields: + int: The position of a bit that is set + """ + _check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + bits_per_ulong = types.unsigned_long_type.sizeof * 8 + + size = size_in_bytes * 8 + idx = 0 + bit = 0 + while size > 0: + ulong = bitmap[idx] + + if ulong != 0: + for off in range(min(size, bits_per_ulong)): + if ulong & 1 != 0: + yield bit + bit += 1 + ulong >>= 1 + else: + bit += bits_per_ulong + + size -= bits_per_ulong + idx += 1 + +def _find_first_set_bit(val: gdb.Value) -> int: + r = 1 + + if val == 0: return 0 - @export - @classmethod - def find_first_set_bit(cls, bitmap: gdb.Value, - size_in_bytes: int=None) -> int: - """ - Return the first set bit in the bitmap - - Args: - bitmap (gdb.Value: - The bitmap to scan - size_in_bytes (int): The size of the bitmap if the type is - unsigned long *. + if (val & 0xffffffff) == 0: + val >>= 32 + r += 32 + + if (val & 0xffff) == 0: + val >>= 16 + r += 16 + + if (val & 0xff) == 0: + val >>= 8 + r += 8 + + if (val & 0xf) == 0: + val >>= 4 + r += 4 + + if (val & 0x3) == 0: + val >>= 2 + r += 2 + + if (val & 0x1) == 0: + val >>= 1 + r += 1 + + return r + +def find_next_zero_bit(bitmap: gdb.Value, start: int, + size_in_bytes: int=None) -> int: + """ + Return the next unset bit in the bitmap starting at position `start', + inclusive. + + Args: + bitmap (gdb.Value: + The bitmap to test + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the first bit that is unset or 0 if all are set + """ + _check_bitmap_type(bitmap) - Returns: - int: The position of the first bit that is set, or 0 if all are - unset - """ - return cls.find_next_set_bit(bitmap, 0, size_in_bytes) + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + elements = size_in_bytes // types.unsigned_long_type.sizeof + + if start > size_in_bytes << 3: + raise IndexError("Element {} is out of range ({} elements)" + .format(start, elements)) + + element = start // (types.unsigned_long_type.sizeof << 3) + offset = start % (types.unsigned_long_type.sizeof << 3) + + for n in range(element, elements): + item = ~bitmap[n] + if item == 0: + continue + + if offset > 0: + item &= ~((1 << offset) - 1) + + v = _find_first_set_bit(item) + if v > 0: + ret = n * (types.unsigned_long_type.sizeof << 3) + v + assert(ret >= start) + return ret + + offset = 0 + + return 0 + +def find_first_zero_bit(bitmap: gdb.Value, size_in_bytes: int=None) -> int: + """ + Return the first unset bit in the bitmap + + Args: + bitmap (gdb.Value: + The bitmap to scan + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + + Returns: + int: The position of the first bit that is unset + """ + return find_next_zero_bit(bitmap, 0, size_in_bytes) + +def find_next_set_bit(bitmap: gdb.Value, start: int, + size_in_bytes: int=None) -> int: + """ + Return the next set bit in the bitmap starting at position `start', + inclusive. + + Args: + bitmap (gdb.Value: + The bitmap to scan + start (int): The bit number to use as a starting position. If + the bit at this position is unset, it will be the first + bit number yielded. + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the next bit that is set, or 0 if all are unset + """ + _check_bitmap_type(bitmap) + + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof + + elements = size_in_bytes // types.unsigned_long_type.sizeof + + if start > size_in_bytes << 3: + raise IndexError("Element {} is out of range ({} elements)" + .format(start, elements)) + + element = start // (types.unsigned_long_type.sizeof << 3) + offset = start % (types.unsigned_long_type.sizeof << 3) + + for n in range(element, elements): + if bitmap[n] == 0: + continue + + item = bitmap[n] + if offset > 0: + item &= ~((1 << offset) - 1) + + v = _find_first_set_bit(item) + if v > 0: + ret = n * (types.unsigned_long_type.sizeof << 3) + v + assert(ret >= start) + return ret + + offset = 0 - @classmethod - def _find_last_set_bit(cls, val: gdb.Value) -> int: - r = cls.unsigned_long_type.sizeof << 3 + return 0 + +def find_first_set_bit(bitmap: gdb.Value, size_in_bytes: int=None) -> int: + """ + Return the first set bit in the bitmap - if val == 0: - return 0 + Args: + bitmap (gdb.Value: + The bitmap to scan + size_in_bytes (int): The size of the bitmap if the type is + unsigned long *. + + Returns: + int: The position of the first bit that is set, or 0 if all are unset + """ + return find_next_set_bit(bitmap, 0, size_in_bytes) + +def _find_last_set_bit(val: gdb.Value) -> int: + r = types.unsigned_long_type.sizeof << 3 + + if val == 0: + return 0 - if (val & 0xffffffff00000000) == 0: - val <<= 32 - r -= 32 + if (val & 0xffffffff00000000) == 0: + val <<= 32 + r -= 32 - if (val & 0xffff000000000000) == 0: - val <<= 16 - r -= 16 + if (val & 0xffff000000000000) == 0: + val <<= 16 + r -= 16 - if (val & 0xff00000000000000) == 0: - val <<= 8 - r -= 8 + if (val & 0xff00000000000000) == 0: + val <<= 8 + r -= 8 - if (val & 0xf000000000000000) == 0: - val <<= 4 - r -= 4 + if (val & 0xf000000000000000) == 0: + val <<= 4 + r -= 4 - if (val & 0xc000000000000000) == 0: - val <<= 2 - r -= 2 + if (val & 0xc000000000000000) == 0: + val <<= 2 + r -= 2 - if (val & 0x8000000000000000) == 0: - val <<= 1 - r -= 1 + if (val & 0x8000000000000000) == 0: + val <<= 1 + r -= 1 - return r + return r - @export - @classmethod - def find_last_set_bit(cls, bitmap: gdb.Value, - size_in_bytes: int=None) -> int: - """ - Return the last set bit in the bitmap +def find_last_set_bit(bitmap: gdb.Value, size_in_bytes: int=None) -> int: + """ + Return the last set bit in the bitmap - Args: - bitmap (gdb.Value: - The bitmap to scan + Args: + bitmap (gdb.Value: + The bitmap to scan - Returns: - int: The position of the last bit that is set, or 0 if all are unset - """ - cls._check_bitmap_type(bitmap) + Returns: + int: The position of the last bit that is set, or 0 if all are unset + """ + _check_bitmap_type(bitmap) - if size_in_bytes is None: - size_in_bytes = bitmap.type.sizeof + if size_in_bytes is None: + size_in_bytes = bitmap.type.sizeof - elements = size_in_bytes // cls.unsigned_long_type.sizeof + elements = size_in_bytes // types.unsigned_long_type.sizeof - for n in range(elements - 1, -1, -1): - if bitmap[n] == 0: - continue + for n in range(elements - 1, -1, -1): + if bitmap[n] == 0: + continue - v = cls._find_last_set_bit(bitmap[n]) - if v > 0: - return n * (cls.unsigned_long_type.sizeof << 3) + v + v = _find_last_set_bit(bitmap[n]) + if v > 0: + return n * (types.unsigned_long_type.sizeof << 3) + v - return 0 + return 0 diff --git a/crash/types/classdev.py b/crash/types/classdev.py index 3b82a5a7cb0..885872651ae 100644 --- a/crash/types/classdev.py +++ b/crash/types/classdev.py @@ -2,15 +2,14 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.infra import CrashBaseClass, export + from crash.types.klist import klist_for_each_entry +from crash.util.symbols import Types -class ClassDeviceClass(CrashBaseClass): - __types__ = [ 'struct device' ] +types = Types(['struct device']) - @export - def for_each_class_device(self, class_struct, subtype=None): - klist = class_struct['p']['klist_devices'] - for dev in klist_for_each_entry(klist, self.device_type, 'knode_class'): - if subtype is None or int(subtype) == int(dev['type']): - yield dev +def for_each_class_device(class_struct, subtype=None): + klist = class_struct['p']['klist_devices'] + for dev in klist_for_each_entry(klist, types.device_type, 'knode_class'): + if subtype is None or int(subtype) == int(dev['type']): + yield dev diff --git a/crash/types/cpu.py b/crash/types/cpu.py index 1a83dee38e2..841619c748b 100644 --- a/crash/types/cpu.py +++ b/crash/types/cpu.py @@ -1,8 +1,10 @@ #!/usr/bin/python3 # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable, List + import gdb -from crash.infra import CrashBaseClass, export +from crash.util.symbols import SymbolCallbacks from crash.types.bitmap import for_each_set_bit from crash.exceptions import DelayedAttributeError @@ -10,69 +12,73 @@ # this wraps no particular type, rather it's a placeholder for # functions to iterate over online cpu's etc. -class TypesCPUClass(CrashBaseClass): - __symbol_callbacks__ = [ ('cpu_online_mask', '_setup_online_mask'), - ('__cpu_online_mask', '_setup_online_mask'), - ('cpu_possible_mask', '_setup_possible_mask'), - ('__cpu_possible_mask', '_setup_possible_mask') ] +class TypesCPUClass(object): cpus_online: List[int] = list() cpus_possible: List[int] = list() + cpu_online_mask: gdb.Value = None + cpu_possible_mask: gdb.Value = None + @classmethod def _setup_online_mask(cls, symbol: gdb.Symbol) -> None: cls.cpu_online_mask = symbol.value() bits = cls.cpu_online_mask["bits"] cls.cpus_online = list(for_each_set_bit(bits)) - @export - def for_each_online_cpu(self) -> Iterable[int]: - """ - Yield CPU numbers of all online CPUs - - Yields: - int: Number of a possible CPU location - """ - for cpu in self.cpus_online: - yield cpu - - @export - def highest_online_cpu_nr(self) -> None: - """ - Return The highest online CPU number - - Returns: - int: The highest online CPU number - """ - if not TypesCPUClass.cpus_online : - raise DelayedAttributeError('cpus_online') - return self.cpus_online[-1] - @classmethod def _setup_possible_mask(cls, cpu_mask: gdb.Symbol) -> None: cls.cpu_possible_mask = cpu_mask.value() bits = cls.cpu_possible_mask["bits"] cls.cpus_possible = list(for_each_set_bit(bits)) - @export - def for_each_possible_cpu(self) -> Iterable[int]: - """ - Yield CPU numbers of all possible CPUs - - Yields: - int: Number of a possible CPU location - """ - for cpu in self.cpus_possible: - yield cpu - - @export - def highest_possible_cpu_nr(self) -> int: - """ - Return The highest possible CPU number - - Returns: - int: The highest possible CPU number - """ - if not self.cpus_possible: - raise DelayedAttributeError('cpus_possible') - return self.cpus_possible[-1] +def for_each_online_cpu() -> Iterable[int]: + """ + Yield CPU numbers of all online CPUs + + Yields: + int: Number of a possible CPU location + """ + for cpu in TypesCPUClass.cpus_online: + yield cpu + +def highest_online_cpu_nr() -> int: + """ + Return The highest online CPU number + + Returns: + int: The highest online CPU number + """ + if not TypesCPUClass.cpus_online: + raise DelayedAttributeError('cpus_online') + return TypesCPUClass.cpus_online[-1] + +def for_each_possible_cpu() -> Iterable[int]: + """ + Yield CPU numbers of all possible CPUs + + Yields: + int: Number of a possible CPU location + """ + for cpu in TypesCPUClass.cpus_possible: + yield cpu + +def highest_possible_cpu_nr() -> int: + """ + Return The highest possible CPU number + + Returns: + int: The highest possible CPU number + """ + if not TypesCPUClass.cpus_possible: + raise DelayedAttributeError('cpus_possible') + return TypesCPUClass.cpus_possible[-1] + +symbol_cbs = SymbolCallbacks([ ('cpu_online_mask', + TypesCPUClass._setup_online_mask), + ('__cpu_online_mask', + TypesCPUClass._setup_online_mask), + ('cpu_possible_mask', + TypesCPUClass._setup_possible_mask), + ('__cpu_possible_mask', + TypesCPUClass._setup_possible_mask) ]) diff --git a/crash/types/klist.py b/crash/types/klist.py index 7ccb49fa7b3..e58b074fa86 100644 --- a/crash/types/klist.py +++ b/crash/types/klist.py @@ -5,35 +5,34 @@ from crash.util import container_of from crash.types.list import list_for_each_entry from crash.exceptions import CorruptedError -from crash.infra import CrashBaseClass, export + +from crash.util.symbols import Types + +types = Types([ 'struct klist_node', 'struct klist' ]) class KlistCorruptedError(CorruptedError): pass -class TypesKlistClass(CrashBaseClass): - __types__ = [ 'struct klist_node', 'struct klist' ] - - @export - def klist_for_each(self, klist): - if klist.type == self.klist_type.pointer(): - klist = klist.dereference() - elif klist.type != self.klist_type: - raise TypeError("klist must be gdb.Value representing 'struct klist' or 'struct klist *' not {}" - .format(klist.type)) - if klist.type is not self.klist_type: - self.klist_type = klist.type +def klist_for_each(klist): + if klist.type == types.klist_type.pointer(): + klist = klist.dereference() + elif klist.type != types.klist_type: + raise TypeError("klist must be gdb.Value representing 'struct klist' or 'struct klist *' not {}" + .format(klist.type)) + if klist.type is not types.klist_type: + types.override('struct klist', klist.type) - for node in list_for_each_entry(klist['k_list'], - self.klist_node_type, 'n_node'): - if node['n_klist'] != klist.address: - raise KlistCorruptedError("Corrupted") - yield node + for node in list_for_each_entry(klist['k_list'], + types.klist_node_type, 'n_node'): + if node['n_klist'] != klist.address: + raise KlistCorruptedError("Corrupted") + yield node - @export - def klist_for_each_entry(self, klist, gdbtype, member): - for node in klist_for_each(klist): - if node.type != self.klist_node_type: - raise TypeError("Type {} found. Expected {}.".format(node.type), self.klist_node_type.pointer()) - if node.type is not self.klist_node_type: - self.klist_node_type = node.type - yield container_of(node, gdbtype, member) +def klist_for_each_entry(klist, gdbtype, member): + for node in klist_for_each(klist): + if node.type != types.klist_node_type: + raise TypeError("Type {} found. Expected {}." + .format(node.type), types.klist_node_type.pointer()) + if node.type is not types.klist_node_type: + types.override('struct klist_node', node.type) + yield container_of(node, gdbtype, member) diff --git a/crash/types/list.py b/crash/types/list.py index 53df2c5cfd1..54c384717b1 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -3,7 +3,7 @@ import gdb from crash.util import container_of -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Types class ListError(Exception): pass @@ -14,117 +14,112 @@ class CorruptListError(ListError): class ListCycleError(CorruptListError): pass -class TypesListClass(CrashBaseClass): - __types__ = [ 'struct list_head' ] - - @export - def list_for_each(self, list_head, include_head=False, reverse=False, - print_broken_links=True, exact_cycles=False): - pending_exception = None - if isinstance(list_head, gdb.Symbol): - list_head = list_head.value() - if not isinstance(list_head, gdb.Value): - raise TypeError("list_head must be gdb.Value representing 'struct list_head' or a 'struct list_head *' not {}" - .format(type(list_head).__name__)) - if list_head.type == self.list_head_type.pointer(): - list_head = list_head.dereference() - elif list_head.type != self.list_head_type: - raise TypeError("Must be struct list_head not {}" - .format(str(list_head.type))) - if list_head.type is not self.list_head_type: - self.list_head_type = list_head.type - fast = None - if int(list_head.address) == 0: - raise CorruptListError("list_head is NULL pointer.") - - next_ = 'next' - prev_ = 'prev' - if reverse: - next_ = 'prev' - prev_ = 'next' - +types = Types([ 'struct list_head' ]) + +def list_for_each(list_head, include_head=False, reverse=False, + print_broken_links=True, exact_cycles=False): + pending_exception = None + if isinstance(list_head, gdb.Symbol): + list_head = list_head.value() + if not isinstance(list_head, gdb.Value): + raise TypeError("list_head must be gdb.Value representing 'struct list_head' or a 'struct list_head *' not {}" + .format(type(list_head).__name__)) + if list_head.type == types.list_head_type.pointer(): + list_head = list_head.dereference() + elif list_head.type != types.list_head_type: + raise TypeError("Must be struct list_head not {}" + .format(str(list_head.type))) + if list_head.type is not types.list_head_type: + types.override('struct list_head', list_head.type) + fast = None + if int(list_head.address) == 0: + raise CorruptListError("list_head is NULL pointer.") + + next_ = 'next' + prev_ = 'prev' + if reverse: + next_ = 'prev' + prev_ = 'next' + + if exact_cycles: + visited = set() + + if include_head: + yield list_head.address + + try: + nxt = list_head[next_] + prev = list_head + if int(nxt) == 0: + raise CorruptListError("{} pointer is NULL".format(next_)) + node = nxt.dereference() + except gdb.error as e: + raise BufferError("Failed to read list_head {:#x}: {}" + .format(int(list_head.address), str(e))) + + while node.address != list_head.address: if exact_cycles: - visited = set() - - if include_head: - yield list_head.address - + if int(node.address) in visited: + raise ListCycleError("Cycle in list detected.") + else: + visited.add(int(node.address)) try: - nxt = list_head[next_] - prev = list_head - if int(nxt) == 0: - raise CorruptListError("{} pointer is NULL".format(next_)) - node = nxt.dereference() + if int(prev.address) != int(node[prev_]): + error = ("broken {} link {:#x} -{}-> {:#x} -{}-> {:#x}" + .format(prev_, int(prev.address), next_, int(node.address), + prev_, int(node[prev_]))) + pending_exception = CorruptListError(error) + if print_broken_links: + print(error) + # broken prev link means there might be a cycle that + # does not include the initial head, so start detecting + # cycles + if not exact_cycles and fast is not None: + fast = node + nxt = node[next_] + # only yield after trying to read something from the node, no + # point in giving out bogus list elements + yield node.address except gdb.error as e: - raise BufferError("Failed to read list_head {:#x}: {}" - .format(int(list_head.address), str(e))) - - while node.address != list_head.address: - if exact_cycles: - if int(node.address) in visited: - raise ListCycleError("Cycle in list detected.") - else: - visited.add(int(node.address)) - try: - if int(prev.address) != int(node[prev_]): - error = ("broken {} link {:#x} -{}-> {:#x} -{}-> {:#x}" - .format(prev_, int(prev.address), next_, int(node.address), - prev_, int(node[prev_]))) - pending_exception = CorruptListError(error) - if print_broken_links: - print(error) - # broken prev link means there might be a cycle that - # does not include the initial head, so start detecting - # cycles - if not exact_cycles and fast is not None: - fast = node - nxt = node[next_] - # only yield after trying to read something from the node, no - # point in giving out bogus list elements - yield node.address - except gdb.error as e: - raise BufferError("Failed to read list_head {:#x} in list {:#x}: {}" - .format(int(node.address), int(list_head.address), str(e))) - - try: - if fast is not None: - # are we detecting cycles? advance fast 2 times and compare - # each with our current node (Floyd's Tortoise and Hare - # algorithm) - for i in range(2): - fast = fast[next_].dereference() - if node.address == fast.address: - raise ListCycleError("Cycle in list detected.") - except gdb.error: - # we hit an unreadable element, so just stop detecting cycles - # and the slow iterator will hit it as well - fast = None - - prev = node - if int(nxt) == 0: - raise CorruptListError("{} -> {} pointer is NULL" - .format(node.address, next_)) - node = nxt.dereference() - - if pending_exception is not None: - raise pending_exception - - @export - def list_for_each_entry(self, list_head, gdbtype, member, - include_head=False, reverse=False, - exact_cycles=False): - for node in list_for_each(list_head, include_head=include_head, - reverse=reverse, exact_cycles=exact_cycles): - if node.type != self.list_head_type.pointer(): - raise TypeError("Type {} found. Expected struct list_head *." - .format(str(node.type))) - yield container_of(node, gdbtype, member) - - @export - def list_empty(self, list_head): - addr = int(list_head.address) - if list_head.type.code == gdb.TYPE_CODE_PTR: - addr = int(list_head) - - return addr == int(list_head['next']) + raise BufferError("Failed to read list_head {:#x} in list {:#x}: {}" + .format(int(node.address), int(list_head.address), str(e))) + try: + if fast is not None: + # are we detecting cycles? advance fast 2 times and compare + # each with our current node (Floyd's Tortoise and Hare + # algorithm) + for i in range(2): + fast = fast[next_].dereference() + if node.address == fast.address: + raise ListCycleError("Cycle in list detected.") + except gdb.error: + # we hit an unreadable element, so just stop detecting cycles + # and the slow iterator will hit it as well + fast = None + + prev = node + if int(nxt) == 0: + raise CorruptListError("{} -> {} pointer is NULL" + .format(node.address, next_)) + node = nxt.dereference() + + if pending_exception is not None: + raise pending_exception + +def list_for_each_entry(list_head, gdbtype, member, + include_head=False, reverse=False, + exact_cycles=False): + for node in list_for_each(list_head, include_head=include_head, + reverse=reverse, exact_cycles=exact_cycles): + if node.type != types.list_head_type.pointer(): + raise TypeError("Type {} found. Expected struct list_head *." + .format(str(node.type))) + yield container_of(node, gdbtype, member) + +def list_empty(list_head): + addr = int(list_head.address) + if list_head.type.code == gdb.TYPE_CODE_PTR: + addr = int(list_head) + + return addr == int(list_head['next']) diff --git a/crash/types/module.py b/crash/types/module.py index 3a17b245345..d787f9f538f 100644 --- a/crash/types/module.py +++ b/crash/types/module.py @@ -4,51 +4,44 @@ from typing import Iterable, Tuple import gdb -from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each_entry +from crash.util.symbols import Symvals, Types -class Module(CrashBaseClass): - __symvals__ = [ 'modules'] - __types__ = [ 'struct module' ] - - @classmethod - @export - def for_each_module(cls) -> Iterable[gdb.Value]: - """ - Iterate over each module in the modules list - - Yields: - gdb.Value(): The next module on the list - - """ - for module in list_for_each_entry(cls.modules, cls.module_type, - 'list'): - yield module - - @classmethod - @export - def for_each_module_section(cls, module: gdb.Value) \ - -> Iterable[Tuple[str, int]]: - """ - Iterate over each ELF section in a loaded module - - This routine iterates over the 'sect_attrs' member of the - 'struct module' already in memory. For ELF sections from the - module at rest, use pyelftools on the module file. - - Args: - module (gdb.Value): The struct module to iterate - - Yields: - (str, int): A 2-tuple containing the name and address - of the section - """ - attrs = module['sect_attrs'] - - for sec in range(0, attrs['nsections']): - attr = attrs['attrs'][sec] - name = attr['name'].string() - if name == '.text': - continue - - yield (name, int(attr['address'])) +symvals = Symvals([ 'modules' ]) +types = Types([ 'struct module' ]) + +def for_each_module() -> Iterable[gdb.Value]: + """ + Iterate over each module in the modules list + + Yields: + gdb.Value(): The next module on the list + + """ + for module in list_for_each_entry(symvals.modules, types.module_type, + 'list'): + yield module + +def for_each_module_section(module: gdb.Value) -> Iterable[Tuple[str, int]]: + """ + Iterate over each ELF section in a loaded module + + This routine iterates over the 'sect_attrs' member of the 'struct module' + already in memory. For ELF sections from the module at rest, use + pyelftools on the module file. + + Args: + module (gdb.Value): The struct module to iterate + + Yields: + (str, int): A 2-tuple containing the name and address of the section + """ + attrs = module['sect_attrs'] + + for sec in range(0, attrs['nsections']): + attr = attrs['attrs'][sec] + name = attr['name'].string() + if name == '.text': + continue + + yield (name, int(attr['address'])) diff --git a/crash/types/node.py b/crash/types/node.py index 1e460d9cbb0..34db2196def 100644 --- a/crash/types/node.py +++ b/crash/types/node.py @@ -2,30 +2,26 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Symbols, Symvals, Types, SymbolCallbacks from crash.util import container_of, find_member_variant, get_symbol_value from crash.types.percpu import get_percpu_var from crash.types.bitmap import for_each_set_bit import crash.types.zone -class TypesNodeUtilsClass(CrashBaseClass): - __symbols__ = [ 'numa_node' ] - __symvals__ = [ 'numa_cpu_lookup_table' ] +symbols = Symbols([ 'numa_node' ]) +symvals = Symvals([ 'numa_cpu_lookup_table', 'node_data' ]) +types = Types([ 'pg_data_t', 'struct zone' ]) - @export - def numa_node_id(self, cpu): - if gdb.current_target().arch.name() == "powerpc:common64": - return int(self.numa_cpu_lookup_table[cpu]) - else: - return int(get_percpu_var(self.numa_node, cpu)) - -class Node(CrashBaseClass): - __types__ = [ 'pg_data_t', 'struct zone' ] +def numa_node_id(cpu): + if gdb.current_target().arch.name() == "powerpc:common64": + return int(symvals.numa_cpu_lookup_table[cpu]) + else: + return int(get_percpu_var(symbols.numa_node, cpu)) +class Node(object): @staticmethod def from_nid(nid): - node_data = gdb.lookup_global_symbol("node_data").value() - return Node(node_data[nid].dereference()) + return Node(symvals.node_data[nid].dereference()) def for_each_zone(self): node_zones = self.gdb_obj["node_zones"] @@ -37,25 +33,21 @@ def for_each_zone(self): # FIXME: gdb seems to lose the alignment padding with plain # node_zones[zid], so we have to simulate it using zone_type.sizeof # which appears to be correct - zone = gdb.Value(ptr).cast(self.zone_type.pointer()).dereference() + zone = gdb.Value(ptr).cast(types.zone_type.pointer()).dereference() yield crash.types.zone.Zone(zone, zid) - ptr += self.zone_type.sizeof + ptr += types.zone_type.sizeof def __init__(self, obj): self.gdb_obj = obj -class Nodes(CrashBaseClass): - - __symbol_callbacks__ = [ ('node_states', 'setup_node_states') ] - +class NodeStates(object): nids_online = None nids_possible = None @classmethod def setup_node_states(cls, node_states_sym): - - node_states = node_states_sym.value() + node_states = node_states_sym.value() enum_node_states = gdb.lookup_type("enum node_states") N_POSSIBLE = enum_node_states["N_POSSIBLE"].enumval @@ -67,23 +59,21 @@ def setup_node_states(cls, node_states_sym): bits = node_states[N_ONLINE]["bits"] cls.nids_online = list(for_each_set_bit(bits)) - @export - def for_each_nid(cls): - for nid in cls.nids_possible: - yield nid - - @export - def for_each_online_nid(cls): - for nid in cls.nids_online: - yield nid - - @export - def for_each_node(cls): - for nid in cls.for_each_nid(): - yield Node.from_nid(nid) - - @export - def for_each_online_node(cls): - for nid in cls.for_each_online_nid(): - yield Node.from_nid(nid) +symbol_cbs = SymbolCallbacks([('node_states', NodeStates.setup_node_states)]) + +def for_each_nid(): + for nid in NodeStates.nids_possible: + yield nid + +def for_each_online_nid(): + for nid in NodeStates.nids_online: + yield nid + +def for_each_node(): + for nid in for_each_nid(): + yield Node.from_nid(nid) + +def for_each_online_node(): + for nid in for_each_online_nid(): + yield Node.from_nid(nid) diff --git a/crash/types/page.py b/crash/types/page.py index 54487aa2bd7..2ea07853e76 100644 --- a/crash/types/page.py +++ b/crash/types/page.py @@ -3,28 +3,18 @@ from math import log, ceil import gdb -import types -from crash.infra import CrashBaseClass, export from crash.util import container_of, find_member_variant +from crash.util.symbols import Types, Symvals, TypeCallbacks, SymbolCallbacks from crash.cache.syscache import config #TODO debuginfo won't tell us, depends on version? PAGE_MAPPING_ANON = 1 -class Page(CrashBaseClass): - __types__ = [ 'unsigned long', 'struct page', 'enum pageflags', - 'enum zone_type', 'struct mem_section'] - __type_callbacks__ = [ ('struct page', 'setup_page_type' ), - ('enum pageflags', 'setup_pageflags' ), - ('enum zone_type', 'setup_zone_type' ), - ('struct mem_section', 'setup_mem_section') ] - __symvals__ = [ 'mem_section' ] - # TODO: this should better be generalized to some callback for - # "config is available" without refering to the symbol name here - __symbol_callbacks__ = [ ('kernel_config_data', 'setup_nodes_width' ), - ('vmemmap_base', 'setup_vmemmap_base' ), - ('page_offset_base', 'setup_directmap_base' ) ] +types = Types([ 'unsigned long', 'struct page', 'enum pageflags', + 'enum zone_type', 'struct mem_section']) +symvals = Symvals([ 'mem_section' ]) +class Page(object): slab_cache_name = None slab_page_name = None compound_head_name = None @@ -85,10 +75,10 @@ def pfn_to_page(cls, pfn): section_nr = pfn >> (cls.SECTION_SIZE_BITS - cls.PAGE_SHIFT) root_idx = section_nr / cls.SECTIONS_PER_ROOT offset = section_nr & (cls.SECTIONS_PER_ROOT - 1) - section = cls.mem_section[root_idx][offset] + section = symvals.mem_section[root_idx][offset] pagemap = section["section_mem_map"] & ~3 - return (pagemap.cast(cls.page_type.pointer()) + pfn).dereference() + return (pagemap.cast(types.page_type.pointer()) + pfn).dereference() else: return cls.vmemmap[pfn] @@ -110,7 +100,7 @@ def setup_vmemmap_base(cls, symbol): # setup_page_type() was first and used the hardcoded initial value, # we have to update if cls.vmemmap is not None: - cls.vmemmap = gdb.Value(cls.vmemmap_base).cast(cls.page_type.pointer()) + cls.vmemmap = gdb.Value(cls.vmemmap_base).cast(types.page_type.pointer()) @classmethod def setup_directmap_base(cls, symbol): @@ -132,7 +122,7 @@ def setup_nodes_width(cls, symbol): cls.NODES_WIDTH = 8 # piggyback on this callback because type callback doesn't seem to work # for unsigned long - cls.BITS_PER_LONG = cls.unsigned_long_type.sizeof * 8 + cls.BITS_PER_LONG = types.unsigned_long_type.sizeof * 8 @classmethod def setup_pageflags_finish(cls): @@ -149,8 +139,8 @@ def setup_pageflags_finish(cls): @staticmethod def from_page_addr(addr): - page_ptr = gdb.Value(addr).cast(Page.page_type.pointer()) - pfn = (addr - Page.vmemmap_base) / Page.page_type.sizeof + page_ptr = gdb.Value(addr).cast(types.page_type.pointer()) + pfn = (addr - Page.vmemmap_base) / types.page_type.sizeof return Page(page_ptr.dereference(), pfn) def __is_tail_flagcombo(self): @@ -201,37 +191,41 @@ def compound_head(self): return self return Page.from_page_addr(self.__compound_head()) - + def __init__(self, obj, pfn): self.gdb_obj = obj self.pfn = pfn self.flags = int(obj["flags"]) -class Pages(CrashBaseClass): +type_cbs = TypeCallbacks([ ('struct page', Page.setup_page_type ), + ('enum pageflags', Page.setup_pageflags ), + ('enum zone_type', Page.setup_zone_type ), + ('struct mem_section', Page.setup_mem_section) ]) + +# TODO: this should better be generalized to some callback for +# "config is available" without refering to the symbol name here +symbol_cbs = SymbolCallbacks([ ('kernel_config_data', Page.setup_nodes_width ), + ('vmemmap_base', Page.setup_vmemmap_base ), + ('page_offset_base', Page.setup_directmap_base ) ]) - @export - def pfn_to_page(cls, pfn): - return Page(Page.pfn_to_page(pfn), pfn) - - @export - def page_from_addr(cls, addr): - pfn = (addr - Page.directmap_base) / Page.PAGE_SIZE - return pfn_to_page(pfn) - - @export - def page_from_gdb_obj(cls, gdb_obj): - pfn = (int(gdb_obj.address) - Page.vmemmap_base) / Page.page_type.sizeof - return Page(gdb_obj, pfn) - - @export - def for_each_page(): - # TODO works only on x86? - max_pfn = int(gdb.lookup_global_symbol("max_pfn").value()) - for pfn in range(max_pfn): - try: - yield Page.pfn_to_page(pfn) - except gdb.error: - # TODO: distinguish pfn_valid() and report failures for those? - pass +def pfn_to_page(pfn): + return Page(Page.pfn_to_page(pfn), pfn) +def page_from_addr(addr): + pfn = (addr - Page.directmap_base) / Page.PAGE_SIZE + return pfn_to_page(pfn) + +def page_from_gdb_obj(gdb_obj): + pfn = (int(gdb_obj.address) - Page.vmemmap_base) / types.page_type.sizeof + return Page(gdb_obj, pfn) + +def for_each_page(): + # TODO works only on x86? + max_pfn = int(gdb.lookup_global_symbol("max_pfn").value()) + for pfn in range(max_pfn): + try: + yield Page.pfn_to_page(pfn) + except gdb.error: + # TODO: distinguish pfn_valid() and report failures for those? + pass diff --git a/crash/types/percpu.py b/crash/types/percpu.py index bbc65c96dec..2f9e4ba4fe8 100644 --- a/crash/types/percpu.py +++ b/crash/types/percpu.py @@ -4,8 +4,9 @@ from typing import Dict, Union, List, Tuple import gdb -from crash.infra import CrashBaseClass, export from crash.util import array_size, struct_has_member +from crash.util.symbols import Types, Symvals, MinimalSymvals, MinimalSymbols +from crash.util.symbols import MinimalSymbolCallbacks, SymbolCallbacks from crash.types.list import list_for_each_entry from crash.types.module import for_each_module from crash.exceptions import DelayedAttributeError @@ -19,9 +20,15 @@ class PerCPUError(TypeError): def __init__(self, var): super().__init__(self.fmt.format(var)) +types = Types([ 'void *', 'char *', 'struct pcpu_chunk', + 'struct percpu_counter' ]) +symvals = Symvals([ '__per_cpu_offset', 'pcpu_base_addr', 'pcpu_slot', + 'pcpu_nr_slots', 'pcpu_group_offsets' ]) +msymvals = MinimalSymvals( ['__per_cpu_start', '__per_cpu_end' ]) + SymbolOrValue = Union[gdb.Value, gdb.Symbol] -class TypesPerCPUClass(CrashBaseClass): +class PerCPUState(object): """ Per-cpus come in a few forms: - "Array" of objects @@ -33,16 +40,6 @@ class TypesPerCPUClass(CrashBaseClass): pointer to a percpu but we don't want to dereference a percpu pointer. """ - - __types__ = [ 'void *', 'char *', 'struct pcpu_chunk', - 'struct percpu_counter' ] - __symvals__ = [ '__per_cpu_offset', 'pcpu_base_addr', 'pcpu_slot', - 'pcpu_nr_slots', 'pcpu_group_offsets' ] - __minsymvals__ = ['__per_cpu_start', '__per_cpu_end' ] - __minsymbol_callbacks__ = [ ('__per_cpu_start', '_setup_per_cpu_size'), - ('__per_cpu_end', '_setup_per_cpu_size') ] - __symbol_callbacks__ = [ ('__per_cpu_offset', '_setup_nr_cpus') ] - dynamic_offset_cache: List[Tuple[int, int]] = list() static_ranges: Dict[int, int] = dict() module_ranges: Dict[int, int] = dict() @@ -52,13 +49,13 @@ class TypesPerCPUClass(CrashBaseClass): @classmethod def _setup_per_cpu_size(cls, symbol: gdb.Symbol) -> None: try: - size = cls.__per_cpu_end - cls.__per_cpu_start + size = msymvals['__per_cpu_end'] - msymvals['__per_cpu_start'] except DelayedAttributeError: pass cls.static_ranges[0] = size - if cls.__per_cpu_start != 0: - cls.static_ranges[cls.__per_cpu_start] = size + if msymvals['__per_cpu_start'] != 0: + cls.static_ranges[msymvals['__per_cpu_start']] = size try: # This is only an optimization so we don't return NR_CPUS values @@ -69,7 +66,7 @@ def _setup_per_cpu_size(cls, symbol: gdb.Symbol) -> None: @classmethod def _setup_nr_cpus(cls, ignored: gdb.Symbol) -> None: - cls.nr_cpus = array_size(cls.__per_cpu_offset) + cls.nr_cpus = array_size(symvals['__per_cpu_offset']) if cls.last_cpu == -1: cls.last_cpu = cls.nr_cpus @@ -84,9 +81,8 @@ def _setup_module_ranges(cls, modules: gdb.Symbol) -> None: size = int(module['percpu_size']) cls.module_ranges[start] = size - @classmethod - def _add_to_offset_cache(cls, base: int, start: int, end: int) -> None: - cls.dynamic_offset_cache.append((base + start, base + end)) + def _add_to_offset_cache(self, base: int, start: int, end: int) -> None: + self.dynamic_offset_cache.append((base + start, base + end)) @classmethod def dump_ranges(cls) -> None: @@ -97,14 +93,12 @@ def dump_ranges(cls) -> None: print(f"static start={start:#x}, size={size:#x}") for (start, size) in cls.module_ranges.items(): print(f"module start={start:#x}, size={size:#x}") - if cls.dynamic_offset_cache: - for (start, end) in cls.dynamic_offset_cache: - print(f"dynamic start={start:#x}, end={end:#x}") + for (start, end) in cls.dynamic_offset_cache: + print(f"dynamic start={start:#x}, end={end:#x}") - @classmethod - def _setup_dynamic_offset_cache_area_map(cls, chunk: gdb.Value) -> None: + def _setup_dynamic_offset_cache_area_map(self, chunk: gdb.Value) -> None: used_is_negative = None - chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) + chunk_base = int(chunk["base_addr"]) - int(symvals.pcpu_base_addr) off = 0 start = None @@ -139,11 +133,11 @@ def _setup_dynamic_offset_cache_area_map(cls, chunk: gdb.Value) -> None: start = off else: if start is not None: - cls._add_to_offset_cache(chunk_base, start, off) + self._add_to_offset_cache(chunk_base, start, off) start = None off += abs(val) if start is not None: - cls._add_to_offset_cache(chunk_base, start, off) + self._add_to_offset_cache(chunk_base, start, off) else: for i in range(map_used): off = int(_map[i]) @@ -153,35 +147,33 @@ def _setup_dynamic_offset_cache_area_map(cls, chunk: gdb.Value) -> None: start = off else: if start is not None: - cls._add_to_offset_cache(chunk_base, start, off) + self._add_to_offset_cache(chunk_base, start, off) start = None if start is not None: off = int(_map[map_used]) - 1 - cls._add_to_offset_cache(chunk_base, start, off) + self._add_to_offset_cache(chunk_base, start, off) - @classmethod - def _setup_dynamic_offset_cache_bitmap(cls, chunk: gdb.Value) -> None: - group_offset = int(cls.pcpu_group_offsets[0]) + def _setup_dynamic_offset_cache_bitmap(self, chunk: gdb.Value) -> None: + group_offset = int(symvals.pcpu_group_offsets[0]) size_in_bytes = int(chunk['nr_pages']) * Page.PAGE_SIZE size_in_bits = size_in_bytes << 3 start = -1 end = 0 - chunk_base = int(chunk["base_addr"]) - int(cls.pcpu_base_addr) - cls._add_to_offset_cache(chunk_base, 0, size_in_bytes) + chunk_base = int(chunk["base_addr"]) - int(symvals.pcpu_base_addr) + self._add_to_offset_cache(chunk_base, 0, size_in_bytes) - @classmethod - def _setup_dynamic_offset_cache(cls) -> None: + def _setup_dynamic_offset_cache(self) -> None: # TODO: interval tree would be more efficient, but this adds no 3rd # party module dependency... - use_area_map = struct_has_member(cls.pcpu_chunk_type, 'map') - for slot in range(cls.pcpu_nr_slots): - for chunk in list_for_each_entry(cls.pcpu_slot[slot], cls.pcpu_chunk_type, 'list'): + use_area_map = struct_has_member(types.pcpu_chunk_type, 'map') + for slot in range(symvals.pcpu_nr_slots): + for chunk in list_for_each_entry(symvals.pcpu_slot[slot], types.pcpu_chunk_type, 'list'): if use_area_map: - cls._setup_dynamic_offset_cache_area_map(chunk) + self._setup_dynamic_offset_cache_area_map(chunk) else: - cls._setup_dynamic_offset_cache_bitmap(chunk) + self._setup_dynamic_offset_cache_bitmap(chunk) def _is_percpu_var_dynamic(self, var: int) -> bool: try: @@ -203,7 +195,7 @@ def _is_static_percpu_address(self, addr: int) -> bool: for start in self.static_ranges: size = self.static_ranges[start] for cpu in range(0, self.last_cpu): - offset = int(__per_cpu_offset[cpu]) + start + offset = int(symvals['__per_cpu_offset'][cpu]) + start if addr >= offset and addr < offset + size: return True return False @@ -233,7 +225,7 @@ def is_static_percpu_var(self, addr: int) -> bool: # loading debuginfo but not when debuginfo is embedded. def _relocated_offset(self, var): addr=int(var) - start = self.__per_cpu_start + start = msymvals['__per_cpu_start'] size = self.static_ranges[start] if addr >= start and addr < start + size: return addr - start @@ -257,7 +249,6 @@ def is_module_percpu_var(self, addr: int) -> bool: return True return False - @export def is_percpu_var(self, var: SymbolOrValue) -> bool: """ Returns whether the provided value or symbol falls within @@ -311,16 +302,15 @@ def _get_percpu_var(self, var: SymbolOrValue, cpu: int) -> gdb.Value: if cpu < 0: raise ValueError("cpu must be >= 0") - addr = self.__per_cpu_offset[cpu] + addr = symvals['__per_cpu_offset'][cpu] if addr > 0: addr += self._relocated_offset(var) val = gdb.Value(addr).cast(var.type) - if var.type != self.void_p_type: + if var.type != types.void_p_type: val = val.dereference() return val - @export def get_percpu_var(self, var: SymbolOrValue, cpu: int) -> gdb.Value: """ Retrieve a per-cpu variable for a single CPU @@ -342,7 +332,6 @@ def get_percpu_var(self, var: SymbolOrValue, cpu: int) -> gdb.Value: var = self._resolve_percpu_var(var) return self._get_percpu_var(var, cpu) - @export def get_percpu_vars(self, var: SymbolOrValue, nr_cpus: int=None) -> Dict[int, gdb.Value]: """ @@ -377,30 +366,94 @@ def get_percpu_vars(self, var: SymbolOrValue, vals[cpu] = self._get_percpu_var(var, cpu) return vals - @export - def percpu_counter_sum(self, var: SymbolOrValue) -> int: - """ - Returns the sum of a percpu counter +msym_cbs = MinimalSymbolCallbacks([ ('__per_cpu_start', + PerCPUState._setup_per_cpu_size), + ('__per_cpu_end', + PerCPUState._setup_per_cpu_size) ]) +symbol_cbs = SymbolCallbacks([ ('__per_cpu_offset', PerCPUState._setup_nr_cpus), + ('modules', PerCPUState._setup_module_ranges) ]) - Args: - var (gdb.Value or gdb.Symbol): The percpu counter to sum +_state = PerCPUState() - Returns: - int: the sum of all components of the percpu counter - """ - if isinstance(var, gdb.Symbol): - var = var.value() +def is_percpu_var(var: SymbolOrValue) -> bool: + """ + Returns whether the provided value or symbol falls within + any of the percpu ranges + + Args: + var: (gdb.Value or gdb.Symbol): The value to query + + Returns: + bool: whether the value belongs to any percpu range + """ + return _state.is_percpu_var(var) + +def get_percpu_var(var: SymbolOrValue, cpu: int) -> gdb.Value: + """ + Retrieve a per-cpu variable for a single CPU + + Args: + var (gdb.Symbol, gdb.MinSymbol, gdb.Value): + The value to use to resolve the percpu location + cpu (int): The cpu for which to return the per-cpu value. + + Returns: + gdb.Value: If cpu is specified, the value corresponding to + the specified CPU. + + Raises: + TypeError: var is not gdb.Symbol or gdb.Value + PerCPUError: var does not fall into any percpu range + ValueError: cpu is less than 0 + """ + return _state.get_percpu_var(var, cpu) + +def get_percpu_vars(var: SymbolOrValue, + nr_cpus: int=None) -> Dict[int, gdb.Value]: + """ + Retrieve a per-cpu variable for all CPUs + + Args: + var (gdb.Symbol, gdb.MinSymbol, gdb.Value): + The value to use to resolve the percpu location + nr_cpus(int, optional, default=None): The number of CPUs to + return results for. None (or unspecified) will use + the highest possible CPU count. + + Returns: + dict(int, gdb.Value): The values corresponding to every CPU + in a dictionary indexed by CPU number. + + Raises: + TypeError: var is not gdb.Symbol or gdb.Value + PerCPUError: var does not fall into any percpu range + ValueError: nr_cpus is <= 0 + """ + return _state.get_percpu_vars(var, nr_cpus) + +def percpu_counter_sum(var: SymbolOrValue) -> int: + """ + Returns the sum of a percpu counter + + Args: + var (gdb.Value or gdb.Symbol): The percpu counter to sum + + Returns: + int: the sum of all components of the percpu counter + """ + if isinstance(var, gdb.Symbol): + var = var.value() - if not (var.type == self.percpu_counter_type or - (var.type.code == gdb.TYPE_CODE_PTR and - var.type.target() == self.percpu_counter_type)): - raise TypeError("var must be gdb.Symbol or gdb.Value describing `{}' not `{}'" - .format(self.percpu_counter_type, var.type)) + if not (var.type == types.percpu_counter_type or + (var.type.code == gdb.TYPE_CODE_PTR and + var.type.target() == types.percpu_counter_type)): + raise TypeError("var must be gdb.Symbol or gdb.Value describing `{}' not `{}'" + .format(types.percpu_counter_type, var.type)) - total = int(var['count']) + total = int(var['count']) - v = get_percpu_vars(var['counters']) - for cpu in v: - total += int(v[cpu]) + v = get_percpu_vars(var['counters']) + for cpu in v: + total += int(v[cpu]) - return total + return total diff --git a/crash/types/slab.py b/crash/types/slab.py index bbf96a4da91..9b4b312cc9f 100644 --- a/crash/types/slab.py +++ b/crash/types/slab.py @@ -7,8 +7,8 @@ import traceback from crash.util import container_of, find_member_variant, get_symbol_value from crash.util import safe_get_symbol_value +from crash.util.symbols import Types, TypeCallbacks, SymbolCallbacks from crash.types.percpu import get_percpu_var -from crash.infra import CrashBaseClass, export from crash.types.list import list_for_each, list_for_each_entry from crash.types.page import Page, page_from_gdb_obj, page_from_addr from crash.types.node import for_each_nid @@ -34,14 +34,9 @@ def col_error(msg): def col_bold(msg): return "\033[1;37;40m {}\033[0;37;40m ".format(msg) +types = Types([ 'kmem_cache', 'struct kmem_cache' ]) -class Slab(CrashBaseClass): - __types__ = [ 'struct slab', 'struct page', 'kmem_cache', 'kmem_bufctl_t', - 'freelist_idx_t' ] - __type_callbacks__ = [ ('struct page', 'check_page_type'), - ('struct slab', 'check_slab_type'), - ('kmem_bufctl_t', 'check_bufctl_type'), - ('freelist_idx_t', 'check_bufctl_type') ] +class Slab(object): slab_list_head = None page_slab = None @@ -90,7 +85,7 @@ def from_list_head(cls, list_head, kmem_cache): def __add_free_obj_by_idx(self, idx): objs_per_slab = self.kmem_cache.objs_per_slab bufsize = self.kmem_cache.buffer_size - + if (idx >= objs_per_slab): self.__error(": free object index %d overflows %d" % (idx, objs_per_slab)) @@ -102,13 +97,13 @@ def __add_free_obj_by_idx(self, idx): return False else: self.free.add(obj_addr) - + return True def __populate_free(self): if self.free: return - + self.free = set() bufsize = self.kmem_cache.buffer_size objs_per_slab = self.kmem_cache.objs_per_slab @@ -142,7 +137,7 @@ def __populate_free(self): def find_obj(self, addr): bufsize = self.kmem_cache.buffer_size objs_per_slab = self.kmem_cache.objs_per_slab - + if int(addr) < self.s_mem: return None @@ -168,7 +163,7 @@ def contains_obj(self, addr): return (False, obj_addr, ac[obj_addr]) return (True, obj_addr, None) - + def __error(self, msg, misplaced = False): msg = col_error("cache %s slab %x%s" % (self.kmem_cache.name, int(self.gdb_obj.address), msg)) @@ -177,7 +172,7 @@ def __error(self, msg, misplaced = False): self.misplaced_error = msg else: print(msg) - + def __free_error(self, list_name): self.misplaced_list = list_name self.__error(": is on list %s, but has %d of %d objects allocated" % @@ -216,7 +211,7 @@ def check(self, slabtype, nid): elif struct_slab_cache != self.kmem_cache.off_slab_cache: self.__error(": OFF_SLAB struct slab is in a wrong cache %s" % struct_slab_cache) - + struct_slab_obj = struct_slab_slab.contains_obj(self.gdb_obj.address) if not struct_slab_obj[0]: self.__error(": OFF_SLAB struct slab is not allocated") @@ -228,7 +223,7 @@ def check(self, slabtype, nid): if self.inuse + num_free != max_free: self.__error(": inuse=%d free=%d adds up to %d (should be %d)" % (self.inuse, num_free, self.inuse + num_free, max_free)) - + if slabtype == slab_free: if num_free != max_free: self.__free_error("slab_free") @@ -242,7 +237,7 @@ def check(self, slabtype, nid): if self.page_slab: slab_nid = self.page.get_nid() if nid != slab_nid: - self.__error(": slab is on nid %d instead of %d" % + self.__error(": slab is on nid %d instead of %d" % (slab_nid, nid)) print("free objects %d" % num_free) @@ -264,7 +259,7 @@ def check(self, slabtype, nid): last_page_addr = int(page.gdb_obj.address) if page.get_nid() != nid: - self.__error(": obj %x is on nid %d instead of %d" % + self.__error(": obj %x is on nid %d instead of %d" % (obj, page.get_nid(), nid)) if not page.is_slab(): self.__error(": obj %x is not on PageSlab page" % obj) @@ -300,11 +295,7 @@ def __init__(self, gdb_obj, kmem_cache, error=False): self.inuse = int(gdb_obj["inuse"]) self.s_mem = int(gdb_obj["s_mem"]) -class KmemCache(CrashBaseClass): - __types__ = [ 'struct kmem_cache', 'struct alien_cache' ] - __type_callbacks__ = [ ('struct kmem_cache', 'check_kmem_cache_type'), - ('struct alien_cache', 'setup_alien_cache_type') ] - +class KmemCache(object): buffer_size_name = None nodelists_name = None percpu_name = None @@ -326,7 +317,7 @@ def setup_alien_cache_type(cls, gdbtype): def __get_nodelist(self, node): return self.gdb_obj[KmemCache.nodelists_name][node] - + def __get_nodelists(self): for nid in for_each_nid(): node = self.__get_nodelist(nid) @@ -345,7 +336,7 @@ def __init__(self, name, gdb_obj): self.name = name self.gdb_obj = gdb_obj self.array_caches = None - + self.objs_per_slab = int(gdb_obj["num"]) self.buffer_size = int(gdb_obj[KmemCache.buffer_size_name]) @@ -377,7 +368,7 @@ def __fill_array_cache(self, acache, ac_type, nid_src, nid_tgt): print(col_error("WARNING: array cache duplicity detected!")) else: self.array_caches[ptr] = cache_dict - + page = page_from_addr(ptr) obj_nid = page.get_nid() @@ -430,7 +421,7 @@ def __fill_all_array_caches(self): shared_cache = node["shared"] if int(shared_cache) != 0: self.__fill_array_cache(shared_cache.dereference(), AC_SHARED, nid, nid) - + self.__fill_alien_caches(node, nid) def get_array_caches(self): @@ -533,7 +524,7 @@ def ___check_slabs(self, node, slabtype, nid, reverse=False): print(col_error("Unrecoverable error when traversing {} slab list: {}".format( slab_list_name[slabtype], e))) check_ok = False - + if errors['num_ok'] > 0: print("{} slab objects were ok between {:#x} and {:#x}". format(errors['num_ok'], errors['first_ok'], errors['last_ok'])) @@ -545,7 +536,7 @@ def ___check_slabs(self, node, slabtype, nid, reverse=False): return (check_ok, slabs, free) def __check_slabs(self, node, slabtype, nid): - + slab_list = node[slab_list_fullname[slabtype]] print("checking {} slab list {:#x}".format(slab_list_name[slabtype], @@ -562,7 +553,7 @@ def __check_slabs(self, node, slabtype, nid): slabtype, nid, reverse=True) slabs += slabs_rev free += free_rev - + #print("checked {} slabs in {} slab list".format( # slabs, slab_list_name[slabtype])) @@ -606,53 +597,56 @@ def check_all(self): (nid, free_declared, free_counted))) self.check_array_caches() -class KmemCaches(CrashBaseClass): - - __symbol_callbacks__ = [ ('slab_caches', 'setup_slab_caches'), - (' cache_chain', 'setup_slab_caches') ] - - kmem_caches = None - kmem_caches_by_addr = None - - @classmethod - def setup_slab_caches(cls, slab_caches): - cls.kmem_caches = dict() - cls.kmem_caches_by_addr = dict() - - list_caches = slab_caches.value() - - for cache in list_for_each_entry(list_caches, - KmemCache.kmem_cache_type, - KmemCache.head_name): - name = cache["name"].string() - kmem_cache = KmemCache(name, cache) - - cls.kmem_caches[name] = kmem_cache - cls.kmem_caches_by_addr[int(cache.address)] = kmem_cache - - @export - def kmem_cache_from_addr(cls, addr): - try: - return cls.kmem_caches_by_addr[addr] - except KeyError: - return None - - @export - def kmem_cache_from_name(cls, name): - try: - return cls.kmem_caches[name] - except KeyError: - return None - - @export - def kmem_cache_get_all(cls): - return cls.kmem_caches.values() - - @export - def slab_from_obj_addr(cls, addr): - page = page_from_addr(addr).compound_head() - if not page.is_slab(): - return None - - return Slab.from_page(page) - +kmem_caches = None +kmem_caches_by_addr = None + +def setup_slab_caches(slab_caches): + global kmem_caches + global kmem_caches_by_addr + + kmem_caches = dict() + kmem_caches_by_addr = dict() + + list_caches = slab_caches.value() + + for cache in list_for_each_entry(list_caches, + types.kmem_cache_type, + KmemCache.head_name): + name = cache["name"].string() + kmem_cache = KmemCache(name, cache) + + kmem_caches[name] = kmem_cache + kmem_caches_by_addr[int(cache.address)] = kmem_cache + +def kmem_cache_from_addr(addr): + try: + return kmem_caches_by_addr[addr] + except KeyError: + return None + +def kmem_cache_from_name(name): + try: + return kmem_caches[name] + except KeyError: + return None + +def kmem_cache_get_all(): + return kmem_caches.values() + +def slab_from_obj_addr(addr): + page = page_from_addr(addr).compound_head() + if not page.is_slab(): + return None + + return Slab.from_page(page) + +type_cbs = TypeCallbacks([ ('struct page', Slab.check_page_type), + ('struct slab', Slab.check_slab_type), + ('kmem_bufctl_t', Slab.check_bufctl_type), + ('freelist_idx_t', Slab.check_bufctl_type), + ('struct kmem_cache', + KmemCache.check_kmem_cache_type), + ('struct alien_cache', + KmemCache.setup_alien_cache_type) ]) +symbol_cbs = SymbolCallbacks([ ('slab_caches', setup_slab_caches), + (' cache_chain', setup_slab_caches) ]) diff --git a/crash/types/task.py b/crash/types/task.py index 9a87111d6a8..b8adbedc020 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -3,8 +3,7 @@ import gdb from crash.util import array_size, struct_has_member -from crash.infra import CrashBaseClass -from crash.infra.lookup import DelayedValue, ClassProperty, get_delayed_lookup +from crash.util.symbols import Types, Symvals, SymbolCallbacks PF_EXITING = 0x4 @@ -13,16 +12,15 @@ def get_value(symname): if sym[0]: return sym[0].value() +types = Types(['struct task_struct', 'struct mm_struct', 'atomic_long_t' ]) +symvals = Symvals([ 'task_state_array' ]) + # This is pretty painful. These are all #defines so none of them end # up with symbols in the kernel. The best approximation we have is # task_state_array which doesn't include all of them. All we can do # is make some assumptions based on the changes upstream. This will # be fragile. -class TaskStateFlags(CrashBaseClass): - __types__ = [ 'char *', 'struct task_struct' ] - __symvals__ = [ 'task_state_array' ] - __symbol_callbacks__ = [ ('task_state_array', '_task_state_flags_callback') ] - +class TaskStateFlags(object): TASK_RUNNING = 0 TASK_FLAG_UNINITIALIZED = -1 @@ -51,11 +49,11 @@ def has_flag(cls, flagname): @classmethod def _task_state_flags_callback(cls, symbol): - count = array_size(cls.task_state_array) + count = array_size(symvals.task_state_array) bit = 0 for i in range(count): - state = cls.task_state_array[i].string() + state = symvals.task_state_array[i].string() state_strings = { '(running)' : 'TASK_RUNNING', '(sleeping)' : 'TASK_INTERRUPTIBLE', @@ -153,6 +151,9 @@ def _check_state_bits(cls): raise RuntimeError("Missing required task states: {}" .format(",".join(missing))) +symbol_cbs = SymbolCallbacks([ ('task_state_array', + TaskStateFlags._task_state_flags_callback) ]) + TF = TaskStateFlags class BadTaskError(TypeError): @@ -178,8 +179,8 @@ def __init__(self, task_struct, active=False, cpu=None, regs=None): raise TypeError("cpu must be integer or None") if not (isinstance(task_struct, gdb.Value) and - (task_struct.type == self.task_struct_type or - task_struct.type == self.task_struct_type.pointer())): + (task_struct.type == types.task_struct_type or + task_struct.type == types.task_struct_type.pointer())): raise BadTaskError(task_struct) self.task_struct = task_struct @@ -200,7 +201,7 @@ def __init__(self, task_struct, active=False, cpu=None, regs=None): @classmethod def init_task_types(cls, task): if not cls.valid: - t = gdb.lookup_type('struct task_struct') + t = types.task_struct_type if task.type != t: raise BadTaskError(task) @@ -209,10 +210,9 @@ def init_task_types(cls, task): # a type resolved from a symbol will be different structures # within gdb. Equality requires a deep comparison rather than # a simple pointer comparison. - cls.task_struct_type = task.type - fields = cls.task_struct_type.fields() + types.task_struct_type = task.type + fields = types.task_struct_type.fields() cls.task_state_has_exit_state = 'exit_state' in fields - cls.mm_struct_type = gdb.lookup_type('struct mm_struct') cls.pick_get_rss() cls.pick_last_run() cls.init_mm = get_value('init_mm') @@ -334,21 +334,21 @@ def get_anon_file_rss_fields(self): # select the proper function and assign it to the class. @classmethod def pick_get_rss(cls): - if struct_has_member(cls.mm_struct_type, 'rss'): + if struct_has_member(types.mm_struct_type, 'rss'): cls.get_rss = cls.get_rss_field - elif struct_has_member(cls.mm_struct_type, '_rss'): + elif struct_has_member(types.mm_struct_type, '_rss'): cls.get_rss = cls.get__rss_field - elif struct_has_member(cls.mm_struct_type, 'rss_stat'): + elif struct_has_member(types.mm_struct_type, 'rss_stat'): cls.MM_FILEPAGES = get_value('MM_FILEPAGES') cls.MM_ANONPAGES = get_value('MM_ANONPAGES') cls.get_rss = cls.get_rss_stat_field else: cls.anon_file_rss_fields = [] - if struct_has_member(cls.mm_struct_type, '_file_rss'): + if struct_has_member(types.mm_struct_type, '_file_rss'): cls.anon_file_rss_fields.append('_file_rss') - if struct_has_member(cls.mm_struct_type, '_anon_rss'): + if struct_has_member(types.mm_struct_type, '_anon_rss'): cls.anon_file_rss_fields.append('_anon_rss') cls.atomic_long_type = gdb.lookup_type('atomic_long_t') @@ -368,9 +368,9 @@ def last_run__last_arrival(self): @classmethod def pick_last_run(cls): - fields = cls.task_struct_type.keys() + fields = types.task_struct_type.keys() if ('sched_info' in fields and - 'last_arrival' in cls.task_struct_type['sched_info'].type.keys()): + 'last_arrival' in types.task_struct_type['sched_info'].type.keys()): cls.last_run = cls.last_run__last_arrival elif 'last_run' in fields: diff --git a/crash/types/vmstat.py b/crash/types/vmstat.py index 6d2cc77fd29..1f2c77aae38 100644 --- a/crash/types/vmstat.py +++ b/crash/types/vmstat.py @@ -2,17 +2,16 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.infra import CrashBaseClass, export from crash.util import container_of, find_member_variant +from crash.util.symbols import Types, TypeCallbacks, Symbols import crash.types.node from crash.types.percpu import get_percpu_var from crash.types.cpu import for_each_online_cpu -class VmStat(CrashBaseClass): - __types__ = ['enum zone_stat_item', 'enum vm_event_item'] - __symbols__ = [ 'vm_event_states' ] - __type_callbacks__ = [ ('enum zone_stat_item', 'check_enum_type'), - ('enum vm_event_item', 'check_enum_type') ] + +class VmStat(object): + types = Types(['enum zone_stat_item', 'enum vm_event_item']) + symbols = Symbols([ 'vm_event_states' ]) nr_stat_items = None nr_event_items = None @@ -22,12 +21,14 @@ class VmStat(CrashBaseClass): @classmethod def check_enum_type(cls, gdbtype): - if gdbtype == cls.enum_zone_stat_item_type: - (items, names) = cls.__populate_names(gdbtype, 'NR_VM_ZONE_STAT_ITEMS') + if gdbtype == cls.types.enum_zone_stat_item_type: + (items, names) = cls.__populate_names(gdbtype, + 'NR_VM_ZONE_STAT_ITEMS') cls.nr_stat_items = items cls.vm_stat_names = names - elif gdbtype == cls.enum_vm_event_item_type: - (items, names) = cls.__populate_names(gdbtype, 'NR_VM_EVENT_ITEMS') + elif gdbtype == cls.types.enum_vm_event_item_type: + (items, names) = cls.__populate_names(gdbtype, + 'NR_VM_EVENT_ITEMS') cls.nr_event_items = items cls.vm_event_names = names else: @@ -45,22 +46,27 @@ def __populate_names(cls, enum_type, items_name): return (nr_items, names) - @staticmethod - def get_stat_names(): - return VmStat.vm_stat_names + @classmethod + def get_stat_names(cls): + return cls.vm_stat_names - @staticmethod - def get_event_names(): - return VmStat.vm_event_names + @classmethod + def get_event_names(cls): + return cls.vm_event_names @classmethod - def get_events(): - nr = VmStat.nr_event_items + def get_events(cls): + nr = cls.nr_event_items events = [0] * nr for cpu in for_each_online_cpu(): - states = get_percpu_var(cls.vm_event_states, cpu) + states = get_percpu_var(cls.symbols.vm_event_states, cpu) for item in range(0, nr): events[item] += int(states["event"][item]) return events + +type_cbs = TypeCallbacks([ ('enum zone_stat_item', + VmStat.check_enum_type), + ('enum vm_event_item', + VmStat.check_enum_type) ]) diff --git a/crash/types/zone.py b/crash/types/zone.py index 64f515a5aea..836ab682e9f 100644 --- a/crash/types/zone.py +++ b/crash/types/zone.py @@ -2,19 +2,17 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb -from crash.infra import CrashBaseClass, export from crash.util import container_of, find_member_variant, array_for_each +from crash.util.symbols import Types import crash.types.node from crash.types.percpu import get_percpu_var from crash.types.vmstat import VmStat from crash.types.cpu import for_each_online_cpu from crash.types.list import list_for_each_entry -def getValue(sym): - return gdb.lookup_symbol(sym, None)[0].value() +class Zone(object): -class Zone(CrashBaseClass): - __types__ = [ 'struct zone', 'struct page' ] + types = Types([' struct page' ]) def __init__(self, obj, zid): self.gdb_obj = obj @@ -52,7 +50,9 @@ def _check_free_area(self, area, is_pcp): nr_free = 0 list_array_name = "lists" if is_pcp else "free_list" for free_list in array_for_each(area[list_array_name]): - for page_obj in list_for_each_entry(free_list, self.page_type, "lru"): + for page_obj in list_for_each_entry(free_list, + self.types.page_type, + "lru"): page = crash.types.page.Page.from_obj(page_obj) nr_free += 1 if page.get_nid() != self.nid or page.get_zid() != self.zid: @@ -72,18 +72,14 @@ def check_free_pages(self): pageset = get_percpu_var(self.gdb_obj["pageset"], cpu) self._check_free_area(pageset["pcp"], True) -class Zones(CrashBaseClass): +def for_each_zone(): + for node in crash.types.node.for_each_node(): + for zone in node.for_each_zone(): + yield zone - @export - def for_each_zone(cls): - for node in crash.types.node.for_each_node(): - for zone in node.for_each_zone(): - yield zone - - @export - def for_each_populated_zone(cls): - #TODO: some filter thing? - for zone in cls.for_each_zone(): - if zone.is_populated(): - yield zone +def for_each_populated_zone(): + #TODO: some filter thing? + for zone in for_each_zone(): + if zone.is_populated(): + yield zone diff --git a/crash/util/__init__.py b/crash/util/__init__.py index bfa1548502d..817180b4054 100644 --- a/crash/util/__init__.py +++ b/crash/util/__init__.py @@ -7,8 +7,7 @@ import uuid from typing import Dict - -from crash.infra import CrashBaseClass, export +from crash.util.symbols import Types from crash.exceptions import MissingTypeError, MissingSymbolError TypeSpecifier = Union [ gdb.Type, gdb.Value, str, gdb.Symbol ] @@ -73,450 +72,418 @@ def __init__(self, member, gdbtype): self.member = member self.type = gdbtype -class TypesUtilClass(CrashBaseClass): - __types__ = [ 'char *', 'uuid_t' ] - - @export - def container_of(self, val, gdbtype, member): - """ - Returns an object that contains the specified object at the given - offset. - - Args: - val (gdb.Value): The value to be converted. It can refer to an - allocated structure or a pointer. - gdbtype (gdb.Type): The type of the object that will be generated - member (str): The name of the member in the target struct that - contains `val`. - - Returns: - gdb.Value: The converted object, of the type specified by - the caller. - Raises: - TypeError: val is not a gdb.Value - """ - if not isinstance(val, gdb.Value): - raise TypeError("container_of expects gdb.Value") - charp = self.char_p_type - if val.type.code != gdb.TYPE_CODE_PTR: - val = val.address - gdbtype = resolve_type(gdbtype) - offset = offsetof(gdbtype, member) - return (val.cast(charp) - offset).cast(gdbtype.pointer()).dereference() - - @export - @staticmethod - def struct_has_member(gdbtype: TypeSpecifier, name: str) -> bool: - """ - Returns whether a structure has a given member name. - - A typical method of determining whether a structure has a member is just - to check the fields list. That generally works but falls apart when - the structure contains an anonymous union or substructure since - it will push the members one level deeper in the namespace. - - This routine provides a simple interface that covers those details. - - Args: - val (gdb.Type, gdb.Value, str, gdb.Symbol): The object for which - to resolve the type to search for the member - name (str): The name of the member to query - - Returns: - bool: Whether the member is present in the specified type - - Raises: - TypeError: An invalid argument has been provided. - - """ - try: - x = TypesUtilClass.offsetof(gdbtype, name) - return True - except InvalidComponentError: - return False - - @export - @staticmethod - def get_symbol_value(symname, block=None, domain=None): - """ - Returns the value associated with a named symbol - - Args: - symname (str): Name of the symbol to resolve - block (gdb.Block, optional, default=None): The block to resolve - the symbol within - domain (gdb.Symbol constant SYMBOL_*_DOMAIN, optional, default=None): - The domain to search for the symbol - Returns: - gdb.Value: The requested value - Raises: - MissingSymbolError: The symbol or value cannot be located - """ - if domain is None: - domain = gdb.SYMBOL_VAR_DOMAIN - sym = gdb.lookup_symbol(symname, block, domain)[0] - if sym: - return sym.value() - raise MissingSymbolError("Cannot locate symbol {}".format(symname)) - - @export - @classmethod - def safe_get_symbol_value(cls, symname, block=None, domain=None): - """ - Returns the value associated with a named symbol - - Args: - symname (str): Name of the symbol to resolve - block (gdb.Block, optional, default=None): The block to resolve - the symbol within - domain (gdb.Symbol constant SYMBOL_*_DOMAIN, optional, default=None): - The domain to search for the symbol - Returns: - gdb.Value: The requested value or - None: if the symbol or value cannot be found - - """ - try: - return cls.get_symbol_value(symname, block, domain) - except MissingSymbolError: - return None - - @export - @staticmethod - def resolve_type(val): - """ - Resolves a gdb.Type given a type, value, string, or symbol - - Args: - val (gdb.Type, gdb.Value, str, gdb.Symbol): The object for which - to resolve the type - - Returns: - gdb.Type: The resolved type - - Raises: - TypeError: The object type of val is not valid - """ - if isinstance(val, gdb.Type): - gdbtype = val - elif isinstance(val, gdb.Value): - gdbtype = val.type - elif isinstance(val, str): - try: - gdbtype = gdb.lookup_type(val) - except gdb.error: - raise MissingTypeError("Could not resolve type {}" - .format(val)) - elif isinstance(val, gdb.Symbol): - gdbtype = val.value().type - else: - raise TypeError("Invalid type {}".format(str(type(val)))) - return gdbtype - - @classmethod - def __offsetof(cls, val, spec, error): - gdbtype = val - offset = 0 - - for member in spec.split('.'): - found = False - if gdbtype.code != gdb.TYPE_CODE_STRUCT and \ - gdbtype.code != gdb.TYPE_CODE_UNION: - raise _InvalidComponentTypeError(field.name, spec) - for field in gdbtype.fields(): - off = field.bitpos >> 3 - if field.name == member: - nexttype = field.type - found = True - break +types = Types([ 'char *', 'uuid_t' ]) + +def container_of(val, gdbtype, member): + """ + Returns an object that contains the specified object at the given + offset. + + Args: + val (gdb.Value): The value to be converted. It can refer to an + allocated structure or a pointer. + gdbtype (gdb.Type): The type of the object that will be generated + member (str): The name of the member in the target struct that + contains `val`. + + Returns: + gdb.Value: The converted object, of the type specified by + the caller. + Raises: + TypeError: val is not a gdb.Value + """ + if not isinstance(val, gdb.Value): + raise TypeError("container_of expects gdb.Value") + charp = types.char_p_type + if val.type.code != gdb.TYPE_CODE_PTR: + val = val.address + gdbtype = resolve_type(gdbtype) + offset = offsetof(gdbtype, member) + return (val.cast(charp) - offset).cast(gdbtype.pointer()).dereference() + +def struct_has_member(gdbtype: TypeSpecifier, name: str) -> bool: + """ + Returns whether a structure has a given member name. + + A typical method of determining whether a structure has a member is just + to check the fields list. That generally works but falls apart when + the structure contains an anonymous union or substructure since + it will push the members one level deeper in the namespace. + + This routine provides a simple interface that covers those details. + + Args: + val (gdb.Type, gdb.Value, str, gdb.Symbol): The object for which + to resolve the type to search for the member + name (str): The name of the member to query + + Returns: + bool: Whether the member is present in the specified type + + Raises: + TypeError: An invalid argument has been provided. + + """ + try: + x = offsetof(gdbtype, name) + return True + except InvalidComponentError: + return False + +def get_symbol_value(symname, block=None, domain=None): + """ + Returns the value associated with a named symbol + + Args: + symname (str): Name of the symbol to resolve + block (gdb.Block, optional, default=None): The block to resolve + the symbol within + domain (gdb.Symbol constant SYMBOL_*_DOMAIN, optional, default=None): + The domain to search for the symbol + Returns: + gdb.Value: The requested value + Raises: + MissingSymbolError: The symbol or value cannot be located + """ + if domain is None: + domain = gdb.SYMBOL_VAR_DOMAIN + sym = gdb.lookup_symbol(symname, block, domain)[0] + if sym: + return sym.value() + raise MissingSymbolError("Cannot locate symbol {}".format(symname)) + +def safe_get_symbol_value(symname, block=None, domain=None): + """ + Returns the value associated with a named symbol + + Args: + symname (str): Name of the symbol to resolve + block (gdb.Block, optional, default=None): The block to resolve + the symbol within + domain (gdb.Symbol constant SYMBOL_*_DOMAIN, optional, default=None): + The domain to search for the symbol + Returns: + gdb.Value: The requested value or + None: if the symbol or value cannot be found + + """ + try: + return get_symbol_value(symname, block, domain) + except MissingSymbolError: + return None - # Step into anonymous structs and unions - if field.name is None: - res = cls.__offsetof(field.type, member, False) - if res is not None: - found = True - off += res[0] - nexttype = res[1] - break - if not found: - if error: - raise _InvalidComponentNameError(member, gdbtype) - else: - return None - gdbtype = nexttype - offset += off - - return (offset, gdbtype) - - @export - @classmethod - def offsetof_type(cls, val, spec, error=True): - """ - Returns the offset and type of a named member of a structure - - Args: - val (gdb.Type, gdb.Symbol, gdb.Value, or str): The type that - contains the specified member, must be a struct or union - spec (str): The member of the member to resolve - error (bool, optional, default=True): Whether to consider lookup - failures an error - - Returns: - Tuple of: - int: The offset of the resolved member - gdb.Type: The type of the resolved member - - Raises: - InvalidArgumentError: val is not a valid type - InvalidComponentError: spec is not valid for the type - """ - gdbtype = None - try: - gdbtype = resolve_type(val) - except MissingTypeError as e: - pass - except TypeError as e: - pass +def resolve_type(val): + """ + Resolves a gdb.Type given a type, value, string, or symbol - if not isinstance(gdbtype, gdb.Type): - raise InvalidArgumentError(val) + Args: + val (gdb.Type, gdb.Value, str, gdb.Symbol): The object for which + to resolve the type - # We'll be friendly and accept pointers as the initial type - if gdbtype.code == gdb.TYPE_CODE_PTR: - gdbtype = gdbtype.target() + Returns: + gdb.Type: The resolved type + Raises: + TypeError: The object type of val is not valid + """ + if isinstance(val, gdb.Type): + gdbtype = val + elif isinstance(val, gdb.Value): + gdbtype = val.type + elif isinstance(val, str): + try: + gdbtype = gdb.lookup_type(val) + except gdb.error: + raise MissingTypeError("Could not resolve type {}" + .format(val)) + elif isinstance(val, gdb.Symbol): + gdbtype = val.value().type + else: + raise TypeError("Invalid type {}".format(str(type(val)))) + return gdbtype + +def __offsetof(val, spec, error): + gdbtype = val + offset = 0 + + for member in spec.split('.'): + found = False if gdbtype.code != gdb.TYPE_CODE_STRUCT and \ gdbtype.code != gdb.TYPE_CODE_UNION: - raise InvalidArgumentTypeError(gdbtype) - - try: - return cls.__offsetof(gdbtype, spec, error) - except _InvalidComponentBaseError as e: + raise _InvalidComponentTypeError(field.name, spec) + for field in gdbtype.fields(): + off = field.bitpos >> 3 + if field.name == member: + nexttype = field.type + found = True + break + + # Step into anonymous structs and unions + if field.name is None: + res = __offsetof(field.type, member, False) + if res is not None: + found = True + off += res[0] + nexttype = res[1] + break + if not found: if error: - raise InvalidComponentError(gdbtype, spec, e.message) + raise _InvalidComponentNameError(member, gdbtype) else: return None + gdbtype = nexttype + offset += off + + return (offset, gdbtype) - @export - @classmethod - def offsetof(cls, val, spec, error=True): - """ - Returns the offset of a named member of a structure +def offsetof_type(val, spec, error=True): + """ + Returns the offset and type of a named member of a structure - Args: - val (gdb.Type, gdb.Symbol, gdb.Value, or str): The type that - contains the specified member, must be a struct or union - spec (str): The member of the member to resolve - error (bool, optional, default=True): Whether to consider lookup - failures an error + Args: + val (gdb.Type, gdb.Symbol, gdb.Value, or str): The type that + contains the specified member, must be a struct or union + spec (str): The member of the member to resolve + error (bool, optional, default=True): Whether to consider lookup + failures an error - Returns: + Returns: + Tuple of: int: The offset of the resolved member - None: The member could not be resolved - - Raises: - InvalidArgumentError: val is not a valid type - InvalidComponentError: spec is not valid for the type - """ - res = cls.offsetof_type(val, spec, error) - if res: - return res[0] + gdb.Type: The type of the resolved member + + Raises: + InvalidArgumentError: val is not a valid type + InvalidComponentError: spec is not valid for the type + """ + gdbtype = None + try: + gdbtype = resolve_type(val) + except MissingTypeError as e: + pass + except TypeError as e: + pass + + if not isinstance(gdbtype, gdb.Type): + raise InvalidArgumentError(val) + + # We'll be friendly and accept pointers as the initial type + if gdbtype.code == gdb.TYPE_CODE_PTR: + gdbtype = gdbtype.target() + + if gdbtype.code != gdb.TYPE_CODE_STRUCT and \ + gdbtype.code != gdb.TYPE_CODE_UNION: + raise InvalidArgumentTypeError(gdbtype) + + try: + return __offsetof(gdbtype, spec, error) + except _InvalidComponentBaseError as e: + if error: + raise InvalidComponentError(gdbtype, spec, e.message) + else: + return None + +def offsetof(val, spec, error=True): + """ + Returns the offset of a named member of a structure + + Args: + val (gdb.Type, gdb.Symbol, gdb.Value, or str): The type that + contains the specified member, must be a struct or union + spec (str): The member of the member to resolve + error (bool, optional, default=True): Whether to consider lookup + failures an error + + Returns: + int: The offset of the resolved member + None: The member could not be resolved + + Raises: + InvalidArgumentError: val is not a valid type + InvalidComponentError: spec is not valid for the type + """ + res = offsetof_type(val, spec, error) + if res: + return res[0] + return None + +def find_member_variant(gdbtype, variants): + """ + Examines the given type and returns the first found member name + + Over time, structure member names may change. This routine + allows the caller to provide a list of potential names and returns + the first one found. + + Args: + gdbtype (gdb.Type): The type of structure or union to examine + variants (list of str): The names of members to search + + Returns: + str: The first member name found + + Raises: + TypeError: No named member could be found + """ + for v in variants: + if offsetof(gdbtype, v, False) is not None: + return v + raise TypeError("Unrecognized '{}': could not find member '{}'" + .format(str(gdbtype), variants[0])) + +def safe_lookup_type(name, block=None): + """ + Looks up a gdb.Type without throwing an exception on failure + + Args: + name (str): The name of the type to look up + + Returns: + gdb.Type for requested type or None if it could not be found + """ + try: + return gdb.lookup_type(name, block) + except gdb.error: return None - @export - @classmethod - def find_member_variant(cls, gdbtype, variants): - """ - Examines the given type and returns the first found member name - - Over time, structure member names may change. This routine - allows the caller to provide a list of potential names and returns - the first one found. - - Args: - gdbtype (gdb.Type): The type of structure or union to examine - variants (list of str): The names of members to search - - Returns: - str: The first member name found - - Raises: - TypeError: No named member could be found - """ - for v in variants: - if cls.offsetof(gdbtype, v, False) is not None: - return v - raise TypeError("Unrecognized '{}': could not find member '{}'" - .format(str(gdbtype), variants[0])) - - @export - @staticmethod - def safe_lookup_type(name, block=None): - """ - Looks up a gdb.Type without throwing an exception on failure - - Args: - name (str): The name of the type to look up - - Returns: - gdb.Type for requested type or None if it could not be found - """ +def array_size(value): + """ + Returns the number of elements in an array + + Args: + value (gdb.Value): The array to size + """ + return value.type.sizeof // value[0].type.sizeof + +def get_typed_pointer(val, gdbtype): + """ + Returns a pointer to the requested type at the given address + + Args: + val (gdb.Value, str, or int): The address for which to provide + a casted pointer + gdbtype (gdb.Type): The type of the pointer to return + + Returns: + gdb.Value: The casted pointer of the requested type + """ + if gdbtype.code != gdb.TYPE_CODE_PTR: + gdbtype = gdbtype.pointer() + if isinstance(val, gdb.Value): + if (val.type != gdbtype and + val.type != gdbtype.target()): + raise TypeError("gdb.Value must refer to {} not {}" + .format(gdbtype, val.type)) + elif isinstance(val, str): try: - return gdb.lookup_type(name, block) - except gdb.error: - return None - - @export - @staticmethod - def array_size(value): - """ - Returns the number of elements in an array - - Args: - value (gdb.Value): The array to size - """ - return value.type.sizeof // value[0].type.sizeof - - @export - @staticmethod - def get_typed_pointer(val, gdbtype): - """ - Returns a pointer to the requested type at the given address - - Args: - val (gdb.Value, str, or int): The address for which to provide - a casted pointer - gdbtype (gdb.Type): The type of the pointer to return - - Returns: - gdb.Value: The casted pointer of the requested type - """ - if gdbtype.code != gdb.TYPE_CODE_PTR: - gdbtype = gdbtype.pointer() - if isinstance(val, gdb.Value): - if (val.type != gdbtype and - val.type != gdbtype.target()): - raise TypeError("gdb.Value must refer to {} not {}" - .format(gdbtype, val.type)) - elif isinstance(val, str): + val = int(val, 16) + except TypeError as e: + print(e) + raise TypeError("string must describe hex address: ".format(e)) + if isinstance(val, int): + val = gdb.Value(val).cast(gdbtype).dereference() + + return val + +def array_for_each(value): + size = array_size(value) + for i in range(array_size(value)): + yield value[i] + +def decode_flags(value: gdb.Value, names: Dict[int, str], + separator: str="|") -> str: + """ + Present a bitfield of individual flags in a human-readable format. + + Args: + value (gdb.Value): + The value containing the flags to be decoded. + names (dict of int->str): + A dictionary containing mappings for each bit number to + a human-readable name. Any flags found that do not have + a matching value in the dict will be displayed as FLAG_. + separator (str, defaults to "|"): + The string to use as a separator between each flag name in the + output. + + Returns: + str: A human-readable string displaying the flag values. + + Raises: + TypeError: value is not gdb.Value or names is not dict. + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if not isinstance(names, dict): + raise TypeError("names must be a dictionary of int -> str") + + flags_val = int(value) + flags = [] + for n in range(0, value.type.sizeof << 3): + if flags_val & (1 << n): try: - val = int(val, 16) - except TypeError as e: - print(e) - raise TypeError("string must describe hex address: ".format(e)) - if isinstance(val, int): - val = gdb.Value(val).cast(gdbtype).dereference() - - return val - - @export - @staticmethod - def array_for_each(value): - size = array_size(value) - for i in range(array_size(value)): - yield value[i] - - @export - @staticmethod - def decode_flags(value: gdb.Value, names: Dict[int, str], - separator: str="|") -> str: - """ - Present a bitfield of individual flags in a human-readable format. - - Args: - value (gdb.Value): - The value containing the flags to be decoded. - names (dict of int->str): - A dictionary containing mappings for each bit number to - a human-readable name. Any flags found that do not have - a matching value in the dict will be displayed as FLAG_. - separator (str, defaults to "|"): - The string to use as a separator between each flag name in the - output. - - Returns: - str: A human-readable string displaying the flag values. - - Raises: - TypeError: value is not gdb.Value or names is not dict. - """ - if not isinstance(value, gdb.Value): - raise TypeError("value must be gdb.Value") - - if not isinstance(names, dict): - raise TypeError("names must be a dictionary of int -> str") - - flags_val = int(value) - flags = [] - for n in range(0, value.type.sizeof << 3): - if flags_val & (1 << n): - try: - flags.append(names[1 << n]) - except KeyError: - flags.append("FLAG_{}".format(n)) - - return separator.join(flags) - - @export - @classmethod - def decode_uuid(cls, value: gdb.Value) -> uuid.UUID: - """ - Decode an array of bytes that describes a UUID into a Python-style - UUID object - - Args: - value (gdb.Value): The UUID to decode - - Returns: - uuid.UUID: The UUID object that describes the value - - Raises: - TypeError: value is not gdb.Value or does not describe a 16-byte array. - - """ - if not isinstance(value, gdb.Value): - raise TypeError("value must be gdb.Value") - - if (value.type.code != gdb.TYPE_CODE_ARRAY or - value[0].type.sizeof != 1 or - value.type.sizeof != 16): - raise TypeError("value must describe an array of 16 bytes") - - u = 0 - for i in range(0, 16): - u <<= 8 - u += int(value[i]) - - return uuid.UUID(int=u) - - @export - @classmethod - def decode_uuid_t(cls, value: gdb.Value) -> uuid.UUID: - """ - Decode a Linux kernel uuid_t into a Python-style UUID object - - Args: - value (gdb.Value): The uuid_t to be decoded - - Returns: - uuid.UUID: The UUID object that describes the value - - Raises: - TypeError: value is not gdb.Value - """ - if not isinstance(value, gdb.Value): - raise TypeError("value must be gdb.Value") - - if value.type != self.uuid_t_type: - if (value.type.code == gdb.TYPE_CODE_PTR and - value.type.target() == self.uuid_t_type): - value = value.dereference() - else: - raise TypeError("value must describe a uuid_t") + flags.append(names[1 << n]) + except KeyError: + flags.append("FLAG_{}".format(n)) + + return separator.join(flags) + +def decode_uuid(value: gdb.Value) -> uuid.UUID: + """ + Decode an array of bytes that describes a UUID into a Python-style + UUID object + + Args: + value (gdb.Value): The UUID to decode - if 'b' in cls.uuid_t_type: - member = 'b' + Returns: + uuid.UUID: The UUID object that describes the value + + Raises: + TypeError: value is not gdb.Value or does not describe a 16-byte array. + + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if (value.type.code != gdb.TYPE_CODE_ARRAY or + value[0].type.sizeof != 1 or value.type.sizeof != 16): + raise TypeError("value must describe an array of 16 bytes") + + u = 0 + for i in range(0, 16): + u <<= 8 + u += int(value[i]) + + return uuid.UUID(int=u) + +def decode_uuid_t(value: gdb.Value) -> uuid.UUID: + """ + Decode a Linux kernel uuid_t into a Python-style UUID object + + Args: + value (gdb.Value): The uuid_t to be decoded + + Returns: + uuid.UUID: The UUID object that describes the value + + Raises: + TypeError: value is not gdb.Value + """ + if not isinstance(value, gdb.Value): + raise TypeError("value must be gdb.Value") + + if value.type != types.uuid_t_type: + if (value.type.code == gdb.TYPE_CODE_PTR and + value.type.target() == types.uuid_t_type): + value = value.dereference() else: - member = '__u_bits' + raise TypeError("value must describe a uuid_t") + + if struct_has_member(types.uuid_t_type, 'b'): + member = 'b' + else: + member = '__u_bits' - return cls.decode_uuid(value[member]) + return decode_uuid(value[member]) diff --git a/tests/test_infra.py b/tests/test_infra.py deleted file mode 100644 index 3c3c5d50d60..00000000000 --- a/tests/test_infra.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: - -import unittest -import gdb - -from crash.infra import CrashBaseClass, export - -# The delayed init tests check for presence of an attribute in the instance -# dict (or class dict for class attributes) since hasattr() will call -# __getattr__, causing delayed initialization to occur. - -class TestInfra(unittest.TestCase): - def test_exporter_baseline(self): - class test_class(CrashBaseClass): - inited = False - def __init__(self): - self.retval = 1020 - setattr(self.__class__, 'inited', True) - @export - def test_func(self): - return self.retval - - x = test_class() - self.assertTrue(x.inited) - - self.assertTrue(test_class.inited) - self.assertTrue(test_func() == 1020) - self.assertTrue(test_class.inited) - - def test_export_normal(self): - class test_class(CrashBaseClass): - @export - def test_func(self): - return 104 - - self.assertTrue(test_func() == 104) - - def test_static_export(self): - class test_class(CrashBaseClass): - @staticmethod - @export - def test_func(): - return 1050 - - self.assertTrue(test_func() == 1050) - - def test_export_static(self): - class test_class(CrashBaseClass): - @export - @staticmethod - def test_func(): - return 105 - - self.assertTrue(test_func() == 105) - - def test_export_class(self): - class test_class(CrashBaseClass): - @classmethod - @export - def test_func(self): - return 106 - - self.assertTrue(test_func() == 106) - - def test_export_multiple_exports_one_instance(self): - class test_class(CrashBaseClass): - instances = 0 - def __init__(self): - setattr(self.__class__, 'instances', self.instances + 1) - - @export - def test_func(self): - return 1060 - @export - def test_func2(self): - return 1061 - - self.assertTrue(test_class.instances == 0) - self.assertTrue(test_func() == 1060) - self.assertTrue(test_class.instances == 1) - self.assertTrue(test_func() == 1060) - self.assertTrue(test_class.instances == 1) - self.assertTrue(test_func2() == 1061) - self.assertTrue(test_class.instances == 1) - self.assertTrue(test_func2() == 1061) - self.assertTrue(test_class.instances == 1) diff --git a/tests/test_infra_lookup.py b/tests/test_infra_lookup.py index a1acb52ecb5..8f79ccdd7e8 100644 --- a/tests/test_infra_lookup.py +++ b/tests/test_infra_lookup.py @@ -3,16 +3,14 @@ import unittest import gdb -from crash.infra import CrashBaseClass from crash.exceptions import DelayedAttributeError from crash.infra.callback import ObjfileEventCallback from crash.infra.lookup import SymbolCallback, TypeCallback from crash.infra.lookup import MinimalSymbolCallback -from crash.infra.lookup import DelayedLookups, ClassProperty from crash.infra.lookup import DelayedType, DelayedSymbol, DelayedSymval from crash.infra.lookup import DelayedMinimalSymbol, DelayedMinimalSymval -class TestDelayedLookupSetup(unittest.TestCase): +class TestTypeNameResolution(unittest.TestCase): def test_resolve_struct_normal(self): spec = 'struct test' @@ -69,137 +67,6 @@ def test_resolve_char_pointer(self): self.assertTrue(attrname == 'char_p_type') self.assertTrue(pointer) - def test_name_collision_attrs(self): - class test_data(object): - def __init__(self): - self.name = 'foo' - def get(self): - pass - def set(self, value): - pass - d = {'__delayed_lookups__' : {}} - attr = test_data() - DelayedLookups.add_lookup('TestClass', d, 'foo', attr) - with self.assertRaises(NameError): - DelayedLookups.add_lookup('TestClass', d, 'foo', attr) - - def test_name_collision_reserved(self): - d = {'__delayed_lookups__' : {}} - with self.assertRaises(NameError): - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - - def test_type_setup(self): - d = {'__types__' : [ 'void *', 'struct test' ] } - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - self.assertFalse('__types__' in d) - self.assertTrue('void_p_type' in d) - self.assertTrue(isinstance(d['void_p_type'], ClassProperty)) - self.assertTrue('void_p_type' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['void_p_type'], - DelayedType)) - self.assertTrue('test_type' in d) - self.assertTrue(isinstance(d['test_type'], ClassProperty)) - self.assertTrue('test_type' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['test_type'], - DelayedType)) - def test_symbol_setup(self): - d = {'__symbols__' : [ 'main' ]} - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - self.assertFalse('__symbols__' in d) - self.assertTrue('main' in d) - self.assertTrue(isinstance(d['main'], ClassProperty)) - self.assertTrue('main' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['main'], - DelayedSymbol)) - - def test_symval_setup(self): - d = {'__symvals__' : [ 'main' ]} - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - self.assertFalse('__symvals__' in d) - self.assertTrue('main' in d) - self.assertTrue(isinstance(d['main'], ClassProperty)) - self.assertTrue('main' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['main'], - DelayedSymval)) - - def test_symval_setup_bad(self): - d = {'__symvals__' : 'main' } - with self.assertRaises(TypeError): - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - - def test_minsymbol_setup(self): - d = {'__minsymbols__' : [ 'main' ]} - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - self.assertFalse('__minsymbols__' in d) - self.assertTrue('main' in d) - self.assertTrue(isinstance(d['main'], ClassProperty)) - self.assertTrue('main' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['main'], - DelayedMinimalSymbol)) - def test_minsymval_setup(self): - d = {'__minsymvals__' : [ 'main' ]} - DelayedLookups.setup_delayed_lookups_for_class('TestClass', d) - self.assertFalse('__minsymvals__' in d) - self.assertTrue('main' in d) - self.assertTrue(isinstance(d['main'], ClassProperty)) - self.assertTrue('main' in d['__delayed_lookups__']) - self.assertTrue(isinstance(d['__delayed_lookups__']['main'], - DelayedMinimalSymval)) - - def get_callback_class(self): - class TestClass(DelayedLookups): - @classmethod - def main_callback(self, value): - self.main_value = value - - @classmethod - def voidp_callback(self, value): - self.voidp_value = value - - return TestClass - - def test_type_callback_setup(self): - TestClass = self.get_callback_class() - d = {'__type_callbacks__' : [ ('void *', 'voidp_callback') ], - '__delayed_lookups__' : {} } - DelayedLookups.setup_named_callbacks(TestClass, d) - self.assertFalse('__type_callbacks__' in d) - - def test_type_callback_setup_bad(self): - TestClass = self.get_callback_class() - d = {'__type_callbacks__' : [ 'void *', 'voidp_callback' ], - '__delayed_lookups__' : {} } - with self.assertRaises(ValueError): - DelayedLookups.setup_named_callbacks(TestClass, d) - - def test_symbol_callback_setup(self): - TestClass = self.get_callback_class() - d = {'__symbol_callbacks__' : [ ('main', 'main_callback') ], - '__delayed_lookups__' : {} } - DelayedLookups.setup_named_callbacks(TestClass, d) - self.assertFalse('__symbol_callbacks__' in d) - - def test_symbol_callback_setup_bad(self): - TestClass = self.get_callback_class() - d = {'__symbol_callbacks__' : [ 'main', 'main_callback' ], - '__delayed_lookups__' : {} } - with self.assertRaises(ValueError): - DelayedLookups.setup_named_callbacks(TestClass, d) - - def test_minsymbol_callback_setup(self): - TestClass = self.get_callback_class() - d = {'__minsymbol_callbacks__' : [ ('main', 'main_callback') ], - '__delayed_lookups__' : {} } - DelayedLookups.setup_named_callbacks(TestClass, d) - self.assertFalse('__minsymbol_callbacks__' in d) - - def test_minsymbol_callback_setup_bad(self): - TestClass = self.get_callback_class() - d = {'__minsymbol_callbacks__' : [ 'main', 'main_callback' ], - '__delayed_lookups__' : {} } - with self.assertRaises(ValueError): - DelayedLookups.setup_named_callbacks(TestClass, d) - class TestMinimalSymbolCallback(unittest.TestCase): def setUp(self): gdb.execute("file") @@ -355,230 +222,3 @@ def test_type_not_found_in_early_load_then_found_after_load(self): self.load_file() self.assertTrue(x.found) self.assertTrue(isinstance(x.gdbtype, gdb.Type)) - -class TestDelayedLookup(unittest.TestCase): - def setUp(self): - gdb.execute("file") - - def load_file(self): - gdb.execute("file tests/test-util") - - def msymbol_test(self): - class Test(CrashBaseClass): - __minsymbols__ = [ 'test_struct' ] - return Test - - def test_bad_msymbol_name(self): - test = self.msymbol_test() - x = test - with self.assertRaises(AttributeError): - y = x.bad_symbol_name - - def test_msymbol_unavailable_at_start(self): - test = self.msymbol_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - - def test_msymbol_available_on_load(self): - test = self.msymbol_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - self.load_file() - self.assertTrue(isinstance(x.test_struct, gdb.MinSymbol)) - - def test_msymbol_available_at_start(self): - test = self.msymbol_test() - self.load_file() - - x = test() - self.assertTrue(isinstance(x.test_struct, gdb.MinSymbol)) - - def symbol_test(self): - class Test(CrashBaseClass): - __symbols__ = [ 'test_struct' ] - return Test - - def test_bad_symbol_name(self): - test = self.symbol_test() - x = test - with self.assertRaises(AttributeError): - y = x.bad_symbol_name - - def test_symbol_unavailable_at_start(self): - test = self.symbol_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - - def test_symbol_available_on_load(self): - test = self.symbol_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - self.load_file() - self.assertTrue(isinstance(x.test_struct, gdb.Symbol)) - - def test_symbol_available_at_start(self): - test = self.symbol_test() - self.load_file() - - x = test() - self.assertTrue(isinstance(x.test_struct, gdb.Symbol)) - - def symval_test(self): - class Test(CrashBaseClass): - __symvals__ = [ 'test_struct' ] - return Test - - def test_bad_symval_name(self): - test = self.symval_test() - x = test - with self.assertRaises(AttributeError): - y = x.bad_symval_name - - def test_symval_unavailable_at_start(self): - test = self.symval_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - - def test_symval_available_on_load(self): - test = self.symval_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_struct - self.load_file() - self.assertTrue(isinstance(x.test_struct, gdb.Value)) - - def test_symval_available_at_start(self): - test = self.symval_test() - self.load_file() - - x = test() - self.assertTrue(isinstance(x.test_struct, gdb.Value)) - - def type_test(self): - class Test(CrashBaseClass): - __types__ = [ 'struct test' ] - return Test - - def test_bad_type_name(self): - test = self.type_test() - x = test - with self.assertRaises(AttributeError): - y = x.bad_type_name - - def test_type_unavailable_at_start(self): - test = self.type_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_type - - def test_type_available_on_load(self): - test = self.type_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_type - self.load_file() - y = x.test_type - self.assertTrue(isinstance(y, gdb.Type)) - - def test_type_available_at_start(self): - test = self.type_test() - self.load_file() - - x = test() - y = x.test_type - self.assertTrue(isinstance(y, gdb.Type)) - - def ptype_test(self): - class Test(CrashBaseClass): - __types__ = [ 'struct test *' ] - return Test - - def test_bad_ptype_name(self): - test = self.ptype_test() - x = test - with self.assertRaises(AttributeError): - y = x.bad_ptype_name - - def test_p_type_unavailable_at_start(self): - test = self.ptype_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_p_type - - def test_p_type_available_on_load(self): - test = self.ptype_test() - x = test() - with self.assertRaises(DelayedAttributeError): - y = x.test_p_type - self.load_file() - y = x.test_p_type - self.assertTrue(isinstance(y, gdb.Type)) - - def test_p_type_available_at_start(self): - test = self.ptype_test() - self.load_file() - - x = test() - y = x.test_p_type - self.assertTrue(isinstance(y, gdb.Type)) - - def type_callback_test(self): - class Test(CrashBaseClass): - __type_callbacks__ = [ - ('unsigned long', 'check_ulong') - ] - ulong_valid = False - @classmethod - def check_ulong(cls, gdbtype): - cls.ulong_valid = True - - return Test - - def test_type_callback_nofile(self): - test = self.type_callback_test() - x = test() - self.assertFalse(test.ulong_valid) - with self.assertRaises(AttributeError): - y = x.unsigned_long_type - - def test_type_callback(self): - test = self.type_callback_test() - x = test() - self.load_file() - self.assertTrue(test.ulong_valid) - with self.assertRaises(AttributeError): - y = x.unsigned_long_type - - def type_callback_test_multi(self): - class Test(CrashBaseClass): - __types__ = [ 'unsigned long' ] - __type_callbacks__ = [ - ('unsigned long', 'check_ulong') - ] - ulong_valid = False - @classmethod - def check_ulong(cls, gdbtype): - cls.ulong_valid = True - - return Test - - def test_type_callback_nofile_multi(self): - test = self.type_callback_test_multi() - x = test() - self.assertFalse(test.ulong_valid) - with self.assertRaises(DelayedAttributeError): - y = x.unsigned_long_type - - def test_type_callback_multi(self): - test = self.type_callback_test_multi() - x = test() - self.load_file() - self.assertTrue(test.ulong_valid) - y = x.unsigned_long_type - self.assertTrue(isinstance(y, gdb.Type)) - self.assertTrue(y.sizeof > 4) diff --git a/tests/test_syscache.py b/tests/test_syscache.py index 2f719090f67..980df67280e 100644 --- a/tests/test_syscache.py +++ b/tests/test_syscache.py @@ -111,7 +111,7 @@ def test_get_uptime_value(self): from crash.cache.syscache import CrashConfigCache, CrashKernelCache config = CrashConfigCache() kernel = CrashKernelCache(config) - kernel.jiffies = 27028508 + kernel.set_jiffies(27028508) kernel.adjust_jiffies = False x = kernel.uptime uptime = str(x) From 30dd42bdb9c76543db894e3320b7c2e5a9674f84 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 20 May 2019 13:16:07 -0400 Subject: [PATCH 46/75] crash.util: add more documentation No code changes. This just updates the API documentation for crash.util. Signed-off-by: Jeff Mahoney --- crash/util/__init__.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crash/util/__init__.py b/crash/util/__init__.py index 817180b4054..eaef267fdb5 100644 --- a/crash/util/__init__.py +++ b/crash/util/__init__.py @@ -82,9 +82,10 @@ def container_of(val, gdbtype, member): Args: val (gdb.Value): The value to be converted. It can refer to an allocated structure or a pointer. - gdbtype (gdb.Type): The type of the object that will be generated - member (str): The name of the member in the target struct that - contains `val`. + gdbtype (gdb.Type, gdb.Value, str, gdb.Symbol): + The type of the object that will be generated + member (str): + The name of the member in the target struct that contains `val`. Returns: gdb.Value: The converted object, of the type specified by @@ -185,6 +186,7 @@ def resolve_type(val): Raises: TypeError: The object type of val is not valid + MissingTypeError: could not resolve the type from string argument """ if isinstance(val, gdb.Type): gdbtype = val @@ -337,6 +339,8 @@ def safe_lookup_type(name, block=None): Args: name (str): The name of the type to look up + block (gdb.Block, optional, default=None): + The block to use to resolve the type Returns: gdb.Type for requested type or None if it could not be found @@ -352,6 +356,9 @@ def array_size(value): Args: value (gdb.Value): The array to size + + Returns: + int: The number of elements in the array """ return value.type.sizeof // value[0].type.sizeof @@ -366,6 +373,9 @@ def get_typed_pointer(val, gdbtype): Returns: gdb.Value: The casted pointer of the requested type + + Raises: + TypeError: string value for val does not describe a hex address """ if gdbtype.code != gdb.TYPE_CODE_PTR: gdbtype = gdbtype.pointer() @@ -386,6 +396,15 @@ def get_typed_pointer(val, gdbtype): return val def array_for_each(value): + """ + Yields each element in an array separately + + Args: + value (gdb.Value): The array to iterate + + Yields: + gdb.Value: One element in the array at a time + """ size = array_size(value) for i in range(array_size(value)): yield value[i] From 65a7e06c79a76ec3ff7b7bb283bbad45fdbdebe8 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Tue, 7 May 2019 16:30:58 -0400 Subject: [PATCH 47/75] crash: factor out task iteration This commit moves the open-coded iteration over the task lists in crash.kernel to crash.types.task. This will allow it to be unit tested one we have kernel testing in place. Signed-off-by: Jeff Mahoney --- crash/kernel.py | 12 ++---------- crash/types/task.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/crash/kernel.py b/crash/kernel.py index c8f410400d0..04809e1485d 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -13,6 +13,7 @@ from crash.types.percpu import get_percpu_vars from crash.types.list import list_for_each_entry from crash.types.module import for_each_module, for_each_module_section +from crash.types.task import for_each_all_tasks import crash.cache.tasks import crash.cache.syscache from crash.types.task import LinuxTask @@ -609,21 +610,12 @@ def setup_tasks(self) -> None: sys.stdout.flush() task_count = 0 - tasks = [] - for taskg in list_for_each_entry(task_list, self.symvals.init_task.type, - 'tasks', include_head=True): - tasks.append(taskg) - for task in list_for_each_entry(taskg['thread_group'], - self.symvals.init_task.type, - 'thread_group'): - tasks.append(task) - try: crashing_cpu = int(get_symbol_value('crashing_cpu')) except Exception as e: crashing_cpu = -1 - for task in tasks: + for task in for_each_all_tasks(): cpu = None regs = None active = int(task.address) in rqscurrs diff --git a/crash/types/task.py b/crash/types/task.py index b8adbedc020..7562d5d6997 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -4,6 +4,7 @@ import gdb from crash.util import array_size, struct_has_member from crash.util.symbols import Types, Symvals, SymbolCallbacks +from crash.types.list import list_for_each_entry PF_EXITING = 0x4 @@ -13,7 +14,7 @@ def get_value(symname): return sym[0].value() types = Types(['struct task_struct', 'struct mm_struct', 'atomic_long_t' ]) -symvals = Symvals([ 'task_state_array' ]) +symvals = Symvals([ 'task_state_array', 'init_task' ]) # This is pretty painful. These are all #defines so none of them end # up with symbols in the kernel. The best approximation we have is @@ -380,3 +381,21 @@ def pick_last_run(cls): cls.last_run = cls.last_run__timestamp else: raise RuntimeError("No method to retrieve last run from task found.") + +def for_each_thread_group_leader(): + task_list = symvals.init_task['tasks'] + for task in list_for_each_entry(task_list, symvals.init_task.type, + 'tasks', include_head=True): + yield task + +def for_each_thread_in_group(task): + thread_list = task['thread_group'] + for thread in list_for_each_entry(thread_list, symvals.init_task.type, + 'thread_group'): + yield thread + +def for_each_all_tasks(): + for leader in for_each_thread_group_leader(): + yield leader + for task in for_each_thread_in_group(leader): + yield task From abfd2dceefd673d02fabe1f8c0ec2661b1d002af Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 8 May 2019 17:50:13 -0400 Subject: [PATCH 48/75] crash.commands: handle DelayedAttributeError gracefully Command.execute should be catching DelayedAttributeError and displaying an understandable message instead of a stack trace. Signed-off-by: Jeff Mahoney --- crash/commands/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crash/commands/__init__.py b/crash/commands/__init__.py index fac1b28a82b..5a17cf75c88 100644 --- a/crash/commands/__init__.py +++ b/crash/commands/__init__.py @@ -8,6 +8,8 @@ import importlib import argparse +from crash.exceptions import DelayedAttributeError + class CommandError(RuntimeError): pass @@ -48,6 +50,8 @@ def invoke(self, argstr, from_tty=False): except CommandLineError as e: print(f"{self.name}: {str(e)}") self.parser.print_usage() + except DelayedAttributeError as e: + print(f"{self.name}: command unavailable, {str(e)}") except (SystemExit, KeyboardInterrupt): pass From 62bb502fb95f38904a3a1c7f2e1f193fea2dfaeb Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 8 May 2019 19:39:30 -0400 Subject: [PATCH 49/75] crash.commands.dmesg: handling dictionary string encoding better The dictionary handling code was outputting bytes instead of strings, so the output of dmesg -d was littered with b''. This commit properly encodes the dictionary into ASCII. Signed-off-by: Jeff Mahoney --- crash/commands/dmesg.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crash/commands/dmesg.py b/crash/commands/dmesg.py index 18a286cf96a..8d5ba0fcb1e 100644 --- a/crash/commands/dmesg.py +++ b/crash/commands/dmesg.py @@ -200,16 +200,8 @@ def log_from_idx(self, logbuf, idx, dict_needed=False): dict_len = int(msg['dict_len']) d = (msg.cast(types.char_p_type) + types.printk_log_p_type.target().sizeof + textlen) - s = '' - - for i in range(0, dict_len): - if d[i]: - s += chr(d[i]) - else: - msgdict['dict'].append(s) - s = '' - - if s != '': + if dict_len > 0: + s = d.string('ascii', 'backslashreplace', dict_len) msgdict['dict'].append(s) return msgdict @@ -248,7 +240,7 @@ def handle_structured_log(self, args): print('{}{}{}'.format(level, timestamp, line)) for d in msg['dict']: - print('{}'.format(d.encode('string_escape'))) + print(d) def handle_logbuf(self, args): if symvals.log_buf_len and symvals.log_buf: From e7ff8dd79bdec255eec262ddd86a5d1eb78a104e Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 8 May 2019 20:17:16 -0400 Subject: [PATCH 50/75] crash.types.slab: improve address printing The slab code was printing raw tuples with ints in base-10 that represented addresses. This commit ensures those are displayed as hex addresses. Signed-off-by: Jeff Mahoney --- crash/types/slab.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crash/types/slab.py b/crash/types/slab.py index 9b4b312cc9f..fa22e8b8787 100644 --- a/crash/types/slab.py +++ b/crash/types/slab.py @@ -155,14 +155,14 @@ def contains_obj(self, addr): self.__populate_free() if obj_addr in self.free: - return (False, obj_addr, None) + return (False, int(obj_addr), None) ac = self.kmem_cache.get_array_caches() if obj_addr in ac: - return (False, obj_addr, ac[obj_addr]) + return (False, int(obj_addr), ac[obj_addr]) - return (True, obj_addr, None) + return (True, int(obj_addr), None) def __error(self, msg, misplaced = False): msg = col_error("cache %s slab %x%s" % (self.kmem_cache.name, @@ -575,8 +575,9 @@ def check_array_caches(self): print("cached pointer {:#x} in {} is not allocated: {}".format( ac_ptr, acs[ac_ptr], ac_obj_obj)) elif ac_obj_obj[1] != ac_ptr: - print("cached pointer {:#x} in {} has wrong offset: {}".format( - ac_ptr, acs[ac_ptr], ac_obj_obj)) + print("cached pointer {:#x} in {} has wrong offset: ({}, {:#x}, {})" + .format( ac_ptr, acs[ac_ptr], ac_obj_obj[0], + ac_obj_obj[1], ac_obj_obj[2])) def check_all(self): for (nid, node) in self.__get_nodelists(): From 6da88b1d63fb7e77cf940b8deaacc7af3a12a2da Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 8 May 2019 20:22:53 -0400 Subject: [PATCH 51/75] crash.commands.kmem: use exceptions to exit command with error The kmem command currently exits with success even when errors are encountered. This makes testing the code automatically difficult. This commit uses the CommandError and CommandLineError exceptions to handle those conditions. Signed-off-by: Jeff Mahoney --- crash/commands/kmem.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/crash/commands/kmem.py b/crash/commands/kmem.py index 95d5d60fbc3..6437d927b02 100644 --- a/crash/commands/kmem.py +++ b/crash/commands/kmem.py @@ -4,6 +4,7 @@ import gdb import crash from crash.commands import Command, ArgumentParser +from crash.commands import CommandError, CommandLineError from crash.types.slab import kmem_cache_get_all, kmem_cache_from_name, slab_from_obj_addr from crash.types.zone import for_each_zone, for_each_populated_zone from crash.types.vmstat import VmStat @@ -55,8 +56,7 @@ def execute(self, args): print("Checking kmem cache {}".format(cache_name)) cache = kmem_cache_from_name(cache_name) if cache is None: - print("Cache {} not found.".format(cache_name)) - return + raise CommandError(f"Cache {cache_name} not found.") cache.check_all() else: print("Checking all kmem caches...") @@ -68,15 +68,16 @@ def execute(self, args): return if not args.arg: - print("Nothing to do.") - return + raise CommandLineError("no address specified") - addr = int(args.arg[0], 0) + try: + addr = int(args.arg[0], 0) + except ValueError: + raise CommandLineError("address must be numeric") slab = slab_from_obj_addr(addr) if not slab: - print("Address not found in any kmem cache.") - return + raise CommandError("Address not found in any kmem cache.") obj = slab.contains_obj(addr) name = slab.kmem_cache.name @@ -98,9 +99,7 @@ def execute(self, args): elif ac["ac_type"] == "alien": ac_desc = "alien cache of node %d for node %d" % (ac["nid_src"], ac["nid_tgt"]) else: - print("unexpected array cache type") - print(ac) - return + raise CommandError(f"unexpected array cache type {str(ac)}") print("FREE object %x from slab %s (in %s)" % (obj[1], name, ac_desc)) @@ -120,7 +119,7 @@ def print_vmstats(self): try: vm_stat = getValue("vm_stat") except AttributeError: - raise gdb.GdbError("Support for new-style vmstat is unimplemented.") + raise CommandError("Support for new-style vmstat is unimplemented.") print(" VM_STAT:") #TODO put this... where? From 29ea5046cc7f4dbcd769e9ee3d03f64cfc1c3f1e Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 8 May 2019 21:09:59 -0400 Subject: [PATCH 52/75] crash.commands.kmem: fix argument parsing The argument parsing for the kmem command wasn't strict enough and would allow some strange combinations through. This commit fixes the parsing so that it accepts an address only when none of the options are used and that the only option that accepts an (optional) argument is -s [slabname]. Signed-off-by: Jeff Mahoney --- crash/commands/kmem.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crash/commands/kmem.py b/crash/commands/kmem.py index 6437d927b02..2a6312b2812 100644 --- a/crash/commands/kmem.py +++ b/crash/commands/kmem.py @@ -5,7 +5,8 @@ import crash from crash.commands import Command, ArgumentParser from crash.commands import CommandError, CommandLineError -from crash.types.slab import kmem_cache_get_all, kmem_cache_from_name, slab_from_obj_addr +from crash.types.slab import kmem_cache_get_all, kmem_cache_from_name +from crash.types.slab import slab_from_obj_addr from crash.types.zone import for_each_zone, for_each_populated_zone from crash.types.vmstat import VmStat import argparse @@ -34,13 +35,12 @@ def __init__(self, name): parser = ArgumentParser(prog=name) group = parser.add_mutually_exclusive_group() - group.add_argument('-s', action='store_true', default=False) + group.add_argument('-s', nargs='?', const=True, default=False, + dest='slabname') group.add_argument('-z', action='store_true', default=False) group.add_argument('-V', action='store_true', default=False) + group.add_argument('address', nargs='?') - parser.add_argument('arg', nargs=argparse.REMAINDER) - - parser.format_usage = lambda : "kmem [-s] [addr | slabname]\n" super().__init__(name, parser) def execute(self, args): @@ -50,28 +50,28 @@ def execute(self, args): elif args.V: self.print_vmstats() return - elif args.s: - if args.arg: - cache_name = args.arg[0] - print("Checking kmem cache {}".format(cache_name)) - cache = kmem_cache_from_name(cache_name) - if cache is None: - raise CommandError(f"Cache {cache_name} not found.") - cache.check_all() - else: + elif args.slabname: + if args.slabname is True: print("Checking all kmem caches...") for cache in kmem_cache_get_all(): print(cache.name) cache.check_all() + else: + cache_name = args.slabname + print(f"Checking kmem cache {cache_name}") + cache = kmem_cache_from_name(cache_name) + if cache is None: + raise CommandError(f"Cache {cache_name} not found.") + cache.check_all() print("Checking done.") return - if not args.arg: + if not args.address: raise CommandLineError("no address specified") try: - addr = int(args.arg[0], 0) + addr = int(args.address[0], 0) except ValueError: raise CommandLineError("address must be numeric") slab = slab_from_obj_addr(addr) From f2142e5372ec1137c0f05281bb1791950a80acdd Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 8 May 2019 21:12:15 -0400 Subject: [PATCH 53/75] crash.commands.kmem: clean up some trivial things - Don't use a private version of get_symbol_value - Don't import re and argparse when they're not used Signed-off-by: Jeff Mahoney --- crash/commands/kmem.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crash/commands/kmem.py b/crash/commands/kmem.py index 2a6312b2812..38f8cc32a7c 100644 --- a/crash/commands/kmem.py +++ b/crash/commands/kmem.py @@ -9,11 +9,8 @@ from crash.types.slab import slab_from_obj_addr from crash.types.zone import for_each_zone, for_each_populated_zone from crash.types.vmstat import VmStat -import argparse -import re - -def getValue(sym): - return gdb.lookup_symbol(sym, None)[0].value() +from crash.util import get_symbol_value +from crash.exceptions import MissingSymbolError class KmemCommand(Command): """ kernel memory inspection @@ -117,8 +114,8 @@ def __print_vmstat(self, vmstat, diffs): def print_vmstats(self): try: - vm_stat = getValue("vm_stat") - except AttributeError: + vm_stat = get_symbol_value("vm_stat") + except MissingSymbolError: raise CommandError("Support for new-style vmstat is unimplemented.") print(" VM_STAT:") From 08749dd03f95cb4fcb3e438d62a6c0c1fd6129c6 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Thu, 9 May 2019 20:42:10 -0400 Subject: [PATCH 54/75] crash.commands.ps: uncomment -G mode The -G argument for ps was commented out. This adds the test for whether a task is a thread group leader to crash.types.task and enables the -G argument in ps. Signed-off-by: Jeff Mahoney --- crash/commands/ps.py | 3 ++- crash/types/task.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crash/commands/ps.py b/crash/commands/ps.py index 4b4bb919482..3717a1073ca 100755 --- a/crash/commands/ps.py +++ b/crash/commands/ps.py @@ -578,7 +578,8 @@ def execute(self, argv): # Only show thread group leaders -# if argv.G and task.pid != int(task.task_struct['tgid']): + if argv.G and not task.is_thread_group_leader(): + continue task.update_mem_usage() self.print_one(argv, thread) diff --git a/crash/types/task.py b/crash/types/task.py index 7562d5d6997..e1a6866f8a0 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -263,6 +263,9 @@ def is_exiting(self): def is_zombie(self): return self.task_state() & TF.EXIT_ZOMBIE + def is_thread_group_leader(self): + return int(self.task_struct['exit_signal']) >= 0 + def update_mem_usage(self): if self.mem_valid: return From 004ad940d697ff80b370beedc715760b44d96cf4 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 7 May 2019 17:52:33 -0400 Subject: [PATCH 55/75] crash: add kernel testing against vmcores One of the things that has had a serious negative effect on the quality of crash-python is the inability to do automated testing on a broad variety of kernels and vmcores. Often, we have to run our tests by hand. More often, we get reports from new users that something broke unexpectedly. This commit adds real unit testing against real kernels and vmcores. It includes a few test cases that will need to be extended further. The policy moving forward will be that new features will require a matching test case that passes across a variety of kernels. Signed-off-by: Jeff Mahoney --- INTERNALS.md | 19 + TESTING.md | 61 ++++ crash/commands/__init__.py | 2 +- kernel-tests/decorators.py | 62 ++++ kernel-tests/test_commands_dmesg.py | 75 ++++ kernel-tests/test_commands_kmem.py | 129 +++++++ kernel-tests/test_commands_mount.py | 40 +++ kernel-tests/test_commands_ps.py | 528 ++++++++++++++++++++++++++++ kernel-tests/test_types_bitmap.py | 76 ++++ kernel-tests/test_types_cpu.py | 31 ++ kernel-tests/test_types_module.py | 30 ++ kernel-tests/test_types_node.py | 41 +++ kernel-tests/test_types_percpu.py | 19 + kernel-tests/test_types_task.py | 40 +++ kernel-tests/test_types_zone.py | 27 ++ kernel-tests/unittest-bootstrap.py | 42 +++ kernel-tests/unittest-prepare.py | 49 +++ setup.py | 2 +- test-all.sh | 28 ++ 19 files changed, 1299 insertions(+), 2 deletions(-) create mode 100644 INTERNALS.md create mode 100644 TESTING.md create mode 100644 kernel-tests/decorators.py create mode 100644 kernel-tests/test_commands_dmesg.py create mode 100644 kernel-tests/test_commands_kmem.py create mode 100644 kernel-tests/test_commands_mount.py create mode 100644 kernel-tests/test_commands_ps.py create mode 100644 kernel-tests/test_types_bitmap.py create mode 100644 kernel-tests/test_types_cpu.py create mode 100644 kernel-tests/test_types_module.py create mode 100644 kernel-tests/test_types_node.py create mode 100644 kernel-tests/test_types_percpu.py create mode 100644 kernel-tests/test_types_task.py create mode 100644 kernel-tests/test_types_zone.py create mode 100644 kernel-tests/unittest-bootstrap.py create mode 100644 kernel-tests/unittest-prepare.py diff --git a/INTERNALS.md b/INTERNALS.md new file mode 100644 index 00000000000..871cac2b4b9 --- /dev/null +++ b/INTERNALS.md @@ -0,0 +1,19 @@ + +# GDB + +## Python contexts within GDB + +Each time gdb enters the Python interpreter it establishes a context. +Part of the context includes what architecture gdb believes it is +debugging ('gdbarch') and that is passed into the context. If anything +changes the gdbarch in that Python context, it won't be visible to any +subsequent Python code until a new session is established. + +When gdb starts up on x86_64, it uses a gdbarch of i386 -- with 32-bit words +and pointers. Only when we load an executable or target does it switch +to i386:x86_64. + +The effect of this is that any code that relys on type information *must* +be executed in a separate context from the one that loaded the executable +and/or taret. Otherwise, any built-in types that are pointers or `long` +based will use the 32-bit sizes. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 00000000000..95764818a2f --- /dev/null +++ b/TESTING.md @@ -0,0 +1,61 @@ +# Testing + +## Summary + +There are unit tests in the tests/ dir that are standalone and useful for +testing basic functionality. + +There are unit tests in the kernel-tests dir that require configuration, +kernel images, debuginfo, and vmcores to use. + +## Configuration + +The configuration for each kernel/vmcore to be tested goes in a .ini file +with the following format. All fields except kernel and vmcore are +optional, and defaults will be used. A kernel missing debuginfo cannot +be used for testing. Missing modules will mean module-specific tests +will be skipped. + +```[test] +kernel=/path/to/kernel +vmcore=/path/to/vmcore +vmlinux_debuginfo=/path/to/vmlinux-debuginfo +modules=/path/to/modules +module_debuginfo_path=/path/to/module/debuginfo +root=/root/for/tree/searches``` + +The optional fields match those defined in crash.kernel.CrashKernel. + +Example 1: +```[test] +kernel=/var/crash/2019-04-23-11:35/vmlinux-4.12.14-150.14-default.gz +vmcore=/var/crash/2019-04-23-11:35/vmcore``` + +In this example, the kernel and debuginfo packages are installed in the +default locations and will be searched automatically. + +Example 2: +```[test] +kernel=/var/crash/2019-04-23-11:35/vmlinux-4.12.14-150.14-default.gz +vmcore=/var/crash/2019-04-23-11:35/vmcore +root=/var/cache/crash-setup/leap15/4.12.14-150.14-default +``` + +In this example, the kernel and debuginfo packages are installed under +/var/cache/crash-setup/leap15/4.12.14-150.14-default and so we only +specify a root directory. + +## Running + +The script `test-all.sh` when run with no options will execute only +the standalone tests. The script takes a list of the .ini files +described above and will execute the kernel tests against those +configurations immediately after the standalone tests. + +Example: +```sh test-all.sh kernel-test-configs/4.12.14-150.14-default.ini kernel-test-configs/5.1.0-rc7-vanilla.ini``` +or +```sh test-all.sh kernel-test-configs/*.ini``` + +Each configuration will execute independently from one another. + diff --git a/crash/commands/__init__.py b/crash/commands/__init__.py index 5a17cf75c88..8d90438d1d5 100644 --- a/crash/commands/__init__.py +++ b/crash/commands/__init__.py @@ -37,7 +37,7 @@ def __init__(self, name, parser=None): self.commands[self.name] = self gdb.Command.__init__(self, self.name, gdb.COMMAND_USER) - def invoke_uncaught(self, argstr, from_tty): + def invoke_uncaught(self, argstr, from_tty=False): argv = gdb.string_to_argv(argstr) args = self.parser.parse_args(argv) self.execute(args) diff --git a/kernel-tests/decorators.py b/kernel-tests/decorators.py new file mode 100644 index 00000000000..db205bfb2e0 --- /dev/null +++ b/kernel-tests/decorators.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +def skip_with_type(typename): + try: + gdbtype = gdb.lookup_type(typename) + return unittest.skip(f"found type {typename}") + except gdb.error: + pass + + return lambda func: func + +def skip_without_type(typename): + try: + gdbtype = gdb.lookup_type(typename) + except gdb.error: + return unittest.skip(f"missing type {typename}") + + return lambda func: func + +def skip_with_symbol(symname): + symbol = gdb.lookup_symbol(symname, None)[0] + if symbol is not None: + return unittest.skip(f"found symbol {symname}") + + return lambda func: func + +def skip_without_symbol(symname): + symbol = gdb.lookup_symbol(symname, None)[0] + if symbol is None: + return unittest.skip(f"missing symbol {symname}") + + return lambda func: func + +def has_super_blocks(name): + from crash.subsystem.filesystem import for_each_super_block + for sb in for_each_super_block(): + if sb['s_type']['name'].string() == name: + return True + return False + +can_test = {} + +def skip_with_supers(name): + if not name in can_test: + can_test[name] = has_super_blocks(name) + + if not can_test[name]: + return lambda func: func + + return unittest.skip(f"{name} file systems in image") + +def skip_without_supers(name): + if not name in can_test: + can_test[name] = has_super_blocks(name) + + if can_test[name]: + return lambda func: func + + return unittest.skip(f"no {name} file systems in image") diff --git a/kernel-tests/test_commands_dmesg.py b/kernel-tests/test_commands_dmesg.py new file mode 100644 index 00000000000..ba31719522a --- /dev/null +++ b/kernel-tests/test_commands_dmesg.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from crash.commands.dmesg import LogCommand +from crash.commands import CommandLineError + +class TestCommandsLog(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + sys.stdout = io.StringIO() + self.command = LogCommand("dmesg") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return sys.stdout.getvalue() + + def test_dmesg(self): + """`dmesg' produces valid output""" + self.command.invoke_uncaught("") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_bad_option(self): + """`dmesg -x` raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-x") + + def test_dmesg_t(self): + """`dmesg' produces valid output""" + self.command.invoke_uncaught("-t") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_d(self): + """`dmesg -d' produces valid output""" + self.command.invoke_uncaught("-d") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_m(self): + """`dmesg -m ' produces valid output""" + self.command.invoke_uncaught("-m") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_tm(self): + """`dmesg -t -m' produces valid output""" + self.command.invoke_uncaught("-t -m") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_td(self): + """`dmesg -t -d' produces valid output""" + self.command.invoke_uncaught("-t -d") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_dm(self): + """`dmesg -m -d' produces valid output""" + self.command.invoke_uncaught("-m -d") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_dmesg_tdm(self): + """`dmesg -t -d -m' produces valid output""" + self.command.invoke_uncaught("-t -d -m") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + diff --git a/kernel-tests/test_commands_kmem.py b/kernel-tests/test_commands_kmem.py new file mode 100644 index 00000000000..269d7d137fc --- /dev/null +++ b/kernel-tests/test_commands_kmem.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from decorators import skip_without_symbol +from decorators import skip_with_symbol + +from crash.commands.kmem import KmemCommand +from crash.commands import CommandLineError, CommandError + +class TestCommandsKmem(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + sys.stdout = io.StringIO() + self.command = KmemCommand("kmem") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return sys.stdout.getvalue() + + def test_kmem_empty(self): + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("") + + def test_kmem_invalid(self): + """`kmem' returns error""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("invalid") + + def test_kmem_s(self): + """`kmem -s' produces valid output""" + self.command.invoke_uncaught("-s") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_kmem_s_inode_cache(self): + """`kmem -s inode_cache' produces valid output""" + self.command.invoke_uncaught("-s inode_cache") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_kmem_s_unknown_cache(self): + """`kmem -s unknown_cache' raises CommandError""" + with self.assertRaises(CommandError): + self.command.invoke_uncaught("-s unknown_cache") + + def test_kmem_sz(self): + """`kmem -s -z' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s -z") + + def test_kmem_sz_valid_cache(self): + """`kmem -s -z' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s inode_cache -z") + + def test_kmem_sz_invalid_cache(self): + """`kmem -s unknown_cache -z' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s unknown_cache -z") + + def test_kmem_sv(self): + """`kmem -s -V' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s -V") + + def test_kmem_sv_valid_cache(self): + """`kmem -s inode_cache -V' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s inode_cache -V") + + def test_kmem_sv_invalid_cache(self): + """`kmem -s unknown_cache -V' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-s unknown_cache -V") + + def test_kmem_z(self): + """`kmem -z' produces valid output""" + self.command.invoke_uncaught("-z") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_kmem_z_invalid(self): + """`kmem -z invalid' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-z invalid") + + @skip_without_symbol('vm_stat') + def test_kmem_v(self): + """`kmem -V' produces valid output""" + self.command.invoke_uncaught("-V") + output = self.output() + self.assertTrue(len(output.split("\n")) > 0) + + @skip_with_symbol('vm_stat') + def test_kmem_v_unimplemented(self): + """`kmem -V' raises CommandError due to missing symbol""" + with self.assertRaises(CommandError): + self.command.invoke_uncaught("-V") + + def test_kmem_v_invalid(self): + """`kmem -V invalid' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-V invalid") + + def test_kmem_vz(self): + """`kmem -V -z' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-V -z") + + def test_kmem_svz(self): + """`kmem -V -z -s' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-V -z -s") + + def test_kmem_svz_valid_cache(self): + """`kmem -V -z -s inode_cache' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-V -z -s inode_cache") + + def test_kmem_svz_invalid_cache(self): + """`kmem -V -z -s unknown_cache' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("-V -z -s unknown_cache") diff --git a/kernel-tests/test_commands_mount.py b/kernel-tests/test_commands_mount.py new file mode 100644 index 00000000000..e3af005db8f --- /dev/null +++ b/kernel-tests/test_commands_mount.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from crash.commands.mount import MountCommand + +class TestCommandsMount(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + sys.stdout = io.StringIO() + self.command = MountCommand("mount") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return sys.stdout.getvalue() + + def test_mount(self): + self.command.invoke("") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_mount_f(self): + self.command.invoke("-f") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_mount_v(self): + self.command.invoke("-v") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_mount_d(self): + self.command.invoke("-d") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) diff --git a/kernel-tests/test_commands_ps.py b/kernel-tests/test_commands_ps.py new file mode 100644 index 00000000000..9c76a074c59 --- /dev/null +++ b/kernel-tests/test_commands_ps.py @@ -0,0 +1,528 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +import sys +import io +import re +import fnmatch + +from crash.commands import CommandError, CommandLineError +from crash.commands.ps import PSCommand +import crash.types.task as tasks + +def bad_command_line(fn, ignored=True): + """Marks test to expect CommandLineError for unimplemented options""" + def test_decorator(fn): + def test_decorated(self, *args, **kwargs): + self.assertRaises(CommandLineError, fn, self, *args, **kwargs) + return test_decorated + test_decorator.__doc__ = fn.__doc__ + " (bad command line raises CommandLineError)" + return test_decorator + +def unimplemented(fn, ignored=True): + """Marks test to expect CommandError for unimplemented options""" + def test_decorator(fn): + def test_decorated(self, *args, **kwargs): + self.assertRaises(CommandError, fn, self, *args, **kwargs) + return test_decorated + test_decorator.__doc__ = fn.__doc__ + " (unimplemented command raises CommandError)" + return test_decorator + +PF_KTHREAD = 0x200000 + +class TestCommandsPs(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + self.redirected = io.StringIO() + sys.stdout = self.redirected + self.command = PSCommand() + self.do_output = False + + def tearDown(self): + sys.stdout = self.stdout + if self.do_output: + print(self._output()) + + def _output(self): + return self.redirected.getvalue() + + def output(self): + try: + return self.output_list + except AttributeError: + self.output_list = self._output().split("\n") + return self.output_list + + def output_lines(self): + output = self.output() + return len(output) - 1 + + def get_wildcard_regex(self, wildcard): + return re.compile(fnmatch.translate(wildcard)) + + def check_line_count(self, count): + self.assertTrue(self.output_lines() == count) + + def check_header(self, expected): + header = self.output()[0] + self.assertTrue(re.match(expected, header) is not None) + + def check_task_header(self): + regex = "\s+PID\s+PPID\s+CPU\s+TASK\s+ST\s+%MEM\s+VSZ\s+RSS\s+COMM" + self.check_header(regex) + + def check_kstack_header(self): + regex = "\s+PID\s+PPID\s+CPU\s+KSTACK\s+ST\s+%MEM\s+VSZ\s+RSS\s+COMM" + self.check_header(regex) + + def check_threadnum_header(self): + regex = "\s+PID\s+PPID\s+CPU\s+THREAD#\s+ST\s+%MEM\s+VSZ\s+RSS\s+COMM" + self.check_header(regex) + + def check_body(self, regex, start=1): + comp = re.compile(regex) + lines = 0 + for line in self.output()[start:-1]: + self.assertTrue(comp.match(line) is not None) + lines += 1 + + self.assertTrue(lines > 0) + + def check_threadnum_output(self): + regex = ">?\s+\d+\s+\d+\s+\d+\s+\d+\s+[A-Z]+\s+[\d\.]+\s+\d+\s+\d+\s+.*" + + self.check_body(regex) + + + def check_normal_output(self): + regex = ">?\s+\d+\s+\d+\s+\d+\s+[\d+a-f]+\s+[A-Z]+\s+[\d\.]+\s+\d+\s+\d+\s+.*" + + self.check_body(regex) + + def check_last_run_output(self): + regex = "\[\d+\]\s+\[[A-Z][A-Z]\]\s+PID:\s+\d+\s+TASK:\s+[\da-f]+\s+CPU:\s+\d+\s+COMMAND: \".*\"" + self.check_body(regex, 0) + + def check_no_matches_output(self): + self.check_header('No matches for.*') + lines = self.output_lines() + self.assertTrue(lines == 1) + + def is_kernel_thread(self, task): + return (int(task['flags']) & PF_KTHREAD) + + def is_user_task(self, task): + return not self.is_kernel_thread(task) + + def task_name(self, task_struct): + return task_struct['comm'].string() + + def count_tasks(self, test=None, regex=None): + count = 0 + for task in tasks.for_each_all_tasks(): + if test is not None and not test(task): + continue + if regex is None or regex.match(self.task_name(task)): + count += 1 + + return count + + def count_kernel_tasks(self, regex=None): + return self.count_tasks(self.is_kernel_thread, regex) + + def count_user_tasks(self, regex=None): + return self.count_tasks(self.is_user_task, regex) + + def count_thread_group_leaders(self, regex=None): + count = 0 + for task in tasks.for_each_thread_group_leader(): + if regex is None or regex.match(self.task_name(task)): + count += 1 + + return count + + def test_ps_empty(self): + self.command.invoke_uncaught("") + self.assertTrue(self.output_lines() > 1) + + def test_ps_wildcard(self): + self.command.invoke_uncaught("*worker*") + + regex = self.get_wildcard_regex("*worker*") + self.check_line_count(self.count_tasks(regex=regex) + 1) + + def test_ps_bad_wildcard(self): + """Test `ps *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("*BaDWiLdCaRd2019*") + self.check_no_matches_output() + + def test_ps_k(self): + """Test `ps -k' outputs all (and only) kernel threads""" + self.command.invoke_uncaught("-k") + lines = self.output_lines() + + self.check_task_header() + self.check_normal_output() + + self.check_line_count(self.count_kernel_tasks() + 1) + + def test_ps_k_wildcard(self): + """Test `ps -k *wonder*' outputs only matching kernel threads""" + self.command.invoke_uncaught("-k *worker*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*worker*") + + self.check_task_header() + self.check_normal_output() + self.check_line_count(self.count_kernel_tasks(regex) + 1) + + def test_ps_k_bad_wildcard(self): + """Test `ps -k *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-k *BaDWiLdCaRd2019*") + self.check_no_matches_output() + + def test_ps_u(self): + """Test `ps -u' outputs all (and only) user tasks""" + self.command.invoke_uncaught("-u") + + self.check_task_header() + self.check_normal_output() + + self.check_line_count(self.count_user_tasks() + 1) + + def test_ps_u_wildcard(self): + """Test `ps -u *wonder*' outputs only matching user tasks""" + self.command.invoke_uncaught("-u *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.check_task_header() + self.check_normal_output() + + self.check_line_count(self.count_user_tasks(regex) + 1) + + def test_ps_u_bad_wildcard(self): + """Test `ps -u *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-u *BaDWiLdCaRd2019*") + self.check_no_matches_output() + + def test_ps_g(self): + """Test `ps -G' outputs all (and only) thread group leaders""" + self.command.invoke_uncaught("-G") + + self.check_task_header() + self.check_normal_output() + + self.check_line_count(self.count_thread_group_leaders() + 1) + + def test_ps_g_wildcard(self): + """Test `ps -G *nscd*' outputs only matching thread group leaders""" + self.command.invoke_uncaught("-G *nscd*") + + regex = self.get_wildcard_regex("*nscd*") + + self.check_task_header() + self.check_normal_output() + + self.check_line_count(self.count_thread_group_leaders() + 1) + + def test_ps_g_bad_wildcard(self): + """Test `ps -G *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-G *BaDWiLdCaRd2019*") + self.check_no_matches_output() + + @bad_command_line + def test_ps_uk(self): + """Test `ps -u -k'""" + self.command.invoke_uncaught("-u -k") + + @bad_command_line + def test_ps_uk_wildcard(self): + """Test `ps -u -k *'""" + self.command.invoke_uncaught("-u -k *") + + @bad_command_line + def test_ps_uG(self): + """Test `ps -u -G'""" + self.command.invoke_uncaught("-u -k") + + @bad_command_line + def test_ps_uG_wildcard(self): + """Test `ps -u -G *'""" + self.command.invoke_uncaught("-u -k *") + + @bad_command_line + def test_ps_kG(self): + """Test `ps -k -G'""" + self.command.invoke_uncaught("-k -G") + + @bad_command_line + def test_ps_kG_wildcard(self): + """Test `ps -k -G *'""" + self.command.invoke_uncaught("-k -G *") + + @bad_command_line + def test_ps_ukG(self): + """Test `ps -u -k -G'""" + self.command.invoke_uncaught("-u -k -G") + + @bad_command_line + def test_ps_ukG_wildcard(self): + """Test `ps -u -k -G *'""" + self.command.invoke_uncaught("-u -k -G *") + + def test_ps_s(self): + """Test `ps -s'""" + self.command.invoke_uncaught("-s") + + self.check_kstack_header() + self.check_normal_output() + + self.check_line_count(self.count_tasks() + 1) + + def test_ps_s_wildcard(self): + """Test `ps -s *nscd*'""" + self.command.invoke_uncaught("-s *nscd*") + + self.check_kstack_header() + self.check_normal_output() + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_tasks(regex=regex) + 1) + + def test_ps_s_bad_wildcard(self): + """Test `ps -s *BaDWiLdCaRd2019*'""" + self.command.invoke_uncaught("-s *BaDWiLdCaRd2019*") + + self.check_no_matches_output() + + def test_ps_n(self): + """Test `ps -n'""" + self.command.invoke_uncaught("-n") + + self.check_threadnum_header() + self.check_threadnum_output() + + self.check_line_count(self.count_tasks() + 1) + + def test_ps_n_wildcard(self): + """Test `ps -n *nscd*'""" + self.command.invoke_uncaught("-n *nscd*") + + self.check_threadnum_header() + self.check_threadnum_output() + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_tasks(regex=regex) + 1) + + def test_ps_n_bad_wildcard(self): + """Test `ps -n *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-n *BaDWiLdCaRd2019*") + + self.check_no_matches_output() + + def test_ps_nu(self): + """Test `ps -n -u'""" + self.command.invoke_uncaught("-n -u") + + self.check_threadnum_header() + self.check_threadnum_output() + + self.check_line_count(self.count_user_tasks() + 1) + + def test_ps_nu_wildcard(self): + """Test `ps -n -u *nscd*'""" + self.command.invoke_uncaught("-n -u *nscd*") + + self.check_threadnum_header() + self.check_threadnum_output() + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_user_tasks(regex) + 1) + + def test_ps_nu_bad_wildcard(self): + """Test `ps -n -u *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-n -u *BaDWiLdCaRd2019*") + + self.check_no_matches_output() + + def test_ps_nk(self): + """Test `ps -n -k'""" + self.command.invoke_uncaught("-n -k") + + self.check_threadnum_header() + self.check_threadnum_output() + + self.check_line_count(self.count_kernel_tasks() + 1) + + def test_ps_nk_wildcard(self): + """Test `ps -n -k *worker*'""" + self.command.invoke_uncaught("-n -k *worker*") + + self.check_threadnum_header() + self.check_threadnum_output() + + regex = self.get_wildcard_regex("*worker*") + self.check_line_count(self.count_kernel_tasks(regex) + 1) + + def test_ps_nk_bad_wildcard(self): + """Test `ps -n -k *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-n -k *BaDWiLdCaRd2019*") + + self.check_no_matches_output() + + def test_ps_nG(self): + """Test `ps -n -G'""" + self.command.invoke_uncaught("-n -G") + + self.check_threadnum_header() + self.check_threadnum_output() + + self.check_line_count(self.count_thread_group_leaders() + 1) + + def test_ps_nG_wildcard(self): + """Test `ps -n -G *nscd*'""" + self.command.invoke_uncaught("-n -G *nscd*") + + self.check_threadnum_header() + self.check_threadnum_output() + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_thread_group_leaders(regex) + 1) + + def test_ps_nG_bad_wildcard(self): + """Test `ps -n -G *BaDWiLdCaRd2019*' returns no matches output""" + self.command.invoke_uncaught("-n -G *BaDWiLdCaRd2019*") + + self.check_no_matches_output() + + @unimplemented + def test_ps_t(self): + """Test `ps -t'""" + self.command.invoke_uncaught("-t") + + # Check format + + self.check_line_count(self.count_tasks()) + + @unimplemented + def test_ps_t_wildcard(self): + """Test `ps -t *nscd*'""" + self.command.invoke_uncaught("-t *nscd*") + + # Check format + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_tasks(regex=regex)) + + def test_ps_l(self): + """Test `ps -l'""" + self.command.invoke_uncaught("-l") + + # No header to test + self.check_last_run_output() + self.check_line_count(self.count_tasks()) + + def test_ps_l_wildcard(self): + """Test `ps -l *nscd*'""" + self.command.invoke_uncaught("-l *nscd*") + + # No header to test + self.check_last_run_output() + + regex = self.get_wildcard_regex("*nscd*") + self.check_line_count(self.count_tasks(regex=regex)) + + @unimplemented + def test_ps_p(self): + """Test `ps -p'""" + self.command.invoke_uncaught("-p") + lines = self.output_lines() + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_p_wildcard(self): + """Test `ps -p *nscd*'""" + self.command.invoke_uncaught("-p *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_c(self): + """Test `ps -c'""" + self.command.invoke_uncaught("-c") + lines = self.output_lines() + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_c_wildcard(self): + """Test `ps -c *nscd*'""" + self.command.invoke_uncaught("-c *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_a(self): + """Test `ps -a'""" + self.command.invoke_uncaught("-a") + lines = self.output_lines() + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_a_wildcard(self): + """Test `ps -a *nscd*'""" + self.command.invoke_uncaught("-a *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_g(self): + """Test `ps -g'""" + self.command.invoke_uncaught("-g") + lines = self.output_lines() + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_g_wildcard(self): + """Test `ps -g *nscd*'""" + self.command.invoke_uncaught("-g *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_r(self): + """Test `ps -r'""" + self.command.invoke_uncaught("-r") + lines = self.output_lines() + + self.assertTrue(lines > 1) + + @unimplemented + def test_ps_r_wildcard(self): + """Test `ps -r *nscd*'""" + self.command.invoke_uncaught("-r *nscd*") + lines = self.output_lines() + + regex = self.get_wildcard_regex("*nscd*") + + self.assertTrue(lines > 1) diff --git a/kernel-tests/test_types_bitmap.py b/kernel-tests/test_types_bitmap.py new file mode 100644 index 00000000000..4e0c938e89c --- /dev/null +++ b/kernel-tests/test_types_bitmap.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +import crash.types.bitmap as bitmaps + +class TestBitmap(unittest.TestCase): + def test_for_each_set_bit(self): + sym = gdb.lookup_symbol('cpu_online_mask', None)[0] + if sym is None: + sym = gdb.lookup_symbol('__cpu_online_mask', None)[0] + + self.assertTrue(sym is not None) + + bitmap = sym.value()['bits'] + + count = 0 + for bit in bitmaps.for_each_set_bit(bitmap): + self.assertTrue(type(bit) is int) + count += 1 + + self.assertTrue(count > 0) + + def test_find_first_set_bit(self): + sym = gdb.lookup_symbol('cpu_online_mask', None)[0] + if sym is None: + sym = gdb.lookup_symbol('__cpu_online_mask', None)[0] + + self.assertTrue(sym is not None) + + bitmap = sym.value()['bits'] + + count = 0 + bit = bitmaps.find_first_set_bit(bitmap) + self.assertTrue(type(bit) is int) + + def test_find_next_set_bit(self): + sym = gdb.lookup_symbol('cpu_online_mask', None)[0] + if sym is None: + sym = gdb.lookup_symbol('__cpu_online_mask', None)[0] + + self.assertTrue(sym is not None) + + bitmap = sym.value()['bits'] + + count = 0 + bit = bitmaps.find_next_set_bit(bitmap, 1) + self.assertTrue(type(bit) is int) + + def test_find_first_zero_bit(self): + sym = gdb.lookup_symbol('cpu_online_mask', None)[0] + if sym is None: + sym = gdb.lookup_symbol('__cpu_online_mask', None)[0] + + self.assertTrue(sym is not None) + + bitmap = sym.value()['bits'] + + count = 0 + bit = bitmaps.find_first_zero_bit(bitmap) + self.assertTrue(type(bit) is int) + + def test_find_next_zero_bit(self): + sym = gdb.lookup_symbol('cpu_online_mask', None)[0] + if sym is None: + sym = gdb.lookup_symbol('__cpu_online_mask', None)[0] + + self.assertTrue(sym is not None) + + bitmap = sym.value()['bits'] + + count = 0 + bit = bitmaps.find_next_zero_bit(bitmap, 10) + self.assertTrue(type(bit) is int) + diff --git a/kernel-tests/test_types_cpu.py b/kernel-tests/test_types_cpu.py new file mode 100644 index 00000000000..803d18fa872 --- /dev/null +++ b/kernel-tests/test_types_cpu.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +import crash.types.cpu as cpus + +class TestCPU(unittest.TestCase): + def test_online_cpu_iteration(self): + count = 0 + for cpu in cpus.for_each_online_cpu(): + self.assertTrue(type(cpu) is int) + count += 1 + + self.assertTrue(count > 0) + + def test_highest_online_cpu(self): + cpu = cpus.highest_online_cpu_nr() + self.assertTrue(type(cpu) is int) + + def test_possible_cpu_iteration(self): + count = 0 + for cpu in cpus.for_each_possible_cpu(): + self.assertTrue(type(cpu) is int) + count += 1 + + self.assertTrue(count > 0) + + def test_highest_possible_cpu(self): + cpu = cpus.highest_possible_cpu_nr() + self.assertTrue(type(cpu) is int) diff --git a/kernel-tests/test_types_module.py b/kernel-tests/test_types_module.py new file mode 100644 index 00000000000..2954901906b --- /dev/null +++ b/kernel-tests/test_types_module.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +gdbinit = """ +set build-id-verbose 0 +set python print-stack full +set prompt py-crash> +set height 0 +set print pretty on""" + +class TestModules(unittest.TestCase): + def test_for_each_module(self): + from crash.types.module import for_each_module + + modtype = gdb.lookup_type('struct module') + + for mod in for_each_module(): + self.assertTrue(mod.type == modtype) + + def test_for_each_module_section(self): + from crash.types.module import for_each_module_section + from crash.types.module import for_each_module + + for mod in for_each_module(): + for section in for_each_module_section(mod): + self.assertTrue(type(section) is tuple) + self.assertTrue(type(section[0]) is str) + self.assertTrue(type(section[1]) is int) diff --git a/kernel-tests/test_types_node.py b/kernel-tests/test_types_node.py new file mode 100644 index 00000000000..3f3a9aa63de --- /dev/null +++ b/kernel-tests/test_types_node.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + + +import crash.types.node as numa_node + +class TestNumaNode(unittest.TestCase): + def test_for_each_node(self): + count = 0 + for node in numa_node.for_each_node(): + self.assertTrue(type(node) is numa_node.Node) + count += 1 + self.assertTrue(count > 0) + + def test_for_each_online_node(self): + count = 0 + for node in numa_node.for_each_online_node(): + self.assertTrue(type(node) is numa_node.Node) + count += 1 + self.assertTrue(count > 0) + + def test_for_each_nid(self): + count = 0 + for nid in numa_node.for_each_nid(): + self.assertTrue(type(nid) is int) + count += 1 + self.assertTrue(count > 0) + + def test_for_each_online_nid(self): + count = 0 + for nid in numa_node.for_each_online_nid(): + self.assertTrue(type(nid) is int) + count += 1 + self.assertTrue(count > 0) + + def test_numa_node_id(self): + nid = numa_node.numa_node_id(0) + self.assertTrue(type(nid) is int) + diff --git a/kernel-tests/test_types_percpu.py b/kernel-tests/test_types_percpu.py new file mode 100644 index 00000000000..ac3df699ff9 --- /dev/null +++ b/kernel-tests/test_types_percpu.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +from crash.types.percpu import get_percpu_vars, is_percpu_var + +class TestPerCPU(unittest.TestCase): + def test_runqueues(self): + rqs = gdb.lookup_symbol('runqueues', None)[0] + rq_type = gdb.lookup_type('struct rq') + + self.assertTrue(rqs.type == rq_type) + + pcpu = get_percpu_vars(rqs) + for (cpu, rq) in pcpu.items(): + self.assertTrue(type(cpu) is int) + self.assertTrue(type(rq) is gdb.Value) + self.assertTrue(rq.type == rq_type) diff --git a/kernel-tests/test_types_task.py b/kernel-tests/test_types_task.py new file mode 100644 index 00000000000..5099e295cfa --- /dev/null +++ b/kernel-tests/test_types_task.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +import crash.types.task as tasks + +class TestTasks(unittest.TestCase): + def setUp(self): + self.task_struct_type = gdb.lookup_type('struct task_struct') + + def test_thread_group_leader_iteration(self): + count = 0 + for leader in tasks.for_each_thread_group_leader(): + self.assertTrue(type(leader) is gdb.Value) + self.assertTrue(leader.type == self.task_struct_type) + self.assertTrue(int(leader['exit_signal']) >= 0) + count += 1 + + self.assertTrue(count > 0) + + def test_thread_group_iteration(self): + count = 0 + for leader in tasks.for_each_thread_group_leader(): + for thread in tasks.for_each_thread_in_group(leader): + self.assertTrue(type(thread) is gdb.Value) + self.assertTrue(thread.type == self.task_struct_type) + self.assertTrue(int(thread['exit_signal']) < 0) + count += 1 + + self.assertTrue(count > 0) + + def test_iterate_all_tasks(self): + count = 0 + for task in tasks.for_each_all_tasks(): + self.assertTrue(type(task) is gdb.Value) + self.assertTrue(task.type == self.task_struct_type) + count += 1 + + self.assertTrue(count > 0) diff --git a/kernel-tests/test_types_zone.py b/kernel-tests/test_types_zone.py new file mode 100644 index 00000000000..45d65fd489e --- /dev/null +++ b/kernel-tests/test_types_zone.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + + +import crash.types.node as numa_node +import crash.types.zone as mmzone + +class TestNumaNode(unittest.TestCase): + def test_for_each_zone(self): + count = 0 + for node in numa_node.for_each_node(): + for zone in node.for_each_zone(): + self.assertTrue(type(zone) is mmzone.Zone) + count += 1 + + self.assertTrue(count > 0) + + def test_for_each_populated_zone(self): + count = 0 + for zone in mmzone.for_each_populated_zone(): + self.assertTrue(type(zone) is mmzone.Zone) + count += 1 + + self.assertTrue(count > 0) + diff --git a/kernel-tests/unittest-bootstrap.py b/kernel-tests/unittest-bootstrap.py new file mode 100644 index 00000000000..35542c5f395 --- /dev/null +++ b/kernel-tests/unittest-bootstrap.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import unittest +import sys +import os +import os.path +import configparser +import gzip +import shutil + +config = configparser.ConfigParser() +filename = os.environ['CRASH_PYTHON_TESTFILE'] +try: + f = open(filename) + config.read_file(f) +except FileNotFoundError as e: + print(f"{str(e)}") + sys.exit(1) + +try: + vmcore = config['test']['vmcore'] +except KeyError: + print(f"{filename} doesn't contain the required sections.") + sys.exit(1) + +roots = config['test'].get('root', None) +vmlinux_debuginfo = config['test'].get('vmlinux_debuginfo', None) +module_path = config['test'].get('module_path', None) +module_debuginfo_path = config['test'].get('module_debuginfo_path', None) + +from crash.kernel import CrashKernel +kernel = CrashKernel(roots=roots, vmlinux_debuginfo=vmlinux_debuginfo, + module_path=module_path, + module_debuginfo_path=module_debuginfo_path) + +kernel.setup_tasks() +kernel.load_modules() + +test_loader = unittest.TestLoader() +test_suite = test_loader.discover('kernel-tests', pattern='test_*.py') +unittest.TextTestRunner(verbosity=2).run(test_suite) diff --git a/kernel-tests/unittest-prepare.py b/kernel-tests/unittest-prepare.py new file mode 100644 index 00000000000..42b558e2933 --- /dev/null +++ b/kernel-tests/unittest-prepare.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import sys +import os +import os.path +import configparser +import gzip +import shutil + +config = configparser.ConfigParser() +filename = os.environ['CRASH_PYTHON_TESTFILE'] +try: + f = open(filename) + config.read_file(f) +except FileNotFoundError as e: + print(f"{str(e)}") + sys.exit(1) + +try: + vmlinux = config['test']['kernel'] + vmcore = config['test']['vmcore'] +except KeyError as e: + print(f"{filename} doesn't contain the required sections `{str(e)}.") + sys.exit(1) + +roots = config['test'].get('root', None) +vmlinux_debuginfo = config['test'].get('vmlinux_debuginfo', None) +module_path = config['test'].get('module_path', None) +module_debuginfo_path = config['test'].get('module_debuginfo_path', None) + +if vmlinux.endswith(".gz"): + vmlinux_gz = vmlinux + testdir = os.environ['CRASH_PYTHON_TESTDIR'] + base = os.path.basename(vmlinux)[:-3] + vmlinux = os.path.join(testdir, base) + + with gzip.open(vmlinux_gz, 'r') as f_in, open(vmlinux, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + f_out.close() + f_in.close() + +gdb.execute(f"file {vmlinux}") + +from kdump.target import Target +target = Target(debug=False) + +gdb.execute(f"target kdumpfile {vmcore}") diff --git a/setup.py b/setup.py index 36e66af06eb..3a8db1515ce 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name = "crash", version = "0.1", - packages = find_packages(exclude=['tests']), + packages = find_packages(exclude=['tests', 'kernel-tests']), package_data = { '' : [ "*.dist" "*.txt" ], }, diff --git a/test-all.sh b/test-all.sh index 6e9eaeb2bf0..4bfc47b10ec 100755 --- a/test-all.sh +++ b/test-all.sh @@ -1,5 +1,17 @@ #!/bin/sh +set -e + +cleanup() { + test -n "$DIR" && rm -rf "$DIR" +} + +trap cleanup EXIT + +DIR=$(mktemp -d "/tmp/crash-python-tests.XXXXXX") + +export CRASH_PYTHON_TESTDIR="$DIR" + rm -rf build/lib/crash python3 setup.py -q build make -C tests -s @@ -28,3 +40,19 @@ if has_mypy; then echo "OK" fi fi + +cat << END > $DIR/gdbinit +python sys.path.insert(0, 'build/lib') +set build-id-verbose 0 +set python print-stack full +set prompt py-crash> +set height 0 +set print pretty on +source kernel-tests/unittest-prepare.py +source kernel-tests/unittest-bootstrap.py +END + +for f in "$@"; do + export CRASH_PYTHON_TESTFILE="$f" + crash-python-gdb -nx -batch -x $DIR/gdbinit +done From cfac8fb2cd9e590bd4b665c26def7e344be0a76c Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 7 May 2019 17:52:48 -0400 Subject: [PATCH 56/75] crash.types.classdev: update classdev iteration to use device_private With Linux v5.1-rc1, knode_class was moved from struct device to struct device_private. This commit updates for_each_class_device to use the implementation that matches the kernel. Signed-off-by: Jeff Mahoney --- crash/types/classdev.py | 31 +++++++++++++++++++++++++---- kernel-tests/test_types_classdev.py | 21 +++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 kernel-tests/test_types_classdev.py diff --git a/crash/types/classdev.py b/crash/types/classdev.py index 885872651ae..8f8c40ece4e 100644 --- a/crash/types/classdev.py +++ b/crash/types/classdev.py @@ -3,13 +3,36 @@ import gdb -from crash.types.klist import klist_for_each_entry -from crash.util.symbols import Types +from crash.types.klist import klist_for_each +from crash.util import struct_has_member, container_of +from crash.util.symbols import Types, TypeCallbacks -types = Types(['struct device']) +types = Types(['struct device', 'struct device_private']) + +class ClassdevState(object): + class_is_private = True + + #v5.1-rc1 moved knode_class from struct device to struct device_private + @classmethod + def setup_iterator_type(cls, gdbtype): + if struct_has_member(gdbtype, 'knode_class'): + cls.class_is_private = False + + +type_cbs = TypeCallbacks([ ('struct device', + ClassdevState.setup_iterator_type) ]) def for_each_class_device(class_struct, subtype=None): klist = class_struct['p']['klist_devices'] - for dev in klist_for_each_entry(klist, types.device_type, 'knode_class'): + + container_type = types.device_type + if ClassdevState.class_is_private: + container_type = types.device_private_type + + for knode in klist_for_each(klist): + dev = container_of(knode, container_type, 'knode_class') + if ClassdevState.class_is_private: + dev = dev['device'].dereference() + if subtype is None or int(subtype) == int(dev['type']): yield dev diff --git a/kernel-tests/test_types_classdev.py b/kernel-tests/test_types_classdev.py new file mode 100644 index 00000000000..409c926aeb7 --- /dev/null +++ b/kernel-tests/test_types_classdev.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb + +import crash.types.classdev as classdevs + +class TestClassdev(unittest.TestCase): + def setUp(self): + self.device_type = gdb.lookup_type('struct device') + + def test_classdev_iteration(self): + count = 0 + block_class = gdb.lookup_symbol('block_class', None)[0].value() + for dev in classdevs.for_each_class_device(block_class): + self.assertTrue(type(dev) is gdb.Value) + self.assertTrue(dev.type == self.device_type) + count += 1 + + self.assertTrue(count > 0) + From 39131e15777ca16f5eb6f1d06d1f26bc972d598c Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Thu, 9 May 2019 16:32:34 -0400 Subject: [PATCH 57/75] crash.subsystem.filesystems: add is_fstype_super and is_fstype_inode These two helpers will take a generic superblock or inode and determine whether it belongs to the given file system. It's a naive implementation that uses a string comparison. This is intentional so the comparison can be made without symbol resolution that may require module loading. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index ee2f2dbf8bb..714b18a933e 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -142,3 +142,35 @@ def get_super_block(desc: AddressSpecifier, force: bool=False) -> gdb.Value: raise gdb.NotAvailableError(f"no superblock available at `{desc}'") return sb + +def is_fstype_super(super_block: gdb.Value, name: str) -> bool: + """ + Tests whether the super_block belongs to a particular file system type. + + This uses a naive string comparison so modules are not required. + + Args: + super_block (gdb.Value): + The struct super_block to test + name (str): The name of the file system type + + Returns: + bool: whether the super_block belongs to the specified file system + """ + return super_fstype(super_block) == name + +def is_fstype_inode(inode: gdb.Value, name: str) -> bool: + """ + Tests whether the inode belongs to a particular file system type. + + Args: + inode (gdb.Value): The struct inode to test + name (str): The name of the file system type + + Returns: + bool: whether the inode belongs to the specified file system + + Raises: + gdb.NotAvailableError: The target value was not available. + """ + return is_fstype_super(inode['i_sb'], name) From bfd2628fe32ebc4030881f011a12c747f2e94bd2 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 15 May 2019 18:12:44 -0400 Subject: [PATCH 58/75] crash.subsystem.filesystem: document gdb.NotAvailableError The helper routines can be passed bad pointers, so document that each can raise gdb.NotAvailableError. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crash/subsystem/filesystem/__init__.py b/crash/subsystem/filesystem/__init__.py index 714b18a933e..bfc9ded270f 100644 --- a/crash/subsystem/filesystem/__init__.py +++ b/crash/subsystem/filesystem/__init__.py @@ -86,6 +86,9 @@ def super_fstype(sb: gdb.Value) -> str: Returns: str: The file system type's name + + Raises: + gdb.NotAvailableError: The target value was not available. """ return sb['s_type']['name'].string() @@ -100,6 +103,8 @@ def super_flags(sb: gdb.Value) -> str: Returns: str: The flags field in human-readable form. + Raises: + gdb.NotAvailableError: The target value was not available. """ return decode_flags(sb['s_flags'], SB_FLAGS) @@ -112,6 +117,9 @@ def for_each_super_block() -> Iterable[gdb.Value]: Yields: gdb.Value + + Raises: + gdb.NotAvailableError: The target value was not available. """ for sb in list_for_each_entry(symvals.super_blocks, types.super_block_type, 's_list'): @@ -156,6 +164,9 @@ def is_fstype_super(super_block: gdb.Value, name: str) -> bool: Returns: bool: whether the super_block belongs to the specified file system + + Raises: + gdb.NotAvailableError: The target value was not available. """ return super_fstype(super_block) == name From c5691bf7437a058070927fa442fc51525832e2b8 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Thu, 9 May 2019 16:33:50 -0400 Subject: [PATCH 59/75] crash.subsytem.filesystem.mount: add API documentation This commit adds documentation for the mount API and makes private some methods/functions that are meant to be internal. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/mount.py | 135 +++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 20 deletions(-) diff --git a/crash/subsystem/filesystem/mount.py b/crash/subsystem/filesystem/mount.py index 44ebda4d080..3194c8e2109 100644 --- a/crash/subsystem/filesystem/mount.py +++ b/crash/subsystem/filesystem/mount.py @@ -48,11 +48,16 @@ def for_each_mount_impl(cls, task): @classmethod def for_each_mount_nsproxy(cls, task): + """ + An implementation of for_each_mount that uses the task's + nsproxy to locate the mount namespace. See for_each_mount + for more details. + """ return list_for_each_entry(task['nsproxy']['mnt_ns']['list'], types.mount_type, 'mnt_list') @classmethod - def check_task_interface(cls, symval): + def _check_task_interface(cls, symval): try: nsproxy = symvals.init_task['nsproxy'] cls.for_each_mount_impl = cls.for_each_mount_nsproxy @@ -60,7 +65,7 @@ def check_task_interface(cls, symval): print("check_task_interface called but no init_task?") pass -def check_mount_type(gdbtype): +def _check_mount_type(gdbtype): try: types.mount_type = gdb.lookup_type('struct mount') except gdb.error: @@ -68,36 +73,77 @@ def check_mount_type(gdbtype): types.mount_type = types.vfsmount_type def for_each_mount(task=None): + """ + Iterate over each mountpoint in the namespace of the specified task + + If no task is given, the init_task is used. + + The type of the mount structure returned depends on whether + 'struct mount' exists on the kernel version being debugged. + + Args: + task (gdb.Value, default=): + The task which contains the namespace to iterate. + + Yields: + gdb.Value: + A mountpoint attached to the namespace. + + """ if task is None: task = symvals.init_task return Mount.for_each_mount_impl(task) -def real_mount(vfsmnt): - if (vfsmnt.type == types.mount_type or - vfsmnt.type == types.mount_type.pointer()): - t = vfsmnt.type - if t.code == gdb.TYPE_CODE_PTR: - t = t.target() - if t is not types.mount_type: - types.mount_type = t - return vfsmnt - return container_of(vfsmnt, types.mount_type, 'mnt') +def mount_flags(mnt: gdb.Value, show_hidden: bool=False) -> str: + """ + Returns the human-readable flags of the mount structure + + Args: + mnt (gdb.Value): + The mount structure for which to return flags -def mount_flags(mnt, show_hidden=False): + show_hidden (bool, default=False): + Whether to return hidden flags + + Returns: + str: The mount flags in human-readable form + """ if struct_has_member(mnt, 'mnt'): mnt = mnt['mnt'] if show_hidden: return decode_flags(mnt['mnt_flags'], MNT_FLAGS_HIDDEN, ",") return decode_flags(mnt['mnt_flags'], MNT_FLAGS, ",") -def mount_super(mnt): +def mount_super(mnt: gdb.Value) -> gdb.Value: + """ + Returns the struct super_block associated with a mount + + Args: + mnt: gdb.Value: + The mount structure for which to return the super_block + + Returns: + gdb.Value: + The super_block associated with the mount + """ try: sb = mnt['mnt']['mnt_sb'] except gdb.error: sb = mnt['mnt_sb'] return sb -def mount_root(mnt): +def mount_root(mnt: gdb.Value) -> gdb.Value: + """ + Returns the struct dentry corresponding to the root of a mount + + Args: + mnt: gdb.Value: + The mount structure for which to return the root dentry + + Returns: + gdb.Value: + The dentry that corresponds to the root of the mount + """ try: mnt = mnt['mnt'] except gdb.error: @@ -105,16 +151,65 @@ def mount_root(mnt): return mnt['mnt_root'] -def mount_fstype(mnt): +def mount_fstype(mnt: gdb.Value) -> str: + """ + Returns the file system type of the mount + + Args: + mnt (gdb.Value): + The mount structure for which to return the file system tyoe + + Returns: + str: The file system type of the mount in string form + """ return super_fstype(mount_super(mnt)) -def mount_device(mnt): +def mount_device(mnt: gdb.Value) -> str: + """ + Returns the device name that this mount is using + + Args: + gdb.Value: + The mount structure for which to get the device name + + Returns: + str: The device name in string form + + """ devname = mnt['mnt_devname'].string() if devname is None: devname = "none" return devname +def _real_mount(vfsmnt): + if (vfsmnt.type == types.mount_type or + vfsmnt.type == types.mount_type.pointer()): + t = vfsmnt.type + if t.code == gdb.TYPE_CODE_PTR: + t = t.target() + if t is not types.mount_type: + types.mount_type = t + return vfsmnt + return container_of(vfsmnt, types.mount_type, 'mnt') + def d_path(mnt, dentry, root=None): + """ + Returns a file system path described by a mount and dentry + + Args: + mnt (gdb.Value): + The mount for the start of the path + + dentry (gdb.Value): + The dentry for the start of the path + + root (gdb.Value, default=None): + The mount at which to stop resolution. If None, + the current root of the namespace. + + Returns: + str: The path in string form + """ if root is None: root = symvals.init_task['fs']['root'] @@ -124,7 +219,7 @@ def d_path(mnt, dentry, root=None): if mnt.type.code != gdb.TYPE_CODE_PTR: mnt = mnt.address - mount = real_mount(mnt) + mount = _real_mount(mnt) if mount.type.code != gdb.TYPE_CODE_PTR: mount = mount.address @@ -157,5 +252,5 @@ def d_path(mnt, dentry, root=None): name = '/' return name -type_cbs = TypeCallbacks([ ('struct vfsmount', check_mount_type ) ]) -symbols_cbs = SymbolCallbacks([ ('init_task', Mount.check_task_interface ) ]) +type_cbs = TypeCallbacks([ ('struct vfsmount', _check_mount_type ) ]) +symbols_cbs = SymbolCallbacks([ ('init_task', Mount._check_task_interface ) ]) From 4e60c1cd263fda571dfe457bf1cb0e7d15c06711 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 29 Apr 2019 15:08:53 -0400 Subject: [PATCH 60/75] crash.subsystem.filesystem.btrfs: add helpers for super, inode, and uuids This adds helpers to: - export the fsid and metadata uuid from btrfs file systems - test whether a generic super block belongs to btrfs - test whether a generic inode belongs to btrfs We also document the APIs of existing helpers. Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/btrfs.py | 117 ++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/crash/subsystem/filesystem/btrfs.py b/crash/subsystem/filesystem/btrfs.py index d42790dee39..b5e33ba7bc5 100644 --- a/crash/subsystem/filesystem/btrfs.py +++ b/crash/subsystem/filesystem/btrfs.py @@ -2,13 +2,47 @@ # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: import gdb +import uuid +from crash.util import decode_uuid, struct_has_member, container_of from crash.util.symbols import Types +from crash.subsystem.filesystem import is_fstype_super types = Types([ 'struct btrfs_inode', 'struct btrfs_fs_info *', - 'struct btrfs_fs_info' ]) + 'struct btrfs_fs_info' ]) -def btrfs_inode(vfs_inode): +def is_btrfs_super(super_block: gdb.Value) -> bool: + """ + Tests whether a super_block belongs to btrfs. + + Args: + super_block (gdb.Value): The struct super_block + to test + + Returns: + bool: Whether the super_block belongs to btrfs + + Raises: + gdb.NotAvailableError: The target value was not available. + """ + return is_fstype_super(super_block, "btrfs") + +def is_btrfs_inode(vfs_inode: gdb.Value) -> bool: + """ + Tests whether a inode belongs to btrfs. + + Args: + vfs_inode (gdb.Value): The struct inode to test + + Returns: + bool: Whether the inode belongs to btrfs + + Raises: + gdb.NotAvailableError: The target value was not available. + """ + return is_btrfs_super(vfs_inode['i_sb']) + +def btrfs_inode(vfs_inode: gdb.Value, force: bool=False ) -> gdb.Value: """ Converts a VFS inode to a btrfs inode @@ -18,23 +52,90 @@ def btrfs_inode(vfs_inode): vfs_inode (gdb.Value): The struct inode to convert to a struct btrfs_inode + force (bool): Ignore type checking. + Returns: gdb.Value: The converted struct btrfs_inode + + Raises: + TypeError: the inode does not belong to btrfs + gdb.NotAvailableError: The target value was not available. """ + if not force and not is_btrfs_inode(vfs_inode): + raise TypeError("inode does not belong to btrfs") + return container_of(vfs_inode, types.btrfs_inode_type, 'vfs_inode') -def btrfs_fs_info(super_block): +def btrfs_fs_info(super_block: gdb.Value, force: bool=False) -> gdb.Value: """ - Converts a VFS superblock to a btrfs fs_info + Resolves a btrfs_fs_info from a VFS superblock - This method converts a struct super_block to a struct btrfs_fs_info + This method resolves a struct btrfs_fs_info from a struct super_block Args: super_block (gdb.Value): The struct super_block - to convert to a struct btrfs_fs_info. + to use to resolve a struct btrfs_fs_info. A pointer to a + struct super_block is also acceptable. + + force (bool): Ignore type checking. Returns: - gdb.Value: The converted struct + gdb.Value: The resolved struct btrfs_fs_info + + Raises: + TypeError: the super_block does not belong to btrfs + gdb.NotAvailableError: The target value was not available. + """ + if not force and not is_btrfs_super(super_block): + raise TypeError("super_block does not belong to btrfs") + + fs_info = super_block['s_fs_info'].cast(types.btrfs_fs_info_p_type) + return fs_info.dereference() + +def btrfs_fsid(super_block: gdb.Value, force: bool=False) -> uuid.UUID: + """ + Returns the btrfs fsid (UUID) for the specified superblock. + + Args: + super_block (gdb.Value): The struct super_block + for which to return the btrfs fsid. + + force (bool): Ignore type checking. + + Returns: + uuid.UUID: The Python UUID Object for the btrfs fsid + + Raises: + TypeError: the super_block does not belong to btrfs + gdb.NotAvailableError: The target value was not available. + """ + fs_info = btrfs_fs_info(super_block, force) + if struct_has_member(types.btrfs_fs_info_type, 'fsid'): + return decode_uuid(fs_info['fsid']) + return decode_uuid(fs_info['fs_devices']['fsid']) + +def btrfs_metadata_uuid(sb: gdb.Value, force: bool=False) -> uuid.UUID: + """ + Returns the btrfs metadata uuid for the specified superblock. + + Args: + super_block (gdb.Value): The struct super_block + for which to return the btrfs metadata uuid. + + force (bool): Ignore type checking. + + Returns: + uuid.UUID: The Python UUID Object for the btrfs fsid + + Raises: + TypeError: the super_block does not belong to btrfs + gdb.NotAvailableError: The target value was not available. """ - return super_block['s_fs_info'].cast(types.btrfs_fs_info_p_type) + fs_info = btrfs_fs_info(sb, force) + if struct_has_member(types.btrfs_fs_info_type, 'metadata_uuid'): + return decode_uuid(fs_info['metadata_uuid']) + elif struct_has_member(fs_info['fs_devices'].type, 'metadata_uuid'): + return decode_uuid(fs_info['fs_devices']['metadata_uuid']) + else: + return btrfs_fsid(sb, force) From 7e077de120dde3e280d842cc598f0956a9ded51d Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 18 Sep 2018 05:38:53 -0400 Subject: [PATCH 61/75] crash.commands.btrfs: add basic btrfs command This adds an `btrfs' command to display some details of btrfs file systems. Included subcommands are: - 'list' -- list all mounted btrfs file systems, including device and uuid. Signed-off-by: Jeff Mahoney --- crash/commands/btrfs.py | 60 +++++++++++++++++++++++++ kernel-tests/test_commands_btrfs.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 crash/commands/btrfs.py create mode 100644 kernel-tests/test_commands_btrfs.py diff --git a/crash/commands/btrfs.py b/crash/commands/btrfs.py new file mode 100644 index 00000000000..e32c609ab23 --- /dev/null +++ b/crash/commands/btrfs.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb + +from argparse import Namespace +from crash.commands import Command, ArgumentParser +from crash.commands import CommandLineError +from crash.exceptions import DelayedAttributeError +from crash.subsystem.filesystem import for_each_super_block, super_fstype +from crash.subsystem.filesystem.btrfs import btrfs_fsid, btrfs_metadata_uuid + +class BtrfsCommand(Command): + """display Btrfs internal data structures + +NAME + btrfs - display Btrfs internal data structures + +SYNOPSIS + btrfs + +COMMANDS + btrfs list [-m] - list all btrfs file systems (-m to show metadata uuid)""" + + def __init__(self, name): + parser = ArgumentParser(prog=name) + subparsers = parser.add_subparsers(help="sub-command help") + list_parser = subparsers.add_parser('list', help='list help') + list_parser.set_defaults(subcommand=self.list_btrfs) + list_parser.add_argument('-m', action='store_true', default=False) + + parser.format_usage = lambda: 'btrfs [args...]\n' + Command.__init__(self, name, parser) + + def list_btrfs(self, args: Namespace) -> None: + print_header = True + count = 0 + for sb in for_each_super_block(): + if super_fstype(sb) == "btrfs": + if args.m: + u = btrfs_metadata_uuid(sb) + which_fsid = "METADATA UUID" + else: + u = btrfs_fsid(sb) + which_fsid = "FSID" + if print_header: + print("SUPER BLOCK\t\tDEVICE\t\t{}".format(which_fsid)) + print_header = False + print("{}\t{}\t\t{}".format(sb.address, sb['s_id'].string(), u)) + count += 1 + if count == 0: + print("No btrfs file systems were mounted.") + + def execute(self, args): + if hasattr(args, 'subcommand'): + args.subcommand(args) + else: + raise CommandLineError("no command specified") + +BtrfsCommand("btrfs") diff --git a/kernel-tests/test_commands_btrfs.py b/kernel-tests/test_commands_btrfs.py new file mode 100644 index 00000000000..8c217b9a2b8 --- /dev/null +++ b/kernel-tests/test_commands_btrfs.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from decorators import skip_without_supers, skip_with_supers, skip_without_type + +from crash.commands.btrfs import BtrfsCommand +from crash.commands import CommandLineError +from crash.exceptions import DelayedAttributeError + +class TestCommandsBtrfs(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + self.redirected = io.StringIO() + sys.stdout = self.redirected + self.command = BtrfsCommand("btrfs") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return self.redirected.getvalue() + + def output_lines(self): + output = self.output() + return len(output.split("\n")) - 1 + + def test_btrfs_empty(self): + """`btrfs` raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("") + + @skip_without_supers('btrfs') + def test_btrfs_list(self): + """`btrfs list` produces valid output""" + self.command.invoke_uncaught("list") + self.assertTrue(self.output_lines() > 0) + + @skip_without_supers('btrfs') + def test_btrfs_list_m(self): + """`btrfs list -m` produces valid output""" + self.command.invoke_uncaught("list -m") + self.assertTrue(self.output_lines() > 0) + + @skip_with_supers('btrfs') + def test_btrfs_list_without_supers(self): + """`btrfs list` without supers produces one-line status""" + self.command.invoke_uncaught("list") + self.assertTrue(self.output_lines() == 1) + + @skip_with_supers('btrfs') + def test_btrfs_list_m_without_supers(self): + """`btrfs list -m` without supers produces one-line status""" + self.command.invoke_uncaught("list -m") + self.assertTrue(self.output_lines() == 1) + + def test_btrfs_list_invalid(self): + """`btrfs list -invalid` raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("list -invalid") + + def test_btrfs_invalid_command(self): + """`btrfs invalid command` raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("invalid command") From 575d1ef8d9414395a05d8bedfe27c1c8b721de7b Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 18 Sep 2018 05:41:46 -0400 Subject: [PATCH 62/75] crash.subsystem.filesystem.xfs: Add basic XFS infra This commit adds a basic xfs file system system module. Included are: - Python variables for flags - Mappings from flags to flag names - Decoding for xfs_bufs and inodes - Helpers for mount flags, superblock version, and uuid - AIL iterators including item decoding Signed-off-by: Jeff Mahoney --- crash/subsystem/filesystem/xfs.py | 551 ++++++++++++++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 crash/subsystem/filesystem/xfs.py diff --git a/crash/subsystem/filesystem/xfs.py b/crash/subsystem/filesystem/xfs.py new file mode 100644 index 00000000000..8c40d68a7a5 --- /dev/null +++ b/crash/subsystem/filesystem/xfs.py @@ -0,0 +1,551 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +import uuid + +from typing import Union, Iterable + +from crash.types.list import list_for_each_entry +from crash.util import container_of, decode_uuid_t, decode_flags +from crash.util import struct_has_member +from crash.util.symbols import Types, TypeCallbacks +from crash.subsystem.filesystem import is_fstype_super, is_fstype_inode +from crash.subsystem.storage import block_device_name +from crash.subsystem.storage.decoders import Decoder + +# XFS inode locks +XFS_IOLOCK_EXCL = 0x01 +XFS_IOLOCK_SHARED = 0x02 +XFS_ILOCK_EXCL = 0x04 +XFS_ILOCK_SHARED = 0x08 +XFS_MMAPLOCK_EXCL = 0x10 +XFS_MMAPLOCK_SHARED = 0x20 + +XFS_LOCK_MASK = 0x3f + +XFS_LOCK_FLAGS = { + XFS_IOLOCK_EXCL : "XFS_IOLOCK_EXCL", + XFS_IOLOCK_SHARED : "XFS_IOLOCK_SHARED", + XFS_ILOCK_EXCL : "XFS_ILOCK_EXCL", + XFS_ILOCK_SHARED : "XFS_ILOCK_SHARED", + XFS_MMAPLOCK_EXCL : "XFS_MMAPLOCK_EXCL", + XFS_MMAPLOCK_SHARED : "XFS_MMAPLOCK_SHARED", +} + +XFS_LI_EFI = 0x1236 +XFS_LI_EFD = 0x1237 +XFS_LI_IUNLINK = 0x1238 +XFS_LI_INODE = 0x123b # aligned ino chunks, var-size ibufs +XFS_LI_BUF = 0x123c # v2 bufs, variable sized inode bufs +XFS_LI_DQUOT = 0x123d +XFS_LI_QUOTAOFF = 0x123e + +XFS_LI_TYPES = { + XFS_LI_EFI : "XFS_LI_EFI", + XFS_LI_EFD : "XFS_LI_EFD", + XFS_LI_IUNLINK : "XFS_LI_IUNLINK", + XFS_LI_INODE : "XFS_LI_INODE", + XFS_LI_BUF : "XFS_LI_BUF", + XFS_LI_EFI : "XFS_LI_EFI", + XFS_LI_DQUOT : "XFS_LI_DQUOT", + XFS_LI_QUOTAOFF : "XFS_LI_QUOTAOFF", +} + +XFS_BLI_HOLD = 0x01 +XFS_BLI_DIRTY = 0x02 +XFS_BLI_STALE = 0x04 +XFS_BLI_LOGGED = 0x08 +XFS_BLI_INODE_ALLOC_BUF = 0x10 +XFS_BLI_STALE_INODE = 0x20 +XFS_BLI_INODE_BUF = 0x40 + +XFS_BLI_FLAGS = { + XFS_BLI_HOLD : "HOLD", + XFS_BLI_DIRTY : "DIRTY", + XFS_BLI_STALE : "STALE", + XFS_BLI_LOGGED : "LOGGED", + XFS_BLI_INODE_ALLOC_BUF : "INODE_ALLOC", + XFS_BLI_STALE_INODE : "STALE_INODE", + XFS_BLI_INODE_BUF : "INODE_BUF", +} + +XBF_READ = (1 << 0) # buffer intended for reading from device +XBF_WRITE = (1 << 1) # buffer intended for writing to device +XBF_MAPPED = (1 << 2) # buffer mapped (b_addr valid) +XBF_ASYNC = (1 << 4) # initiator will not wait for completion +XBF_DONE = (1 << 5) # all pages in the buffer uptodate +XBF_DELWRI = (1 << 6) # buffer has dirty pages +XBF_STALE = (1 << 7) # buffer has been staled, do not find it +XBF_ORDERED = (1 << 11) # use ordered writes +XBF_READ_AHEAD = (1 << 12) # asynchronous read-ahead +XBF_LOG_BUFFER = (1 << 13) # this is a buffer used for the log + +# flags used only as arguments to access routines +XBF_LOCK = (1 << 14) # lock requested +XBF_TRYLOCK = (1 << 15) # lock requested, but do not wait +XBF_DONT_BLOCK = (1 << 16) # do not block in current thread + +# flags used only internally +_XBF_PAGES = (1 << 18) # backed by refcounted pages +_XBF_RUN_QUEUES = (1 << 19) # run block device task queue +_XBF_KMEM = (1 << 20) # backed by heap memory +_XBF_DELWRI_Q = (1 << 21) # buffer on delwri queue +_XBF_LRU_DISPOSE = (1 << 24) # buffer being discarded + +XFS_BUF_FLAGS = { + XBF_READ : "READ", + XBF_WRITE : "WRITE", + XBF_MAPPED : "MAPPED", + XBF_ASYNC : "ASYNC", + XBF_DONE : "DONE", + XBF_DELWRI : "DELWRI", + XBF_STALE : "STALE", + XBF_ORDERED : "ORDERED", + XBF_READ_AHEAD : "READ_AHEAD", + XBF_LOCK : "LOCK", # should never be set + XBF_TRYLOCK : "TRYLOCK", # ditto + XBF_DONT_BLOCK : "DONT_BLOCK", # ditto + _XBF_PAGES : "PAGES", + _XBF_RUN_QUEUES : "RUN_QUEUES", + _XBF_KMEM : "KMEM", + _XBF_DELWRI_Q : "DELWRI_Q", + _XBF_LRU_DISPOSE : "LRU_DISPOSE", +} + +XFS_ILOG_CORE = 0x001 +XFS_ILOG_DDATA = 0x002 +XFS_ILOG_DEXT = 0x004 +XFS_ILOG_DBROOT = 0x008 +XFS_ILOG_DEV = 0x010 +XFS_ILOG_UUID = 0x020 +XFS_ILOG_ADATA = 0x040 +XFS_ILOG_AEXT = 0x080 +XFS_ILOG_ABROOT = 0x100 +XFS_ILOG_DOWNER = 0x200 +XFS_ILOG_AOWNER = 0x400 +XFS_ILOG_TIMESTAMP = 0x4000 + +XFS_ILI_FLAGS = { + XFS_ILOG_CORE : "CORE", + XFS_ILOG_DDATA : "DDATA", + XFS_ILOG_DEXT : "DEXT", + XFS_ILOG_DBROOT : "DBROOT", + XFS_ILOG_DEV : "DEV", + XFS_ILOG_UUID : "UUID", + XFS_ILOG_ADATA : "ADATA", + XFS_ILOG_AEXT : "AEXT", + XFS_ILOG_ABROOT : "ABROOT", + XFS_ILOG_DOWNER : "DOWNER", + XFS_ILOG_AOWNER : "AOWNER", + XFS_ILOG_TIMESTAMP : "TIMESTAMP", +} + +XFS_MOUNT_WSYNC = (1 << 0) +XFS_MOUNT_UNMOUNTING = (1 << 1) +XFS_MOUNT_DMAPI = (1 << 2) +XFS_MOUNT_WAS_CLEAN = (1 << 3) +XFS_MOUNT_FS_SHUTDOWN = (1 << 4) +XFS_MOUNT_DISCARD = (1 << 5) +XFS_MOUNT_NOALIGN = (1 << 7) +XFS_MOUNT_ATTR2 = (1 << 8) +XFS_MOUNT_GRPID = (1 << 9) +XFS_MOUNT_NORECOVERY = (1 << 10) +XFS_MOUNT_DFLT_IOSIZE = (1 << 12) +XFS_MOUNT_SMALL_INUMS = (1 << 14) +XFS_MOUNT_32BITINODES = (1 << 15) +XFS_MOUNT_NOUUID = (1 << 16) +XFS_MOUNT_BARRIER = (1 << 17) +XFS_MOUNT_IKEEP = (1 << 18) +XFS_MOUNT_SWALLOC = (1 << 19) +XFS_MOUNT_RDONLY = (1 << 20) +XFS_MOUNT_DIRSYNC = (1 << 21) +XFS_MOUNT_COMPAT_IOSIZE = (1 << 22) +XFS_MOUNT_FILESTREAMS = (1 << 24) +XFS_MOUNT_NOATTR2 = (1 << 25) + +XFS_MOUNT_FLAGS = { + XFS_MOUNT_WSYNC : "WSYNC", + XFS_MOUNT_UNMOUNTING : "UNMOUNTING", + XFS_MOUNT_DMAPI : "DMAPI", + XFS_MOUNT_WAS_CLEAN : "WAS_CLEAN", + XFS_MOUNT_FS_SHUTDOWN : "FS_SHUTDOWN", + XFS_MOUNT_DISCARD : "DISCARD", + XFS_MOUNT_NOALIGN : "NOALIGN", + XFS_MOUNT_ATTR2 : "ATTR2", + XFS_MOUNT_GRPID : "GRPID", + XFS_MOUNT_NORECOVERY : "NORECOVERY", + XFS_MOUNT_DFLT_IOSIZE : "DFLT_IOSIZE", + XFS_MOUNT_SMALL_INUMS : "SMALL_INUMS", + XFS_MOUNT_32BITINODES : "32BITINODES", + XFS_MOUNT_NOUUID : "NOUUID", + XFS_MOUNT_BARRIER : "BARRIER", + XFS_MOUNT_IKEEP : "IKEEP", + XFS_MOUNT_SWALLOC : "SWALLOC", + XFS_MOUNT_RDONLY : "RDONLY", + XFS_MOUNT_DIRSYNC : "DIRSYNC", + XFS_MOUNT_COMPAT_IOSIZE : "COMPAT_IOSIZE", + XFS_MOUNT_FILESTREAMS : "FILESTREAMS", + XFS_MOUNT_NOATTR2 : "NOATTR2", +} + +class XFSBufDecoder(Decoder): + """ + Decodes a struct xfs_buf into human-readable form + """ + + def __init__(self, xfsbuf): + super(XFSBufDecoder, self).__init__() + self.xfsbuf = xfsbuf + + def __str__(self): + return xfs_format_xfsbuf(self.xfsbuf) + +class XFSBufBioDecoder(Decoder): + """ + Decodes a bio with an xfsbuf ->bi_end_io + """ + description = "{:x} bio: xfs buffer on {}" + __endio__ = 'xfs_buf_bio_end_io' + types = Types([ 'struct xfs_buf *' ]) + + def __init__(self, bio): + super(XFSBufBioDecoder, self).__init__() + self.bio = bio + + def interpret(self): + self.xfsbuf = bio['bi_private'].cast(cls.types.xfs_buf_p_type) + self.devname = block_device_name(bio['bi_bdev']) + + def __next__(self): + return XFSBufDecoder(xfs.xfsbuf) + + def __str__(self): + return self.description.format(self.bio, self.devname) + +XFSBufBioDecoder.register() + +types = Types([ 'struct xfs_log_item', 'struct xfs_buf_log_item', + 'struct xfs_inode_log_item', 'struct xfs_efi_log_item', + 'struct xfs_efd_log_item', 'struct xfs_dq_logitem', + 'struct xfs_qoff_logitem', 'struct xfs_inode', + 'struct xfs_mount *', 'struct xfs_buf *' ]) + +class XFS(object): + """ + XFS File system state class. Not meant to be instantiated directly. + """ + ail_head_name = None + + @classmethod + def _detect_ail_version(cls, gdbtype): + if struct_has_member(gdbtype, 'ail_head'): + cls.ail_head_name = 'ail_head' + else: + cls.ail_head_name = 'xa_ail' + +def is_xfs_super(super_block: gdb.Value) -> bool: + """ + Tests whether a super_block belongs to XFS. + + Args: + super_block (gdb.Value): + The struct super_block to test + + Returns: + bool: Whether the super_block belongs to XFS + """ + return is_fstype_super(super_block, "xfs") + +def is_xfs_inode(vfs_inode: gdb.Value) -> bool: + """ + Tests whether a generic VFS inode belongs to XFS + + Args: + vfs_inode (gdb.value(): + The struct inode to test whether it belongs to XFS + + Returns: + bool: Whether the inode belongs to XFS + """ + + return is_fstype_inode(vfs_inode, "xfs") + +def xfs_inode(vfs_inode: gdb.Value, force: bool=False) -> gdb.Value: + """ + Converts a VFS inode to a xfs inode + + This method converts a struct inode to a struct xfs_inode. + + Args: + vfs_inode (gdb.Value): + The struct inode to convert to a struct xfs_inode + + force (bool): ignore type checking + + Returns: + gdb.Value: The converted struct xfs_inode + """ + if not force and not is_xfs_inode(vfs_inode): + raise TypeError("inode does not belong to xfs") + + return container_of(vfs_inode, types.xfs_inode, 'i_vnode') + +def xfs_mount(sb: gdb.Value, force: bool=False) -> gdb.Value: + """ + Converts a VFS superblock to a xfs mount + + This method converts a struct super_block to a struct xfs_mount * + + Args: + super_block (gdb.Value): + The struct super_block to convert to a struct xfs_fs_info. + + Returns: + gdb.Value: The converted struct xfs_mount + """ + if not force and not is_xfs_super(sb): + raise TypeError("superblock does not belong to xfs") + + return sb['s_fs_info'].cast(types.xfs_mount_p_type) + +def xfs_mount_flags(mp: gdb.Value) -> str: + """ + Return the XFS-internal mount flags in string form + + Args: + mp (gdb.Value): + The struct xfs_mount for the file system + + Returns: + str: The mount flags in string form + """ + return decode_flags(mp['m_flags'], XFS_MOUNT_FLAGS) + +def xfs_mount_uuid(mp: gdb.Value) -> uuid.UUID: + """ + Return the UUID for an XFS file system in string form + + Args: + mp gdb.Value(): + The struct xfs_mount for the file system + + Returns: + uuid.UUID: The Python UUID object that describes the xfs UUID + """ + return decode_uuid_t(mp['m_sb']['sb_uuid']) + +def xfs_mount_version(mp: gdb.Value) -> int: + return int(mp['m_sb']['sb_versionnum']) & 0xf + +def xfs_for_each_ail_entry(ail: gdb.Value) -> Iterable[gdb.Value]: + """ + Iterates over the XFS Active Item Log and returns each item + + Args: + ail (gdb.Value): The XFS AIL to iterate + + Yields: + gdb.Value + """ + head = ail[XFS.ail_head_name] + for item in list_for_each_entry(head, types.xfs_log_item_type, 'li_ail'): + yield item + +def xfs_for_each_ail_log_item(mp: gdb.Value) -> Iterable[gdb.Value]: + """ + Iterates over the XFS Active Item Log and returns each item + + Args: + mp (gdb.Value): The XFS mount to iterate + + Yields: + gdb.Value + """ + for item in xfs_for_each_ail_entry(mp['m_ail']): + yield item + +def item_to_buf_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_buf_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_BUF + """ + if item['li_type'] != XFS_LI_BUF: + raise TypeError("item is not a buf log item") + return container_of(item, types.xfs_buf_log_item_type, 'bli_item') + +def item_to_inode_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_inode_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_INODE + """ + if item['li_type'] != XFS_LI_INODE: + raise TypeError("item is not an inode log item") + return container_of(item, types.xfs_inode_log_item_type, 'ili_item') + +def item_to_efi_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_efi_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_EFI + """ + if item['li_type'] != XFS_LI_EFI: + raise TypeError("item is not an EFI log item") + return container_of(item, types.xfs_efi_log_item_type, 'efi_item') + +def item_to_efd_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_efd_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_EFD + """ + if item['li_type'] != XFS_LI_EFD: + raise TypeError("item is not an EFD log item") + return container_of(item, types.xfs_efd_log_item_type, 'efd_item') + +def item_to_dquot_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_dquot_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_DQUOT + """ + if item['li_type'] != XFS_LI_DQUOT: + raise TypeError("item is not an DQUOT log item") + return container_of(item, types.xfs_dq_logitem_type, 'qli_item') + +def item_to_quotaoff_log_item(item: gdb.Value) -> gdb.Value: + """ + Converts an xfs_log_item to an xfs_quotaoff_log_item + + Args: + item (gdb.Value): The log item to convert + + Returns: + gdb.Value + + Raises: + TypeError: The type of log item is not XFS_LI_QUOTAOFF + """ + if item['li_type'] != XFS_LI_QUOTAOFF: + raise TypeError("item is not an QUOTAOFF log item") + return container_of(item, types.xfs_qoff_logitem_type, 'qql_item') + +def xfs_log_item_typed(item:gdb.Value) -> gdb.Value: + """ + Returns the log item converted from the generic type to the actual type + + Args: + item (gdb.Value): The struct xfs_log_item to + convert. + + Returns: + Depending on the item type, one of: + gdb.Value + gdb.Value + gdb.Value + gdb.Value + gdb.Value + gdb.Value (for UNLINK item) + + Raises: + RuntimeError: An unexpected item type was encountered + """ + li_type = int(item['li_type']) + if li_type == XFS_LI_BUF: + return item_to_buf_log_item(item) + elif li_type == XFS_LI_INODE: + return item_to_inode_log_item(item) + elif li_type == XFS_LI_EFI: + return item_to_efi_log_item(item) + elif li_type == XFS_LI_EFD: + return item_to_efd_log_item(item) + elif li_type == XFS_LI_IUNLINK: + # There isn't actually any type information for this + return item['li_type'] + elif li_type == XFS_LI_DQUOT: + return item_to_dquot_log_item(item) + elif li_type == XFS_LI_QUOTAOFF: + return item_to_quotaoff_log_item(item) + + raise RuntimeError("Unknown AIL item type {:x}".format(li_type)) + +def xfs_format_xfsbuf(buf: gdb.Value) -> str: + """ + Returns a human-readable format of struct xfs_buf + + Args: + buf (gdb.Value): + The struct xfs_buf to decode + + Returns: + str: The human-readable representation of the struct xfs_buf + """ + state = "" + bflags = decode_flags(buf['b_flags'], XFS_BUF_FLAGS) + + if buf['b_pin_count']['counter']: + state += "P" + if buf['b_sema']['count'] >= 0: + state += "L" + + return f"{int(buf):x} xfsbuf: logical offset {buf['b_bn']:d}, " \ + f"size {buf['b_buffer_len']:d}, block number {buf['b_bn']:d}, " \ + f"flags {bflags}, state {state}" + +def xfs_for_each_ail_log_item_typed(mp: gdb.Value) -> gdb.Value: + """ + Iterates over the XFS Active Item Log and returns each item, resolved + to the specific type. + + Args: + mp (gdb.Value): The XFS mount to iterate + + Yields: + Depending on the item type, one of: + gdb.Value + gdb.Value + gdb.Value + gdb.Value + gdb.Value + """ + for item in types.xfs_for_each_ail_log_item(mp): + yield types.xfs_log_item_typed(item) + +type_cbs = TypeCallbacks([ ('struct xfs_ail', XFS._detect_ail_version) ]) From 15220124a3a939bef3fe7fc73a9a3834e5bca783 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 18 Sep 2018 05:38:36 -0400 Subject: [PATCH 63/75] crash.commands.xfs: add basic xfs command This adds an `xfs' command to display some details of xfs file systems. Included subcommands are: - 'list' -- list all mounted xfs file systems, including device and uuid - 'show' -- show details of a single xfs file system - 'dump-ail' -- dump contents of the AIL for one file system - 'dump-buft' -- dump contents of the bt_delwrite_queue for one file system Signed-off-by: Jeff Mahoney --- crash/commands/xfs.py | 196 ++++++++++++++++++++++++++++++ kernel-tests/test_commands_xfs.py | 77 ++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 crash/commands/xfs.py create mode 100644 kernel-tests/test_commands_xfs.py diff --git a/crash/commands/xfs.py b/crash/commands/xfs.py new file mode 100644 index 00000000000..1add4eee09d --- /dev/null +++ b/crash/commands/xfs.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +import os.path +import argparse +import re + +from argparse import Namespace +from crash.commands import Command, ArgumentParser +from crash.commands import CommandLineError, CommandError +from crash.exceptions import DelayedAttributeError +from crash.types.list import list_for_each_entry, list_empty +from crash.subsystem.filesystem import for_each_super_block, get_super_block +from crash.subsystem.filesystem import super_flags +from crash.subsystem.filesystem.xfs import xfs_mount +from crash.subsystem.filesystem.xfs import xfs_for_each_ail_log_item +from crash.subsystem.filesystem.xfs import xfs_log_item_typed +from crash.subsystem.filesystem.xfs import xfs_format_xfsbuf +from crash.subsystem.filesystem.xfs import XFS_LI_TYPES +from crash.subsystem.filesystem.xfs import XFS_LI_EFI, XFS_LI_EFD +from crash.subsystem.filesystem.xfs import XFS_LI_IUNLINK, XFS_LI_INODE +from crash.subsystem.filesystem.xfs import XFS_LI_BUF, XFS_LI_DQUOT +from crash.subsystem.filesystem.xfs import XFS_LI_QUOTAOFF, XFS_BLI_FLAGS +from crash.subsystem.filesystem.xfs import xfs_mount_flags, xfs_mount_uuid +from crash.subsystem.filesystem.xfs import xfs_mount_version + +from crash.util.symbols import Types + +types = Types(['struct xfs_buf *']) + +class XFSCommand(Command): + """display XFS internal data structures + +NAME + xfs - display XFS internal data structures + +SYNOPSIS + xfs [arguments ...] + +COMMANDS + xfs list + xfs show + xfs dump-ail + xfs dump-buft + """ + + def __init__(self, name): + parser = ArgumentParser(prog=name) + subparsers = parser.add_subparsers(help="sub-command help") + show_parser = subparsers.add_parser('show', help='show help') + show_parser.set_defaults(subcommand=self.show_xfs) + show_parser.add_argument('addr') + list_parser = subparsers.add_parser('list', help='list help') + list_parser.set_defaults(subcommand=self.list_xfs) + ail_parser = subparsers.add_parser('dump-ail', help='ail help') + ail_parser.set_defaults(subcommand=self.dump_ail) + ail_parser.add_argument('addr') + buft_parser = subparsers.add_parser('dump-buft', help='buft help') + buft_parser.set_defaults(subcommand=self.dump_buftargs) + buft_parser.add_argument('addr') + + Command.__init__(self, name, parser) + + def list_xfs(self, args: Namespace) -> None: + count = 0 + print_header = True + for sb in for_each_super_block(): + if sb['s_type']['name'].string() == "xfs": + mp = xfs_mount(sb) + u = xfs_mount_uuid(mp) + if print_header: + print_header = False + print("SUPER BLOCK\t\t\tDEVICE\t\tUUID") + + print("{}\t{}\t{}".format(sb.address, sb['s_id'].string(), u)) + count += 1 + + if count == 0: + print("No xfs file systems are mounted.") + + def show_xfs(self, args: Namespace) -> None: + try: + sb = get_super_block(args.addr) + except gdb.NotAvailableError as e: + raise CommandError(str(e)) + + mp = xfs_mount(sb) + + print("Device: {}".format(sb['s_id'].string())) + print("UUID: {}".format(xfs_mount_uuid(mp))) + print("VFS superblock flags: {}".format(super_flags(sb))) + print("Flags: {}".format(xfs_mount_flags(mp))) + print("Version: {}".format(xfs_mount_version(mp))) + if list_empty(mp['m_ail']['xa_ail']): + print("AIL is empty") + else: + print("AIL has items queued") + + def dump_ail(self, args: Namespace) -> None: + try: + sb = get_super_block(args.addr) + except gdb.NotAvailableError as e: + raise CommandError(str(e)) + + mp = xfs_mount(sb) + ail = mp['m_ail'] + itemno = 0 + print("AIL @ {:x}".format(int(ail))) + print("target={} last_pushed_lsn={} log_flush=" + .format(int(ail['xa_target']), int(ail['xa_last_pushed_lsn'])), + end='') + try: + print("{}".format(int(ail['xa_log_flush']))) + except: + print("[N/A]") + + for bitem in xfs_for_each_ail_log_item(mp): + li_type = int(bitem['li_type']) + lsn = int(bitem['li_lsn']) + item = xfs_log_item_typed(bitem) + print("{}: item={:x} lsn={} {} " + .format(itemno, int(bitem.address), lsn, + XFS_LI_TYPES[li_type][7:]), end='') + if li_type == XFS_LI_BUF: + buf = item['bli_buf'] + flags = [] + bli_flags = int(item['bli_flags']) + + for flag in XFS_BLI_FLAGS.keys(): + if flag & bli_flags: + flags.append(XFS_BLI_FLAGS[flag]) + + print(" buf@{:x} bli_flags={}" + .format(int(buf), "|".join(flags))) + + print(" {}".format(xfs_format_xfsbuf(buf))) + elif li_type == XFS_LI_INODE: + ili_flags = int(item['ili_lock_flags']) + flags = [] + xfs_inode = item['ili_inode'] + print("inode@{:x} i_ino={} ili_lock_flags={:x} " + .format(int(xfs_inode['i_vnode'].address), + int(xfs_inode['i_ino']), ili_flags)) + elif li_type == XFS_LI_EFI: + efi = item['efi_format'] + print("efi@{:x} size={}, nextents={}, id={:x}" + .format(int(item.address), int(efi['efi_size']), + int(efi['efi_nextents']), int(efi['efi_id']))) + elif li_type == XFS_LI_EFI: + efd = item['efd_format'] + print("efd@{:x} size={}, nextents={}, id={:x}" + .format(int(item.address), int(efd['efd_size']), + int(efd['efd_nextents']), int(efd['efd_id']))) + elif li_type == XFS_LI_DQUOT: + dquot = item['qli_dquot'] + print("dquot@{:x}".format(int(dquot), int(dquot['dq_flags']))) + elif li_type == XFS_LI_QUOTAOFF: + qoff = item['qql_format'] + print("qoff@{:x} type={} size={} flags={}" + .format(int(qoff), int(qoff['qf_type']), + int(qoff['qf_size']), int(qoff['qf_flags']))) + else: + print("item@{:x}".format(int(item.address))) + itemno += 1 + + @classmethod + def dump_buftarg(cls, targ: gdb.Value) -> None: + for buf in list_for_each_entry(targ['bt_delwrite_queue'], + types.xfs_buf_p_type.target(), 'b_list'): + print("{:x} {}".format(int(buf.address), xfs_format_xfsbuf(buf))) + + @classmethod + def dump_buftargs(cls, args: Namespace): + try: + sb = get_super_block(args.addr) + except gdb.NotAvailableError as e: + raise CommandError(str(e)) + mp = xfs_mount(sb) + ddev = mp['m_ddev_targp'] + ldev = mp['m_logdev_targp'] + + print("Data device queue @ {:x}:".format(int(ddev))) + cls.dump_buftarg(ddev) + + if int(ddev) != int(ldev): + print("Log device queue:") + cls.dump_buftarg(ldev) + + def execute(self, args): + if hasattr(args, 'subcommand'): + args.subcommand(args) + else: + raise CommandLineError("no command specified") + +XFSCommand("xfs") diff --git a/kernel-tests/test_commands_xfs.py b/kernel-tests/test_commands_xfs.py new file mode 100644 index 00000000000..0737eca01c5 --- /dev/null +++ b/kernel-tests/test_commands_xfs.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from decorators import skip_without_supers, skip_with_supers + +from crash.commands.xfs import XFSCommand +from crash.exceptions import DelayedAttributeError +from crash.commands import CommandLineError, CommandError + +class TestCommandsXFS(unittest.TestCase): + """ + These tests require that the xfs file system be built-in or loaded as + a module. If the test vmcore doesn't have the xfs module loaded or + modules haven't been provided, most of these tests will be skipped. + """ + + def setUp(self): + self.stdout = sys.stdout + self.redirected = io.StringIO() + sys.stdout = self.redirected + self.command = XFSCommand("xfs") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return self.redirected.getvalue() + + def output_lines(self): + return len(self.output().split("\n")) - 1 + + def test_empty_command(self): + """`xfs' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("") + + def test_invalid_command(self): + """`xfs invalid command' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("invalid command") + + @skip_without_supers('xfs') + def test_xfs_list(self): + """`xfs list' produces valid output""" + self.command.invoke_uncaught("list") + self.assertTrue(self.output_lines() > 0) + + @skip_with_supers('xfs') + def test_xfs_list_no_mounts(self): + """`xfs list' produces one-line status with no mounts""" + self.command.invoke_uncaught("list") + self.assertTrue(self.output_lines() == 1) + + def test_xfs_list_invalid(self): + """`xfs list invalid' raises CommandLineError""" + with self.assertRaises(CommandLineError): + self.command.invoke_uncaught("list invalid") + + def test_xfs_show_null(self): + """`xfs show 0' raises CommandError""" + with self.assertRaises(CommandError): + self.command.invoke_uncaught("show 0") + + def test_xfs_dump_ail_null(self): + """`xfs dump-ail 0' raises CommandError""" + with self.assertRaises(CommandError): + self.command.invoke_uncaught("dump-ail 0") + + def test_xfs_dump_buft_null(self): + """`xfs dump-buft 0' raises CommandError""" + with self.assertRaises(CommandError): + self.command.invoke_uncaught("dump-buft 0") + From 5483d9f9deb9fbc6b17424ba9cddd49a4b821ba0 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 22 Apr 2019 18:14:32 -0400 Subject: [PATCH 64/75] crash.subsystem.storage.blocksq: add per-queue requests_in_flight call This adds a helper to pass back the requests in flight for a particular queue (block single-queue only). Signed-off-by: Jeff Mahoney --- crash/subsystem/storage/blocksq.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crash/subsystem/storage/blocksq.py b/crash/subsystem/storage/blocksq.py index 53533c0c7b0..52a517f39d4 100644 --- a/crash/subsystem/storage/blocksq.py +++ b/crash/subsystem/storage/blocksq.py @@ -48,3 +48,14 @@ def request_age_ms(request): current jiffies in milliseconds. """ return kernel.jiffies_to_msec(kernel.jiffies - request['start_time']) + +def requests_in_flight(queue): + """ + Report how many requests are in flight for this queue + + This method returns a 2-tuple of ints. The first value + is the number of read requests in flight. The second + value is the number of write requests in flight. + """ + return (int(queue['in_flight'][0]), + int(queue['in_flight'][1])) From 538acd1efb8966f0ef8fc8df165aa7e3beaa2711 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Wed, 1 May 2019 11:20:33 -0400 Subject: [PATCH 65/75] crash.commands.lsmod: add basic lsmod command This commit adds a basic `lsmod' command. By default, it will display the module name, core address, size, and users of it. With the -p option, it will display the percpu base and size. With -p , it will display the percpu base for the given CPU number. Signed-off-by: Jeff Mahoney --- crash/commands/lsmod.py | 126 ++++++++++++++++++++++++++++ kernel-tests/test_commands_lsmod.py | 40 +++++++++ 2 files changed, 166 insertions(+) create mode 100644 crash/commands/lsmod.py create mode 100644 kernel-tests/test_commands_lsmod.py diff --git a/crash/commands/lsmod.py b/crash/commands/lsmod.py new file mode 100644 index 00000000000..8eaca535381 --- /dev/null +++ b/crash/commands/lsmod.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +import re +import fnmatch +import argparse + +from crash.commands import Command, ArgumentParser +from crash.types.module import for_each_module +from crash.util import struct_has_member +from crash.types.list import list_for_each_entry +from crash.types.percpu import get_percpu_var +import crash.types.percpu + +class ModuleCommand(Command): + """display module information + +NAME + lsmod - display module information + +SYNOPSIS + lsmod [-p [n]] [name-wildcard] + +DESCRIPTION + This command displays information about loaded modules. + + The default output will show all loaded modules, the core address, + its size, and any users of the module. By specifying [name-wildcard], + the results can be filtered to modules matching the wildcard. + + The following options are available: + -p display the percpu base for the module and the size of its region + -p CPU# display the percpu base for the module and the size of its region + for the specified CPU number + +""" + def __init__(self): + parser = ArgumentParser(prog="lsmod") + + parser.add_argument('-p', nargs='?', const=-1, default=None, type=int) + parser.add_argument('args', nargs=argparse.REMAINDER) + + parser.format_usage = lambda: "lsmod [-p] [regex] ...\n" + + Command.__init__(self, "lsmod", parser) + + self.module_use_type = gdb.lookup_type('struct module_use') + + def print_module_percpu(self, mod, cpu=-1): + cpu = int(cpu) + addr = int(mod['percpu']) + if addr == 0: + return + + if cpu != -1: + addr = get_percpu_var(mod['percpu'], cpu) + tabs = "\t\t" + else: + tabs = "\t\t\t" + + size = int(mod['percpu_size']) + print("{:16s}\t{:#x}{}{:d}".format(mod['name'].string(), int(addr), + tabs, size)) + + + def execute(self, argv): + regex = None + show_deps = True + print_header = True + if argv.args: + regex = re.compile(fnmatch.translate(argv.args[0])) + + if argv.p is not None: + show_deps = False + + core_layout = None + + for mod in for_each_module(): + if core_layout is None: + core_layout = struct_has_member(mod.type, 'core_layout') + + modname = mod['name'].string() + if regex: + m = regex.match(modname) + if m is None: + continue + + if argv.p is not None: + if print_header: + print_header = False + if argv.p == -1: + print("Module\t\t\tPercpu Base\t\tSize") + else: + print("Module\t\t\tPercpu Base@CPU{:d}\t\tSize" + .format(argv.p)) + self.print_module_percpu(mod, argv.p) + continue + + if print_header: + print_header = False + print("Module\t\t\tAddress\t\t\tSize\tUsed by") + + if core_layout: + addr = int(mod['core_layout']['base']) + size = int(mod['core_layout']['size']) + else: + addr = int(mod['module_core']) + size = int(core_size['module_core']) + + module_use = "" + count = 0 + for use in list_for_each_entry(mod['source_list'], + self.module_use_type, + 'source_list'): + if module_use == "": + module_use += " " + else: + module_use += "," + module_use += use['source']['name'].string() + count += 1 + + print("{:16s}\t{:#x}\t{:d}\t{:d}{}" + .format(modname, addr, size, count, module_use)) + +ModuleCommand() diff --git a/kernel-tests/test_commands_lsmod.py b/kernel-tests/test_commands_lsmod.py new file mode 100644 index 00000000000..610409a29ff --- /dev/null +++ b/kernel-tests/test_commands_lsmod.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from crash.commands.lsmod import ModuleCommand + +class TestCommandsLsmod(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + sys.stdout = io.StringIO() + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return sys.stdout.getvalue() + + def test_lsmod(self): + ModuleCommand().invoke("") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_lsmod_wildcard(self): + ModuleCommand().invoke("*") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + + def test_lsmod_p(self): + ModuleCommand().invoke("-p") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) + print(output) + + def test_lsmod_p_0(self): + ModuleCommand().invoke("-p 0") + output = self.output() + self.assertTrue(len(output.split("\n")) > 2) From e7789d4fbb0a2dbf5af60d4099ff7342ebac1e80 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 15 May 2019 14:49:16 -0400 Subject: [PATCH 66/75] crash.types.task: get_stack_pointer should take thread_struct The arch-specific part of get_stack_pointer just needs to interpret the arch's thread_struct. Pass it that and avoid confusion. Signed-off-by: Jeff Mahoney --- crash/arch/x86_64.py | 6 +++--- crash/types/task.py | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/crash/arch/x86_64.py b/crash/arch/x86_64.py index b025c0672c7..4d72f1e9285 100644 --- a/crash/arch/x86_64.py +++ b/crash/arch/x86_64.py @@ -53,6 +53,7 @@ def fetch_register_scheduled_inactive(self, thread, register): task = thread.info.task_struct rsp = task['thread']['sp'].cast(ulong_type.pointer()) + thread.registers['rsp'].value = rsp frame = rsp.cast(self.inactive_task_frame_type.pointer()).dereference() @@ -62,7 +63,6 @@ def fetch_register_scheduled_inactive(self, thread, register): if register == 16: return True - thread.registers['rsp'].value = rsp thread.registers['rbp'].value = frame['bp'] thread.registers['rbx'].value = frame['bx'] thread.registers['r12'].value = frame['r12'] @@ -114,7 +114,7 @@ def fetch_register_scheduled_thread_return(self, thread, register): thread.info.valid_stack = True @classmethod - def get_stack_pointer(cls, thread): - return int(thread.registers['rsp'].value) + def get_stack_pointer(cls, thread_struct): + return thread_struct['sp'] register(x86_64Architecture) diff --git a/crash/types/task.py b/crash/types/task.py index e1a6866f8a0..6a10d79f2ec 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -302,12 +302,8 @@ def is_kernel_task(self): def set_get_stack_pointer(cls, fn): cls.get_stack_pointer_fn = fn - @classmethod - def get_stack_pointer(cls): - # This unbinds the function from the task object so we don't - # pass self to the function. - fn = cls.get_stack_pointer_fn - return fn(self.thread) + def get_stack_pointer(self): + return self.get_stack_pointer_fn(self.task_struct['thread']) def get_rss_field(self): return int(self.task_struct['mm']['rss'].value()) From 1d22d73fd85cd8eac7a2fb62a5bac40dfedce1dc Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 15 May 2019 14:51:56 -0400 Subject: [PATCH 67/75] crash.types.task: add accessor-helpers Every consumer of a task shouldn't need to drill down into the structure just to get the task name, pid, etc. Signed-off-by: Jeff Mahoney --- crash/types/task.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crash/types/task.py b/crash/types/task.py index 6a10d79f2ec..8464a243aa6 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -283,6 +283,22 @@ def update_mem_usage(self): self.pgd_addr = int(mm['pgd']) self.mem_valid = True + def task_name(self, brackets=False): + name = self.task_struct['comm'].string() + if brackets and self.is_kernel_task(): + return f"[{name}]" + else: + return name + + def task_pid(self): + return int(self.task_struct['pid']) + + def parent_pid(self): + return int(self.task_struct['parent']['pid']) + + def task_address(self): + return int(self.task_struct.address) + def is_kernel_task(self): if self.task_struct['pid'] == 0: return True From af12734750f8b8ae3f2f6d02e0d29b65a0e77b52 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Wed, 15 May 2019 14:58:04 -0400 Subject: [PATCH 68/75] crash.commands.ps: factor out formatting This is a big commit that pulls the formatting of the output out of the command. The idea is that we can implement this more cleanly by adding methods to the formatting class. Signed-off-by: Jeff Mahoney --- crash/commands/ps.py | 281 ++++++++++++++++++++++++++----------------- 1 file changed, 168 insertions(+), 113 deletions(-) diff --git a/crash/commands/ps.py b/crash/commands/ps.py index 3717a1073ca..ee170d8d476 100755 --- a/crash/commands/ps.py +++ b/crash/commands/ps.py @@ -7,9 +7,152 @@ import re from crash.commands import Command, ArgumentParser -from crash.commands import CommandLineError +from crash.commands import CommandLineError, CommandError from crash.types.task import LinuxTask, TaskStateFlags as TF +class TaskFormat(object): + """ + This class is responsible for converting the arguments into formatting + rules. + """ + def __init__(self, argv, regex): + self.sort = lambda x: x.info.task_pid() + self._filter = lambda x: True + self._format_one_task = self._format_common_line + self._regex = regex + + if argv.s: + self._format_header = self._format_stack_header + self._format_column4 = self._format_stack_address + elif argv.n: + self._format_header = self._format_threadnum_header + self._format_column4 = self._format_thread_num + else: + self._format_header = self._format_task_header + self._format_column4 = self._format_task_address + + if argv.k: + self._filter = self._is_kernel_thread + elif argv.u: + self._filter = self._is_user_task + elif argv.G: + self._filter = self._is_thread_group_leader + + if argv.l: + self.sort = lambda x: -x.info.last_run() + self._format_one_task = self._format_last_run + self._format_header = lambda : "" + + def _format_generic_header(self, col4name: str, col4width: int) -> str: + header = f" PID PPID CPU {col4name:^{col4width}} ST %MEM " + header += "VSZ RSS COMM" + + return header + + def _format_stack_header(self) -> str: + return self._format_generic_header("KSTACK", 16) + + def _format_stack_address(self, task: LinuxTask) -> str: + addr = int(task.get_stack_pointer()) + return f"{addr:16x}" + + def _format_task_header(self) ->str: + return self._format_generic_header("TASK", 16) + + def _format_task_address(self, task: LinuxTask) -> str: + addr = int(task.task_struct.address) + return f"{addr:16x}" + + def _format_threadnum_header(self) -> str: + return self._format_generic_header("THREAD#", 7) + + def _format_thread_num(self, task: LinuxTask) -> str: + return f"{task.thread.num:7d}" + + def _is_kernel_thread(self, task: LinuxTask) -> bool: + return task.is_kernel_task() + + def _is_user_task(self, task: LinuxTask) -> bool: + return not self._is_kernel_thread(task) + + def _is_thread_group_leader(self, task: LinuxTask) -> bool: + return task.is_thread_group_leader() + + def _format_common_line(self, task: LinuxTask, state: str) -> str: + pid = task.task_pid() + parent_pid = task.parent_pid() + last_cpu = task.get_last_cpu() + name = task.task_name() + + # This needs adaptation for page size != 4k + total_vm = task.total_vm * 4096 // 1024 + rss = task.rss * 4096 // 1024 + + if task.active: + active = ">" + else: + active = " " + + line = f"{active} {pid:>5} {parent_pid:>5} {last_cpu:>3} " + line += self._format_column4(task) + line += f" {state:3} {0:.1f} {total_vm:7d} {rss:6d} {name}" + + return line + + def _format_last_run(self, task: LinuxTask, state: str) -> str: + pid = task.task_pid() + addr = task.task_address() + cpu = task.get_last_cpu() + name = task.task_name() + if task.active: + cpu = task.cpu + + line = f"[{task.last_run():d}] [{state}] PID: {pid:-5d} " + line += f"TASK: {addr:x} CPU: {cpu:>2d} COMMAND: \"{name}\"" + + return line + + def should_print_task(self, task: LinuxTask) -> bool: + """ + Given optional filters and regex as part of the parent + object, return whether a task passes the criteria to be + printed. + + Args: + task (LinuxTask): The task under consideration + + Returns: + bool: Whether this task should be printed + """ + if self._filter(task) is False: + return False + + if self._regex and not self._regex.match(task.task_name()): + return False + + return True + + def format_one_task(self, task: LinuxTask, state: str) -> str: + """ + Given the formatting rules, produce the output line for this task. + + Args: + task (LinuxTask): The task to be printed + + Returns: + str: The ps output line for this task + """ + return self._format_one_task(task, state) + + def format_header(self) -> str: + """ + Return the header for this output object + + Returns: + str: The header for this type of ps output + """ + return self._format_header() + class PSCommand(Command): """display process status information @@ -410,21 +553,6 @@ def __init__(self): Command.__init__(self, "ps", parser) - self.header_template = " PID PPID CPU {1:^{0}} ST %MEM " \ - "VSZ RSS COMM" - -# PID PPID CPU TASK ST %MEM VSZ RSS COMM -# 1 0 3 ffff88033aa780c8 RU 0.0 0 0 [systemd] -#> 17080 16749 6 ffff8801db5ae040 RU 0.0 8168 1032 less -# PID PPID CPU TASK ST %MEM VSZ RSS COMM -#> 0 0 0 ffffffff81c13460 RU 0.0 0 0 [swapper/0] -# 17077 16749 0 ffff8800b956b848 RU 0.0 0 0 [less] - self.line_template = "{0} {1:>5} {2:>5} {3:>3} {4:{5}x} {6:3} {7:.1f}" - self.line_template += " {8:7d} {9:6d} {10:.{11}}{12}{13:.{14}}" - - self.num_line_template = "{0} {1:>5} {2:>5} {3:>3} {4:{5}d} {6:3} {7:.1f}" - self.num_line_template += " {8:7d} {9:6d} {10:.{11}}{12}{13:.{14}}" - def task_state_string(self, task): state = task.task_state() buf = None @@ -453,72 +581,6 @@ def task_state_string(self, task): return buf - @classmethod - def task_header(cls, task): - task_struct = task.task_struct - template = "PID: {0:-5d} TASK: {1:x} CPU: {2:>2d} COMMAND: \"{3}\"" - cpu = task.get_last_cpu() - if task.active: - cpu = task.cpu - return template.format(int(task_struct['pid']), - int(task_struct.address), cpu, - task_struct['comm'].string()) - - def print_last_run(self, task): - radix = 10 - if radix == 10: - radix_string = "d" - else: - radix_string = "x" - template = "[{0:{1}}] [{2}] {3}" - print(template.format(task.last_run(), radix_string, - self.task_state_string(task), - self.task_header(task))) - - def print_one(self, argv, thread): - task = thread.info - specified = argv.args is None - task_struct = task.task_struct - - pointer = task_struct.address - if argv.s: - pointer = task.get_stack_pointer() - - if argv.n: - pointer = thread.num - - if argv.l: - self.print_last_run(task) - return - - try: - parent_pid = task_struct['parent']['pid'] - except KeyError: - # This can happen on live systems where pids have gone - # away - print("Couldn't locate task at address {:#x}" - .format(task_struct.parent.address)) - return - - if task.active: - active = ">" - else: - active = " " - line = self.line_template - width = 16 - if argv.n: - line = self.num_line_template - width = 7 - - print(line.format(active, int(task_struct['pid']), int(parent_pid), - int(task.get_last_cpu()), int(pointer), - width, self.task_state_string(task), 0, - task.total_vm * 4096 // 1024, - task.rss * 4096 // 1024, - "[", int(task.is_kernel_task()), - task_struct['comm'].string(), - "]", int(task.is_kernel_task()))) - def setup_task_states(self): self.task_states = { TF.TASK_RUNNING : "RU", @@ -538,49 +600,42 @@ def setup_task_states(self): self.task_states[TF.TASK_IDLE] = "ID" def execute(self, argv): - sort_by_pid = lambda x: x.info.task_struct['pid'] - sort_by_last_run = lambda x: -x.info.last_run() + # Unimplemented + if argv.p or argv.c or argv.t or argv.a or argv.g or argv.r: + raise CommandError("Support for the -p, -c, -t, -a, -g, and -r options is unimplemented.") if not hasattr(self, 'task_states'): self.setup_task_states() - sort_by = sort_by_pid - if argv.l: - sort_by = sort_by_last_run - else: - if argv.s: - col4name = "KSTACK" - width = 16 - elif argv.n: - col4name = " THREAD#" - width = 7 - else: - col4name = "TASK" - width = 16 - print(self.header_template.format(width, col4name)) - regex = None if argv.args: regex = re.compile(fnmatch.translate(argv.args[0])) - for thread in sorted(gdb.selected_inferior().threads(), key=sort_by): + taskformat = TaskFormat(argv, regex) + + count = 0 + header = taskformat.format_header() + for thread in sorted(gdb.selected_inferior().threads(), + key=taskformat.sort): task = thread.info if task: - if argv.k and not task.is_kernel_task(): + if not taskformat.should_print_task(task): continue - if argv.u and task.is_kernel_task(): - continue - - if regex is not None: - m = regex.match(task.task_struct['comm'].string()) - if m is None: - continue - - # Only show thread group leaders - if argv.G and not task.is_thread_group_leader(): - continue + if header: + print(header) + header = None task.update_mem_usage() - self.print_one(argv, thread) + state = self.task_state_string(task) + line = taskformat.format_one_task(task, state) + print(line) + count += 1 + + if count == 0: + if regex: + print(f"No matches for {argv.args[0]}.") + else: + raise CommandError("Unfiltered output has no matches. BUG?") + PSCommand() From 550c3374ac2b9ef5cc4290f1eec21ebe64831e31 Mon Sep 17 00:00:00 2001 From: Jeffrey Mahoney Date: Thu, 9 May 2019 12:59:37 -0400 Subject: [PATCH 69/75] crash.cache.syscache: update config parsing Kernel v5.1-rc1 moved the compressed config data into .rodata using asm .globl variables to mark the bounds. This commit updates crash.cache.syscache to handle the new variables and cleans up the code a bit. Signed-off-by: Jeff Mahoney --- crash/cache/syscache.py | 95 +++++++++++++++++++++---------- crash/types/page.py | 11 ++-- kernel-tests/test_commands_sys.py | 32 +++++++++++ 3 files changed, 103 insertions(+), 35 deletions(-) create mode 100644 kernel-tests/test_commands_sys.py diff --git a/crash/cache/syscache.py b/crash/cache/syscache.py index b5d9a666115..2012ce8ae61 100644 --- a/crash/cache/syscache.py +++ b/crash/cache/syscache.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Dict + from builtins import round import gdb @@ -11,9 +13,12 @@ from crash.exceptions import DelayedAttributeError from crash.cache import CrashCache from crash.util import array_size -from crash.util.symbols import Types, Symvals, SymbolCallbacks +from crash.util.symbols import Types, Symvals, SymbolCallbacks, MinimalSymvals from crash.infra.lookup import DelayedValue + +ImageLocation = Dict[str, Dict[str, int]] + class CrashUtsnameCache(CrashCache): symvals = Symvals([ 'init_uts_ns' ]) @@ -45,6 +50,8 @@ def __getattr__(self, name): class CrashConfigCache(CrashCache): types = Types([ 'char *' ]) symvals = Symvals([ 'kernel_config_data' ]) + msymvals = MinimalSymvals([ 'kernel_config_data', + 'kernel_config_data_end' ]) def __getattr__(self, name): if name == 'config_buffer': @@ -53,51 +60,77 @@ def __getattr__(self, name): return self._parse_config() return getattr(self.__class__, name) - @staticmethod - def read_buf(address, size): + def read_buf(self, address: int, size: int) -> memoryview: return gdb.selected_inferior().read_memory(address, size) - @staticmethod - def read_buf_str(address, size): - buf = gdb.selected_inferior().read_memory(address, size) - if isinstance(buf, memoryview): - return buf.tobytes().decode('utf-8') - else: - return str(buf) - - def decompress_config_buffer(self): - MAGIC_START = 'IKCFG_ST' - MAGIC_END = 'IKCFG_ED' - - # Must cast it to char * to do the pointer arithmetic correctly - data_addr = self.symvals.kernel_config_data.address.cast(self.types.char_p_type) - data_len = self.symvals.kernel_config_data.type.sizeof + def read_buf_bytes(self, address: int, size: int) -> bytes: + return self.read_buf(address, size).tobytes() + + def locate_config_buffer_section(self) -> ImageLocation: + data_start = int(self.msymvals.kernel_config_data) + data_end = int(self.msymvals.kernel_config_data_end) + + return { + 'data' : { + 'start' : data_start, + 'size' : data_end - data_start, + }, + 'magic' : { + 'start' : data_start - 8, + 'end' : data_end, + }, + } + + def locate_config_buffer_typed(self) -> ImageLocation: + start = int(self.symvals.kernel_config_data.address) + end = start + self.symvals.kernel_config_data.type.sizeof + + return { + 'data' : { + 'start' : start + 8, + 'size' : end - start - 2*8 - 1, + }, + 'magic' : { + 'start' : start, + 'end' : end - 8 - 1, + }, + } + + def verify_image(self, location: ImageLocation) -> None: + MAGIC_START = b'IKCFG_ST' + MAGIC_END = b'IKCFG_ED' buf_len = len(MAGIC_START) - buf = self.read_buf_str(data_addr, buf_len) + buf = self.read_buf_bytes(location['magic']['start'], buf_len) if buf != MAGIC_START: - raise IOError("Missing MAGIC_START in kernel_config_data.") + raise IOError(f"Missing MAGIC_START in kernel_config_data. Got `{buf}'") buf_len = len(MAGIC_END) - buf = self.read_buf_str(data_addr + data_len - buf_len - 1, buf_len) + buf = self.read_buf_bytes(location['magic']['end'], buf_len) if buf != MAGIC_END: - raise IOError("Missing MAGIC_END in kernel_config_data.") + raise IOError("Missing MAGIC_END in kernel_config_data. Got `{buf}'") + + def decompress_config_buffer(self) -> str: + try: + location = self.locate_config_buffer_section() + except DelayedAttributeError: + location = self.locate_config_buffer_typed() + + self.verify_image(location) # Read the compressed data - buf_len = data_len - len(MAGIC_START) - len(MAGIC_END) - buf = self.read_buf(data_addr + len(MAGIC_START), buf_len) - self.config_buffer = zlib.decompress(buf, 16 + zlib.MAX_WBITS) - if (isinstance(self.config_buffer, bytes)): - self.config_buffer = str(self.config_buffer.decode('utf-8')) - else: - self.config_buffer = str(self.config_buffer) + buf = self.read_buf_bytes(location['data']['start'], + location['data']['size']) + + decompressed = zlib.decompress(buf, 16 + zlib.MAX_WBITS) + self.config_buffer = str(decompressed.decode('utf-8')) return self.config_buffer def __str__(self): return self.config_buffer - def _parse_config(self): - self.ikconfig_cache = {} + def _parse_config(self) -> Dict[str, str]: + self.ikconfig_cache: Dict[str, str] = dict() for line in self.config_buffer.splitlines(): # bin comments diff --git a/crash/types/page.py b/crash/types/page.py index 2ea07853e76..9385c9f2e0a 100644 --- a/crash/types/page.py +++ b/crash/types/page.py @@ -4,7 +4,8 @@ from math import log, ceil import gdb from crash.util import container_of, find_member_variant -from crash.util.symbols import Types, Symvals, TypeCallbacks, SymbolCallbacks +from crash.util.symbols import Types, Symvals, TypeCallbacks +from crash.util.symbols import SymbolCallbacks, MinimalSymbolCallbacks from crash.cache.syscache import config #TODO debuginfo won't tell us, depends on version? @@ -201,12 +202,14 @@ def __init__(self, obj, pfn): ('enum pageflags', Page.setup_pageflags ), ('enum zone_type', Page.setup_zone_type ), ('struct mem_section', Page.setup_mem_section) ]) +msymbol_cbs = MinimalSymbolCallbacks([ ('kernel_config_data', + Page.setup_nodes_width ) ]) # TODO: this should better be generalized to some callback for # "config is available" without refering to the symbol name here -symbol_cbs = SymbolCallbacks([ ('kernel_config_data', Page.setup_nodes_width ), - ('vmemmap_base', Page.setup_vmemmap_base ), - ('page_offset_base', Page.setup_directmap_base ) ]) +symbol_cbs = SymbolCallbacks([ ('vmemmap_base', Page.setup_vmemmap_base ), + ('page_offset_base', + Page.setup_directmap_base ) ]) def pfn_to_page(pfn): diff --git a/kernel-tests/test_commands_sys.py b/kernel-tests/test_commands_sys.py new file mode 100644 index 00000000000..e246722abe1 --- /dev/null +++ b/kernel-tests/test_commands_sys.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +import unittest +import gdb +import io +import sys + +from crash.commands.syscmd import SysCommand + +class TestCommandsSys(unittest.TestCase): + def setUp(self): + self.stdout = sys.stdout + self.redirect = io.StringIO() + sys.stdout = self.redirect + self.command = SysCommand("sys") + + def tearDown(self): + sys.stdout = self.stdout + + def output(self): + return self.redirect.getvalue() + + def output_lines(self): + return len(self.output().split("\n")) + + def test_sys(self): + self.command.invoke_uncaught("") + self.assertTrue(self.output_lines() > 2) + + def test_sys_config(self): + self.command.invoke_uncaught("config") + self.assertTrue(self.output_lines() > 2) From d04844b5ee7e6bf07a7ee168e43a3c8e1113b594 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Fri, 17 May 2019 15:20:13 -0400 Subject: [PATCH 70/75] tests: silence as much noise as possible The test output is littered with 'broken link' reports during tests that are specifically testing that behavior. We can tidy up a bit by adding a print_broken_links option that defaults to True but can be set to False by the test cases. Signed-off-by: Jeff Mahoney --- crash/types/list.py | 5 ++++- tests/test_list.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/crash/types/list.py b/crash/types/list.py index 54c384717b1..dae232f6e9a 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -109,9 +109,12 @@ def list_for_each(list_head, include_head=False, reverse=False, def list_for_each_entry(list_head, gdbtype, member, include_head=False, reverse=False, + print_broken_links=True, exact_cycles=False): for node in list_for_each(list_head, include_head=include_head, - reverse=reverse, exact_cycles=exact_cycles): + reverse=reverse, + print_broken_links=print_broken_links, + exact_cycles=exact_cycles): if node.type != types.list_head_type.pointer(): raise TypeError("Type {} found. Expected struct list_head *." .format(str(node.type))) diff --git a/tests/test_list.py b/tests/test_list.py index d78bba53ee1..83d80770cf9 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -80,7 +80,8 @@ def test_corrupt_list(self): expected_count = short_list.type.sizeof // short_list[0].type.sizeof count = 0 with self.assertRaises(CorruptListError): - for node in list_for_each(normal_list, exact_cycles=True): + for node in list_for_each(normal_list, exact_cycles=True, + print_broken_links=False): count += 1 def test_normal_container_list_with_string(self): @@ -113,7 +114,8 @@ def test_cycle_container_list_with_string(self): count = 0 with self.assertRaises(ListCycleError): for node in list_for_each_entry(cycle_list, 'struct container', - 'list', exact_cycles=True): + 'list', exact_cycles=True, + print_broken_links=False): count += 1 def test_cycle_container_list_with_type(self): @@ -125,7 +127,8 @@ def test_cycle_container_list_with_type(self): count = 0 with self.assertRaises(ListCycleError): for node in list_for_each_entry(cycle_list, struct_container, - 'list', exact_cycles=True): + 'list', exact_cycles=True, + print_broken_links=False): count += 1 def test_bad_container_list_with_string(self): @@ -136,7 +139,7 @@ def test_bad_container_list_with_string(self): count = 0 with self.assertRaises(CorruptListError): for node in list_for_each_entry(bad_list, 'struct container', - 'list'): + 'list', print_broken_links=False): count += 1 def test_bad_container_list_with_type(self): @@ -148,5 +151,5 @@ def test_bad_container_list_with_type(self): count = 0 with self.assertRaises(CorruptListError): for node in list_for_each_entry(bad_list, struct_container, - 'list'): + 'list', print_broken_links=False): count += 1 From cb870bbc38c8e91c833eebbd860cd2268e47feec Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 20 May 2019 09:36:58 -0400 Subject: [PATCH 71/75] crash.util: fix get_typed_pointer semantics Despite being called 'get_typed_pointer', we were dereferencing the pointer before turning. Also, we were refusing to take the address of a value that wasn't already the type we were targeting, which is silly since that would just return the object back. Signed-off-by: Jeff Mahoney --- crash/util/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crash/util/__init__.py b/crash/util/__init__.py index eaef267fdb5..8b23e5c3c86 100644 --- a/crash/util/__init__.py +++ b/crash/util/__init__.py @@ -366,6 +366,10 @@ def get_typed_pointer(val, gdbtype): """ Returns a pointer to the requested type at the given address + If the val is passed as a gdb.Value, it will be casted to + the expected type. If it is not a pointer, the address of the + value will be used instead. + Args: val (gdb.Value, str, or int): The address for which to provide a casted pointer @@ -380,10 +384,8 @@ def get_typed_pointer(val, gdbtype): if gdbtype.code != gdb.TYPE_CODE_PTR: gdbtype = gdbtype.pointer() if isinstance(val, gdb.Value): - if (val.type != gdbtype and - val.type != gdbtype.target()): - raise TypeError("gdb.Value must refer to {} not {}" - .format(gdbtype, val.type)) + if val.type.code != gdb.TYPE_CODE_PTR: + val = val.address elif isinstance(val, str): try: val = int(val, 16) @@ -391,7 +393,9 @@ def get_typed_pointer(val, gdbtype): print(e) raise TypeError("string must describe hex address: ".format(e)) if isinstance(val, int): - val = gdb.Value(val).cast(gdbtype).dereference() + val = gdb.Value(val).cast(gdbtype) + else: + val = val.cast(gdbtype) return val From 0f70ab143911c8f6e9da6ca39c37e86d089b4ffa Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Mon, 20 May 2019 13:53:32 -0400 Subject: [PATCH 72/75] crash.types.task: add documentation and static typing hints This commit adds API documentation and static typing hints to tasks. There are some minor code changes to make mypy happy with the result. Signed-off-by: Jeff Mahoney --- crash/types/task.py | 254 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 207 insertions(+), 47 deletions(-) diff --git a/crash/types/task.py b/crash/types/task.py index 8464a243aa6..23da9043dbd 100644 --- a/crash/types/task.py +++ b/crash/types/task.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable, Callable + import gdb from crash.util import array_size, struct_has_member from crash.util.symbols import Types, Symvals, SymbolCallbacks @@ -169,12 +171,10 @@ def __init__(self, task): class LinuxTask(object): task_struct_type = None mm_struct_fields = None - get_rss = None - get_stack_pointer_fn = None valid = False def __init__(self, task_struct, active=False, cpu=None, regs=None): - self.init_task_types(task_struct) + self._init_task_types(task_struct) if cpu is not None and not isinstance(cpu, int): raise TypeError("cpu must be integer or None") @@ -200,7 +200,7 @@ def __init__(self, task_struct, active=False, cpu=None, regs=None): self.pgd_addr = 0 @classmethod - def init_task_types(cls, task): + def _init_task_types(cls, task): if not cls.valid: t = types.task_struct_type if task.type != t: @@ -214,35 +214,78 @@ def init_task_types(cls, task): types.task_struct_type = task.type fields = types.task_struct_type.fields() cls.task_state_has_exit_state = 'exit_state' in fields - cls.pick_get_rss() - cls.pick_last_run() + cls._pick_get_rss() + cls._pick_last_run() cls.init_mm = get_value('init_mm') cls.valid = True - def attach_thread(self, thread): + def attach_thread(self, thread: gdb.InferiorThread) -> None: + """ + Associate a gdb thread with this task + + Args: + thread (gdb.InferiorThread): + The gdb thread to associate with this task + """ if not isinstance(thread, gdb.InferiorThread): raise TypeError("Expected gdb.InferiorThread") self.thread = thread - def set_thread_info(self, thread_info): + def set_thread_info(self, thread_info: gdb.Value) -> None: + """ + Set the thread info for this task + + The thread info structure is architecture specific. This method + allows the architecture code to assign its thread info structure + to this task. + + Args: + gdb.Value: + The struct thread_info to be associated with this task + """ self.thread_info = thread_info - def get_thread_info(self): + def get_thread_info(self) -> gdb.Value: + """ + Get the thread info for this task + + The thread info structure is architecture specific and so this + method abstracts its retreival. + + Returns: + gdb.Value: + The struct thread_info associated with this task + """ return self.thread_info - def get_last_cpu(self): - try: - return int(self.task_struct['cpu']) - except gdb.error as e: - return int(self.thread_info['cpu']) + def get_last_cpu(self) -> int: + """ + Returns the last cpu this task was scheduled to execute on + + Returns: + int: the last cpu this task was scheduled to execute on + """ + if struct_has_member(self.task_struct, 'cpu'): + cpu = self.task_struct['cpu'] + else: + cpu = self.thread_info['cpu'] + return int(cpu) + # Hrm. This seems broken since we're combining flags from + # two fields. def task_state(self): state = int(self.task_struct['state']) if self.task_state_has_exit_state: state |= int(self.task_struct['exit_state']) return state - def maybe_dead(self): + def maybe_dead(self) -> bool: + """ + Returns whether this task is dead + + Returns: + bool: whether this task is dead + """ state = self.task_state() known = TF.TASK_INTERRUPTIBLE @@ -254,19 +297,49 @@ def maybe_dead(self): known |= TF.TASK_SWAPPING return (state & known) == 0 - def task_flags(self): + def task_flags(self) -> int: + """ + Returns the flags for this task + + Returns: + int: the flags for this task + """ return int(self.task_struct['flags']) - def is_exiting(self): - return self.task_flags() & PF_EXITING + def is_exiting(self) -> bool: + """ + Returns whether a task is exiting + + Returns: + bool: whether the task is exiting + """ + return (self.task_flags() & PF_EXITING) != 0 + + def is_zombie(self) -> bool: + """ + Returns whether a task is in Zombie state - def is_zombie(self): - return self.task_state() & TF.EXIT_ZOMBIE + Returns: + bool: whether the task is in zombie state + """ + return (self.task_state() & TF.EXIT_ZOMBIE) != 0 - def is_thread_group_leader(self): + def is_thread_group_leader(self) -> bool: + """ + Returns whether a task is a thread group leader + + Returns: + bool: whether the task is a thread group leader + """ return int(self.task_struct['exit_signal']) >= 0 - def update_mem_usage(self): + def update_mem_usage(self) -> None: + """ + Update the memory usage for this task + + Tasks are created initially without their memory statistics. This + method explicitly updates them. + """ if self.mem_valid: return @@ -283,20 +356,48 @@ def update_mem_usage(self): self.pgd_addr = int(mm['pgd']) self.mem_valid = True - def task_name(self, brackets=False): + def task_name(self, brackets: bool=False) -> str: + """ + Returns the `comm' field of this task + + Args: + brackets: If this task is a kernel thread, surround the name + in square brackets + + Returns: + str: the comm field of this task a python string + """ name = self.task_struct['comm'].string() if brackets and self.is_kernel_task(): return f"[{name}]" else: return name - def task_pid(self): + def task_pid(self) -> int: + """ + Returns the pid of this task + + Returns: + int: the pid of this task + """ return int(self.task_struct['pid']) - def parent_pid(self): + def parent_pid(self) -> int: + """ + Returns the pid of this task's parent + + Returns: + int: the pid of this task's parent + """ return int(self.task_struct['parent']['pid']) - def task_address(self): + def task_address(self) -> int: + """ + Returns the address of the task_struct for this task + + Returns: + int: the address of the task_struct + """ return int(self.task_struct.address) def is_kernel_task(self): @@ -314,20 +415,31 @@ def is_kernel_task(self): return False + def get_stack_pointer_fn(self, task_struct: gdb.Value) -> int: + try: + fn = getattr(self, '_get_stack_pointer_fn') + except AttributeError as e: + raise NotImplementedError("Architecture hasn't provided stack pointer callback") + + return fn(task_struct) + @classmethod - def set_get_stack_pointer(cls, fn): - cls.get_stack_pointer_fn = fn + def set_get_stack_pointer(cls, fn: Callable[[gdb.Value], int]): + setattr(cls, '_get_stack_pointer_fn', fn) - def get_stack_pointer(self): - return self.get_stack_pointer_fn(self.task_struct['thread']) + def get_stack_pointer(self) -> int: + """ + Get the stack pointer for this task + """ + return int(self.get_stack_pointer_fn(self.task_struct['thread'])) - def get_rss_field(self): + def _get_rss_field(self): return int(self.task_struct['mm']['rss'].value()) - def get__rss_field(self): + def _get__rss_field(self): return int(self.task_struct['mm']['_rss'].value()) - def get_rss_stat_field(self): + def _get_rss_stat_field(self): stat = self.task_struct['mm']['rss_stat']['count'] stat0 = self.task_struct['mm']['rss_stat']['count'][0] rss = 0 @@ -335,7 +447,7 @@ def get_rss_stat_field(self): rss += int(stat[i]['counter']) return rss - def get_anon_file_rss_fields(self): + def _get_anon_file_rss_fields(self): mm = self.task_struct['mm'] rss = 0 for name in cls.anon_file_rss_fields: @@ -349,15 +461,15 @@ def get_anon_file_rss_fields(self): # dynamically. We may do that eventually, but for now we can just # select the proper function and assign it to the class. @classmethod - def pick_get_rss(cls): + def _pick_get_rss(cls): if struct_has_member(types.mm_struct_type, 'rss'): - cls.get_rss = cls.get_rss_field + cls._get_rss = cls._get_rss_field elif struct_has_member(types.mm_struct_type, '_rss'): - cls.get_rss = cls.get__rss_field + cls._get_rss = cls._get__rss_field elif struct_has_member(types.mm_struct_type, 'rss_stat'): cls.MM_FILEPAGES = get_value('MM_FILEPAGES') cls.MM_ANONPAGES = get_value('MM_ANONPAGES') - cls.get_rss = cls.get_rss_stat_field + cls._get_rss = cls._get_rss_stat_field else: cls.anon_file_rss_fields = [] @@ -368,48 +480,96 @@ def pick_get_rss(cls): cls.anon_file_rss_fields.append('_anon_rss') cls.atomic_long_type = gdb.lookup_type('atomic_long_t') - cls.get_rss = cls.get_anon_file_rss_fields + cls._get_rss = cls._get_anon_file_rss_fields if len(cls.anon_file_rss_fields): raise RuntimeError("No method to retrieve RSS from task found.") - def last_run__last_run(self): + def _get_rss(self) -> int: + raise NotImplementedError("_get_rss not implemented") + + def get_rss(self): + """ + Return the resident set for this task + + Returns: + int: the size of the resident memory set for this task + """ + return self._get_rss() + + def _last_run__last_run(self): return int(self.task_struct['last_run']) - def last_run__timestamp(self): + def _last_run__timestamp(self): return int(self.task_struct['timestamp']) - def last_run__last_arrival(self): + def _last_run__last_arrival(self): return int(self.task_struct['sched_info']['last_arrival']) + def _get_last_run(self) -> int: + raise NotImplementedError("_get_last_run not implemented") + @classmethod - def pick_last_run(cls): + def _pick_last_run(cls): fields = types.task_struct_type.keys() if ('sched_info' in fields and 'last_arrival' in types.task_struct_type['sched_info'].type.keys()): - cls.last_run = cls.last_run__last_arrival + cls._get_last_run = cls._last_run__last_arrival elif 'last_run' in fields: - cls.last_run = cls.last_run__last_run + cls._get_last_run = cls._last_run__last_run elif 'timestamp' in fields: - cls.last_run = cls.last_run__timestamp + cls._get_last_run = cls._last_run__timestamp else: raise RuntimeError("No method to retrieve last run from task found.") + def last_run(self) -> int: + """ + The timestamp of when this task was last run + + Returns: + int: The timestamp of when this task was last run + """ + return self._get_last_run() + def for_each_thread_group_leader(): + """ + Iterate the task list and yield each thread group leader + + Yields: + gdb.Value: The next task on the list + """ task_list = symvals.init_task['tasks'] for task in list_for_each_entry(task_list, symvals.init_task.type, 'tasks', include_head=True): yield task def for_each_thread_in_group(task): + """ + Iterate a thread group leader's thread list and + yield each struct task_struct + + Args: + gdb.Value: + The task_struct that is the thread group leader. + + Yields: + gdb.Value: The next task on the list + """ thread_list = task['thread_group'] for thread in list_for_each_entry(thread_list, symvals.init_task.type, 'thread_group'): yield thread -def for_each_all_tasks(): +def for_each_all_tasks() -> Iterable[gdb.Value]: + """ + Iterate the task list and yield each task including any associated + thread tasks + + Yields: + gdb.Value: The next task on the list + """ for leader in for_each_thread_group_leader(): yield leader for task in for_each_thread_in_group(leader): From d6bc98bf16ca2b761f00707c09dbaca3639dd981 Mon Sep 17 00:00:00 2001 From: Jeff Mahoney Date: Tue, 21 May 2019 17:34:47 -0400 Subject: [PATCH 73/75] crash: more documentation and typing This commit is the leftover bits for typing and documentation. Signed-off-by: Jeff Mahoney --- crash/arch/__init__.py | 4 +- crash/arch/ppc64.py | 4 +- crash/arch/x86_64.py | 15 ++-- crash/commands/__init__.py | 4 +- crash/commands/mount.py | 5 +- crash/subsystem/storage/__init__.py | 20 ++--- crash/subsystem/storage/blocksq.py | 8 +- crash/types/classdev.py | 9 ++- crash/types/klist.py | 33 +++++++- crash/types/list.py | 67 ++++++++++++++-- crash/types/node.py | 118 ++++++++++++++++++++++++---- crash/types/page.py | 6 +- crash/util/__init__.py | 29 ++++--- 13 files changed, 257 insertions(+), 65 deletions(-) diff --git a/crash/arch/__init__.py b/crash/arch/__init__.py index a1eb80c9364..51f033395b9 100644 --- a/crash/arch/__init__.py +++ b/crash/arch/__init__.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import List + import gdb class CrashArchitecture(object): ident = "base-class" - aliases = None + aliases: List[str] = list() def __init__(self): pass diff --git a/crash/arch/ppc64.py b/crash/arch/ppc64.py index 3ab8eb7e159..1d361f6ef97 100644 --- a/crash/arch/ppc64.py +++ b/crash/arch/ppc64.py @@ -18,13 +18,13 @@ def __init__(self): # Stop stack traces with addresses below this self.filter = KernelFrameFilter(0xffff000000000000) - def setup_thread_info(self, thread): + def setup_thread_info(self, thread: gdb.InferiorThread) -> None: task = thread.info.task_struct thread_info = task['stack'].cast(self.thread_info_p_type) thread.info.set_thread_info(thread_info) @classmethod - def get_stack_pointer(cls, thread_struct): + def get_stack_pointer(cls, thread_struct: gdb.Value) -> gdb.Value: return thread_struct['ksp'] register(Powerpc64Architecture) diff --git a/crash/arch/x86_64.py b/crash/arch/x86_64.py index 4d72f1e9285..7febdd2ab79 100644 --- a/crash/arch/x86_64.py +++ b/crash/arch/x86_64.py @@ -33,12 +33,13 @@ def __init__(self): # Stop stack traces with addresses below this self.filter = KernelFrameFilter(0xffff000000000000) - def setup_thread_info(self, thread): + def setup_thread_info(self, thread: gdb.InferiorThread) -> None: task = thread.info.task_struct thread_info = task['stack'].cast(self.thread_info_p_type) thread.info.set_thread_info(thread_info) - def fetch_register_active(self, thread, register): + def fetch_register_active(self, thread: gdb.InferiorThread, + register: gdb.Register) -> None: task = thread.info for reg in task.regs: if reg == "rip" and (register != 16 and register != -1): @@ -48,7 +49,8 @@ def fetch_register_active(self, thread, register): except KeyError as e: pass - def fetch_register_scheduled_inactive(self, thread, register): + def fetch_register_scheduled_inactive(self, thread: gdb.InferiorThread, + register: gdb.Register) -> None: ulong_type = self.ulong_type task = thread.info.task_struct @@ -61,7 +63,7 @@ def fetch_register_scheduled_inactive(self, thread, register): if register == 16 or register == -1: thread.registers['rip'].value = frame['ret_addr'] if register == 16: - return True + return thread.registers['rbp'].value = frame['bp'] thread.registers['rbx'].value = frame['bx'] @@ -75,7 +77,8 @@ def fetch_register_scheduled_inactive(self, thread, register): thread.info.stack_pointer = rsp thread.info.valid_stack = True - def fetch_register_scheduled_thread_return(self, thread, register): + def fetch_register_scheduled_thread_return(self, thread: gdb.InferiorThread, + register: gdb.Register): ulong_type = self.ulong_type task = thread.info.task_struct @@ -114,7 +117,7 @@ def fetch_register_scheduled_thread_return(self, thread, register): thread.info.valid_stack = True @classmethod - def get_stack_pointer(cls, thread_struct): + def get_stack_pointer(cls, thread_struct: gdb.Value) -> gdb.Value: return thread_struct['sp'] register(x86_64Architecture) diff --git a/crash/commands/__init__.py b/crash/commands/__init__.py index 8d90438d1d5..f96394d1566 100644 --- a/crash/commands/__init__.py +++ b/crash/commands/__init__.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Dict + import gdb import os @@ -21,7 +23,7 @@ def error(self, message): raise CommandLineError(message) class Command(gdb.Command): - commands = {} + commands: Dict[str, gdb.Command] = dict() def __init__(self, name, parser=None): self.name = "py" + name if parser is None: diff --git a/crash/commands/mount.py b/crash/commands/mount.py index 7600a508fb6..f3cdedb0d2a 100644 --- a/crash/commands/mount.py +++ b/crash/commands/mount.py @@ -3,7 +3,9 @@ import gdb +from argparse import Namespace from crash.commands import Command, ArgumentParser +from crash.types.task import LinuxTask from crash.subsystem.filesystem.mount import MNT_NOSUID, MNT_NODEV, MNT_NOEXEC from crash.subsystem.filesystem.mount import MNT_NOATIME, MNT_NODIRATIME from crash.subsystem.filesystem.mount import MNT_RELATIME, MNT_READONLY @@ -49,7 +51,8 @@ def execute(self, args): for mnt in for_each_mount(): self.show_one_mount(mnt, args) - def show_one_mount(self, mnt, args, task=None): + def show_one_mount(self, mnt: gdb.Value, args: Namespace, + task: LinuxTask=None) -> None: if mnt.type.code == gdb.TYPE_CODE_PTR: mnt = mnt.dereference() diff --git a/crash/subsystem/storage/__init__.py b/crash/subsystem/storage/__init__.py index 37cf821d8d9..47dd0d05c60 100644 --- a/crash/subsystem/storage/__init__.py +++ b/crash/subsystem/storage/__init__.py @@ -40,7 +40,7 @@ def for_each_bio_in_stack(bio: gdb.Value) -> Iterable[decoders.Decoder]: yield decoder decoder = next(decoder) -def dev_to_gendisk(dev): +def dev_to_gendisk(dev: gdb.Value) -> gdb.Value: """ Converts a struct device that is embedded in a struct gendisk back to the struct gendisk. @@ -55,7 +55,7 @@ def dev_to_gendisk(dev): """ return container_of(dev, types.gendisk_type, 'part0.__dev') -def dev_to_part(dev): +def dev_to_part(dev: gdb.Value) -> gdb.Value: """ Converts a struct device that is embedded in a struct hd_struct back to the struct hd_struct. @@ -71,7 +71,7 @@ def dev_to_part(dev): """ return container_of(dev, types.hd_struct_type, '__dev') -def gendisk_to_dev(gendisk): +def gendisk_to_dev(gendisk: gdb.Value) -> gdb.Value: """ Converts a struct gendisk that embeds a struct device to the struct device. @@ -87,7 +87,7 @@ def gendisk_to_dev(gendisk): return gendisk['part0']['__dev'].address -def part_to_dev(part): +def part_to_dev(part: gdb.Value) -> gdb.Value: """ Converts a struct hd_struct that embeds a struct device to the struct device. @@ -147,7 +147,7 @@ def for_each_block_device(subtype: gdb.Value=None) -> Iterable[gdb.Value]: raise RuntimeError("Encountered unexpected device type {}" .format(dev['type'])) -def for_each_disk(): +def for_each_disk() -> Iterable[gdb.Value]: """ Iterates over each block device registered with the block class that corresponds to an entire disk. @@ -157,7 +157,7 @@ def for_each_disk(): return for_each_block_device(symvals.disk_type) -def gendisk_name(gendisk): +def gendisk_name(gendisk: gdb.Value) -> str: """ Returns the name of the provided block device. @@ -189,7 +189,7 @@ def gendisk_name(gendisk): .format(types.gendisk_type, types.hd_struct_type, gendisk.type.unqualified())) -def block_device_name(bdev): +def block_device_name(bdev: gdb.Value) -> str: """ Returns the name of the provided block device. @@ -205,7 +205,7 @@ def block_device_name(bdev): """ return gendisk_name(bdev['bd_disk']) -def is_bdev_inode(inode): +def is_bdev_inode(inode: gdb.Value) -> bool: """ Tests whether the provided struct inode describes a block device @@ -221,7 +221,7 @@ def is_bdev_inode(inode): """ return inode['i_sb'] == symvals.blockdev_superblock -def inode_to_block_device(inode): +def inode_to_block_device(inode: gdb.Value) -> gdb.Value: """ Returns the block device associated with this inode. @@ -243,7 +243,7 @@ def inode_to_block_device(inode): raise TypeError("inode does not correspond to block device") return container_of(inode, types.bdev_inode_type, 'vfs_inode')['bdev'] -def inode_on_bdev(inode): +def inode_on_bdev(inode: gdb.Value) -> gdb.Value: """ Returns the block device associated with this inode. diff --git a/crash/subsystem/storage/blocksq.py b/crash/subsystem/storage/blocksq.py index 52a517f39d4..34e7d14827b 100644 --- a/crash/subsystem/storage/blocksq.py +++ b/crash/subsystem/storage/blocksq.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable, Tuple + import gdb from crash.util.symbols import Types @@ -12,7 +14,7 @@ class NoQueueError(RuntimeError): types = Types([ 'struct request' ]) -def for_each_request_in_queue(queue): +def for_each_request_in_queue(queue: gdb.Value) -> Iterable[gdb.Value]: """ Iterates over each struct request in request_queue @@ -32,7 +34,7 @@ def for_each_request_in_queue(queue): return list_for_each_entry(queue['queue_head'], types.request_type, 'queuelist') -def request_age_ms(request): +def request_age_ms(request: gdb.Value) -> int: """ Returns the age of the request in milliseconds @@ -49,7 +51,7 @@ def request_age_ms(request): """ return kernel.jiffies_to_msec(kernel.jiffies - request['start_time']) -def requests_in_flight(queue): +def requests_in_flight(queue: gdb.Value) -> Tuple[int, int]: """ Report how many requests are in flight for this queue diff --git a/crash/types/classdev.py b/crash/types/classdev.py index 8f8c40ece4e..d1f1e8a65ab 100644 --- a/crash/types/classdev.py +++ b/crash/types/classdev.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable + import gdb from crash.types.klist import klist_for_each @@ -14,15 +16,16 @@ class ClassdevState(object): #v5.1-rc1 moved knode_class from struct device to struct device_private @classmethod - def setup_iterator_type(cls, gdbtype): + def _setup_iterator_type(cls, gdbtype): if struct_has_member(gdbtype, 'knode_class'): cls.class_is_private = False type_cbs = TypeCallbacks([ ('struct device', - ClassdevState.setup_iterator_type) ]) + ClassdevState._setup_iterator_type) ]) -def for_each_class_device(class_struct, subtype=None): +def for_each_class_device(class_struct: gdb.Value, + subtype: gdb.Value=None) -> Iterable[gdb.Value]: klist = class_struct['p']['klist_devices'] container_type = types.device_type diff --git a/crash/types/klist.py b/crash/types/klist.py index e58b074fa86..c1b8ab24ff4 100644 --- a/crash/types/klist.py +++ b/crash/types/klist.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable + import gdb -from crash.util import container_of +from crash.util import container_of, TypeSpecifier from crash.types.list import list_for_each_entry from crash.exceptions import CorruptedError @@ -13,7 +15,17 @@ class KlistCorruptedError(CorruptedError): pass -def klist_for_each(klist): +def klist_for_each(klist: gdb.Value) -> Iterable[gdb.Value]: + """ + Iterate over a klist and yield each node + + Args: + klist (gdb.Value): + The list to iterate + + Yields: + gdb.Value: The next node in the list + """ if klist.type == types.klist_type.pointer(): klist = klist.dereference() elif klist.type != types.klist_type: @@ -28,7 +40,22 @@ def klist_for_each(klist): raise KlistCorruptedError("Corrupted") yield node -def klist_for_each_entry(klist, gdbtype, member): +def klist_for_each_entry(klist: gdb.Value, gdbtype: TypeSpecifier, + member: str) -> gdb.Value: + """ + Iterate over a klist and yield each node's containing object + + Args: + klist (gdb.Value): + The list to iterate + gdbtype (gdb.Type, gdb.Value, str, gdb.Symbol): + The type of the containing object (see crash.util::container_of) + member (str): The name of the member in the containing object that + corresponds to the klist_node + + Yields: + gdb.Value: The next node in the list + """ for node in klist_for_each(klist): if node.type != types.klist_node_type: raise TypeError("Type {} found. Expected {}." diff --git a/crash/types/list.py b/crash/types/list.py index dae232f6e9a..0e6c6fa27ad 100644 --- a/crash/types/list.py +++ b/crash/types/list.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterator, Set + import gdb -from crash.util import container_of +from crash.util import container_of, TypeSpecifier from crash.util.symbols import Types class ListError(Exception): @@ -16,8 +18,33 @@ class ListCycleError(CorruptListError): types = Types([ 'struct list_head' ]) -def list_for_each(list_head, include_head=False, reverse=False, - print_broken_links=True, exact_cycles=False): +def list_for_each(list_head: gdb.Value, include_head: bool=False, + reverse: bool=False, print_broken_links: bool=True, + exact_cycles: bool=False) -> Iterator[gdb.Value]: + """ + Iterate over a list and yield each node + + Args: + list_head (gdb.Value): + The list to iterate + include_head (bool, optional, default=False): + Include the head of the list in iteration - useful + for lists with no anchors + reverse (bool, optional, default=False): + Iterate the list in reverse order (follow the prev links) + print_broken_links (bool, optional, default=True): + Print warnings about broken links + exact_cycles (bool, optional, default=False): + Detect and raise an exception if a cycle is detected in the list + + Yields: + gdb.Value: The next node in the list + + Raises: + CorruptListError: the list is corrupted + ListCycleError: the list contains cycles + BufferError: portions of the list cannot be read + """ pending_exception = None if isinstance(list_head, gdb.Symbol): list_head = list_head.value() @@ -42,7 +69,7 @@ def list_for_each(list_head, include_head=False, reverse=False, prev_ = 'next' if exact_cycles: - visited = set() + visited: Set[int] = set() if include_head: yield list_head.address @@ -107,10 +134,34 @@ def list_for_each(list_head, include_head=False, reverse=False, if pending_exception is not None: raise pending_exception -def list_for_each_entry(list_head, gdbtype, member, - include_head=False, reverse=False, - print_broken_links=True, - exact_cycles=False): +def list_for_each_entry(list_head: gdb.Value, gdbtype: TypeSpecifier, + member: str, include_head: bool=False, + reverse: bool=False, print_broken_links: bool=True, + exact_cycles: bool=False) -> Iterator[gdb.Value]: + """ + Iterate over a list and yield each node's containing object + + Args: + list_head (gdb.Value): + The list to iterate + gdbtype (gdb.Type, gdb.Value, str, gdb.Symbol): + The type of the containing object (see crash.util::container_of) + member (str): The name of the member in the containing object that + corresponds to the list_head + include_head (bool, optional, default=False): + Include the head of the list in iteration - useful for + lists with no anchors + reverse (bool, optional, default=False): + Iterate the list in reverse order (follow the prev links) + print_broken_links (bool, optional, default=True): + Print warnings about broken links + exact_cycles (bool, optional, default=False): + Detect and raise an exception if a cycle is detected in the list + + Yields: + gdb.Value: The next node in the list + """ + for node in list_for_each(list_head, include_head=include_head, reverse=reverse, print_broken_links=print_broken_links, diff --git a/crash/types/node.py b/crash/types/node.py index 34db2196def..2a452106db6 100644 --- a/crash/types/node.py +++ b/crash/types/node.py @@ -1,29 +1,60 @@ #!/usr/bin/python3 # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Iterable, List, Type, TypeVar + import gdb from crash.util.symbols import Symbols, Symvals, Types, SymbolCallbacks from crash.util import container_of, find_member_variant, get_symbol_value from crash.types.percpu import get_percpu_var from crash.types.bitmap import for_each_set_bit import crash.types.zone +from crash.exceptions import DelayedAttributeError symbols = Symbols([ 'numa_node' ]) symvals = Symvals([ 'numa_cpu_lookup_table', 'node_data' ]) types = Types([ 'pg_data_t', 'struct zone' ]) -def numa_node_id(cpu): +def numa_node_id(cpu: int) -> int: + """ + Return the NUMA node ID for a given CPU + + Args: + cpu (int): The CPU number to obtain the NUMA node ID + Returns: + int: The NUMA node ID for the specified CPU. + """ if gdb.current_target().arch.name() == "powerpc:common64": return int(symvals.numa_cpu_lookup_table[cpu]) else: return int(get_percpu_var(symbols.numa_node, cpu)) +NodeType = TypeVar('NodeType', bound='Node') + class Node(object): - @staticmethod - def from_nid(nid): - return Node(symvals.node_data[nid].dereference()) + """ + A wrapper around the Linux kernel 'struct node' structure + """ + @classmethod + def from_nid(cls: Type[NodeType], nid: int) -> NodeType: + """ + Obtain a Node using the NUMA Node ID (nid) + + Args: + nid (int): The NUMA Node ID - def for_each_zone(self): + Returns: + Node: the Node wrapper for the struct node for this NID + """ + return cls(symvals.node_data[nid].dereference()) + + def for_each_zone(self) -> Iterable[crash.types.zone.Zone]: + """ + Iterate over each zone contained in this NUMA node + + Yields: + Zone: The next Zone in this Node + """ node_zones = self.gdb_obj["node_zones"] ptr = int(node_zones[0].address) @@ -37,15 +68,22 @@ def for_each_zone(self): yield crash.types.zone.Zone(zone, zid) ptr += types.zone_type.sizeof - def __init__(self, obj): + def __init__(self, obj: gdb.Value): + """ + Initialize a Node using the gdb.Value for the struct node + + Args: + obj: gdb.Value: + The node for which to construct a wrapper + """ self.gdb_obj = obj class NodeStates(object): - nids_online = None - nids_possible = None + nids_online: List[int] = list() + nids_possible: List[int] = list() @classmethod - def setup_node_states(cls, node_states_sym): + def _setup_node_states(cls, node_states_sym): node_states = node_states_sym.value() enum_node_states = gdb.lookup_type("enum node_states") @@ -59,21 +97,73 @@ def setup_node_states(cls, node_states_sym): bits = node_states[N_ONLINE]["bits"] cls.nids_online = list(for_each_set_bit(bits)) -symbol_cbs = SymbolCallbacks([('node_states', NodeStates.setup_node_states)]) + def for_each_nid(self) -> Iterable[int]: + """ + Iterate over each NUMA Node ID + + Yields: + int: The next NUMA Node ID + """ + if not self.nids_possible: + raise DelayedAttributeError('node_states') + + for nid in self.nids_possible: + yield nid + + def for_each_online_nid(self) -> Iterable[int]: + """ + Iterate over each online NUMA Node ID + + Yields: + int: The next NUMA Node ID + """ + if not self.nids_online: + raise DelayedAttributeError('node_states') + + for nid in self.nids_online: + yield nid + +symbol_cbs = SymbolCallbacks([('node_states', NodeStates._setup_node_states)]) + +_state = NodeStates() def for_each_nid(): - for nid in NodeStates.nids_possible: + """ + Iterate over each NUMA Node ID + + Yields: + int: The next NUMA Node ID + """ + for nid in _state.for_each_nid(): yield nid def for_each_online_nid(): - for nid in NodeStates.nids_online: + """ + Iterate over each online NUMA Node ID + + Yields: + int: The next NUMA Node ID + """ + for nid in _state.for_each_online_nid(): yield nid -def for_each_node(): +def for_each_node() -> Iterable[Node]: + """ + Iterate over each NUMA Node + + Yields: + int: The next NUMA Node + """ for nid in for_each_nid(): yield Node.from_nid(nid) -def for_each_online_node(): +def for_each_online_node() -> Iterable[Node]: + """ + Iterate over each Online NUMA Node + + Yields: + int: The next NUMA Node + """ for nid in for_each_online_nid(): yield Node.from_nid(nid) diff --git a/crash/types/page.py b/crash/types/page.py index 9385c9f2e0a..7d72d9828e1 100644 --- a/crash/types/page.py +++ b/crash/types/page.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: +from typing import Dict + from math import log, ceil import gdb from crash.util import container_of, find_member_variant @@ -22,7 +24,7 @@ class Page(object): vmemmap_base = 0xffffea0000000000 vmemmap = None directmap_base = 0xffff880000000000 - pageflags = dict() + pageflags: Dict[str, int] = dict() PG_tail = None PG_slab = None @@ -37,6 +39,8 @@ class Page(object): # TODO have arch provide this? BITS_PER_LONG = None + PAGE_SIZE = 4096 + sparsemem = False @classmethod diff --git a/crash/util/__init__.py b/crash/util/__init__.py index 8b23e5c3c86..7293a33fc29 100644 --- a/crash/util/__init__.py +++ b/crash/util/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: -from typing import Union +from typing import Union, Tuple, List, Iterator, Dict import gdb import uuid @@ -11,6 +11,7 @@ from crash.exceptions import MissingTypeError, MissingSymbolError TypeSpecifier = Union [ gdb.Type, gdb.Value, str, gdb.Symbol ] +AddressSpecifier = Union [ gdb.Value, str, int ] class OffsetOfError(Exception): """Generic Exception for offsetof errors""" @@ -74,7 +75,7 @@ def __init__(self, member, gdbtype): types = Types([ 'char *', 'uuid_t' ]) -def container_of(val, gdbtype, member): +def container_of(val: gdb.Value, gdbtype: TypeSpecifier, member) -> gdb.Value: """ Returns an object that contains the specified object at the given offset. @@ -131,7 +132,8 @@ def struct_has_member(gdbtype: TypeSpecifier, name: str) -> bool: except InvalidComponentError: return False -def get_symbol_value(symname, block=None, domain=None): +def get_symbol_value(symname: str, block: gdb.Block=None, + domain: int=None) -> gdb.Value: """ Returns the value associated with a named symbol @@ -153,7 +155,8 @@ def get_symbol_value(symname, block=None, domain=None): return sym.value() raise MissingSymbolError("Cannot locate symbol {}".format(symname)) -def safe_get_symbol_value(symname, block=None, domain=None): +def safe_get_symbol_value(symname: str, block: gdb.Block=None, + domain: int=None) -> gdb.Value: """ Returns the value associated with a named symbol @@ -173,7 +176,7 @@ def safe_get_symbol_value(symname, block=None, domain=None): except MissingSymbolError: return None -def resolve_type(val): +def resolve_type(val: TypeSpecifier) -> gdb.Type: """ Resolves a gdb.Type given a type, value, string, or symbol @@ -238,7 +241,8 @@ def __offsetof(val, spec, error): return (offset, gdbtype) -def offsetof_type(val, spec, error=True): +def offsetof_type(val: TypeSpecifier, spec: str, + error: bool=True) -> Union[Tuple[int, gdb.Type], None]: """ Returns the offset and type of a named member of a structure @@ -285,7 +289,8 @@ def offsetof_type(val, spec, error=True): else: return None -def offsetof(val, spec, error=True): +def offsetof(val: TypeSpecifier, spec: str, + error: bool=True) -> Union[int, None]: """ Returns the offset of a named member of a structure @@ -309,7 +314,7 @@ def offsetof(val, spec, error=True): return res[0] return None -def find_member_variant(gdbtype, variants): +def find_member_variant(gdbtype: TypeSpecifier, variants: List[str]) -> str: """ Examines the given type and returns the first found member name @@ -333,7 +338,7 @@ def find_member_variant(gdbtype, variants): raise TypeError("Unrecognized '{}': could not find member '{}'" .format(str(gdbtype), variants[0])) -def safe_lookup_type(name, block=None): +def safe_lookup_type(name: str, block: gdb.Block=None) -> Union[gdb.Type, None]: """ Looks up a gdb.Type without throwing an exception on failure @@ -350,7 +355,7 @@ def safe_lookup_type(name, block=None): except gdb.error: return None -def array_size(value): +def array_size(value: gdb.Value) -> int: """ Returns the number of elements in an array @@ -362,7 +367,7 @@ def array_size(value): """ return value.type.sizeof // value[0].type.sizeof -def get_typed_pointer(val, gdbtype): +def get_typed_pointer(val: AddressSpecifier, gdbtype: gdb.Type) -> gdb.Type: """ Returns a pointer to the requested type at the given address @@ -399,7 +404,7 @@ def get_typed_pointer(val, gdbtype): return val -def array_for_each(value): +def array_for_each(value: gdb.Value) -> Iterator[gdb.Value]: """ Yields each element in an array separately From 28344703a5e09c5725ae1209c272979ed24f7339 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Thu, 9 May 2019 12:14:08 -0700 Subject: [PATCH 74/75] WIP: Add initial support for "live" debugging This change adds a new "crash-kcore.sh" script that enables the use of this repository for debugging a "live" system via the "/proc/kcore" interface, rather than reading from a kernel crash dump. Current functionality includes: * Ability to print global variables * Ability to run existing crash-python commands * Ability to print backtraces with "bt" Caveats: * Thread information is read once at startup, and never updated. As a result, when listing the backtraces for thread, they may not reflect the current state of the system; they'll reflect the state of the system during crash-python initialization. * We cannot (as far as I know) completely disable the caching done by GDB for the "core" target. Thus, when printing small amounts of data repeatedly (e.g. calling "p jiffies_64" repeatedly), the value shown may not reflect the current state of the system, it'll reflect the value when it was first read and cached. Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Tom Caputi --- crash-kcore.sh | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ crash/kernel.py | 11 +++++--- 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100755 crash-kcore.sh diff --git a/crash-kcore.sh b/crash-kcore.sh new file mode 100755 index 00000000000..5d7ebe9ecc3 --- /dev/null +++ b/crash-kcore.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +if [[ "$EUID" -ne "0" ]]; then + echo "ERROR: Must be run as root." >&2 + exit 1 +fi + +for gdb in crash-python-gdb gdb; do + if $gdb -v >/dev/null 2>/dev/null; then + GDB=$gdb + break + fi +done + +if [[ -z "$GDB" ]]; then + echo "ERROR: gdb is not available." >&2 + exit 1 +fi + +GDBINIT=$(mktemp) +trap "rm '$GDBINIT'" EXIT + +VMLINUX="/usr/lib/debug/boot/vmlinux-$(uname -r)" +STEXT_KALLSYMS=$(awk '$3 == "_stext" { print $1 }' /proc/kallsyms) +STEXT_VMLINUX=$(nm "$VMLINUX" | awk '$3 == "_stext" { print $1 }') + +# +# Due to the KASLR done by the kernel, the symbol addresses contained in +# the "vmlinux" file are not exactly what's used by the running system. +# To translate the addresses in the "vmlinux" file, to the addresses +# being used on the live system, we have to offset all of the "vmlinux" +# addresses by the KASLR offset. Here we determine the KASLR offset by +# determining the difference between the address of the "_stext" symbol +# as reported by "/proc/kallsyms" and the "vminlinux" file; this offset +# is then later fed into GDB when loading the "vmlinux" symbols. +# +OFFSET=$(python -c "print(int('$STEXT_KALLSYMS', 16) - int('$STEXT_VMLINUX', 16))") + +DIR="$(dirname $0)" +if [[ -e "$DIR/setup.py" ]]; then + pushd $DIR >/dev/null + rm -rf build/lib/crash + python3 setup.py build >/dev/null + echo "python sys.path.insert(0, '$DIR/build/lib')" >>"$GDBINIT" + popd >/dev/null +fi + +cat <>"$GDBINIT" +set python print-stack full +set height 0 +set print pretty on + +add-symbol-file $VMLINUX -o $OFFSET +target core /proc/kcore + +# +# Since we're readying from /proc/kcore and the contents of that can +# change, we disable as much of GDB's caching as we can. +# +set stack-cache off +set code-cache off +set dcache size 1 +set dcache line-size 2 +set non-stop on + +python +import crash.session +x = crash.session.Session(None, None, None) +end +EOF + +$GDB -nh -q -x "$GDBINIT" diff --git a/crash/kernel.py b/crash/kernel.py index 04809e1485d..9c50d6417a1 100644 --- a/crash/kernel.py +++ b/crash/kernel.py @@ -301,9 +301,10 @@ def __init__(self, roots: PathSpecifier=None, self.arch = archclass() self.target = gdb.current_target() - self.vmcore = self.target.kdump + if self.target.shortname == 'kdumpfile': + self.vmcore = self.target.kdump + self.target.fetch_registers = self.fetch_registers - self.target.fetch_registers = self.fetch_registers self.crashing_thread = None # When working without a symbol table, we still need to be able @@ -619,7 +620,7 @@ def setup_tasks(self) -> None: cpu = None regs = None active = int(task.address) in rqscurrs - if active: + if active and self.target.shortname == 'kdumpfile': cpu = rqscurrs[int(task.address)] regs = self.vmcore.attr.cpu[cpu].reg @@ -636,6 +637,10 @@ def setup_tasks(self) -> None: self.crashing_thread = thread self.arch.setup_thread_info(thread) + + if not active: + self.arch.fetch_register_scheduled_inactive(thread, -1) + ltask.attach_thread(thread) ltask.set_get_stack_pointer(self.arch.get_stack_pointer) From 14529d13cd9117f34e691b0541274dce399010e1 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 20 May 2019 12:09:49 -0700 Subject: [PATCH 75/75] WIP: Add new "kcore" target using "drgn" backend This adds a new "kcore" GDB target which uses "drgn" as the backend for reading from "/proc/kcore"; very similar to the existing "kdump" target which uses "kdumpfile" for reading from a crash dump. The benefit of using this new "kcore" target as opposed to GDB's existing "core" target is twofold: 1. The caching done by the "core" target is not what we want when inspecting a live, running kernel. By moving to a new python based target, we avoid all of this; each read of memory from GDB will call into our target, so we have much more control. 2. The "core" target is unable to properly fetch registers when it's used with "/proc/kcore". By using a python target, we can override the "fetch_registers" function, and do the right thing for fetching registers of kernel threads. Additionally, the code to do this is the same for "/proc/kcore" and a kernel crash dump, so the same code can be used for both the new "kcore" and existing "kdump" targets. Unfortunately though, this new "kcore" target does not solve the issue of thread information being read and cached during startup, meaning we still do not have "live" thread information when using this new target. See also: https://github.com/osandov/drgn Co-authored-by: Prakash Surya --- crash-kcore.sh | 32 ++++------------- kcore/__init__.py | 0 kcore/target.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 kcore/__init__.py create mode 100644 kcore/target.py diff --git a/crash-kcore.sh b/crash-kcore.sh index 5d7ebe9ecc3..bcc271d06c0 100755 --- a/crash-kcore.sh +++ b/crash-kcore.sh @@ -21,20 +21,6 @@ GDBINIT=$(mktemp) trap "rm '$GDBINIT'" EXIT VMLINUX="/usr/lib/debug/boot/vmlinux-$(uname -r)" -STEXT_KALLSYMS=$(awk '$3 == "_stext" { print $1 }' /proc/kallsyms) -STEXT_VMLINUX=$(nm "$VMLINUX" | awk '$3 == "_stext" { print $1 }') - -# -# Due to the KASLR done by the kernel, the symbol addresses contained in -# the "vmlinux" file are not exactly what's used by the running system. -# To translate the addresses in the "vmlinux" file, to the addresses -# being used on the live system, we have to offset all of the "vmlinux" -# addresses by the KASLR offset. Here we determine the KASLR offset by -# determining the difference between the address of the "_stext" symbol -# as reported by "/proc/kallsyms" and the "vminlinux" file; this offset -# is then later fed into GDB when loading the "vmlinux" symbols. -# -OFFSET=$(python -c "print(int('$STEXT_KALLSYMS', 16) - int('$STEXT_VMLINUX', 16))") DIR="$(dirname $0)" if [[ -e "$DIR/setup.py" ]]; then @@ -50,22 +36,18 @@ set python print-stack full set height 0 set print pretty on -add-symbol-file $VMLINUX -o $OFFSET -target core /proc/kcore +python +from kcore.target import Target +target = Target("$VMLINUX", debug=False) +end -# -# Since we're readying from /proc/kcore and the contents of that can -# change, we disable as much of GDB's caching as we can. -# -set stack-cache off -set code-cache off -set dcache size 1 -set dcache line-size 2 -set non-stop on +target drgn /proc/kcore python import crash.session x = crash.session.Session(None, None, None) +target.unregister() +del target end EOF diff --git a/kcore/__init__.py b/kcore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/kcore/target.py b/kcore/target.py new file mode 100644 index 00000000000..8587515305c --- /dev/null +++ b/kcore/target.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: + +import gdb +import drgn +import sys + + +class Target(gdb.Target): + def __init__(self, vmlinux, debug=False): + super(Target, self).__init__() + self.debug = debug + self.shortname = "drgn" + self.longname = "Use Linux's /proc/kcore as a target through drgn" + self.vmlinux = vmlinux + self.register() + + def open(self, filename, from_tty): + if filename != "/proc/kcore": + raise gdb.GdbError("incorrect file target - should be /proc/kcore") + + self.drgn = drgn.Program() + self.drgn.set_kernel() + + try: + self.drgn.load_default_debug_info() + except drgn.MissingDebugInfoError as e: + print(str(e), file=sys.stderr) + + vmcoreinfo = dict( + [ + line.split("=") + for line in self.drgn["vmcoreinfo_data"].string_().decode().splitlines() + ] + ) + offset = int(vmcoreinfo["KERNELOFFSET"], base=16) + gdb.execute("add-symbol-file {} -o {:#x}".format(self.vmlinux, offset)) + gdb.execute("file {}".format(self.vmlinux)) + + def close(self): + try: + self.unregister() + except: + pass + del self.drgn + + @classmethod + def report_error(cls, addr, length, error): + print( + "Error while reading {:d} bytes from {:#x}: {}".format( + length, addr, str(error) + ), + file=sys.stderr, + ) + + def xfer_partial(self, obj, annex, readbuf, writebuf, offset, ln): + ret = -1 + if obj == self.TARGET_OBJECT_MEMORY: + try: + r = self.drgn.read(offset, ln) + readbuf[:] = r + ret = len(r) + except drgn.FaultError as e: + if self.debug: + self.report_error(offset, ln, e) + raise gdb.TargetXferUnavailable(str(e)) + else: + raise IOError("Unknown obj type") + return ret + + def thread_alive(self, ptid): + return True + + def pid_to_str(self, ptid): + return "pid {:d}".format(ptid[1]) + + def fetch_registers(self, register): + return False + + def prepare_to_store(self, thread): + pass + + # We don't need to store anything; The regcache is already written. + def store_registers(self, thread): + pass + + def has_execution(self, ptid): + return False