crash-python · prakashsurya · Apr 29, 2019 · Apr 29, 2019 · Sep 19, 2018 · Apr 29, 2019
diff --git a/INTERNALS.md b/INTERNALS.md
@@ -0,0 +1,19 @@
+
+# GDB
+
+## Python contexts within GDB
+
+Each time gdb enters the Python interpreter it establishes a context.
+Part of the context includes what architecture gdb believes it is
+debugging ('gdbarch') and that is passed into the context.  If anything
+changes the gdbarch in that Python context, it won't be visible to any
+subsequent Python code until a new session is established.
+
+When gdb starts up on x86_64, it uses a gdbarch of i386 -- with 32-bit words
+and pointers.  Only when we load an executable or target does it switch
+to i386:x86_64.
+
+The effect of this is that any code that relys on type information *must*
+be executed in a separate context from the one that loaded the executable
+and/or taret.  Otherwise, any built-in types that are pointers or `long`
+based will use the 32-bit sizes.
diff --git a/TESTING.md b/TESTING.md
@@ -0,0 +1,61 @@
+# Testing
+
+## Summary
+
+There are unit tests in the tests/ dir that are standalone and useful for
+testing basic functionality.
+
+There are unit tests in the kernel-tests dir that require configuration,
+kernel images, debuginfo, and vmcores to use.
+
+## Configuration
+
+The configuration for each kernel/vmcore to be tested goes in a .ini file
+with the following format.  All fields except kernel and vmcore are
+optional, and defaults will be used.  A kernel missing debuginfo cannot
+be used for testing.  Missing modules will mean module-specific tests
+will be skipped.
+
+```[test]
+kernel=/path/to/kernel
+vmcore=/path/to/vmcore
+vmlinux_debuginfo=/path/to/vmlinux-debuginfo
+modules=/path/to/modules
+module_debuginfo_path=/path/to/module/debuginfo
+root=/root/for/tree/searches```
+
+The optional fields match those defined in crash.kernel.CrashKernel.
+
+Example 1:
+```[test]
+kernel=/var/crash/2019-04-23-11:35/vmlinux-4.12.14-150.14-default.gz
+vmcore=/var/crash/2019-04-23-11:35/vmcore```
+
+In this example, the kernel and debuginfo packages are installed in the
+default locations and will be searched automatically.
+
+Example 2:
+```[test]
+kernel=/var/crash/2019-04-23-11:35/vmlinux-4.12.14-150.14-default.gz
+vmcore=/var/crash/2019-04-23-11:35/vmcore
+root=/var/cache/crash-setup/leap15/4.12.14-150.14-default
+```
+
+In this example, the kernel and debuginfo packages are installed under
+/var/cache/crash-setup/leap15/4.12.14-150.14-default and so we only
+specify a root directory.
+
+## Running
+
+The script `test-all.sh` when run with no options will execute only
+the standalone tests.  The script takes a list of the .ini files
+described above and will execute the kernel tests against those
+configurations immediately after the standalone tests.
+
+Example:
+```sh test-all.sh kernel-test-configs/4.12.14-150.14-default.ini kernel-test-configs/5.1.0-rc7-vanilla.ini```
+or
+```sh test-all.sh kernel-test-configs/*.ini```
+
+Each configuration will execute independently from one another.
+
diff --git a/crash-kcore.sh b/crash-kcore.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+if [[ "$EUID" -ne "0" ]]; then
+	echo "ERROR: Must be run as root." >&2
+	exit 1
+fi
+
+for gdb in crash-python-gdb gdb; do
+	if $gdb -v >/dev/null 2>/dev/null; then
+		GDB=$gdb
+		break
+	fi
+done
+
+if [[ -z "$GDB" ]]; then
+	echo "ERROR: gdb is not available." >&2
+	exit 1
+fi
+
+GDBINIT=$(mktemp)
+trap "rm '$GDBINIT'" EXIT
+
+VMLINUX="/usr/lib/debug/boot/vmlinux-$(uname -r)"
+
+DIR="$(dirname $0)"
+if [[ -e "$DIR/setup.py" ]]; then
+	pushd $DIR >/dev/null
+	rm -rf build/lib/crash
+	python3 setup.py build >/dev/null
+	echo "python sys.path.insert(0, '$DIR/build/lib')" >>"$GDBINIT"
+	popd >/dev/null
+fi
+
+cat <<EOF >>"$GDBINIT"
+set python print-stack full
+set height 0
+set print pretty on
+
+python
+from kcore.target import Target
+target = Target("$VMLINUX", debug=False)
+end
+
+target drgn /proc/kcore
+
+python
+import crash.session
+x = crash.session.Session(None, None, None)
+target.unregister()
+del target
+end
+EOF
+
+$GDB -nh -q -x "$GDBINIT"
diff --git a/crash.sh b/crash.sh
@@ -3,38 +3,98 @@

 usage() {
 cat <<END >&2
-usage: $(basename $0) [-d|--search-dir <debuginfo/module dir>] <vmlinux> <vmcore>
+usage: $(basename $0) [options] <vmlinux> <vmcore>

-Debugging options:
--gdb           Run the embedded gdb underneath a separate gdb instance.
-                This is useful for debugging issues in gdb that are seen
-                while running crash-python.
--valgrind      Run the embedded gdb underneath valgrind.
-                This is useful for debugging memory leaks in gdb patches.
--nofiles       Start up without loading any object files.
-                This is useful for testing delayed lookup error handling.
+Options:
+-r <dir> | --root <dir>
+    Use the specified directory as the root for all file searches.  When
+    using properly configured .build-id symbolic links, this is the
+    best method to use as the debuginfo will be loaded automatically via
+    gdb without searching for filenames.
+
+-m <dir> | --modules <dir>
+    Use the specified directory to search for modules
+
+-d <dir> | --modules-debuginfo <dir>
+    Use the specified directory to search for module debuginfo

+-D <dir> | --vmlinux-debuginfo <dir>
+    Use the specified directory to search for vmlinux debuginfo
+
+-b <dir> | --build-dir <dir>
+    Use the specified directory as the root for all file searches.  This
+    directory should be the root of a built kernel source tree.  This is
+    shorthand for "-r <dir> -m . -d . -D ." and will override preceding
+    options.
+
+Debugging options:
+--debug
+    Enable noisy output for debugging the debugger
+-v | --verbose
+    Enable verbose output for debugging the debugger
+--gdb
+    Run the embedded gdb underneath a separate gdb instance.  This is useful
+    for debugging issues in gdb that are seen while running crash-python.
+--valgrind
+    Run the embedded gdb underneath valgrind.  This is useful
+    for debugging memory leaks in gdb patches.
 END
 exit 1
 }

-TEMP=$(getopt -o 'd:h' --long 'search-dir:,gdb,valgrind,nofiles,help' -n "$(basename $0)" -- "$@")
+TEMP=$(getopt -o 'vr:d:m:D:b:h' --long 'verbose,root:,modules-debuginfo:,modules:,vmlinux-debuginfo:,build-dir:,debug,gdb,valgrind,help' -n "$(basename $0)" -- "$@")

 if [ $? -ne 0 ]; then
-    echo "Terminating." >&2
-    exit 1
+    usage
 fi

 eval set -- "$TEMP"
 unset TEMP

+VERBOSE=False
+DEBUG=False
+
 while true; do
    case "$1" in
-        '-d'|'--search-dir')
-            SEARCHDIRS="$SEARCHDIRS $2"
+        '-r'|'--root')
+            SEARCH_DIRS="$SEARCH_DIRS $2"
+            shift 2
+            continue
+        ;;
+        '-m'|'--modules')
+            MODULES="$MODULES $2"
            shift 2
            continue
        ;;
+        '-d'|'--modules-debuginfo')
+            MODULES_DEBUGINFO="$MODULES_DEBUGINFO $2"
+            shift 2
+            continue
+        ;;
+        '-D'|'--vmlinux-debuginfo')
+            VMLINUX_DEBUGINFO="$VMLINUX_DEBUGINFO $2"
+            shift 2
+            continue
+        ;;
+        '-b'|'--build-dir')
+            SEARCH_DIRS="$2"
+            VMLINUX_DEBUGINFO="."
+            MODULES="."
+            MODULES_DEBUGINFO="."
+            shift 2
+            continue
+            ;;
+        '-v'|'--verbose')
+            VERBOSE="True"
+            shift
+            continue
+        ;;
+        '--debug')
+            DEBUG="True"
+            shift
+            continue
+        ;;
+
        '--gdb')
            DEBUGMODE=gdb
            shift
@@ -45,11 +105,6 @@ while true; do
            shift
            continue
            ;;
-        '--nofiles')
-            NOFILES=yes
-            shift
-            continue
-            ;;
        '-h'|'--help')
            usage ;;
        '--')
@@ -63,7 +118,7 @@ while true; do
    esac
 done

-if [ "$#" -ne 2 -a -z "$NOFILES" ]; then
+if [ "$#" -ne 2 ]; then
    usage
 fi

@@ -111,26 +166,70 @@ else
 fi

 VMCORE=$2
+for path in $SEARCH_DIRS; do
+    if test -n "$DFD"; then
+        DFD="$DFD:$path"
+    else
+        DFD="$path"
+    fi
+done
 cat << EOF >> $GDBINIT
+set debug-file-directory $DFD:/usr/lib/debug
 set build-id-verbose 0
 set python print-stack full
 set prompt py-crash> 
 set height 0
 set print pretty on

+file "$KERNEL"
+
+python
+from kdump.target import Target
+target = Target(debug=False)
+end
+
+target kdumpfile $VMCORE
+
 python
 import sys
 import traceback
 try:
    import crash.session
+    from crash.kernel import CrashKernel
 except RuntimeError as e:
    print("crash-python: {}, exiting".format(str(e)), file=sys.stderr)
    traceback.print_exc()
    sys.exit(1)
-path = "$SEARCHDIRS".split(' ')
+
+roots = None
+module_path = None
+module_debuginfo_path = None
+vmlinux_debuginfo = None
+verbose=$VERBOSE
+debug=$DEBUG
+
+s = "$SEARCH_DIRS"
+if len(s) > 0:
+    roots = s.split(" ")
+
+s = "$VMLINUX_DEBUGINFO"
+if len(s) > 0:
+    vmlinux_debuginfo = s.split(" ")
+
+s = "$MODULES"
+if len(s) > 0:
+    module_path = s.split(" ")
+
+s = "$MODULES_DEBUGINFO"
+if len(s) > 0:
+    module_debuginfo_path = s.split(" ")
+
 try:
-   x = crash.session.Session("$KERNEL", "$VMCORE", "$ZKERNEL", path)
-   print("The 'pyhelp' command will list the command extensions.")
+    kernel = CrashKernel(roots, vmlinux_debuginfo, module_path,
+                         module_debuginfo_path, verbose, debug)
+
+    x = crash.session.Session(kernel, verbose=verbose, debug=debug)
+    print("The 'pyhelp' command will list the command extensions.")
 except gdb.error as e:
    print("crash-python: {}, exiting".format(str(e)), file=sys.stderr)
    traceback.print_exc()
@@ -140,16 +239,19 @@ except RuntimeError as e:
          file=sys.stderr)
    traceback.print_exc()
    sys.exit(1)
+
+target.unregister()
+del target
 EOF

 # This is how we debug gdb problems when running crash
 if [ "$DEBUGMODE" = "gdb" ]; then
-    RUN="run -nh -q -x $GDBINIT"
+    RUN="run -nx -q -x $GDBINIT"

-    echo $RUN > /tmp/gdbinit
-    gdb $GDB -nh -q -x /tmp/gdbinit
+    echo $RUN > $TMPDIR/gdbinit-debug
+    gdb $GDB -nx -q -x $TMPDIR/gdbinit-debug
 elif [ "$DEBUGMODE" = "valgrind" ]; then
    valgrind --keep-stacktraces=alloc-and-free $GDB -nh -q -x $GDBINIT
 else
-    $GDB -nh -q -x $GDBINIT
+    $GDB -nx -q -x $GDBINIT
 fi