lrdp-v2: Add symbol parsing for loadable modules

Current Linux Ram Dump Paser scripts are only able to load kernel
symbols by default. No loadable modules' symbols will be parsed
at startup time, which may result in 'unknown symbols' in some
cases, especially when checking SLAB info on dual-wifi platforms.

With this change, loadable modules' symbols can be parsed on
script startup by the following configurations:

1) Put all symbols under a certain dir, such as:

<dump_location>/symbols/wlan.ko
<dump_location>/symbols/wlan_sdio.ko

2) Add --sym_path when invoking the scripts, such as:

python %RAM_DUMP_PARSER_PATH%\ramparse.py --sym_path
<dump_location>/symbols

With the above actions, all loadable modules' symbols will be
parsed and stored for subsequent information dumping. Symbol
loading commands will also be added into the Trace32 startup
script.

Change-Id: Idda9c9a08cfd24c19bf4021e80fee5187cd031b9

linux-ramdump-parser-v2/gdbmi.py

-# Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+# Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 and
@@ -12,6 +12,8 @@
 import sys
 import re
 import subprocess
+import module_table
+from print_out import print_out_str
 
 GDB_SENTINEL = '(gdb) '
 GDB_DATA_LINE = '~'
@@ -62,6 +64,7 @@ class GdbMI(object):
         self.kaslr_offset = kaslr_offset
         self._cache = {}
         self._gdbmi = None
+        self.mod_table = None
 
     def open(self):
         """Open the connection to the ``gdbmi`` backend. Not needed if using
@@ -95,6 +98,14 @@ class GdbMI(object):
             if line == GDB_SENTINEL:
                 break
 
+    def setup_module_table(self, module_table):
+        self.mod_table = module_table
+        d = {"\\":"\\\\"}
+        for mod_tbl_ent in self.mod_table.module_table:
+            load_mod_sym_cmd = 'add-symbol-file %s 0x%x' % (mod_tbl_ent.get_sym_path(), mod_tbl_ent.module_offset)
+            gdb_cmd = ''.join(d.get(c, c) for c in load_mod_sym_cmd)
+            self._run(gdb_cmd)
+
     def _run(self, cmd, skip_cache=False, save_in_cache=True):
         """Runs a gdb command and returns a GdbMIResult of the result. Results
         are cached (unless skip_cache=True) for quick future lookups.

linux-ramdump-parser-v2/module_table.py

+# Copyright (c) 2017, The Linux Foundation. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 and
+# only version 2 as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+import os
+from print_out import print_out_str
+
+class module_table_entry():
+
+    def __init__(self):
+        self.name = ''
+        self.module_offset = 0
+        self.sym_lookup_table = []
+        self.sym_path = ''
+
+    def num_symbols(self):
+        return len(self.sym_lookup_table)
+
+    def set_sym_path(self, sym_path):
+        if os.path.isfile(sym_path):
+            self.sym_path = sym_path
+            return True
+        else:
+            print_out_str('sym_path: ' + sym_path + ' not valid or file doesn\'t exist')
+            return False
+
+    def get_sym_path(self):
+        return self.sym_path
+
+class module_table_class():
+
+    def __init__(self):
+        self.module_table = []
+        self.sym_path = ''
+
+    def add_entry(self, new_entry):
+        self.module_table.append(new_entry)
+
+    def num_modules(self):
+        return len(self.module_table)
+
+    def setup_sym_path(self, sym_path):
+        if sym_path is None:
+            print_out_str('sym_path: not specified!')
+            self.sym_path = ''
+            return False
+        elif not os.path.exists(sym_path):
+            print_out_str('sym_path: ' + sym_path + ' not valid or directory doesn\'t exist')
+            self.sym_path = ''
+            return False
+        else:
+            self.sym_path = sym_path
+            return True
+
+    def sym_path_exists(self):
+        return self.sym_path != ''
+
+

linux-ramdump-parser-v2/ramdump.py

@@ -29,6 +29,7 @@ from mmu import Armv7MMU, Armv7LPAEMMU, Armv8MMU
 import parser_util
 import minidump_util
 from importlib import import_module
+import module_table
 
 FP = 11
 SP = 13
@@ -551,6 +552,8 @@ class RamDump():
                 print "Oops, missing required library for minidump. Check README"
                 sys.exit(1)
         self.ram_addr = options.ram_addr
+        self.module_table = module_table.module_table_class()
+        self.module_table.setup_sym_path(options.sym_path)
 
         if options.ram_addr is not None:
             # TODO sanity check to make sure the memory regions don't overlap
@@ -713,6 +716,9 @@ class RamDump():
                 print_out_str('!!! Some features may be disabled!')
 
         self.unwind = self.Unwinder(self)
+        if self.module_table.sym_path_exists():
+            self.setup_module_symbols()
+            self.gdbmi.setup_module_table(self.module_table)
 
     def __del__(self):
         self.gdbmi.close()
@@ -1029,8 +1035,18 @@ class RamDump():
                     'task.config /opt/t32/demo/arm/kernel/linux/linux.t32\n'.encode('ascii', 'ignore'))
                 startup_script.write(
                     'menu.reprogram /opt/t32/demo/arm/kernel/linux/linux.men\n'.encode('ascii', 'ignore'))
+
+        for mod_tbl_ent in self.module_table.module_table:
+            mod_sym_path = mod_tbl_ent.get_sym_path()
+            if mod_sym_path != '':
+                where = os.path.abspath(mod_sym_path)
+                where += ' 0x{0:x}'.format(mod_tbl_ent.module_offset)
+                dloadelf = 'data.load.elf {} /nocode /noclear\n'.format(where)
+                startup_script.write(dloadelf.encode('ascii', 'ignore'))
+
         if not self.minidump:
             startup_script.write('task.dtask\n'.encode('ascii', 'ignore'))
+
         startup_script.write(
             'v.v  %ASCII %STRING linux_banner\n'.encode('ascii', 'ignore'))
         if os.path.exists(out_path + '/regs_panic.cmm'):
@@ -1221,6 +1237,54 @@ class RamDump():
                                          s[2].rstrip()))
         stream.close()
 
+    def retrieve_modules(self):
+        mod_list = self.address_of('modules')
+        next_offset = self.field_offset('struct list_head', 'next')
+        list_offset = self.field_offset('struct module', 'list')
+        name_offset = self.field_offset('struct module', 'name')
+
+        if self.kernel_version > (4, 4, 0):
+            module_core_offset = self.field_offset('struct module', 'core_layout.base')
+        else:
+            module_core_offset = self.field_offset('struct module', 'module_core')
+
+        next_list_ent = self.read_pointer(mod_list + next_offset)
+        while next_list_ent != mod_list:
+            mod_tbl_ent = module_table.module_table_entry()
+            module = next_list_ent - list_offset
+            name_ptr = module + name_offset
+            mod_tbl_ent.name = self.read_cstring(name_ptr)
+            mod_tbl_ent.module_offset = self.read_pointer(module + module_core_offset)
+            self.module_table.add_entry(mod_tbl_ent)
+            next_list_ent = self.read_pointer(next_list_ent + next_offset)
+
+    def parse_symbols_of_one_module(self, mod_tbl_ent, sym_path):
+        if not mod_tbl_ent.set_sym_path( os.path.join(sym_path, mod_tbl_ent.name + '.ko') ):
+            return
+        stream = os.popen(self.nm_path + ' -n ' + mod_tbl_ent.get_sym_path())
+        symbols = stream.readlines()
+
+        for line in symbols:
+            s = line.split(' ')
+            if len(s) == 3:
+                mod_tbl_ent.sym_lookup_table.append( ( int(s[0], 16) + mod_tbl_ent.module_offset, s[2].rstrip() ) )
+        stream.close()
+
+    def parse_module_symbols(self):
+        for mod_tbl_ent in self.module_table.module_table:
+            self.parse_symbols_of_one_module(mod_tbl_ent, self.module_table.sym_path)
+
+    def add_symbols_to_global_lookup_table(self):
+        for mod_tbl_ent in self.module_table.module_table:
+            for sym in mod_tbl_ent.sym_lookup_table:
+                self.lookup_table.append(sym)
+        self.lookup_table.sort()
+
+    def setup_module_symbols(self):
+        self.retrieve_modules()
+        self.parse_module_symbols();
+        self.add_symbols_to_global_lookup_table()
+
     def address_of(self, symbol):
         """Returns the address of a symbol.
 
@@ -1302,11 +1366,6 @@ class RamDump():
         if (addr is None):
             return ('(Invalid address)', 0x0)
 
-        # modules are not supported so just print out an address
-        # instead of a confusing symbol
-        if (addr < self.modules_end):
-            return ('(No symbol for address {0:x})'.format(addr), 0x0)
-
         low = 0
         high = len(self.lookup_table)
         # Python now complains about division producing floats

linux-ramdump-parser-v2/ramparse.py

 #!/usr/bin/env python2
 
-# Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
+# Copyright (c) 2012-2017, The Linux Foundation. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 and
@@ -156,6 +156,7 @@ if __name__ == '__main__':
                       help='Parse minidump')
     parser.add_option('', '--ram-elf', dest='ram_elf_addr',
                       help='pass ap_minidump.elf generated by crashscope')
+    parser.add_option('', '--sym_path', dest='sym_path', help='symbol path to all loadable modules')
 
     for p in parser_util.get_parsers():
         parser.add_option(p.shortopt or '',