From 5681eb12f6bd59b8035af13f0a4b45f44260c130 Mon Sep 17 00:00:00 2001
From: Steven Cahail <scahail@codeaurora.org>
Date: Wed, 21 Oct 2015 16:33:55 -0600
Subject: [PATCH] ipc_logging: Fix coding style
The coding style of the IPC Logging extraction script is not compliant
with the Python coding style guide (PEP 0008). In order to be merged
into the Linux Ram Dump Parser, the code must be made compliant with PEP
0008.
Bring the code into compliance with PEP 0008.
Change-Id: Idf587a55670a5b30c7668c78b1625100d6128c9c
---
.../parsers/ipc_logging.py | 3256 +++++++++--------
1 file changed, 1645 insertions(+), 1611 deletions(-)
diff --git a/linux-ramdump-parser-v2/parsers/ipc_logging.py b/linux-ramdump-parser-v2/parsers/ipc_logging.py
index b360c16..e67b14e 100644
--- a/linux-ramdump-parser-v2/parsers/ipc_logging.py
+++ b/linux-ramdump-parser-v2/parsers/ipc_logging.py
@@ -49,59 +49,62 @@ NMAGIC = ctypes.c_uint32(~MAGIC).value
CTX_MAGIC = 0x25874452
CTX_NMAGIC = ctypes.c_uint32(~CTX_MAGIC).value
+
def hex_to_str(chars, width, nLineLimit=None):
- """
- Converts a binary string to a hex dump.
-
- :param chars: The string to convert.
- :param width: The width of the hex output.
- :param nLineLimit: The maximum length of the hex output, in lines.
-
- :return: The hex dump as a string.
- """
- output = []
- offset = 0
- nLines = 0
-
- while chars:
- line = chars[:width]
- chars = chars[width:]
- strBlankHex = ' ' * (width - len(line))
- strHex = ' '.join("%02x" % ord(c) for c in line) + strBlankHex
- strOut = "%08x %s %s" % (offset, strHex, quotechars(line))
- offset += width
- output.append(strOut)
- nLines += 1
- if nLineLimit and nLines > nLineLimit:
- output.append("...")
- break
-
- return '\n'.join(output) + '\n'
+ """
+ Converts a binary string to a hex dump.
+
+ :param chars: The string to convert.
+ :param width: The width of the hex output.
+ :param nLineLimit: The maximum length of the hex output, in lines.
+
+ :return: The hex dump as a string.
+ """
+ output = []
+ offset = 0
+ nLines = 0
+
+ while chars:
+ line = chars[:width]
+ chars = chars[width:]
+ strBlankHex = ' ' * (width - len(line))
+ strHex = ' '.join("%02x" % ord(c) for c in line) + strBlankHex
+ strOut = "%08x %s %s" % (offset, strHex, quotechars(line))
+ offset += width
+ output.append(strOut)
+ nLines += 1
+ if nLineLimit and nLines > nLineLimit:
+ output.append("...")
+ break
+
+ return '\n'.join(output) + '\n'
+
def quotechars(chars):
- return ''.join(['.', c][c.isalnum()] for c in chars)
+ return ''.join(['.', c][c.isalnum()] for c in chars)
+
def str_to_hex(strHex):
- """
- Converts a space-separated string of hex bytes into a binary string.
+ """
+ Converts a space-separated string of hex bytes into a binary string.
- :param strHex: The space-separated string of hex bytes.
+ :param strHex: The space-separated string of hex bytes.
- Example:
- str_to_hex("01 02 0A 0B") ==> "\x01\x02\x0a\x0b"
+ Example:
+ str_to_hex("01 02 0A 0B") ==> "\x01\x02\x0a\x0b"
- :return: The binary string.
- """
- return ''.join([chr(int(x.strip(), 16)) for x in strHex.split()])
+ :return: The binary string.
+ """
+ return ''.join([chr(int(x.strip(), 16)) for x in strHex.split()])
# Element types
-#enum {
-# TSV_TYPE_INVALID,
-# TSV_TYPE_TIMESTAMP,
-# TSV_TYPE_POINTER,
-# TSV_TYPE_INT32,
-# TSV_TYPE_BYTE_ARRAY,
-#};
+# enum {
+# TSV_TYPE_INVALID,
+# TSV_TYPE_TIMESTAMP,
+# TSV_TYPE_POINTER,
+# TSV_TYPE_INT32,
+# TSV_TYPE_BYTE_ARRAY,
+# };
TSV_TYPE_INVALID = 0
TSV_TYPE_TIMESTAMP = 1
TSV_TYPE_POINTER = 2
@@ -109,12 +112,12 @@ TSV_TYPE_INT32 = 3
TSV_TYPE_BYTE_ARRAY = 4
# Message types
-#enum {
-# TSV_TYPE_MSG_START = 1,
-# TSV_TYPE_SKB = TSV_TYPE_MSG_START,
-# TSV_TYPE_STRING,
-# TSV_TYPE_MSG_END = TSV_TYPE_STRING,
-#};
+# enum {
+# TSV_TYPE_MSG_START = 1,
+# TSV_TYPE_SKB = TSV_TYPE_MSG_START,
+# TSV_TYPE_STRING,
+# TSV_TYPE_MSG_END = TSV_TYPE_STRING,
+# };
TSV_TYPE_MSG_START = 1
TSV_TYPE_SKB = TSV_TYPE_MSG_START
TSV_TYPE_STRING = 2
@@ -122,10 +125,10 @@ TSV_TYPE_MSG_END = 2
# TSV values
-#struct tsv_header {
-# unsigned char type;
-# unsigned char size; /* size of data field */
-#};
+# struct tsv_header {
+# unsigned char type;
+# unsigned char size; /* size of data field */
+# };
V0_PAGE_HDR_SIZE = 4*4 + 2*2 + 2*4
V1_PAGE_HDR_SIZE = 0x40
V1_PAGE_HDR_SIZE_64 = 0x48
@@ -138,1581 +141,1614 @@ CTX_READ_SIZE = 64
PAGE_HDR_SIZES = [V0_PAGE_HDR_SIZE, V1_PAGE_HDR_SIZE]
PAGE_HDR_SIZES_64 = [V0_PAGE_HDR_SIZE, V1_PAGE_HDR_SIZE_64]
+
class LogTSV(object):
- """Handles processing message/value headers"""
+ """Handles processing message/value headers"""
- def __init__(self):
- pass
+ def __init__(self):
+ pass
- def unpack(self, data):
- """
- Unpacks a TSV message. This is a wrapper around the
- :func:`~struct.unpack()` function from the :mod:`struct` module.
+ def unpack(self, data):
+ """
+ Unpacks a TSV message. This is a wrapper around the
+ :func:`~struct.unpack()` function from the :mod:`struct` module.
- :param data: The binary data to be unpacked.
+ :param data: The binary data to be unpacked.
- :return: The TSV message with the header stripped off.
- """
- assert len(data) >= 2, "Invalid message size %d" % (len(data))
- self.msg_type, self.msg_size = struct.unpack('BB', data[0:2])
- return data[2:]
+ :return: The TSV message with the header stripped off.
+ """
+ assert len(data) >= 2, "Invalid message size %d" % (len(data))
+ self.msg_type, self.msg_size = struct.unpack('BB', data[0:2])
+ return data[2:]
- def pack(self, msg_type, msg_size):
- """
- Packs a new TSV message structure. This is a wrapper around the
- :func:`~struct.pack()` function from the :mod:`struct` module.
+ def pack(self, msg_type, msg_size):
+ """
+ Packs a new TSV message structure. This is a wrapper around the
+ :func:`~struct.pack()` function from the :mod:`struct` module.
- :param msg_type: The TSV msg_type to be packed into the new \
- TSV message structure.
- :param msg_size: The TSV msg_size to be packed into the new \
- TSV message structure.
+ :param msg_type: The TSV msg_type to be packed into the new
+ TSV message structure.
+ :param msg_size: The TSV msg_size to be packed into the new
+ TSV message structure.
+
+ :return: The TSV message structure packed into a binary string.
+ """
+ return struct.pack('BB', msg_type, msg_size)
- :return: The TSV message structure packed into a binary string.
- """
- return struct.pack('BB', msg_type, msg_size)
class TsvTimestamp(object):
- """Handles processing TSV Timestamps"""
-
- def unpack(self, data):
- """
- Unpacks a TSV timestamp. This is a wrapper around the
- :func:`~struct.unpack()` function from the :mod:`struct` module.
-
- :return: The unpacked timestamp.
-
- **Side Effects**: The unpacked timestamp is saved as an instance \
- variable, ``self.fTimestamp``, which can be accessed as a \
- string through the :meth:`__str__()` method of this class. \
- This method is used by :class:`LogMessageString` to get \
- the timestamp in order to dump it to the logs.
- """
- tsv = LogTSV()
- data = tsv.unpack(data)
- assert tsv.msg_type == TSV_TYPE_TIMESTAMP, "timestamp"
- assert tsv.msg_size == 8, "msg size"
-
- # read 64-bit timestamp
- timestamp_ns, = struct.unpack('<Q', data[0:8])
- data = data[8:]
- self.fTimestamp = timestamp_ns * 1.0E-9
- return data
-
- def pack(self, fTimestamp):
- """
- Packs a timestamp into a new TSV structure. This is a wrapper around the
- :func:`~struct.pack()` function from the :mod:`struct` module.
-
- :param fTimestamp: The TSV timestamp to be packed into a new TSV \
- message structure.
-
- :return: A TSV message structure with *fTimestamp* packed \
- into the binary string.
- """
- self.fTimestamp = fTimestamp
- tsv = LogTSV()
- data = tsv.pack(TSV_TYPE_TIMESTAMP, 8)
- data += struct.pack('<Q', fTimestamp * 1.0E9)
- return data
-
- def __str__(self):
- return '%16.9f' % (self.fTimestamp)
+ """Handles processing TSV Timestamps"""
+
+ def unpack(self, data):
+ """
+ Unpacks a TSV timestamp. This is a wrapper around the
+ :func:`~struct.unpack()` function from the :mod:`struct` module.
+
+ :return: The unpacked timestamp.
+
+ **Side Effects**: The unpacked timestamp is saved as an instance
+ variable, ``self.fTimestamp``, which can be accessed as a
+ string through the :meth:`__str__()` method of this class.
+ This method is used by :class:`LogMessageString` to get
+ the timestamp in order to dump it to the logs.
+ """
+ tsv = LogTSV()
+ data = tsv.unpack(data)
+ assert tsv.msg_type == TSV_TYPE_TIMESTAMP, "timestamp"
+ assert tsv.msg_size == 8, "msg size"
+
+ # read 64-bit timestamp
+ timestamp_ns, = struct.unpack('<Q', data[0:8])
+ data = data[8:]
+ self.fTimestamp = timestamp_ns * 1.0E-9
+ return data
+
+ def pack(self, fTimestamp):
+ """
+ Packs a timestamp into a new TSV structure. This is a wrapper around
+ the :func:`~struct.pack()` function from the :mod:`struct` module.
+
+ :param fTimestamp: The TSV timestamp to be packed into a new TSV
+ message structure.
+
+ :return: A TSV message structure with *fTimestamp* packed
+ into the binary string.
+ """
+ self.fTimestamp = fTimestamp
+ tsv = LogTSV()
+ data = tsv.pack(TSV_TYPE_TIMESTAMP, 8)
+ data += struct.pack('<Q', fTimestamp * 1.0E9)
+ return data
+
+ def __str__(self):
+ return '%16.9f' % (self.fTimestamp)
+
class TsvByteArray(object):
- """Handles processing TSV byte arrays."""
-
- def unpack(self, data):
- """
- Unpacks a TSV byte array. This is a wrapper around the
- :func:`~struct.unpack()` function from the :mod:`struct` module.
-
- :param data: The binary data to be unpacked.
-
- :return: The header data of the TSV message.
-
- **Side Effects**: The byte array data is saved as an instance \
- variable, ``self.strByteArray``, which can be accessed as a \
- string through the :meth:`__str__()` method of this class. \
- This method is used by :class:`LogMessageString` to get \
- the byte array data as a string in order to dump it to the logs.
- """
- tsv = LogTSV()
- data = tsv.unpack(data)
- assert tsv.msg_type == TSV_TYPE_BYTE_ARRAY, \
- "%d != %d" % (tsv.msg_type, TSV_TYPE_BYTE_ARRAY)
-
- # read byte array
- self.strByteArray = data[:tsv.msg_size]
- data = data[tsv.msg_size:]
- return data
-
- def pack(self, data):
- """
- Packs a new TSV message structure, using the TSV byte array in *data*.
- This is a wrapper around the :func:`~struct.pack()` function from the
- :mod:`struct` module.
-
- :param data: The data to be packed into the new TSV message structure.
-
- :return: The new structure as binary data, with *data* appended to the \
- end.
- """
- tsv = LogTSV()
- lst = tsv.pack(TSV_TYPE_BYTE_ARRAY, len(data))
- lst += data
- return lst
-
- def __str__(self):
- return self.strByteArray
+ """Handles processing TSV byte arrays."""
+
+ def unpack(self, data):
+ """
+ Unpacks a TSV byte array. This is a wrapper around the
+ :func:`~struct.unpack()` function from the :mod:`struct` module.
+
+ :param data: The binary data to be unpacked.
+
+ :return: The header data of the TSV message.
+
+ **Side Effects**: The byte array data is saved as an instance
+ variable, ``self.strByteArray``, which can be accessed as a
+ string through the :meth:`__str__()` method of this class.
+ This method is used by :class:`LogMessageString` to get
+ the byte array data as a string in order to dump it to the
+ logs.
+ """
+ tsv = LogTSV()
+ data = tsv.unpack(data)
+ assert tsv.msg_type == TSV_TYPE_BYTE_ARRAY, \
+ "%d != %d" % (tsv.msg_type, TSV_TYPE_BYTE_ARRAY)
+
+ # read byte array
+ self.strByteArray = data[:tsv.msg_size]
+ data = data[tsv.msg_size:]
+ return data
+
+ def pack(self, data):
+ """
+ Packs a new TSV message structure, using the TSV byte array in *data*.
+ This is a wrapper around the :func:`~struct.pack()` function from the
+ :mod:`struct` module.
+
+ :param data: The data to be packed into the new TSV message structure.
+
+ :return: The new structure as binary data, with *data* appended to
+ the end.
+ """
+ tsv = LogTSV()
+ lst = tsv.pack(TSV_TYPE_BYTE_ARRAY, len(data))
+ lst += data
+ return lst
+
+ def __str__(self):
+ return self.strByteArray
+
class LogMessageString(object):
- """Handles processing TSV message strings."""
-
- def unpack(self, data, tsv):
- """
- Unpacks a new TSV message string. To do this, it is first
- necessary to unpack the timestamp using :meth:`TsvTimestamp.unpack()`,
- and proceed to use unpack the byte array from the result using
- :meth:`TsvByteArray.unpack()`. The result of this call is what is
- returned.
-
- :param data: The data containing the TSV string message to be unpacked
- :param tsv: The :class:`LogTSV` object that was used to unpack *data*
-
- :return: The TSV message string data
-
- **Side Effects**: The timestamp and the message extracted from *data* \
- are stored as instance variables, and can be printed using the \
- :meth:`__str__` method of this class. This :meth:`__str__()` \
- method is used to dump the logs in their \
- '[timestamp] <message>' format.
- """
- assert tsv.msg_type == TSV_TYPE_STRING, "invalid message type"
-
- # TSV_TYPE_STRING - message type
- # TSV_TYPE_TIMESTAMP
- # TSV_TYPE_BYTE_ARRAY
- self.timestamp = TsvTimestamp()
- data = self.timestamp.unpack(data)
-
- self.strMessage = TsvByteArray()
- data = self.strMessage.unpack(data)
- return data
-
- def pack(self, fTimestamp, strMessage):
- """
- Packs a new TSV LogMessageString structure.
-
- :param fTimestamp: The timestamp to be packed into the structure
- :param strMessage: The string message to be packed into the structure
-
- :return: The packed TSV LogMessageString structure
- """
- self.timestamp = TsvTimestamp()
- data =self.timestamp.pack(fTimestamp)
- data += TsvByteArray().pack(strMessage)
-
- tsv = LogTSV()
- data = tsv.pack(TSV_TYPE_STRING, len(data)) + data
- self.strMessage = strMessage
- return data
-
- def __str__(self):
- return '[%s] %s' % (self.timestamp, self.strMessage)
+ """Handles processing TSV message strings."""
+
+ def unpack(self, data, tsv):
+ """
+ Unpacks a new TSV message string. To do this, it is first
+ necessary to unpack the timestamp using :meth:`TsvTimestamp.unpack()`,
+ and proceed to use unpack the byte array from the result using
+ :meth:`TsvByteArray.unpack()`. The result of this call is what is
+ returned.
+
+ :param data: The data containing the TSV string message to be unpacked
+ :param tsv: The :class:`LogTSV` object that was used to unpack *data*
+
+ :return: The TSV message string data
+
+ **Side Effects**: The timestamp and the message extracted from *data*
+ are stored as instance variables, and can be printed using
+ the :meth:`__str__` method of this class. This
+ :meth:`__str__()` method is used to dump the logs in their
+ '[timestamp] <message>' format.
+ """
+ assert tsv.msg_type == TSV_TYPE_STRING, "invalid message type"
+
+ # TSV_TYPE_STRING - message type
+ # TSV_TYPE_TIMESTAMP
+ # TSV_TYPE_BYTE_ARRAY
+ self.timestamp = TsvTimestamp()
+ data = self.timestamp.unpack(data)
+
+ self.strMessage = TsvByteArray()
+ data = self.strMessage.unpack(data)
+ return data
+
+ def pack(self, fTimestamp, strMessage):
+ """
+ Packs a new TSV LogMessageString structure.
+
+ :param fTimestamp: The timestamp to be packed into the structure
+ :param strMessage: The string message to be packed into the structure
+
+ :return: The packed TSV LogMessageString structure
+ """
+ self.timestamp = TsvTimestamp()
+ data = self.timestamp.pack(fTimestamp)
+ data += TsvByteArray().pack(strMessage)
+
+ tsv = LogTSV()
+ data = tsv.pack(TSV_TYPE_STRING, len(data)) + data
+ self.strMessage = strMessage
+ return data
+
+ def __str__(self):
+ return '[%s] %s' % (self.timestamp, self.strMessage)
+
class LogContext(object):
- """
- A single context in a version 1 or greater IPC Logging log.
-
- .. code-block:: c
-
- struct ipc_log_context {
- uint32_t magic;
- uint32_t nmagic;
- uint32_t version;
- uint16_t user_version;
- uint16_t header_size;
- uint64_t log_id;
-
- /* Local data structures below ... */
- };
-
- The purpose of the log context is to contain the above information about
- the log. One log consists of one to several pages. Thus, pages and contexts
- are associated by log ID, the one piece of information that is common
- between them in their C structs.
-
- """
-
- headerBinaryFormat = '<IIIHHQ20s'
- headerBinaryFormatSize = struct.calcsize(headerBinaryFormat)
-
- def __init__(self):
- self.data = None
-
- def unpack(self, data):
- """
- Unpacks an ipc_log_context structure, and assigns the contents
- into instance variables of this class. This is a wrapper around
- the :func:`~struct.unpack()` function from the :mod:`struct` module.
-
- :param data: The binary ipc_log_context structure to be unpacked.
-
- :return: :const:`True` if the unpack completes successfully, \
- :const:`False` if not.
-
- **Side Effects**: If the unpack is successful, the contexts are \
- assigned to instance variables. In this way, :meth:`unpack()` \
- acts as a sort of constructor. This is common to all the \
- unpack methods.
- """
- self.data = data
- self.magic, self.nmagic, self.version, self.user_version, \
- self.header_size, self.log_id, self.name = \
- struct.unpack(self.headerBinaryFormat,
- self.data[0:self.headerBinaryFormatSize])
- self.name = self.name.rstrip('\0')
-
- if self.magic == CTX_MAGIC and self.nmagic == CTX_NMAGIC:
- return True
-
- return False
-
- def pack(self, nVersion, nUserVersion, nHeaderSize, nLogId, nName):
- """
- Packs a new ipc_log_context structure using the given elements. This
- function is especially useful in unit testing, as it provides the
- ability to create structures and then unpack them using the
- :meth:`unpack()` method.
-
- :param nVersion: The IPC Logging version
- :param nUserVersion: The user version
- :param nHeaderSize: The header size
- :param nLogId: The log ID
- :param nName: The context name
-
- :return: The packed binary data.
- """
- self.data = struct.pack(self.headerBinaryFormat, CTX_MAGIC, CTX_NMAGIC,
- nVersion, nUserVersion, nHeaderSize, nLogId, nName)
-
- def log_info(self):
- """
- If DEBUG logging is turned on (command line parameter -v), logs the
- instance variables of this LogContext to the output file. See the Python
- :mod:`logging` module documentation and the IPC Logging main function
- for more info on logging.
-
- This method logs the ``version``, ``user_version``, ``log_id``,
- ``header_size``, and ``name`` variables to the logging output file.
- """
- logging.debug("\tversion: %d" % (self.version))
- logging.debug("\tuser version: %d" % (self.user_version))
- logging.debug("\tlog id: %x" % (self.log_id))
- logging.debug("\theader_size: %x" % (self.header_size))
- logging.debug("\tname: %s" % (self.name))
+ """
+ A single context in a version 1 or greater IPC Logging log.
+
+ .. code-block:: c
+
+ struct ipc_log_context {
+ uint32_t magic;
+ uint32_t nmagic;
+ uint32_t version;
+ uint16_t user_version;
+ uint16_t header_size;
+ uint64_t log_id;
+
+ /* Local data structures below ... */
+ };
+
+ The purpose of the log context is to contain the above information about
+ the log. One log consists of one to several pages. Thus, pages and contexts
+ are associated by log ID, the one piece of information that is common
+ between them in their C structs.
+
+ """
+
+ headerBinaryFormat = '<IIIHHQ20s'
+ headerBinaryFormatSize = struct.calcsize(headerBinaryFormat)
+
+ def __init__(self):
+ self.data = None
+
+ def unpack(self, data):
+ """
+ Unpacks an ipc_log_context structure, and assigns the contents
+ into instance variables of this class. This is a wrapper around
+ the :func:`~struct.unpack()` function from the :mod:`struct` module.
+
+ :param data: The binary ipc_log_context structure to be unpacked.
+
+ :return: :const:`True` if the unpack completes successfully,
+ :const:`False` if not.
+
+ **Side Effects**: If the unpack is successful, the contexts are
+ assigned to instance variables. In this way, :meth:`unpack()`
+ acts as a sort of constructor. This is common to all the
+ unpack methods.
+ """
+ self.data = data
+ self.magic, self.nmagic, self.version, self.user_version, \
+ self.header_size, self.log_id, self.name = \
+ struct.unpack(self.headerBinaryFormat,
+ self.data[0:self.headerBinaryFormatSize])
+ self.name = self.name.rstrip('\0')
+
+ if self.magic == CTX_MAGIC and self.nmagic == CTX_NMAGIC:
+ return True
+
+ return False
+
+ def pack(self, nVersion, nUserVersion, nHeaderSize, nLogId, nName):
+ """
+ Packs a new ipc_log_context structure using the given elements. This
+ function is especially useful in unit testing, as it provides the
+ ability to create structures and then unpack them using the
+ :meth:`unpack()` method.
+
+ :param nVersion: The IPC Logging version
+ :param nUserVersion: The user version
+ :param nHeaderSize: The header size
+ :param nLogId: The log ID
+ :param nName: The context name
+
+ :return: The packed binary data.
+ """
+ self.data = struct.pack(self.headerBinaryFormat, CTX_MAGIC, CTX_NMAGIC,
+ nVersion, nUserVersion, nHeaderSize, nLogId,
+ nName)
+
+ def log_info(self):
+ """
+ If DEBUG logging is turned on (command line parameter -v), logs the
+ instance variables of this LogContext to the output file. See the
+ Python :mod:`logging` module documentation and the IPC Logging main
+ function for more info on logging.
+
+ This method logs the ``version``, ``user_version``, ``log_id``,
+ ``header_size``, and ``name`` variables to the logging output file.
+ """
+ logging.debug("\tversion: %d" % (self.version))
+ logging.debug("\tuser version: %d" % (self.user_version))
+ logging.debug("\tlog id: %x" % (self.log_id))
+ logging.debug("\theader_size: %x" % (self.header_size))
+ logging.debug("\tname: %s" % (self.name))
+
class LogPage(object):
- """
- LogPage base class. New versions of IPC Logging will require new descendants
- of this class, as their :meth:`~LogPage_v0.unpack()` methods will change,
- among other things. In IPC Logging version 1, the children of
- :class:`LogPage` include :class:`LogPage_v0` and :class:`LogPage_v1`. There
- is another class, :class:`LogPageVersionUnknown`, which is not in this
- inheritance hierarchy; this is because it does not use the methods or data
- needed by the rest of the :class:`LogPage` classes.
-
- Note that this is not an abstract class. The reason for this is to allow
- new methods to be added here without forcing child classes to define them.
- """
- def __init__(self):
- self.data = None
- self.previousPage = None
- self.previousData = None
- self.nextPage = None
- self.nextData = None
- self.iterated = False
- self.page_header_size = V0_PAGE_HDR_SIZE
- self.unknown_messages_data = None
-
- def pack_continue(self, strPayload):
- """
- Packs a new page structure. The *strPayload* can be empty and
- additional payload can be added using :meth:`pack_continue()`.
-
- Returns: Overflow data (*strPayload* that didn't fit into the page) or
- :const:`None` if all fit.
- """
- self.data += strPayload
-
- if len(self.data) > PAGE_SIZE:
- overflowData = self.data[PAGE_SIZE:]
- self.data = self.data[0:PAGE_SIZE]
- else:
- return None
-
- def __iter__(self):
- """Message iterator"""
- self.iterated = True
- self.previousData = None
-
- logging.debug("Iterating log id %d, page %d" % \
- (self.log_id, self.page_num))
- self.log_info()
-
- if self.read_offset == FULL_PAGE_WRITE_OFFSET \
- and self.write_offset == FULL_PAGE_WRITE_OFFSET:
- # page is either empty or full, but only the page with
- # valid read/write pointers knows if we have data
- self.iter_data = self.data[self.page_header_size:]
- logging.debug("\tfull/empty case")
- elif self.write_offset == FULL_PAGE_WRITE_OFFSET:
- # page is full
- read_iter = self.read_offset + self.page_header_size
- self.iter_data = self.data[read_iter:]
- logging.debug("\tfull case")
- elif self.read_offset == FULL_PAGE_WRITE_OFFSET:
- # page hasn't been read, but has only been partially written
- write_iter = self.write_offset + self.page_header_size
- self.iter_data = self.data[self.page_header_size:write_iter]
- logging.debug("\tpartial write, no read")
- elif self.read_offset >= self.write_offset:
- # almost full buffer
- read_iter = self.read_offset + self.page_header_size
- write_iter = self.write_offset + self.page_header_size
- self.iter_data = self.data[read_iter:]
- self.previousData = self.data[self.page_header_size:write_iter]
-
- if self.page_num == 0 and self.read_offset == self.write_offset:
- self.iter_data = self.data[self.page_header_size:]
- self.previousData = None
- logging.debug("\talmost full - first page offsets equal")
-
- logging.debug("\talmost full")
- else:
- # almost empty buffer
- read_iter = self.read_offset + self.page_header_size
- write_iter = self.write_offset + self.page_header_size
- self.iter_data = self.data[read_iter:write_iter]
- logging.debug("\talmost empty")
-
- # prepend data from previous page
- if self.previousPage and self.previousPage.nextData:
- logging.debug("Pulling previous data len")
- logging.debug(hex_to_str(self.previousPage.nextData, 16, 10))
- self.iter_data = self.previousPage.nextData + self.iter_data
-
- return PageIterator(self)
-
- def next(self):
- # handles full-buffer condition where write pointer
- # is behind the read pointer
- if self.nextPage and self.nextPage.previousData:
- self.iter_data += self.nextPage.previousData
- self.nextPage.previousData = None
-
- if len(self.iter_data) < 2:
- # not enough data to retrieve message header
- logging.debug("Pushing data to continue\n%s" % \
- (hex_to_str(self.iter_data, 16, 10)))
- self.nextData = self.iter_data
- self.iter_data = None
- raise StopIteration
-
- while True:
- try:
- tsv = LogTSV()
- data = tsv.unpack(self.iter_data)
- if tsv.msg_size > len(data):
- # not enough data left to extract entire message
- logging.debug("Pushing data to continue\n%s" % \
- (hex_to_str(self.iter_data, 16, 10)))
- self.nextData = self.iter_data
- self.iter_data = None
- raise StopIteration
-
- # TODO - this needs to be replaced with a dictionary for the
- # message deserialization types for custom deserialization
- # functions
- if tsv.msg_type == 0:
- # no more messages
- raise StopIteration
-
- if tsv.msg_type == TSV_TYPE_STRING:
- self.iter_data = data
- msg = LogMessageString()
- self.iter_data = msg.unpack(self.iter_data, tsv)
- return msg
- else:
- debug_str = "Unknown message type 0x%x\n%s" % \
- (tsv.msg_type, hex_to_str(self.iter_data, 16, 10))
-
- logging.debug(debug_str)
- assert False, debug_str
-
- except StopIteration:
- raise
+ """
+ LogPage base class. New versions of IPC Logging will require new
+ descendants of this class, as their :meth:`~LogPage_v0.unpack()` methods
+ will change, among other things. In IPC Logging version 1, the children of
+ :class:`LogPage` include :class:`LogPage_v0` and :class:`LogPage_v1`. There
+ is another class, :class:`LogPageVersionUnknown`, which is not in this
+ inheritance hierarchy; this is because it does not use the methods or data
+ needed by the rest of the :class:`LogPage` classes.
+
+ Note that this is not an abstract class. The reason for this is to allow
+ new methods to be added here without forcing child classes to define them.
+ """
+ def __init__(self):
+ self.data = None
+ self.previousPage = None
+ self.previousData = None
+ self.nextPage = None
+ self.nextData = None
+ self.iterated = False
+ self.page_header_size = V0_PAGE_HDR_SIZE
+ self.unknown_messages_data = None
+
+ def pack_continue(self, strPayload):
+ """
+ Packs a new page structure. The *strPayload* can be empty and
+ additional payload can be added using :meth:`pack_continue()`.
+
+ Returns: Overflow data (*strPayload* that didn't fit into the page) or
+ :const:`None` if all fit.
+ """
+ self.data += strPayload
+
+ if len(self.data) > PAGE_SIZE:
+ overflowData = self.data[PAGE_SIZE:]
+ self.data = self.data[0:PAGE_SIZE]
+ return overflowData
+ else:
+ return None
+
+ def __iter__(self):
+ """Message iterator"""
+ self.iterated = True
+ self.previousData = None
+
+ logging.debug("Iterating log id %d, page %d" %
+ (self.log_id, self.page_num))
+ self.log_info()
+
+ if self.read_offset == FULL_PAGE_WRITE_OFFSET \
+ and self.write_offset == FULL_PAGE_WRITE_OFFSET:
+ # page is either empty or full, but only the page with
+ # valid read/write pointers knows if we have data
+ self.iter_data = self.data[self.page_header_size:]
+ logging.debug("\tfull/empty case")
+ elif self.write_offset == FULL_PAGE_WRITE_OFFSET:
+ # page is full
+ read_iter = self.read_offset + self.page_header_size
+ self.iter_data = self.data[read_iter:]
+ logging.debug("\tfull case")
+ elif self.read_offset == FULL_PAGE_WRITE_OFFSET:
+ # page hasn't been read, but has only been partially written
+ write_iter = self.write_offset + self.page_header_size
+ self.iter_data = self.data[self.page_header_size:write_iter]
+ logging.debug("\tpartial write, no read")
+ elif self.read_offset >= self.write_offset:
+ # almost full buffer
+ read_iter = self.read_offset + self.page_header_size
+ write_iter = self.write_offset + self.page_header_size
+ self.iter_data = self.data[read_iter:]
+ self.previousData = self.data[self.page_header_size:write_iter]
+
+ if self.page_num == 0 and self.read_offset == self.write_offset:
+ self.iter_data = self.data[self.page_header_size:]
+ self.previousData = None
+ logging.debug("\talmost full - first page offsets equal")
+
+ logging.debug("\talmost full")
+ else:
+ # almost empty buffer
+ read_iter = self.read_offset + self.page_header_size
+ write_iter = self.write_offset + self.page_header_size
+ self.iter_data = self.data[read_iter:write_iter]
+ logging.debug("\talmost empty")
+
+ # prepend data from previous page
+ if self.previousPage and self.previousPage.nextData:
+ logging.debug("Pulling previous data len")
+ logging.debug(hex_to_str(self.previousPage.nextData, 16, 10))
+ self.iter_data = self.previousPage.nextData + self.iter_data
+
+ return PageIterator(self)
+
+ def next(self):
+ # handles full-buffer condition where write pointer
+ # is behind the read pointer
+ if self.nextPage and self.nextPage.previousData:
+ self.iter_data += self.nextPage.previousData
+ self.nextPage.previousData = None
+
+ if len(self.iter_data) < 2:
+ # not enough data to retrieve message header
+ logging.debug("Pushing data to continue\n%s" %
+ (hex_to_str(self.iter_data, 16, 10)))
+ self.nextData = self.iter_data
+ self.iter_data = None
+ raise StopIteration
+
+ while True:
+ try:
+ tsv = LogTSV()
+ data = tsv.unpack(self.iter_data)
+ if tsv.msg_size > len(data):
+ # not enough data left to extract entire message
+ logging.debug("Pushing data to continue\n%s" %
+ (hex_to_str(self.iter_data, 16, 10)))
+ self.nextData = self.iter_data
+ self.iter_data = None
+ raise StopIteration
+
+ # TODO - this needs to be replaced with a dictionary for the
+ # message deserialization types for custom deserialization
+ # functions
+ if tsv.msg_type == 0:
+ # no more messages
+ raise StopIteration
+
+ if tsv.msg_type == TSV_TYPE_STRING:
+ self.iter_data = data
+ msg = LogMessageString()
+ self.iter_data = msg.unpack(self.iter_data, tsv)
+ return msg
+ else:
+ debug_str = "Unknown message type 0x%x\n%s" % \
+ (tsv.msg_type, hex_to_str(self.iter_data, 16, 10))
+
+ logging.debug(debug_str)
+ assert False, debug_str
+
+ except StopIteration:
+ raise
+
class LogPageVersionUnknown(object):
- """
- This class is used before the version of the log being parsed is known. It
- only unpacks the magic numbers and the page number from the data that is
- found by :func:`cmdParse()`, in order to determine the version.
-
- This class can only be used to discover whether the log is a version 0 log
- or a version 1 (or greater) log. See :meth:`isVersionOneOrGreater()` for
- more details.
- """
- def __init__(self):
- self.data = None
- self.version = None
- self.headerMagicFormat = '<II'
- self.headerMagicFormatSize = struct.calcsize(self.headerMagicFormat)
- self.versionDataFormat = '<I'
- self.versionDataFormatSize = struct.calcsize(self.versionDataFormat)
-
- def unpack(self, data):
- """
- Partially unpacks a page structure in order to inspect the magic number.
- This is a wrapper around the :func:`~struct.unpack()` function from the
- :mod:`struct` module.
-
- :param data: The binary data to be unpacked.
-
- :return: :const:`True` if the unpack is successful, :const:`False` if \
- not.
-
- **Side Effects**: The unpacked values are assigned to instance \
- variables of this LogPageVersionUnknown instance.
- """
- self.data=data
-
- self.magic, self.nmagic = \
- struct.unpack(self.headerMagicFormat,
- self.data[0:self.headerMagicFormatSize])
-
- if self.magic == MAGIC and self.nmagic == NMAGIC:
- return True
-
- return False
-
- def isVersionOneOrGreater(self, data):
- """
- This method determines whether the log is version 0, or version 1 or
- greater. The top bit of ``self.version_data`` will be 1 if the log
- version is at least 1, and 0 if not. Note that this function should
- only be called after calling :meth:`unpack()`.
-
- :return: :const:`True` if the version is 1 or greater, \
- :const:`False` otherwise.
-
- **Side Effects**: This method unpacks the version data and stores \
- the result in an instance variable.
- """
- self.data=data
- endSize = self.headerMagicFormatSize + self.versionDataFormatSize
- version_data = \
- struct.unpack(self.versionDataFormat,
- self.data[self.headerMagicFormatSize:endSize])
- self.version_data = version_data[0]
- mask = 0x80000000
- result = (self.version_data & mask) >= 0x80000000
- return result
+ """
+ This class is used before the version of the log being parsed is known. It
+ only unpacks the magic numbers and the page number from the data that is
+ found by :func:`cmdParse()`, in order to determine the version.
+
+ This class can only be used to discover whether the log is a version 0 log
+ or a version 1 (or greater) log. See :meth:`isVersionOneOrGreater()` for
+ more details.
+ """
+ def __init__(self):
+ self.data = None
+ self.version = None
+ self.headerMagicFormat = '<II'
+ self.headerMagicFormatSize = struct.calcsize(self.headerMagicFormat)
+ self.versionDataFormat = '<I'
+ self.versionDataFormatSize = struct.calcsize(self.versionDataFormat)
+
+ def unpack(self, data):
+ """
+ Partially unpacks a page structure in order to inspect the magic
+ number. This is a wrapper around the :func:`~struct.unpack()` function
+ from the :mod:`struct` module.
+
+ :param data: The binary data to be unpacked.
+
+ :return: :const:`True` if the unpack is successful, :const:`False` if
+ not.
+
+ **Side Effects**: The unpacked values are assigned to instance
+ variables of this LogPageVersionUnknown instance.
+ """
+ self.data = data
+
+ self.magic, self.nmagic = \
+ struct.unpack(self.headerMagicFormat,
+ self.data[0:self.headerMagicFormatSize])
+
+ if self.magic == MAGIC and self.nmagic == NMAGIC:
+ return True
+
+ return False
+
+ def isVersionOneOrGreater(self, data):
+ """
+ This method determines whether the log is version 0, or version 1 or
+ greater. The top bit of ``self.version_data`` will be 1 if the log
+ version is at least 1, and 0 if not. Note that this function should
+ only be called after calling :meth:`unpack()`.
+
+ :return: :const:`True` if the version is 1 or greater,
+ :const:`False` otherwise.
+
+ **Side Effects**: This method unpacks the version data and stores
+ the result in an instance variable.
+ """
+ self.data = data
+ endSize = self.headerMagicFormatSize + self.versionDataFormatSize
+ version_data = \
+ struct.unpack(self.versionDataFormat,
+ self.data[self.headerMagicFormatSize:endSize])
+ self.version_data = version_data[0]
+ mask = 0x80000000
+ result = (self.version_data & mask) >= 0x80000000
+ return result
+
class LogPage_v0(LogPage):
- """
- A single page in a version 0 IPC Logging log. This class is a descendant of
- :class:`LogPage`.
-
- .. code-block:: c
-
- struct ipc_log_page_header {
- uint32_t magic;
- uint32_t nmagic; /* inverse of magic number */
- uint32_t log_id; /* owner of log */
- uint32_t page_num;
- uint16_t read_offset;
- uint16_t write_offset;
- struct list_head list;
- };
- """
- def __init__(self):
- # Call the parent constructor
- super(self.__class__, self).__init__()
- self.headerBinaryFormat = '<IIIIHH'
- self.headerBinaryFormatSize = struct.calcsize(self.headerBinaryFormat)
-
- def sortAndLink(self, lstPages, bSort):
- """
- Given a list of pages in ascending page number order,
- sort them chronologically and link the pages together.
- This must be called before iterating over items in a page.
-
- :param lstPages: The list of pages to be sorted
- :param bSort: A boolean indicating whether the logs are full or \
- partially full. If they are full, bSort is :const:`True`.
-
- :return: The sorted and linked list of pages.
-
- **Side Effects**: If the starting page cannot be found, this \
- method will fail an assertion.
- """
- # Convert lstPages deque to a regular Python list
- lstPages = list(lstPages)
-
- # Sort pages chronologically by finding the first active page.
- # Since the pages are filled in order by page-id, the first
- # non-full page is the last page written to.
- if bSort:
- for n in range(len(lstPages)):
- if lstPages[0].write_offset != FULL_PAGE_WRITE_OFFSET:
- break
- lstPages.append(lstPages.pop(0))
- assert lstPages[0].write_offset != FULL_PAGE_WRITE_OFFSET, \
- "Unable to find starting page"
-
- # link pages
- numPages = len(lstPages)
- nMaxPage = 0
- for n in range(numPages):
- page = lstPages[n]
- nMaxPage = max(nMaxPage, page.page_num)
- page.previousPage = lstPages[n - 1]
- page.nextPage = lstPages[(n + 1) % numPages]
- page.iterated = False
- assert (nMaxPage + 1) == numPages, \
- "Missing log pages: max page %d, num pages %d" % \
- (nMaxPage, numPages)
-
- return lstPages
-
- def log_info(self):
- """
- If DEBUG logging is turned on (command line parameter -v), logs the
- instance variables of this LogPage_v0 to the output file. See the Python
- :mod:`logging` module documentation and the IPC Logging main function
- for more info on logging.
-
- This method only logs the ``read_offset`` and
- ``write_offset`` variables as the others are logged elsewhere.
- """
- logging.debug("\tread index: %x" % (self.read_offset))
- logging.debug("\twrite index: %x" % (self.write_offset))
-
- def debug_save_log_pages(self, fIn, filename, data, options):
- """
- Writes a binary file containing the data for the current page. This
- is a helper to :func:`cmdParse`, and is particularly useful in
- debugging.
-
- :param fIn: The input file currently being processed by the caller
- :param filename: The name of the input file currently being processed
- :param data: The binary data for the current page
- :param options: Configuration options containing options.output_dir for
- the output directory
-
- **Side Effects**: Writes an output file containing the binary data \
- for the current page.
- """
- # Get just the filename, instead of the whole path.
- # Otherwise the file open call will fail.
- filename = os.path.split(filename)[1]
-
- strDebugFileName = 'log-%d-%d-%s-%u.bin' % (self.log_id, self.page_num,
- filename, fIn.tell() - PAGE_SIZE)
-
- if options.output_dir:
- strDebugFileName = \
- os.path.join(options.output_dir, strDebugFileName)
-
- logging.info("Writing debug file '%s'\n", strDebugFileName)
- fOut = open(strDebugFileName, 'wb')
- fOut.write(data)
- fOut.close()
-
- def unpack(self, data):
- """
- Unpacks an ipc_log_page version 0 structure, and assigns the contents
- into instance variables of this class. This is a wrapper around
- the :func:`~struct.unpack()` function from the :mod:`struct` module.
-
- :param data: The binary ipc_log_context structure to be unpacked.
-
- :return: :const:`True` if the unpack completes successfully, \
- :const:`False` if not.
-
- **Side Effects**: If the unpack is successful, the contexts are \
- assigned to instance variables. In this way, :meth:`unpack()` \
- acts as a sort of constructor. This is common to all the \
- unpack methods.
- """
- self.data = data
- self.magic, self.nmagic, self.log_id, self.page_num, self.read_offset, \
- self.write_offset = struct.unpack(self.headerBinaryFormat,
- self.data[0:self.headerBinaryFormatSize])
-
- # Mask off the top bit of page_num, which is used to indicate
- # a version 1 or greater page
- mask = 0x7FFFFFFF
- self.page_num = self.page_num & mask
-
- if self.magic == MAGIC and self.nmagic == NMAGIC:
- return True
-
- return False
-
- def pack(self, nLogId, nPageNum, nReadOffset, nWriteOffset, strPayload):
- """
- Packs a new version 0 page structure. *strPayload* can be empty and
- additional payload can be added using :meth:`~LogPage.pack_continue()`.
-
- :param nLogId: The log ID to be packed into the new structure
- :param nPageNum: The page number
- :param nReadOffset: The read offset
- :param nWriteOffset: The write offset
- :param strPayload: The payload of the new structure
-
- :return: Overflow data (*strPayload* that didn't fit into the page) or \
- :const:`None` if all fit.
- """
- data = struct.pack(self.headerBinaryFormat, MAGIC, NMAGIC,
- nLogId, nPageNum, nReadOffset, nWriteOffset)
-
- # add padding for list structures
- data += '\0' * (self.page_header_size - len(data))
-
- # append message payload
- self.data = data + strPayload or ''
-
- if len(self.data) > PAGE_SIZE:
- data = self.data[PAGE_SIZE:]
- self.data = self.data[0:PAGE_SIZE]
- return data
- else:
- return None
+ """
+ A single page in a version 0 IPC Logging log. This class is a descendant of
+ :class:`LogPage`.
+
+ .. code-block:: c
+
+ struct ipc_log_page_header {
+ uint32_t magic;
+ uint32_t nmagic; /* inverse of magic number */
+ uint32_t log_id; /* owner of log */
+ uint32_t page_num;
+ uint16_t read_offset;
+ uint16_t write_offset;
+ struct list_head list;
+ };
+ """
+ def __init__(self):
+ # Call the parent constructor
+ super(self.__class__, self).__init__()
+ self.headerBinaryFormat = '<IIIIHH'
+ self.headerBinaryFormatSize = struct.calcsize(self.headerBinaryFormat)
+
+ def sortAndLink(self, lstPages, bSort):
+ """
+ Given a list of pages in ascending page number order,
+ sort them chronologically and link the pages together.
+ This must be called before iterating over items in a page.
+
+ :param lstPages: The list of pages to be sorted
+ :param bSort: A boolean indicating whether the logs are full or
+ partially full. If they are full, bSort is :const:`True`.
+
+ :return: The sorted and linked list of pages.
+
+ **Side Effects**: If the starting page cannot be found, this
+ method will fail an assertion.
+ """
+ # Convert lstPages deque to a regular Python list
+ lstPages = list(lstPages)
+
+ # Sort pages chronologically by finding the first active page.
+ # Since the pages are filled in order by page-id, the first
+ # non-full page is the last page written to.
+ if bSort:
+ for n in range(len(lstPages)):
+ if lstPages[0].write_offset != FULL_PAGE_WRITE_OFFSET:
+ break
+ lstPages.append(lstPages.pop(0))
+ assert lstPages[0].write_offset != FULL_PAGE_WRITE_OFFSET, \
+ "Unable to find starting page"
+
+ # link pages
+ numPages = len(lstPages)
+ nMaxPage = 0
+ for n in range(numPages):
+ page = lstPages[n]
+ nMaxPage = max(nMaxPage, page.page_num)
+ page.previousPage = lstPages[n - 1]
+ page.nextPage = lstPages[(n + 1) % numPages]
+ page.iterated = False
+ assert (nMaxPage + 1) == numPages, \
+ "Missing log pages: max page %d, num pages %d" % \
+ (nMaxPage, numPages)
+
+ return lstPages
+
+ def log_info(self):
+ """
+ If DEBUG logging is turned on (command line parameter -v), logs the
+ instance variables of this LogPage_v0 to the output file. See the
+ Python :mod:`logging` module documentation and the IPC Logging main
+ function for more info on logging.
+
+ This method only logs the ``read_offset`` and
+ ``write_offset`` variables as the others are logged elsewhere.
+ """
+ logging.debug("\tread index: %x" % (self.read_offset))
+ logging.debug("\twrite index: %x" % (self.write_offset))
+
+ def debug_save_log_pages(self, fIn, filename, data, options):
+ """
+ Writes a binary file containing the data for the current page. This
+ is a helper to :func:`cmdParse`, and is particularly useful in
+ debugging.
+
+ :param fIn: The input file currently being processed by the caller
+ :param filename: The name of the input file currently being processed
+ :param data: The binary data for the current page
+ :param options: Configuration options containing options.output_dir for
+ the output directory
+
+ **Side Effects**: Writes an output file containing the binary data
+ for the current page.
+ """
+ # Get just the filename, instead of the whole path.
+ # Otherwise the file open call will fail.
+ filename = os.path.split(filename)[1]
+
+ strDebugFileName = 'log-%d-%d-%s-%u.bin' % (self.log_id, self.page_num,
+ filename, fIn.tell() -
+ PAGE_SIZE)
+
+ if options.output_dir:
+ strDebugFileName = \
+ os.path.join(options.output_dir, strDebugFileName)
+
+ logging.info("Writing debug file '%s'\n", strDebugFileName)
+ fOut = open(strDebugFileName, 'wb')
+ fOut.write(data)
+ fOut.close()
+
+ def unpack(self, data):
+ """
+ Unpacks an ipc_log_page version 0 structure, and assigns the contents
+ into instance variables of this class. This is a wrapper around
+ the :func:`~struct.unpack()` function from the :mod:`struct` module.
+
+ :param data: The binary ipc_log_context structure to be unpacked.
+
+ :return: :const:`True` if the unpack completes successfully,
+ :const:`False` if not.
+
+ **Side Effects**: If the unpack is successful, the contexts are
+ assigned to instance variables. In this way, :meth:`unpack()`
+ acts as a sort of constructor. This is common to all the
+ unpack methods.
+ """
+ self.data = data
+ self.magic, self.nmagic, self.log_id, self.page_num, \
+ self.read_offset, self.write_offset = \
+ struct.unpack(self.headerBinaryFormat,
+ self.data[0:self.headerBinaryFormatSize])
+
+ # Mask off the top bit of page_num, which is used to indicate
+ # a version 1 or greater page
+ mask = 0x7FFFFFFF
+ self.page_num = self.page_num & mask
+
+ if self.magic == MAGIC and self.nmagic == NMAGIC:
+ return True
+
+ return False
+
+ def pack(self, nLogId, nPageNum, nReadOffset, nWriteOffset, strPayload):
+ """
+ Packs a new version 0 page structure. *strPayload* can be empty and
+ additional payload can be added using :meth:`~LogPage.pack_continue()`.
+
+ :param nLogId: The log ID to be packed into the new structure
+ :param nPageNum: The page number
+ :param nReadOffset: The read offset
+ :param nWriteOffset: The write offset
+ :param strPayload: The payload of the new structure
+
+ :return: Overflow data (*strPayload* that didn't fit into the page)
+ or :const:`None` if all fit.
+ """
+ data = struct.pack(self.headerBinaryFormat, MAGIC, NMAGIC,
+ nLogId, nPageNum, nReadOffset, nWriteOffset)
+
+ # add padding for list structures
+ data += '\0' * (self.page_header_size - len(data))
+
+ # append message payload
+ self.data = data + strPayload or ''
+
+ if len(self.data) > PAGE_SIZE:
+ data = self.data[PAGE_SIZE:]
+ self.data = self.data[0:PAGE_SIZE]
+ return data
+ else:
+ return None
+
class LogPage_v1(LogPage):
- """
- A single page in a version 0 IPC Logging log. This class is a descendant of
- :class:`LogPage`.
-
- .. code-block:: c
-
- struct ipc_log_page_header {
- uint32_t magic;
- uint32_t nmagic;
- uint32_t page_num;
- uint16_t read_offset;
- uint16_t write_offset;
- uint64_t log_id;
- uint64_t start_time;
- uint64_t end_time;
- int64_t context_offset;
-
- /* add local data structures after this point */
- struct list_head list;
- uint16_t nd_read_offset;
- };
- """
- def __init__(self):
- # Call the parent constructor
- super(self.__class__, self).__init__()
- self.headerBinaryFormat = '<IIIHHQQQq'
- self.headerBinaryFormatSize = struct.calcsize(self.headerBinaryFormat)
- self.context = LogContext()
- self.context.header_size = V1_PAGE_HDR_SIZE
- self.page_header_size = self.context.header_size
-
- def sortAndLink(self, lstPages, bSort):
- """
- Given a list of pages in ascending page number order,
- sort them chronologically and link the pages together.
- This must be called before iterating over items in a page.
-
- :param lstPages: The list of pages to be sorted
- :param bSort: A boolean indicating whether the logs are full or \
- partially full. If they are full, bSort is :const:`True`.
-
- :return: The sorted and linked list of pages.
-
- **Side Effects**: If the starting page cannot be found, this \
- method will fail an assertion.
- """
- # Sort pages chronologically by finding the first active page.
- # Since the pages are filled in order by page-id, the first
- # non-full page is the last page written to.
- if bSort:
- # Rotate to lowest non-zero end time
- min_end_time = None
- min_start_time = None
- min_end_position = 0
- min_start_position = 0
- for n in range(len(lstPages)):
- cur_end_time = lstPages[n].get_end_time()
- if cur_end_time == 0:
- continue
-
- if not min_end_time:
- min_end_time = cur_end_time
- min_end_position = n
- continue
-
- if cur_end_time > 0 and cur_end_time < min_end_time:
- min_end_time = cur_end_time
- min_end_position = n
-
- min_position = min_end_position
- if lstPages[min_end_position].read_offset == 0:
- for n in range(len(lstPages)):
- cur_start_time = lstPages[n].get_start_time()
- if cur_start_time == 0:
- continue
-
- if not min_start_time:
- min_start_time = cur_start_time
- min_start_position = n
- continue
-
- if cur_start_time > 0 and cur_start_time < min_start_time:
- min_start_time = cur_start_time
- min_start_position = n
-
- if lstPages[min_start_position].read_offset != 0:
- min_position = min_start_position
-
- lstPages.rotate(-min_position)
- lstPages = list(lstPages)
-
- # link pages
- numPages = len(lstPages)
- nMaxPage = 0
- for n in range(numPages):
- page = lstPages[n]
- nMaxPage = max(nMaxPage, page.page_num)
- page.previousPage = lstPages[n - 1]
- page.nextPage = lstPages[(n + 1) % numPages]
- assert (nMaxPage + 1) == numPages, \
- "Missing log pages: max page %d, num pages %d" % \
- (nMaxPage, numPages)
-
- return lstPages
-
- def log_info(self):
- """
- If DEBUG logging is turned on (command line parameter -v), logs the
- instance variables of this LogPage_v1 to the output file. See the Python
- :mod:`logging` module documentation and the IPC Logging main function
- for more info on logging.
-
- This method logs the ``read_offset``, ``write_offset``, ``start_time``,
- ``end_time``, and ``context_offset`` variables.
- """
- logging.debug("\tread index: %x" % (self.read_offset))
- logging.debug("\twrite index: %x" % (self.write_offset))
- logging.debug("\tstart_time (seconds): %f" % (self.start_time * math.pow(10, -9)))
- logging.debug("\tend_time (seconds): %f" % (self.end_time * math.pow(10, -9)))
- logging.debug("\tcontext_offset: %x" % (self.context_offset))
-
- def debug_save_log_pages(self, fIn, filename, data, name, options):
- """
- If DEBUG logging is turned on, writes a binary file containing the data
- for the current page. This is a helper to :func:`cmdParse`.
-
- :param fIn: The input file currently being processed by the caller
- :param filename: The name of the input file currently being processed
- :param data: The binary data for the current page
- :param options: Configuration options containing options.output_dir for
- the output directory
-
- **Side Effects**: Writes an output file containing the binary data \
- for the current page.
- """
- # Get just the filename, instead of the whole path.
- # Otherwise the file open call will fail.
- filename = os.path.split(filename)[1]
-
- strDebugFileName = 'log-%s-%d-%s-%u.bin' % (name, self.page_num,
- filename, fIn.tell() - PAGE_SIZE)
-
- if options.output_dir:
- strDebugFileName = \
- os.path.join(options.output_dir, strDebugFileName)
-
- logging.info("Writing debug file '%s'\n", strDebugFileName)
- fOut = open(strDebugFileName, 'wb')
- fOut.write(data)
- fOut.close()
-
- def unpack(self, data):
- """
- Unpacks an ipc_log_page version 1 structure, and assigns the contents
- into instance variables of this class. This is a wrapper around
- the :func:`~struct.unpack()` function from the :mod:`struct` module.
-
- :param data: The binary ipc_log_context structure to be unpacked.
-
- :return: :const:`True` if the unpack completes successfully, \
- :const:`False` if not.
-
- **Side Effects**: If the unpack is successful, the contexts are \
- assigned to instance variables. In this way, :meth:`unpack()` \
- acts as a sort of constructor. This is common to all the \
- unpack methods.
- """
- self.data=data
- self.magic, self.nmagic, self.page_num, self.read_offset, \
- self.write_offset, self.log_id, self.start_time, \
- self.end_time, self.context_offset = \
- struct.unpack(self.headerBinaryFormat,
- self.data[0:self.headerBinaryFormatSize])
-
- # Mask off the top bit of page_num, which is used to indicate
- # a version 1 or greater page
- mask = 0x7FFFFFFF
- self.page_num = self.page_num & mask
-
- if self.magic == MAGIC and self.nmagic == NMAGIC:
- return True
-
- return False
-
- def pack(self, nPageNum, nReadOffset, nWriteOffset, nLogId,
- nStartTime, nEndTime, nContextOffset, strPayload):
- """
- Packs a new version 1 page structure. *strPayload* can be empty and
- additional payload can be added using :meth:`~LogPage.pack_continue()`.
-
- :param nPageNum: The page number to be packed into the new structure
- :param nReadOffset: The read offset
- :param nWriteOffset: The write offset
- :param nLogId: The log ID
- :param nStartTime: The start time
- :param nEndTime: The end time
- :param nContextOffset: The context offset
- :param strPayload: The payload of the new structure
-
- :return: Overflow data (*strPayload* that didn't fit into the page) or \
- :const:`None` if all fit.
- """
-
- data = struct.pack(self.headerBinaryFormat, MAGIC, NMAGIC,
- nPageNum, nReadOffset, nWriteOffset, nLogId, nStartTime,
- nEndTime, nContextOffset)
-
- # add padding for list structures
- data += '\0' * (self.page_header_size - len(data))
-
- # append message payload
- self.data = data + strPayload or ''
-
- if len(self.data) > PAGE_SIZE:
- data = self.data[PAGE_SIZE:]
- self.data = self.data[0:PAGE_SIZE]
- return data
- else:
- return None
-
- def setContext(self, context):
- """
- Sets the ``context`` field of this log page to *context*, and sets the
- ``page_header_size`` field to the context header size
- (``context.header_size``). This method should be used whenever a page's
- context is changed in order to make sure the page's information about
- its context is consistent.
-
- :param context: The :class:`LogContext` object to be assigned to this \
- page
- """
- self.context = context
- self.page_header_size = self.context.header_size
-
- #@staticmethod
- def get_end_time(self):
- return self.end_time
-
- #@staticmethod
- def get_start_time(self):
- return self.start_time
+ """
+ A single page in a version 0 IPC Logging log. This class is a descendant of
+ :class:`LogPage`.
+
+ .. code-block:: c
+
+ struct ipc_log_page_header {
+ uint32_t magic;
+ uint32_t nmagic;
+ uint32_t page_num;
+ uint16_t read_offset;
+ uint16_t write_offset;
+ uint64_t log_id;
+ uint64_t start_time;
+ uint64_t end_time;
+ int64_t context_offset;
+
+ /* add local data structures after this point */
+ struct list_head list;
+ uint16_t nd_read_offset;
+ };
+ """
+ def __init__(self):
+ # Call the parent constructor
+ super(self.__class__, self).__init__()
+ self.headerBinaryFormat = '<IIIHHQQQq'
+ self.headerBinaryFormatSize = struct.calcsize(self.headerBinaryFormat)
+ self.context = LogContext()
+ self.context.header_size = V1_PAGE_HDR_SIZE
+ self.page_header_size = self.context.header_size
+
+ def sortAndLink(self, lstPages, bSort):
+ """
+ Given a list of pages in ascending page number order,
+ sort them chronologically and link the pages together.
+ This must be called before iterating over items in a page.
+
+ :param lstPages: The list of pages to be sorted
+ :param bSort: A boolean indicating whether the logs are full or
+ partially full. If they are full, bSort is :const:`True`.
+
+ :return: The sorted and linked list of pages.
+
+ **Side Effects**: If the starting page cannot be found, this
+ method will fail an assertion.
+ """
+ # Sort pages chronologically by finding the first active page.
+ # Since the pages are filled in order by page-id, the first
+ # non-full page is the last page written to.
+ if bSort:
+ # Rotate to lowest non-zero end time
+ min_end_time = None
+ min_start_time = None
+ min_end_position = 0
+ min_start_position = 0
+ for n in range(len(lstPages)):
+ cur_end_time = lstPages[n].get_end_time()
+ if cur_end_time == 0:
+ continue
+
+ if not min_end_time:
+ min_end_time = cur_end_time
+ min_end_position = n
+ continue
+
+ if cur_end_time > 0 and cur_end_time < min_end_time:
+ min_end_time = cur_end_time
+ min_end_position = n
+
+ min_position = min_end_position
+ if lstPages[min_end_position].read_offset == 0:
+ for n in range(len(lstPages)):
+ cur_start_time = lstPages[n].get_start_time()
+ if cur_start_time == 0:
+ continue
+
+ if not min_start_time:
+ min_start_time = cur_start_time
+ min_start_position = n
+ continue
+
+ if cur_start_time > 0 and cur_start_time < min_start_time:
+ min_start_time = cur_start_time
+ min_start_position = n
+
+ if lstPages[min_start_position].read_offset != 0:
+ min_position = min_start_position
+
+ lstPages.rotate(-min_position)
+ lstPages = list(lstPages)
+
+ # link pages
+ numPages = len(lstPages)
+ nMaxPage = 0
+ for n in range(numPages):
+ page = lstPages[n]
+ nMaxPage = max(nMaxPage, page.page_num)
+ page.previousPage = lstPages[n - 1]
+ page.nextPage = lstPages[(n + 1) % numPages]
+ assert (nMaxPage + 1) == numPages, \
+ "Missing log pages: max page %d, num pages %d" % \
+ (nMaxPage, numPages)
+
+ return lstPages
+
+ def log_info(self):
+ """
+ If DEBUG logging is turned on (command line parameter -v), logs the
+ instance variables of this LogPage_v1 to the output file. See the
+ Python :mod:`logging` module documentation and the IPC Logging main
+ function for more info on logging.
+
+ This method logs the ``read_offset``, ``write_offset``, ``start_time``,
+ ``end_time``, and ``context_offset`` variables.
+ """
+ logging.debug("\tread index: %x" % (self.read_offset))
+ logging.debug("\twrite index: %x" % (self.write_offset))
+ logging.debug("\tstart_time (seconds): %f" %
+ (self.start_time * math.pow(10, -9)))
+ logging.debug("\tend_time (seconds): %f" %
+ (self.end_time * math.pow(10, -9)))
+ logging.debug("\tcontext_offset: %x" % (self.context_offset))
+
+ def debug_save_log_pages(self, fIn, filename, data, name, options):
+ """
+ If DEBUG logging is turned on, writes a binary file containing the data
+ for the current page. This is a helper to :func:`cmdParse`.
+
+ :param fIn: The input file currently being processed by the caller
+ :param filename: The name of the input file currently being processed
+ :param data: The binary data for the current page
+ :param options: Configuration options containing options.output_dir for
+ the output directory
+
+ **Side Effects**: Writes an output file containing the binary data
+ for the current page.
+ """
+ # Get just the filename, instead of the whole path.
+ # Otherwise the file open call will fail.
+ filename = os.path.split(filename)[1]
+
+ strDebugFileName = 'log-%s-%d-%s-%u.bin' % (name, self.page_num,
+ filename,
+ fIn.tell() - PAGE_SIZE)
+
+ if options.output_dir:
+ strDebugFileName = \
+ os.path.join(options.output_dir, strDebugFileName)
+
+ logging.info("Writing debug file '%s'\n", strDebugFileName)
+ fOut = open(strDebugFileName, 'wb')
+ fOut.write(data)
+ fOut.close()
+
+ def unpack(self, data):
+ """
+ Unpacks an ipc_log_page version 1 structure, and assigns the contents
+ into instance variables of this class. This is a wrapper around
+ the :func:`~struct.unpack()` function from the :mod:`struct` module.
+
+ :param data: The binary ipc_log_context structure to be unpacked.
+
+ :return: :const:`True` if the unpack completes successfully,
+ :const:`False` if not.
+
+ **Side Effects**: If the unpack is successful, the contexts are
+ assigned to instance variables. In this way, :meth:`unpack()`
+ acts as a sort of constructor. This is common to all the
+ unpack methods.
+ """
+ self.data = data
+ self.magic, self.nmagic, self.page_num, self.read_offset, \
+ self.write_offset, self.log_id, self.start_time, \
+ self.end_time, self.context_offset = \
+ struct.unpack(self.headerBinaryFormat,
+ self.data[0:self.headerBinaryFormatSize])
+
+ # Mask off the top bit of page_num, which is used to indicate
+ # a version 1 or greater page
+ mask = 0x7FFFFFFF
+ self.page_num = self.page_num & mask
+
+ if self.magic == MAGIC and self.nmagic == NMAGIC:
+ return True
+
+ return False
+
+ def pack(self, nPageNum, nReadOffset, nWriteOffset, nLogId,
+ nStartTime, nEndTime, nContextOffset, strPayload):
+ """
+ Packs a new version 1 page structure. *strPayload* can be empty and
+ additional payload can be added using :meth:`~LogPage.pack_continue()`.
+
+ :param nPageNum: The page number to be packed into the new structure
+ :param nReadOffset: The read offset
+ :param nWriteOffset: The write offset
+ :param nLogId: The log ID
+ :param nStartTime: The start time
+ :param nEndTime: The end time
+ :param nContextOffset: The context offset
+ :param strPayload: The payload of the new structure
+
+ :return: Overflow data (*strPayload* that didn't fit into the page)
+ or :const:`None` if all fit.
+ """
+
+ data = struct.pack(self.headerBinaryFormat, MAGIC, NMAGIC,
+ nPageNum, nReadOffset, nWriteOffset, nLogId,
+ nStartTime, nEndTime, nContextOffset)
+
+ # add padding for list structures
+ data += '\0' * (self.page_header_size - len(data))
+
+ # append message payload
+ self.data = data + strPayload or ''
+
+ if len(self.data) > PAGE_SIZE:
+ data = self.data[PAGE_SIZE:]
+ self.data = self.data[0:PAGE_SIZE]
+ return data
+ else:
+ return None
+
+ def setContext(self, context):
+ """
+ Sets the ``context`` field of this log page to *context*, and sets the
+ ``page_header_size`` field to the context header size
+ (``context.header_size``). This method should be used whenever a page's
+ context is changed in order to make sure the page's information about
+ its context is consistent.
+
+ :param context: The :class:`LogContext` object to be assigned to this
+ page
+ """
+ self.context = context
+ self.page_header_size = self.context.header_size
+
+ def get_end_time(self):
+ return self.end_time
+
+ def get_start_time(self):
+ return self.start_time
+
class PageIterator(object):
- """
- An iterator object for use by the :class:`LogPage` iterator method.
- """
-
- def __init__(self, page):
- self.page = page
-
- def __iter__(self):
- # TODO - clear all iteration flags
- logging.debug("Starting iteration through page %d" % \
- (self.page.page_num))
-
- def next(self):
- while True:
- try:
- return self.page.next()
- except StopIteration:
- logging.debug("Trying next page")
- self.page = self.page.nextPage
- if self.page.iterated:
- logging.debug("Page %d already iterated" % \
- (self.page.page_num))
- raise
- logging.debug("Iterating through page %d" % \
- (self.page.page_num))
-
- # This isn't right. The page iterator is just discarded
- # and then we are calling the .next.
- self.page.__iter__()
+ """
+ An iterator object for use by the :class:`LogPage` iterator method.
+ """
+
+ def __init__(self, page):
+ self.page = page
+
+ def __iter__(self):
+ # TODO - clear all iteration flags
+ logging.debug("Starting iteration through page %d" %
+ (self.page.page_num))
+
+ def next(self):
+ while True:
+ try:
+ return self.page.next()
+ except StopIteration:
+ logging.debug("Trying next page")
+ self.page = self.page.nextPage
+ if self.page.iterated:
+ logging.debug("Page %d already iterated" %
+ (self.page.page_num))
+ raise
+ logging.debug("Iterating through page %d" %
+ (self.page.page_num))
+
+ # This isn't right. The page iterator is just discarded
+ # and then we are calling the .next.
+ self.page.__iter__()
+
def dumpLogHelper(fout, pages, numPages, bSort):
- """
- Handles iterating through the TSV messages in each :class:`LogPage` object
- and dumping them to the output file for the current log ID. This function
- is called by :func:`dumpLogWithRetry()`.
-
- :param fout: The output file for the log currently being processed by \
- :func:`dumpLogWithRetry()`.
- :param pages: A list containing the :class:`LogPage` objects for the log \
- currently being processed.
- :param numPages: The number of pages in *pages*.
- :param bSort: A boolean indicating whether all pages in the log are \
- completely filled, or if some are partially full. If they are \
- full, bSort is :const:`True`.
-
- :return: :const:`True` if the messages are processed with no errors, \
- :const:`False` if errors are encountered during iteration or \
- during parsing of the TSV messages within the :class:`LogPage` \
- objects.
- """
- # sort pages chronologically
- try:
- lstPages = collections.deque()
- presentKey = None
-
- for key in pages.keys():
- if not presentKey:
- presentKey = key
- lstPages.append(pages[key])
-
- lstPages = pages[presentKey].sortAndLink(lstPages, bSort)
- except AssertionError as e:
- strMsg = "Exception: %s" % (str(e))
- logging.error(strMsg)
- fout.write(strMsg + '\n')
- return False
-
- # dump data in pages
- try:
- page = lstPages[0]
- logging.debug("Extracting messages from %d" % (page.page_num))
- last_unknown_data = ''
- for x in page:
- if page.unknown_messages_data and \
- page.unknown_messages_data != last_unknown_data:
- fout.write(str(page.unknown_messages_data) + "\n")
- last_unknown_data = page.unknown_messages_data
-
- fout.write("\n" + str(x).strip())
-
- except AssertionError as e:
- strMsg = "Exception: %s" % (str(e))
- logging.debug(strMsg)
- fout.write(strMsg + '\n')
- return False
-
- return True
+ """
+ Handles iterating through the TSV messages in each :class:`LogPage` object
+ and dumping them to the output file for the current log ID. This function
+ is called by :func:`dumpLogWithRetry()`.
+
+ :param fout: The output file for the log currently being processed by
+ :func:`dumpLogWithRetry()`.
+ :param pages: A list containing the :class:`LogPage` objects for the log
+ currently being processed.
+ :param numPages: The number of pages in *pages*.
+ :param bSort: A boolean indicating whether all pages in the log are
+ completely filled, or if some are partially full. If they are
+ full, bSort is :const:`True`.
+
+ :return: :const:`True` if the messages are processed with no errors,
+ :const:`False` if errors are encountered during iteration or
+ during parsing of the TSV messages within the :class:`LogPage`
+ objects.
+ """
+ # sort pages chronologically
+ try:
+ lstPages = collections.deque()
+ presentKey = None
+
+ for key in pages.keys():
+ if not presentKey:
+ presentKey = key
+ lstPages.append(pages[key])
+
+ lstPages = pages[presentKey].sortAndLink(lstPages, bSort)
+ except AssertionError as e:
+ strMsg = "Exception: %s" % (str(e))
+ logging.error(strMsg)
+ fout.write(strMsg + '\n')
+ return False
+
+ # dump data in pages
+ try:
+ page = lstPages[0]
+ logging.debug("Extracting messages from %d" % (page.page_num))
+ last_unknown_data = ''
+ for x in page:
+ if page.unknown_messages_data and \
+ page.unknown_messages_data != last_unknown_data:
+ fout.write(str(page.unknown_messages_data) + "\n")
+ last_unknown_data = page.unknown_messages_data
+
+ fout.write("\n" + str(x).strip())
+
+ except AssertionError as e:
+ strMsg = "Exception: %s" % (str(e))
+ logging.debug(strMsg)
+ fout.write(strMsg + '\n')
+ return False
+
+ return True
+
def dumpLogWithRetry(logId, name, version, pages, numPages, test, options):
- """
- This function is called by :func:`dumpLogs()` and :func:`cmdTest()`. It
- handles creating output files for the parsed logs and writing data to them.
- A significant part of this work is delegated to :func:`dumpLogHelper()`.
-
- :param logId: The log ID currently being processed
- :param name: The name of the log context
- :param version: The version of the log
- :param pages: List of pages to process and dump to the output file
- :param numPages: The number of pages in the current log
- :param test: True if this function was called by cmdTest
- :param options: Configuration options containing options.output_dir for the
- output directory
- """
- if version >= 1:
- strFileName = "ipc-log-%s.txt" % (str(name))
- titleStr = "Log name: %s (%d pages)\n" % (str(name), numPages)
- else:
- strFileName = "ipc-log-%s.txt" % (str(logId))
- titleStr = "Log id %s (%d pages)\n" % (str(logId), numPages)
-
- if test:
- strFileName = "ipc-log-%s.txt" % (str(name))
-
- if options.output_dir:
- strFileName = \
- os.path.join(options.output_dir, strFileName)
-
- fout = open(strFileName, 'w')
- logging.info("Writing log file '%s'" % (strFileName))
-
- fout.write('-'*70 + "\n")
- fout.write(titleStr)
- fout.write('-'*70 + "\n")
-
- # Parse logs assuming they have wrapped (are full)
- fout.write("Parsing as a full log\n")
- bSuccess = dumpLogHelper(fout, pages, numPages, True)
- if not bSuccess:
- # If that fails, try assuming that they are partially-full
- strMsg = "Unable to parse as full log, retrying as partial log"
- logging.info(strMsg)
- fout.write(strMsg + '\n')
- bSuccess = dumpLogHelper(fout, pages, numPages, False)
-
-
- if bSuccess:
- strMsg = "Log extracted successfully"
- logging.info(strMsg)
- fout.write('\n' + strMsg + '\n')
- else:
- strMsg = "Could not extract log successfully. See %s for details." % \
- (strFileName)
- if version == 0:
- v0_expl_str = \
-"""
-You are parsing a v0 log. If you collected logs using debugfs before the system
-crashed, it may have adversely affected the RAM dumps. Collect the logs again
-without using debugfs, or use the newest version of IPC Logging if possible.
-This issue has been resolved in v1 and above.
-"""
- fout.write(v0_expl_str)
- logging.info(strMsg)
+ """
+ This function is called by :func:`dumpLogs()` and :func:`cmdTest()`. It
+ handles creating output files for the parsed logs and writing data to them.
+ A significant part of this work is delegated to :func:`dumpLogHelper()`.
+
+ :param logId: The log ID currently being processed
+ :param name: The name of the log context
+ :param version: The version of the log
+ :param pages: List of pages to process and dump to the output file
+ :param numPages: The number of pages in the current log
+ :param test: True if this function was called by cmdTest
+ :param options: Configuration options containing options.output_dir for the
+ output directory
+ """
+ if version >= 1:
+ strFileName = "ipc-log-%s.txt" % (str(name))
+ titleStr = "Log name: %s (%d pages)\n" % (str(name), numPages)
+ else:
+ strFileName = "ipc-log-%s.txt" % (str(logId))
+ titleStr = "Log id %s (%d pages)\n" % (str(logId), numPages)
+
+ if test:
+ strFileName = "ipc-log-%s.txt" % (str(name))
+
+ if options.output_dir:
+ strFileName = \
+ os.path.join(options.output_dir, strFileName)
+
+ fout = open(strFileName, 'w')
+ logging.info("Writing log file '%s'" % (strFileName))
+
+ fout.write('-'*70 + "\n")
+ fout.write(titleStr)
+ fout.write('-'*70 + "\n")
+
+ # Parse logs assuming they have wrapped (are full)
+ fout.write("Parsing as a full log\n")
+ bSuccess = dumpLogHelper(fout, pages, numPages, True)
+ if not bSuccess:
+ # If that fails, try assuming that they are partially-full
+ strMsg = "Unable to parse as full log, retrying as partial log"
+ logging.info(strMsg)
+ fout.write(strMsg + '\n')
+ bSuccess = dumpLogHelper(fout, pages, numPages, False)
+
+ if bSuccess:
+ strMsg = "Log extracted successfully"
+ logging.info(strMsg)
+ fout.write('\n' + strMsg + '\n')
+ else:
+ strMsg = "Could not extract log successfully. See %s for details." % \
+ (strFileName)
+ if version == 0:
+ v0_expl_str = \
+ """
+ You are parsing a v0 log. If you collected logs using debugfs
+ before the system crashed, it may have adversely affected the
+ RAM dumps. Collect the logs again without using debugfs, or
+ use the newest version of IPC Logging if possible. This issue
+ has been resolved in v1 and above.
+ """
+ fout.write(v0_expl_str)
+ logging.info(strMsg)
+
+ # Add a newline to stdout output
+ logging.info("")
+ fout.close()
- # Add a newline to stdout output
- logging.info("")
- fout.close()
def get_context(start_of_page, fIn, page, dictContexts, lstFiles, fileCount):
- '''
- Called by :func:`cmdParse()` in order the find and read the context for
- the log page currently being processed. The context is found by reading
- *page.context_offset* and using it to find the position of the context in
- the "logical file" created by :func:`get_files_info()`. Once the input file
- and position where the context resides have been calculated, the context can
- be read, assigned to the *page.context* field, and added to the log ID-to-
- context dictionary *dictContexts* using the :func:`read_context()` function.
-
- :param start_of_page: The starting position in the current file of the \
- page currently being processed
- :param fIn: The input file
- :param page: The page currently being processed
- :param dictContexts: The dictionary that maintains the \
- log ID-to-:class:`LogContext` mapping.
- :param lstFiles: The "logical file" data structure, a list of dictionaries \
- containing the name, starting position, and ending position of \
- each input file.
- :param fileCount: An index into *lstFiles*, which indicates the file \
- currently being processed. For example, if *lstFiles* contains \
- three input files and we are processing the second one, fileCount \
- is 1.
-
- :return: A :class:`LogContext` object containing the context that was \
- found. If no context is found, returns :const:`None`.
-
- **Side Effects**: There are some performance side effects in this \
- function, as :meth:`file.seek()`, :meth:`file.read()`, and \
- :meth:`file.tell()` are performed several times in order to \
- extract the context based on *page.context_offset*. Since there \
- may be multiple files and the context may not be in the current \
- file, the other files may need to be opened and closed in order \
- to find the context. \
- '''
- context = LogContext()
- start_of_file = lstFiles[fileCount]['start']
- end_of_file = lstFiles[fileCount]['end']
- file_position = start_of_file + start_of_page + page.context_offset
- data = None
-
- logging.debug("start of file: 0x%x, end of file: 0x%x, " + \
- "start of page: 0x%x, context offset: 0x%x, file position: 0x%x,",
- start_of_file, end_of_file, start_of_page, page.context_offset,
- file_position)
-
- for inFile in lstFiles:
- if inFile['start'] <= file_position <= inFile['end']:
- # If the position is in the current file, don't open and close a
- # temporary file - use fIn
- if inFile['start'] == start_of_file:
- saved_position = fIn.tell()
- fIn.seek(file_position - start_of_file)
- data = fIn.read(CTX_READ_SIZE)
- fIn.seek(saved_position)
- else:
- tempIn = open(inFile['name'], 'rb')
- tempIn.seek(file_position - inFile['start'])
- data = tempIn.read(CTX_READ_SIZE)
- tempIn.close()
- break
-
- assert data is not None, "Context not found"
-
- context = read_context(page, data, dictContexts)
- return context
+ """
+ Called by :func:`cmdParse()` in order the find and read the context for
+ the log page currently being processed. The context is found by reading
+ *page.context_offset* and using it to find the position of the context in
+ the "logical file" created by :func:`get_files_info()`. Once the input file
+ and position where the context resides have been calculated, the context
+ can be read, assigned to the *page.context* field, and added to the log
+ ID-to-context dictionary *dictContexts* using the :func:`read_context()`
+ function.
+
+ :param start_of_page: The starting position in the current file of the
+ page currently being processed
+ :param fIn: The input file
+ :param page: The page currently being processed
+ :param dictContexts: The dictionary that maintains the
+ log ID-to-:class:`LogContext` mapping.
+ :param lstFiles: The "logical file" data structure, a list of dictionaries
+ containing the name, starting position, and ending position of
+ each input file.
+ :param fileCount: An index into *lstFiles*, which indicates the file
+ currently being processed. For example, if *lstFiles* contains
+ three input files and we are processing the second one, fileCount
+ is 1.
+
+ :return: A :class:`LogContext` object containing the context that was
+ found. If no context is found, returns :const:`None`.
+
+ **Side Effects**: There are some performance side effects in this
+ function, as :meth:`file.seek()`, :meth:`file.read()`, and
+ :meth:`file.tell()` are performed several times in order to
+ extract the context based on *page.context_offset*. Since there
+ may be multiple files and the context may not be in the current
+ file, the other files may need to be opened and closed in order
+ to find the context.
+ """
+ context = LogContext()
+ start_of_file = lstFiles[fileCount]['start']
+ end_of_file = lstFiles[fileCount]['end']
+ file_position = start_of_file + start_of_page + page.context_offset
+ data = None
+
+ logging.debug("start of file: 0x%x, end of file: 0x%x, " +
+ "start of page: 0x%x, context offset: 0x%x, " +
+ "file position: 0x%x,",
+ start_of_file, end_of_file, start_of_page,
+ page.context_offset, file_position)
+
+ for inFile in lstFiles:
+ if inFile['start'] <= file_position <= inFile['end']:
+ # If the position is in the current file, don't open and close a
+ # temporary file - use fIn
+ if inFile['start'] == start_of_file:
+ saved_position = fIn.tell()
+ fIn.seek(file_position - start_of_file)
+ data = fIn.read(CTX_READ_SIZE)
+ fIn.seek(saved_position)
+ else:
+ tempIn = open(inFile['name'], 'rb')
+ tempIn.seek(file_position - inFile['start'])
+ data = tempIn.read(CTX_READ_SIZE)
+ tempIn.close()
+ break
+
+ assert data is not None, "Context not found"
+
+ context = read_context(page, data, dictContexts)
+ return context
+
def read_context(page, data, dictContexts):
- '''
- Read a context from *data* and make it the context of *page* using
- :meth:`LogPage_v1.setContext()`. Then update the log ID-to-context
- dictionary (*dictContexts*), and return the context. In short, find
- a log context in *data*, perform some bookkeeping, and return the context.
- This is a helper function to :func:`get_context()`, which is a helper to
- :func:`cmdParse()`.
-
- :param page: The page to which the found context will be assigned
- :param data: The data to be read
- :param dictContexts: The dictionary where the context will be placed, if \
- a new context is read
-
- :return: The new context if successful, None if not.
- '''
- context = LogContext()
-
- if data and len(data) >= CTX_READ_SIZE and context.unpack(data):
- page.setContext(context)
-
- if not context.log_id in dictContexts:
- logging.info("Found log context: %s" % (context.name))
- context.log_info()
-
- # Log a single newline to the output
- logging.info("")
- dictContexts[page.log_id] = context
-
- return context
- else:
- return None
+ """
+ Read a context from *data* and make it the context of *page* using
+ :meth:`LogPage_v1.setContext()`. Then update the log ID-to-context
+ dictionary (*dictContexts*), and return the context. In short, find
+ a log context in *data*, perform some bookkeeping, and return the context.
+ This is a helper function to :func:`get_context()`, which is a helper to
+ :func:`cmdParse()`.
+
+ :param page: The page to which the found context will be assigned
+ :param data: The data to be read
+ :param dictContexts: The dictionary where the context will be placed, if
+ a new context is read
+
+ :return: The new context if successful, None if not.
+ """
+ context = LogContext()
+
+ if data and len(data) >= CTX_READ_SIZE and context.unpack(data):
+ page.setContext(context)
+
+ if context.log_id not in dictContexts:
+ logging.info("Found log context: %s" % (context.name))
+ context.log_info()
+
+ # Log a single newline to the output
+ logging.info("")
+ dictContexts[page.log_id] = context
+
+ return context
+ else:
+ return None
+
def check_log_consistency(dictLogs, dictContexts):
- '''
- Creates two sets:
-
- * The set of log IDs gained from reading each page
-
- * The set of log IDs gained from reading each context
-
- Takes the intersection and makes sure that the intersection
- equals both sets, i.e. that all log IDs found from reading pages
- were found reading contexts, and vice-versa.
-
- :param dictLogs: The dictionary of log IDs to :class:`LogPage` objects
- :param dictContext: The dictionary of log IDs to :class:`LogContext` objects
-
- :return: The difference of the two sets, contained in a :class:`set`.
- '''
- diff = None
- context_log_ids = set(dictContexts.keys())
- page_log_ids = set(dictLogs.keys())
- intersection = context_log_ids.intersection(page_log_ids)
- if intersection == page_log_ids == context_log_ids:
- logging.debug("Logs are consistent - the same log IDs are " + \
- "found in pages and contexts.")
- else:
- logging.error("Logs inconsistent.")
- logging.error("Logs from contexts: %s",
- str([hex(n) for n in context_log_ids]))
- logging.error("Logs from pages: %s",
- str([hex(n) for n in page_log_ids]))
-
- if len(page_log_ids) > len(context_log_ids):
- diff = page_log_ids - context_log_ids
- logging.error("IDs from pages, but not contexts: %s",
- str([hex(n) for n in diff]))
- else:
- diff = context_log_ids - page_log_ids
- logging.error("IDs from contexts, but not pages: %s",
- str([hex(n) for n in diff]))
- return diff
+ """
+ Creates two sets:
+
+ * The set of log IDs gained from reading each page
+
+ * The set of log IDs gained from reading each context
+
+ Takes the intersection and makes sure that the intersection
+ equals both sets, i.e. that all log IDs found from reading pages
+ were found reading contexts, and vice-versa.
+
+ :param dictLogs: The dictionary of log IDs to :class:`LogPage` objects
+ :param dictContext: The dictionary of log IDs to :class:`LogContext`
+ objects
+
+ :return: The difference of the two sets, contained in a :class:`set`.
+ """
+ diff = None
+ context_log_ids = set(dictContexts.keys())
+ page_log_ids = set(dictLogs.keys())
+ intersection = context_log_ids.intersection(page_log_ids)
+ if intersection == page_log_ids == context_log_ids:
+ logging.debug("Logs are consistent - the same log IDs are " +
+ "found in pages and contexts.")
+ else:
+ logging.error("Logs inconsistent.")
+ logging.error("Logs from contexts: %s",
+ str([hex(n) for n in context_log_ids]))
+ logging.error("Logs from pages: %s",
+ str([hex(n) for n in page_log_ids]))
+
+ if len(page_log_ids) > len(context_log_ids):
+ diff = page_log_ids - context_log_ids
+ logging.error("IDs from pages, but not contexts: %s",
+ str([hex(n) for n in diff]))
+ else:
+ diff = context_log_ids - page_log_ids
+ logging.error("IDs from contexts, but not pages: %s",
+ str([hex(n) for n in diff]))
+ return diff
+
def calc_file_size(fIn):
- """
- Calculates the size of an input file.
-
- :param fIn: The input file
-
- :return: The file size, which is also the ending position relative to the \
- start of the input file.
-
- **Side Effects**: This function uses :meth:`file.seek()` and \
- :meth:`file.tell()` to calculate the file size, which can have \
- performance considerations. The original seek position of the file \
- is saved at the beginning of the function and is restored before \
- returning.
- """
- saved_file_position = fIn.tell()
- fIn.seek(0, 2)
- file_size = fIn.tell()
- fIn.seek(saved_file_position)
- return file_size
+ """
+ Calculates the size of an input file.
+
+ :param fIn: The input file
+
+ :return: The file size, which is also the ending position relative to the
+ start of the input file.
+
+ **Side Effects**: This function uses :meth:`file.seek()` and
+ :meth:`file.tell()` to calculate the file size, which can have
+ performance considerations. The original seek position of the file
+ is saved at the beginning of the function and is restored before
+ returning.
+ """
+ saved_file_position = fIn.tell()
+ fIn.seek(0, 2)
+ file_size = fIn.tell()
+ fIn.seek(saved_file_position)
+ return file_size
+
def get_files_info(files):
- """
- Calculates the size of each input file and uses this information to build
- a "logical file" data structure, which is a list of dictionaries, with one
- dictionary per input file. Each dictionary contains the name of the file,
- the starting position of the file, the ending position of the file, and
- the size of the file. The starting position of the second file in the list
- is the ending position of the first file in the list, and so on. Once the
- structure is completely built, it can be used to index any position in any
- of the input files as if they were one big file, without reading this big
- file into memory. This allows the retrieval of contexts located in a
- different file than their pages.
-
- The file size calculation is done using the :func:`calc_file_size()`
- function.
-
- :param files: The list of input files
-
- :return: The "logical file" data structure, a list of dictionaries \
- containing information about each of the input files
- """
- lstFiles = []
- fileCount = 0
- for strFile in files:
- fileStart = 0
-
- if fileCount > 0:
- fileStart = (lstFiles[fileCount-1]['end'])
-
- strFileSplit = strFile.split(':', 1)
- fileName = strFileSplit[0]
- if len(strFileSplit) > 1:
- fileStart = int(strFileSplit[1], 16)
-
- lstFiles.append(\
- {'name': fileName, 'start': fileStart, 'end': 0, 'size': 0})
- fIn = open(fileName, 'rb')
- file_size = calc_file_size(fIn)
-
- lstFiles[fileCount]['end'] = fileStart + file_size
- lstFiles[fileCount]['size'] = file_size
-
- fileCount += 1
-
- return lstFiles
+ """
+ Calculates the size of each input file and uses this information to build
+ a "logical file" data structure, which is a list of dictionaries, with one
+ dictionary per input file. Each dictionary contains the name of the file,
+ the starting position of the file, the ending position of the file, and
+ the size of the file. The starting position of the second file in the list
+ is the ending position of the first file in the list, and so on. Once the
+ structure is completely built, it can be used to index any position in any
+ of the input files as if they were one big file, without reading this big
+ file into memory. This allows the retrieval of contexts located in a
+ different file than their pages.
+
+ The file size calculation is done using the :func:`calc_file_size()`
+ function.
+
+ :param files: The list of input files
+
+ :return: The "logical file" data structure, a list of dictionaries
+ containing information about each of the input files
+ """
+ lstFiles = []
+ fileCount = 0
+ for strFile in files:
+ fileStart = 0
+
+ if fileCount > 0:
+ fileStart = (lstFiles[fileCount-1]['end'])
+
+ strFileSplit = strFile.split(':', 1)
+ fileName = strFileSplit[0]
+ if len(strFileSplit) > 1:
+ fileStart = int(strFileSplit[1], 16)
+
+ lstFiles.append(
+ {'name': fileName, 'start': fileStart, 'end': 0, 'size': 0})
+ fIn = open(fileName, 'rb')
+ file_size = calc_file_size(fIn)
+
+ lstFiles[fileCount]['end'] = fileStart + file_size
+ lstFiles[fileCount]['size'] = file_size
+
+ fileCount += 1
+
+ return lstFiles
+
def cmdParse(options):
- """
- The top-level function, which is called from the main entry point to the
- program if the ``parse`` command-line option is given. This function creates
- two dictionaries, ``dictLogs`` and ``dictContexts``, which have log IDs as
- keys and :class:`LogPage` and :class:`LogContext` objects as values,
- respectively. It then populates these dictionaries by going through
- each input file and reading data page-by-page. After all data has been read,
- this function calls :func:`dumpLogs()` to dump the parsed logs to output
- files.
-
- :param options: Configuration options containing options.output_dir for the
- output directory, options.debug_enabled, a flag for
- discerning whether the extraction script is running in debug
- mode, and options.args, the list of input files. All
- configuration options are passed through to helper
- functions.
- """
- dictLogs = {}
- dictContexts = {}
- fileCount = 0
- version = 0
- versionIsOneOrGreater = False
- lstFiles = get_files_info(options.args)
-
- # scan files for log pages
- for strFile in options.args:
- fileName = strFile.split(':', 1)[0]
- fIn = open(fileName,'rb')
-
- logging.info("Parsing '%s'" % (fileName))
- logging.info("-" * 70)
- sys.stdout.flush()
- page = LogPageVersionUnknown()
- context = LogContext()
- file_size = lstFiles[fileCount]['size']
-
- while True:
- start_of_page = fIn.tell()
- data = fIn.read(16)
-
- # PAGE_SIZE - 16 = 4080
- if not data or len(data) + 4080 < PAGE_SIZE:
- break
-
- if page.unpack(data):
- data += fIn.read(PAGE_SIZE-16)
-
- if page.isVersionOneOrGreater(data):
- versionIsOneOrGreater = True
- page = LogPage_v1()
- else:
- versionIsOneOrGreater = False
- page = LogPage_v0()
-
- if page.unpack(data):
- if versionIsOneOrGreater:
- if not page.log_id in dictContexts:
- try:
- context = get_context(start_of_page, fIn, page,
- dictContexts, lstFiles, fileCount)
- except:
- msg = "Context not found - skipping page " + \
- "and trying to " + \
- "continue with unknown log page version"
- logging.debug(msg)
- page = LogPageVersionUnknown()
- continue
-
- else:
- context = dictContexts[page.log_id]
- page.setContext(context)
-
- if context is None:
- logging.debug("Context at 0x%x had no data, " + \
- "skipping this page", page.context_offset)
- if options.debug_enabled:
- page.log_info()
- page = LogPageVersionUnknown()
- continue
-
- version = context.version
-
- if not page.log_id in dictLogs:
- dictLogs[page.log_id] = {}
-
- if dictLogs[page.log_id].has_key(page.page_num):
- logging.error("Duplicate page found log id 0x%x," + \
- "page %d", page.log_id, page.page_num)
-
- dictLogs[page.log_id][page.page_num] = page
-
- if versionIsOneOrGreater:
- logging.info(\
- "Found log page - context: %s, id: 0x%x, " + \
- "page: %d", page.context.name, \
- page.log_id, page.page_num)
- page.log_info()
- sys.stdout.flush()
- else:
- logging.info("Found log page: id 0x%x, page %d" % \
- (page.log_id, page.page_num))
- page.log_info()
- sys.stdout.flush()
-
- # for debug mode, save the extracted log pages
- if options.debug_enabled:
- if version >= 1:
- page.debug_save_log_pages(fIn, fileName, data,
- context.name, options)
- else:
- page.debug_save_log_pages(fIn, fileName, data,
- options)
-
- page = LogPageVersionUnknown()
-
- fIn.close()
- fileCount += 1
-
- # Check that log_ids received from contexts and from pages match
- if versionIsOneOrGreater:
- check_log_consistency(dictLogs, dictContexts)
- dumpLogs(dictLogs, version, options)
- else:
- dumpLogs(dictLogs, 0, options)
-
- logging.debug("lstFiles: " + str(lstFiles))
+ """
+ The top-level function, which is called from the main entry point to the
+ program if the ``parse`` command-line option is given. This function
+ creates two dictionaries, ``dictLogs`` and ``dictContexts``, which have log
+ IDs as keys and :class:`LogPage` and :class:`LogContext` objects as values,
+ respectively. It then populates these dictionaries by going through each
+ input file and reading data page-by-page. After all data has been read,
+ this function calls :func:`dumpLogs()` to dump the parsed logs to output
+ files.
+
+ :param options: Configuration options containing options.output_dir for the
+ output directory, options.debug_enabled, a flag for
+ discerning whether the extraction script is running in
+ debug mode, and options.args, the list of input files. All
+ configuration options are passed through to helper
+ functions.
+ """
+ dictLogs = {}
+ dictContexts = {}
+ fileCount = 0
+ version = 0
+ versionIsOneOrGreater = False
+ lstFiles = get_files_info(options.args)
+
+ # scan files for log pages
+ for strFile in options.args:
+ fileName = strFile.split(':', 1)[0]
+ fIn = open(fileName, 'rb')
+
+ logging.info("Parsing '%s'" % (fileName))
+ logging.info("-" * 70)
+ sys.stdout.flush()
+ page = LogPageVersionUnknown()
+ context = LogContext()
+
+ while True:
+ start_of_page = fIn.tell()
+ data = fIn.read(16)
+
+ # PAGE_SIZE - 16 = 4080
+ if not data or len(data) + 4080 < PAGE_SIZE:
+ break
+
+ if page.unpack(data):
+ data += fIn.read(PAGE_SIZE-16)
+
+ if page.isVersionOneOrGreater(data):
+ versionIsOneOrGreater = True
+ page = LogPage_v1()
+ else:
+ versionIsOneOrGreater = False
+ page = LogPage_v0()
+
+ if page.unpack(data):
+ if versionIsOneOrGreater:
+ if page.log_id not in dictContexts:
+ try:
+ context = get_context(start_of_page, fIn, page,
+ dictContexts, lstFiles,
+ fileCount)
+ except:
+ msg = "Context not found - skipping page " + \
+ "and trying to continue with " + \
+ "unknown log page version"
+ logging.debug(msg)
+ page = LogPageVersionUnknown()
+ continue
+
+ else:
+ context = dictContexts[page.log_id]
+ page.setContext(context)
+
+ if context is None:
+ logging.debug("Context at 0x%x had no data, " +
+ "skipping this page",
+ page.context_offset)
+ if options.debug_enabled:
+ page.log_info()
+ page = LogPageVersionUnknown()
+ continue
+
+ version = context.version
+
+ if page.log_id not in dictLogs:
+ dictLogs[page.log_id] = {}
+
+ if page.page_num in dictLogs[page.log_id]:
+ logging.error("Duplicate page found log id 0x%x," +
+ "page %d", page.log_id, page.page_num)
+
+ dictLogs[page.log_id][page.page_num] = page
+
+ if versionIsOneOrGreater:
+ logging.info(
+ "Found log page - context: %s, id: 0x%x, " +
+ "page: %d", page.context.name,
+ page.log_id, page.page_num)
+ page.log_info()
+ sys.stdout.flush()
+ else:
+ logging.info("Found log page: id 0x%x, page %d" %
+ (page.log_id, page.page_num))
+ page.log_info()
+ sys.stdout.flush()
+
+ # for debug mode, save the extracted log pages
+ if options.debug_enabled:
+ if version >= 1:
+ page.debug_save_log_pages(fIn, fileName, data,
+ context.name, options)
+ else:
+ page.debug_save_log_pages(fIn, fileName, data,
+ options)
+
+ page = LogPageVersionUnknown()
+
+ fIn.close()
+ fileCount += 1
+
+ # Check that log_ids received from contexts and from pages match
+ if versionIsOneOrGreater:
+ check_log_consistency(dictLogs, dictContexts)
+ dumpLogs(dictLogs, version, options)
+ else:
+ dumpLogs(dictLogs, 0, options)
+
+ logging.debug("lstFiles: " + str(lstFiles))
+
def dumpLogs(dictLogs, version, options):
- """
- Dump logs from the dictionary of log IDs to logs built by
- :func:`cmdParse()`. This is called at the end of :func:`cmdParse()`,
- after all files have been processed and the dictionary has been built.
-
- :param dictLogs: The dictionary built by :func:`cmdParse()`.
- :param version: The IPC Logging version.
- :param options: Configuration options passed through to helper functions
-
- **Side Effects**: The :func:`dumpLogWithRetry()` function is called, \
- which dumps the parsed logs to output files.
- """
- # dump the logs
- logs = dictLogs.keys()
- logs.sort()
- logging.debug("Logs: %s", logs)
- for key in logs:
- pages = dictLogs[key]
- numPages = len(pages.keys())
-
- if version >= 1:
- first_page_in_log = dictLogs[key].keys()[0]
- name = dictLogs[key][first_page_in_log].context.name
- else:
- name = None
-
- dumpLogWithRetry(key, name, version, pages, numPages, False, options)
+ """
+ Dump logs from the dictionary of log IDs to logs built by
+ :func:`cmdParse()`. This is called at the end of :func:`cmdParse()`,
+ after all files have been processed and the dictionary has been built.
+
+ :param dictLogs: The dictionary built by :func:`cmdParse()`.
+ :param version: The IPC Logging version.
+ :param options: Configuration options passed through to helper functions
+
+ **Side Effects**: The :func:`dumpLogWithRetry()` function is called,
+ which dumps the parsed logs to output files.
+ """
+ # dump the logs
+ logs = dictLogs.keys()
+ logs.sort()
+ logging.debug("Logs: %s", logs)
+ for key in logs:
+ pages = dictLogs[key]
+ numPages = len(pages.keys())
+
+ if version >= 1:
+ first_page_in_log = dictLogs[key].keys()[0]
+ name = dictLogs[key][first_page_in_log].context.name
+ else:
+ name = None
+
+ dumpLogWithRetry(key, name, version, pages, numPages, False, options)
+
def cmdTest(options):
- """
- Parses the provided log page files in order and extracts the logs. This is
- useful for testing and for dealing with failure cases (such as duplicate
- logs due to left-over log pages from previous boot cycles).
-
- :param options: Configuration options containing options.output_dir for
- the output directory, and options.args, the list of input
- files, and options.arch_64, the flag indicating if the log
- pages should be interpreted as 64-bit dumps or not. All
- configuration options are passed through to helper
- functions.
- """
- dictPages = {}
- version = 0
- numPages = 0
-
- # load existing files
- lstFiles = options.args
-
- try:
- version = int(lstFiles.pop())
- except ValueError as e:
- strMsg = "Version must be provided! Exiting..."
- logging.error("Exception: %s\n%s\n" % (str(e), strMsg))
- raise
-
- for f in lstFiles:
- data = open(f, 'rb').read()
-
- page = LogPageVersionUnknown()
- assert page.unpack(data) == True, "Unable to open file '%s'" % (f)
-
- if page.isVersionOneOrGreater(data):
- page = LogPage_v1()
- page.unpack(data)
- else:
- page = LogPage_v0()
- page.unpack(data)
-
- numPages += 1
- dictPages[page.page_num] = page
- logging.info("Loaded '%s' log id %d, page %d" % \
- (f, page.log_id, page.page_num))
- page.log_info()
-
- try:
- if options.arch_64:
- page.page_header_size = PAGE_HDR_SIZES_64[version]
- else:
- page.page_header_size = PAGE_HDR_SIZES[version]
- except IndexError as e:
- strMsg = "Invalid version! Exiting..."
- logging.error("Exception: %s\n%s\n" % (str(e), strMsg))
- raise
-
- # Use only the last pathname component, as the rest of the path is
- # added in dumpLogWithRetry()
- if options.output_dir:
- filename = os.path.split(lstFiles[0])[1]
- else:
- filename = lstFiles[0]
-
- # dump the logs (in the same order provided)
- dumpLogWithRetry(page.log_id, 'test-log-%s' % (filename),
- version, dictPages, numPages, True, options)
+ """
+ Parses the provided log page files in order and extracts the logs. This is
+ useful for testing and for dealing with failure cases (such as duplicate
+ logs due to left-over log pages from previous boot cycles).
+
+ :param options: Configuration options containing options.output_dir for
+ the output directory, and options.args, the list of input
+ files, and options.arch_64, the flag indicating if the log
+ pages should be interpreted as 64-bit dumps or not. All
+ configuration options are passed through to helper
+ functions.
+ """
+ dictPages = {}
+ version = 0
+ numPages = 0
+
+ # load existing files
+ lstFiles = options.args
+
+ try:
+ version = int(lstFiles.pop())
+ except ValueError as e:
+ strMsg = "Version must be provided! Exiting..."
+ logging.error("Exception: %s\n%s\n" % (str(e), strMsg))
+ raise
+
+ for f in lstFiles:
+ data = open(f, 'rb').read()
+
+ page = LogPageVersionUnknown()
+ assert page.unpack(data) is True, "Unable to open file '%s'" % (f)
+
+ if page.isVersionOneOrGreater(data):
+ page = LogPage_v1()
+ page.unpack(data)
+ else:
+ page = LogPage_v0()
+ page.unpack(data)
+
+ numPages += 1
+ dictPages[page.page_num] = page
+ logging.info("Loaded '%s' log id %d, page %d" %
+ (f, page.log_id, page.page_num))
+ page.log_info()
+
+ try:
+ if options.arch_64:
+ page.page_header_size = PAGE_HDR_SIZES_64[version]
+ else:
+ page.page_header_size = PAGE_HDR_SIZES[version]
+ except IndexError as e:
+ strMsg = "Invalid version! Exiting..."
+ logging.error("Exception: %s\n%s\n" % (str(e), strMsg))
+ raise
+
+ # Use only the last pathname component, as the rest of the path is
+ # added in dumpLogWithRetry()
+ if options.output_dir:
+ filename = os.path.split(lstFiles[0])[1]
+ else:
+ filename = lstFiles[0]
+
+ # dump the logs (in the same order provided)
+ dumpLogWithRetry(page.log_id, 'test-log-%s' % (filename),
+ version, dictPages, numPages, True, options)
+
class LoggingFormatter(logging.Formatter):
- """
- Custom logging formatter that prints all INFO messages without formatting
- and all other messages with the appropriate level name.
+ """
+ Custom logging formatter that prints all INFO messages without formatting
+ and all other messages with the appropriate level name.
+
+ See :mod:`logging` documentation for more info on Python logging.
+ """
+ infoFormat = '%(message)s'
+ defaultFormat = '%(levelname)s %(module)s:%(lineno)d: %(message)s'
- See :mod:`logging` documentation for more info on Python logging.
- """
- infoFormat = '%(message)s'
- defaultFormat = '%(levelname)s %(module)s:%(lineno)d: %(message)s'
+ def __init__(self, fmt=defaultFormat):
+ logging.Formatter.__init__(self, fmt)
- def __init__(self, fmt=defaultFormat):
- logging.Formatter.__init__(self, fmt)
+ def format(self, record):
+ if record.levelno == logging.INFO:
+ self._fmt = self.infoFormat
+ else:
+ self._fmt = self.defaultFormat
+ return logging.Formatter.format(self, record)
- def format(self, record):
- if record.levelno == logging.INFO:
- self._fmt = self.infoFormat
- else:
- self._fmt = self.defaultFormat
- return logging.Formatter.format(self, record)
def configure_logging(options, stdout):
- """
- Configure the logging options.
-
- :param options: The options object, which contains options.verbosity for
- the logging verbosity and options.debug_enabled to indicate
- whether the extraction script is running in debug mode.
- :param stdout: Whether or not to create a logging handler for stdout
- """
-
- if stdout:
- loggingHandler = logging.StreamHandler(sys.stdout)
- loggingHandler.setFormatter(LoggingFormatter())
- logging.root.addHandler(loggingHandler)
-
- # Set Log level
- LOG_LEVELS = { 3: logging.DEBUG,
- 2: logging.INFO,
- 1: logging.WARNING,
- 0: logging.ERROR,
- }
- assert LOG_LEVELS.has_key(options.verbosity), "Unknown log level %d" % \
- (options.verbosity)
- logging.root.setLevel(LOG_LEVELS[options.verbosity])
-
- if logging.root.getEffectiveLevel >= 2 and options.quiet:
- logging.root.setLevel(LOG_LEVELS[1])
-
- options.debug_enabled = False
- if options.verbosity >= 3:
- options.debug_enabled = True
+ """
+ Configure the logging options.
+
+ :param options: The options object, which contains options.verbosity for
+ the logging verbosity and options.debug_enabled to indicate
+ whether the extraction script is running in debug mode.
+ :param stdout: Whether or not to create a logging handler for stdout
+ """
+
+ if stdout:
+ loggingHandler = logging.StreamHandler(sys.stdout)
+ loggingHandler.setFormatter(LoggingFormatter())
+ logging.root.addHandler(loggingHandler)
+
+ # Set Log level
+ LOG_LEVELS = {3: logging.DEBUG,
+ 2: logging.INFO,
+ 1: logging.WARNING,
+ 0: logging.ERROR}
+
+ assert options.verbosity in LOG_LEVELS, "Unknown log level %d" % \
+ (options.verbosity)
+ logging.root.setLevel(LOG_LEVELS[options.verbosity])
+
+ if logging.root.getEffectiveLevel >= 2 and options.quiet:
+ logging.root.setLevel(LOG_LEVELS[1])
+
+ options.debug_enabled = False
+ if options.verbosity >= 3:
+ options.debug_enabled = True
+
def set_output_directory(options):
- """
- Set up the output directory.
+ """
+ Set up the output directory.
- :param options: Configuration options containing options.output_dir for the
- output directory
+ :param options: Configuration options containing options.output_dir for the
+ output directory
- :return: The output logging handler object, or None if no output directory
- was provided by the user.
- """
- if not options.output_dir:
- return None
+ :return: The output logging handler object, or None if no output directory
+ was provided by the user.
+ """
+ if not options.output_dir:
+ return None
- options.output_dir = \
- os.path.abspath(os.path.expanduser(options.output_dir))
+ options.output_dir = \
+ os.path.abspath(os.path.expanduser(options.output_dir))
- # Create output directory if it does not exist
- if not os.path.isdir(options.output_dir):
- os.makedirs(options.output_dir)
+ # Create output directory if it does not exist
+ if not os.path.isdir(options.output_dir):
+ os.makedirs(options.output_dir)
- # Create logging handler to take output in this directory
- output_logging_handler = \
- logging.FileHandler(\
- os.path.join(options.output_dir, "ipc_extraction_report.txt"),
- "w")
- output_logging_handler.setFormatter(LoggingFormatter())
- logging.root.addHandler(output_logging_handler)
+ # Create logging handler to take output in this directory
+ output_logging_handler = \
+ logging.FileHandler(
+ os.path.join(options.output_dir, "ipc_extraction_report.txt"),
+ "w")
+ output_logging_handler.setFormatter(LoggingFormatter())
+ logging.root.addHandler(output_logging_handler)
- logging.info("Output path: " + options.output_dir)
- return output_logging_handler
+ logging.info("Output path: " + options.output_dir)
+ return output_logging_handler
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
# Main Program Entry Point
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
if __name__ == '__main__':
- # Parse command line
- strUsage = """\
+ # Parse command line
+ strUsage = """\
%prog CMD [OPTIONS]
parse
@@ -1726,13 +1762,12 @@ Examples:
\t%prog parse DDRCS0.BIN DDRCS1.BIN
\t%prog parse DDRCS0_0.BIN DDRCS0_1.BIN DDRCS1_0.BIN DDRCS1_1.BIN
\t%prog parse DDRCS0_0.BIN DDRCS0_1.BIN DDRCS1_0.BIN:0x80000000 DDRCS1_1.BIN
-\t%prog parse DDRCS0_0.BIN:0x00000000 DDRCS0_1.BIN:0x30000000 DDRCS1_0.BIN:0x80000000 DDRCS1_1.BIN:0xb0000000
It is only necessary to provide the starting address of a memory dump file if
-the dumps are not contiguous in memory. Whether or not the dumps are contiguous,
-along with their starting addresses, can be found in the file load.cmm which
-is usually included in crash dumps. Starting addresses must be provided in
-hexadecimal.
+the dumps are not contiguous in memory. Whether or not the dumps are
+contiguous, along with their starting addresses, can be found in the file
+load.cmm which is usually included in crash dumps. Starting addresses must be
+provided in hexadecimal.
test
----
@@ -1764,8 +1799,8 @@ Verbosity
Debug logging can be enabled using the count -v or --verbose option. The
log level based upon the number of occurrences of the -v option will be one of
the following:
- ERROR/WARNING/INFO Default
- DEBUG (-v)
+ ERROR/WARNING/INFO Default
+ DEBUG (-v)
Quiet
-----
@@ -1793,58 +1828,57 @@ Examples:
\t%prog parse -o ~/test_output DDRCS0.BIN DDRCS1.BIN
\t%prog test -v -o ~/test_output LOG_PAGE_FILE(s)
"""
- parser = OptionParser(usage=strUsage)
- if len(sys.argv) < 2:
- parser.error('Command required')
+ parser = OptionParser(usage=strUsage)
+ if len(sys.argv) < 2:
+ parser.error('Command required')
- strCmd = (sys.argv[1]).lower()
+ strCmd = (sys.argv[1]).lower()
- parser.add_option("-v", "--verbose",
- action="count", dest="verbosity", default=2,
- help="Log program status to stdout (-v = debug)"
- )
+ parser.add_option("-v", "--verbose",
+ action="count", dest="verbosity", default=2,
+ help="Log program status to stdout (-v = debug)")
- parser.add_option("-q", "--quiet",
- action="count", dest="quiet", default=False,
- help="Log no program status to stdout (-q = quiet)"
- )
+ parser.add_option("-q", "--quiet",
+ action="count", dest="quiet", default=False,
+ help="Log no program status to stdout (-q = quiet)")
- parser.add_option("-o", "--output",
- action="store", type="string", dest="output_dir",
- help="If the current directory can't be written to, specify \
- a path for an output directory. The path is \
- OS-independent, and can be absolute or relative.")
+ parser.add_option("-o", "--output",
+ action="store", type="string", dest="output_dir",
+ help="If the current directory can't be written to, \
+ specify a path for an output directory. The path \
+ is OS-independent, and can be absolute or \
+ relative.")
- parser.add_option("", "--64-bit",
- action="store_true", dest="arch_64",
- help="For use with the test command. Interpret the log pages as \
- 64-bit dumps.")
+ parser.add_option("", "--64-bit",
+ action="store_true", dest="arch_64",
+ help="For use with the test command. Interpret the log \
+ pages as 64-bit dumps.")
- (options, args) = parser.parse_args(sys.argv[2:])
- options.cmd = strCmd
+ (options, args) = parser.parse_args(sys.argv[2:])
+ options.cmd = strCmd
- if args:
- # extract positions args (args not associated with flags)
- options.args = args
+ if args:
+ # extract positions args (args not associated with flags)
+ options.args = args
- # Configure logging format
- configure_logging(options, True)
+ # Configure logging format
+ configure_logging(options, True)
- if not options.cmd or options.cmd == "help":
- parser.error('No command/help specified')
- exit(1)
+ if not options.cmd or options.cmd == "help":
+ parser.error('No command/help specified')
+ exit(1)
- output_logging_handler = set_output_directory(options)
+ output_logging_handler = set_output_directory(options)
- dictCmds = {'parse': cmdParse,
- 'test': cmdTest,
- }
+ dictCmds = {'parse': cmdParse,
+ 'test': cmdTest,
+ }
- # run the requested command
- if not dictCmds.has_key(options.cmd):
- parser.error("Unknown command '%s'" % (options.cmd))
+ # run the requested command
+ if options.cmd not in dictCmds:
+ parser.error("Unknown command '%s'" % (options.cmd))
- dictCmds[options.cmd](options)
+ dictCmds[options.cmd](options)
- if options.output_dir:
- output_logging_handler.close()
+ if options.output_dir:
+ output_logging_handler.close()
--
GitLab