From 63068ef1868a86a8ece28626ce82655b9879dbfe Mon Sep 17 00:00:00 2001
From: GuangHui Liu <ghliu@google.com>
Date: Wed, 4 Oct 2017 14:03:25 -0700
Subject: [PATCH] A class to parse trace format.
Bug: 36368025
Test: unit tests
Change-Id: I87c8c32bb7baf730a49bfaf730a8714ac1cc59ce
---
.../loganalysis/item/TraceFormatItem.java | 118 +++++++++
.../loganalysis/parser/TraceFormatParser.java | 158 ++++++++++++
.../parser/TraceFormatParserTest.java | 227 ++++++++++++++++++
3 files changed, 503 insertions(+)
create mode 100644 src/com/android/loganalysis/item/TraceFormatItem.java
create mode 100644 src/com/android/loganalysis/parser/TraceFormatParser.java
create mode 100644 tests/src/com/android/loganalysis/parser/TraceFormatParserTest.java
diff --git a/src/com/android/loganalysis/item/TraceFormatItem.java b/src/com/android/loganalysis/item/TraceFormatItem.java
new file mode 100644
index 0000000..82944d5
--- /dev/null
+++ b/src/com/android/loganalysis/item/TraceFormatItem.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.item;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/** A {@link GenericItem} of trace format. */
+public class TraceFormatItem extends GenericItem {
+ private static final String REGEX = "regex";
+ private static final String PARAMS = "params";
+ private static final String NUM_PARAMS = "num_params";
+ private static final String HEX_PARAMS = "hex_params";
+ private static final String STR_PARAMS = "str_params";
+ private static final Set<String> ATTRIBUTES =
+ new HashSet<>(Arrays.asList(REGEX, PARAMS, NUM_PARAMS, HEX_PARAMS, STR_PARAMS));
+
+ /** Create a new {@link TraceFormatItem} */
+ public TraceFormatItem() {
+ super(ATTRIBUTES);
+ }
+
+ @Override
+ /** TraceFormatItem doesn't support merge */
+ public IItem merge(IItem other) throws ConflictingItemException {
+ throw new ConflictingItemException("Trace format items cannot be merged");
+ }
+
+ /** Get a compiled regex that matches output of this trace format */
+ public Pattern getRegex() {
+ return (Pattern) getAttribute(REGEX);
+ }
+
+ /** Set a compiled regex that matches output of this trace format */
+ public void setRegex(Pattern regex) {
+ setAttribute(REGEX, regex);
+ }
+
+ /**
+ * Get all parameters found in this trace format. The parameters were converted to camel case
+ * and match the group names in the regex.
+ */
+ public List<String> getParameters() {
+ return (List<String>) getAttribute(PARAMS);
+ }
+
+ /**
+ * Set all parameters found in this trace format. The parameters were converted to camel case
+ * and match the group names in the regex.
+ */
+ public void setParameters(List<String> parameters) {
+ setAttribute(PARAMS, parameters);
+ }
+
+ /**
+ * Get numeric parameters found in this trace format. The parameters were converted to camel
+ * case and match the group names in the regex.
+ */
+ public List<String> getNumericParameters() {
+ return (List<String>) getAttribute(NUM_PARAMS);
+ }
+
+ /**
+ * Set numeric parameters found in this trace format. The parameters were converted to camel
+ * case and match the group names in the regex.
+ */
+ public void setNumericParameters(List<String> numericParameters) {
+ setAttribute(NUM_PARAMS, numericParameters);
+ }
+
+ /**
+ * Get hexadecimal parameters found in this trace format. The parameters were converted to camel
+ * case and match the group names in the regex.
+ */
+ public List<String> getHexParameters() {
+ return (List<String>) getAttribute(HEX_PARAMS);
+ }
+
+ /**
+ * Set hexadecimal parameters found in this trace format. The parameters were converted to camel
+ * case and match the group names in the regex.
+ */
+ public void setHexParameters(List<String> hexParameters) {
+ setAttribute(HEX_PARAMS, hexParameters);
+ }
+
+ /**
+ * Get string parameters found in this trace format. The parameters were converted to camel case
+ * and match the group names in the regex.
+ */
+ public List<String> getStringParameters() {
+ return (List<String>) getAttribute(STR_PARAMS);
+ }
+
+ /**
+ * Set string parameters found in this trace format. The parameters were converted to camel case
+ * and match the group names in the regex.
+ */
+ public void setStringParameters(List<String> stringParameters) {
+ setAttribute(STR_PARAMS, stringParameters);
+ }
+}
diff --git a/src/com/android/loganalysis/parser/TraceFormatParser.java b/src/com/android/loganalysis/parser/TraceFormatParser.java
new file mode 100644
index 0000000..1c444f4
--- /dev/null
+++ b/src/com/android/loganalysis/parser/TraceFormatParser.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.parser;
+
+import com.android.loganalysis.item.TraceFormatItem;
+
+import com.google.common.base.CaseFormat;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Read trace format and generate a regex that matches output of such format.
+ *
+ * <p>Traces under /d/tracing specify the output format with a printf string. This parser reads such
+ * string, finds all parameters, and generates a regex that matches output of such format. Each
+ * parameter corresponds to a named-capturing group in the regex. The parameter names are converted
+ * to camel case because Java regex group name must contain only letters and numbers.
+ *
+ * <p>An end-to-end example:
+ *
+ * <pre>{@code
+ * List<String> formatLine = Arrays.asList("print fmt: \"foo=%llu, bar:%s\", REC->foo, REC->bar");
+ * TraceFormatItem parsedFormat = new TraceFormatParser.parse(formatLine);
+ * parsedFormat.getParameters(); // "foo", "bar"
+ * parsedFormat.getNumericParameters(); // "foo"
+ * Matcher matcher = parsedFormat.getRegex.matcher("foo=123, bar:enabled");
+ * matcher.matches();
+ * matcher.group("foo") // 123
+ * matcher.group("bar") // "enabled"
+ * }</pre>
+ *
+ * <p>The initial implementation supports some commonly used specifiers: signed and unsigned integer
+ * (with or without long or long long modifier), floating point number (with or without precision),
+ * hexadecimal number (with or without 0's padding), and string (contains only [a-zA-Z_0-9]). It is
+ * assumed no characters found in the format line need to be escaped.
+ *
+ * <p>Some examples of trace format line:
+ *
+ * <p>(thermal/tsens_read)
+ *
+ * <p>print fmt: "temp=%lu sensor=tsens_tz_sensor%u", REC->temp, REC->sensor
+ *
+ * <p>(sched/sched_cpu_hotplug)
+ *
+ * <p>print fmt: "cpu %d %s error=%d", REC->affected_cpu, REC->status ? "online" : "offline",
+ * REC->error
+ *
+ * <p>(mmc/mmc_blk_erase_start)
+ *
+ * <p>print fmt: "cmd=%u,addr=0x%08x,size=0x%08x", REC->cmd, REC->addr, REC->size
+ */
+public class TraceFormatParser implements IParser {
+ // split the raw format line
+ private static final Pattern SPLIT_FORMAT_LINE =
+ Pattern.compile(".*?\"(?<printf>.*?)\"(?<params>.*)");
+ // match parameter names
+ private static final Pattern SPLIT_PARAMS = Pattern.compile("->(?<param>\\w+)");
+ // match and categorize common printf specifiers
+ // use ?: to flag all non-capturing group so any group captured correspond to a specifier
+ private static final Pattern PRINTF_SPECIFIERS =
+ Pattern.compile(
+ "(?<num>%(?:llu|lu|u|lld|ld|d|(?:.\\d*)?f))|(?<hex>%\\d*(?:x|X))|(?<str>%s)");
+
+ // regex building blocks to match simple numeric/hex/string parameters, exposed for unit testing
+ static final String MATCH_NUM = "-?\\\\d+(?:\\\\.\\\\d+)?";
+ static final String MATCH_HEX = "[\\\\da-fA-F]+";
+ static final String MATCH_STR = "[\\\\w]*";
+
+ /** Parse a trace format line and return an {@link TraceFormatItem} */
+ @Override
+ public TraceFormatItem parse(List<String> lines) {
+ // sanity check
+ if (lines == null || lines.size() != 1) {
+ throw new RuntimeException("Cannot parse format line: expect one-line trace format");
+ }
+
+ // split the raw format line
+ Matcher formatLineMatcher = SPLIT_FORMAT_LINE.matcher(lines.get(0));
+ if (!formatLineMatcher.matches()) {
+ throw new RuntimeException("Cannot parse format line: unexpected format");
+ }
+ String printfString = formatLineMatcher.group("printf");
+ String paramsString = formatLineMatcher.group("params");
+
+ // list of parameters, to be populated soon
+ List<String> allParams = new ArrayList<>();
+ List<String> numParams = new ArrayList<>();
+ List<String> hexParams = new ArrayList<>();
+ List<String> strParams = new ArrayList<>();
+
+ // find all parameters and convert them to camel case
+ Matcher paramsMatcher = SPLIT_PARAMS.matcher(paramsString);
+ while (paramsMatcher.find()) {
+ String camelCasedParam =
+ CaseFormat.LOWER_UNDERSCORE.to(
+ CaseFormat.LOWER_CAMEL, paramsMatcher.group("param"));
+ allParams.add(camelCasedParam);
+ }
+
+ // scan the printf string, categorizing parameters and generating a matching regex
+ StringBuffer regexBuilder = new StringBuffer();
+ int paramIndex = 0;
+ String currentParam;
+
+ Matcher printfMatcher = PRINTF_SPECIFIERS.matcher(printfString);
+ while (printfMatcher.find()) {
+ // parameter corresponds to the found specifier
+ currentParam = allParams.get(paramIndex++);
+ if (printfMatcher.group("num") != null) {
+ printfMatcher.appendReplacement(
+ regexBuilder, createNamedRegexGroup(MATCH_NUM, currentParam));
+ numParams.add(currentParam);
+ } else if (printfMatcher.group("hex") != null) {
+ printfMatcher.appendReplacement(
+ regexBuilder, createNamedRegexGroup(MATCH_HEX, currentParam));
+ hexParams.add(currentParam);
+ } else if (printfMatcher.group("str") != null) {
+ printfMatcher.appendReplacement(
+ regexBuilder, createNamedRegexGroup(MATCH_STR, currentParam));
+ strParams.add(currentParam);
+ } else {
+ throw new RuntimeException("Unrecognized specifier: " + printfMatcher.group());
+ }
+ }
+ printfMatcher.appendTail(regexBuilder);
+ Pattern generatedRegex = Pattern.compile(regexBuilder.toString());
+
+ // assemble and return a TraceFormatItem
+ TraceFormatItem item = new TraceFormatItem();
+ item.setRegex(generatedRegex);
+ item.setParameters(allParams);
+ item.setNumericParameters(numParams);
+ item.setHexParameters(hexParams);
+ item.setStringParameters(strParams);
+ return item;
+ }
+
+ /** Helper function to create a regex group with given name. */
+ private static String createNamedRegexGroup(String base, String name) {
+ return String.format("(?<%s>%s)", name, base);
+ }
+}
diff --git a/tests/src/com/android/loganalysis/parser/TraceFormatParserTest.java b/tests/src/com/android/loganalysis/parser/TraceFormatParserTest.java
new file mode 100644
index 0000000..06ce9ec
--- /dev/null
+++ b/tests/src/com/android/loganalysis/parser/TraceFormatParserTest.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.parser;
+
+import static org.junit.Assert.fail;
+
+import com.android.loganalysis.item.TraceFormatItem;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+
+/** Test for {@link TraceFormatParser}. */
+@RunWith(JUnit4.class)
+public class TraceFormatParserTest {
+ private TraceFormatParser mParser;
+
+ // "unwrap" the regex strings so that we can compare with the generated regex
+ private static final String MATCH_NUM_UNESCAPED =
+ TraceFormatParser.MATCH_NUM.replaceAll("\\\\\\\\", "\\\\");
+ private static final String MATCH_HEX_UNESCAPED =
+ TraceFormatParser.MATCH_HEX.replaceAll("\\\\\\\\", "\\\\");
+ private static final String MATCH_STR_UNESCAPED =
+ TraceFormatParser.MATCH_STR.replaceAll("\\\\\\\\", "\\\\");
+
+ @Before
+ public void setUp() {
+ mParser = new TraceFormatParser();
+ }
+
+ @Test
+ public void testParseFormatLine() {
+ List<String> formatLine =
+ Arrays.asList("print fmt: \"foo=%llu, bar=%s\", REC->foo, REC->bar");
+ String expectedRegex =
+ String.format(
+ "foo=(?<foo>%s), bar=(?<bar>%s)", MATCH_NUM_UNESCAPED, MATCH_STR_UNESCAPED);
+ List<String> expectedParameters = Arrays.asList("foo", "bar");
+ List<String> expectedNumericParameters = Arrays.asList("foo");
+ List<String> expectedHexParameters = Arrays.asList();
+ List<String> expectedStringParameters = Arrays.asList("bar");
+ String shouldMatch = "foo=123, bar=enabled";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+ Assert.assertEquals(expectedNumericParameters, parsedItem.getNumericParameters());
+ Assert.assertEquals(expectedHexParameters, parsedItem.getHexParameters());
+ Assert.assertEquals(expectedStringParameters, parsedItem.getStringParameters());
+ Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "123");
+ Assert.assertEquals(m.group("bar"), "enabled");
+ }
+
+ @Test
+ public void testNoParameters() {
+ List<String> formatLine = Arrays.asList("print fmt: \"foo\"");
+ String expectedRegex = "foo";
+ List<String> expectedParameters = Arrays.asList();
+ String shouldMatch = "foo";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+ Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ }
+
+ @Test
+ public void testNullInput() {
+ try {
+ mParser.parse(null);
+ fail("Expected an exception thrown by TraceFormatParser");
+ } catch (RuntimeException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testEmptyInput() {
+ List<String> formatLine = Arrays.asList("");
+ try {
+ mParser.parse(formatLine);
+ fail("Expected an exception thrown by TraceFormatParser");
+ } catch (RuntimeException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMultiLineInput() {
+ List<String> formatLine = Arrays.asList("foo", "bar");
+ try {
+ mParser.parse(formatLine);
+ fail("Expected an exception thrown by TraceFormatParser");
+ } catch (RuntimeException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOneLineInvalidInput() {
+ List<String> formatLine = Arrays.asList("foo bar");
+ try {
+ mParser.parse(formatLine);
+ fail("Expected an exception thrown by TraceFormatParser");
+ } catch (RuntimeException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testQuoteInParams() {
+ List<String> formatLine =
+ Arrays.asList("print fmt: \"foo %s\", REC->foo ? \"online\" : \"offline\"");
+ String expectedRegex = String.format("foo (?<foo>%s)", MATCH_STR_UNESCAPED);
+ String shouldMatch = "foo online";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "online");
+ }
+
+ @Test
+ public void testCategorizeParameters() {
+ List<String> formatLine =
+ Arrays.asList(
+ "print fmt: \"num1=%lu, num2=%f, hex=%08x, str=%s\", REC->num1, REC->num2, REC->hex, REC->str");
+ List<String> expectedNumericParameters = Arrays.asList("num1", "num2");
+ List<String> expectedHexParameters = Arrays.asList("hex");
+ List<String> expectedStringParameters = Arrays.asList("str");
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Assert.assertEquals(expectedNumericParameters, parsedItem.getNumericParameters());
+ Assert.assertEquals(expectedHexParameters, parsedItem.getHexParameters());
+ Assert.assertEquals(expectedStringParameters, parsedItem.getStringParameters());
+ }
+
+ @Test
+ public void testCaseConvertParameterName() {
+ List<String> formatLine = Arrays.asList("print fmt: \"foo_bar=%llu\", REC->foo_bar");
+ List<String> expectedParameters = Arrays.asList("fooBar");
+ String shouldMatch = "foo_bar=123";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("fooBar"), "123");
+ }
+
+ @Test
+ public void testMatchInt() {
+ List<String> formatLine =
+ Arrays.asList("print fmt: \"foo=%d, bar=%lu\", REC->foo, REC->bar");
+ String shouldMatch = "foo=-123, bar=456";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "-123");
+ Assert.assertEquals(m.group("bar"), "456");
+ }
+
+ @Test
+ public void testMatchFloat() {
+ List<String> formatLine =
+ Arrays.asList("print fmt: \"foo=%f, bar=%.2f\", REC->foo, REC->bar");
+ String shouldMatch = "foo=123.4567, bar=456.78";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "123.4567");
+ Assert.assertEquals(m.group("bar"), "456.78");
+ }
+
+ @Test
+ public void testMatchHex() {
+ List<String> formatLine =
+ Arrays.asList(
+ "print fmt: \"foo=0x%04x, bar=0x%08X, baz=%x\", REC->foo, REC->bar, REC->baz");
+ String shouldMatch = "foo=0x007b, bar=0x000001C8, baz=7b";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "007b");
+ Assert.assertEquals(m.group("bar"), "000001C8");
+ Assert.assertEquals(m.group("baz"), "7b");
+ }
+
+ @Test
+ public void testMatchString() {
+ List<String> formatLine =
+ Arrays.asList("print fmt: \"foo=%s, bar=%s\", REC->foo, REC->bar");
+ String shouldMatch = "foo=oof, bar=123";
+
+ TraceFormatItem parsedItem = mParser.parse(formatLine);
+ Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+ Assert.assertTrue(m.matches());
+ Assert.assertEquals(m.group("foo"), "oof");
+ Assert.assertEquals(m.group("bar"), "123");
+ }
+}
--
GitLab