A class to parse trace format.

Bug: 36368025
Test: unit tests
Change-Id: I87c8c32bb7baf730a49bfaf730a8714ac1cc59ce

src/com/android/loganalysis/item/TraceFormatItem.java

+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.item;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/** A {@link GenericItem} of trace format. */
+public class TraceFormatItem extends GenericItem {
+    private static final String REGEX = "regex";
+    private static final String PARAMS = "params";
+    private static final String NUM_PARAMS = "num_params";
+    private static final String HEX_PARAMS = "hex_params";
+    private static final String STR_PARAMS = "str_params";
+    private static final Set<String> ATTRIBUTES =
+            new HashSet<>(Arrays.asList(REGEX, PARAMS, NUM_PARAMS, HEX_PARAMS, STR_PARAMS));
+
+    /** Create a new {@link TraceFormatItem} */
+    public TraceFormatItem() {
+        super(ATTRIBUTES);
+    }
+
+    @Override
+    /** TraceFormatItem doesn't support merge */
+    public IItem merge(IItem other) throws ConflictingItemException {
+        throw new ConflictingItemException("Trace format items cannot be merged");
+    }
+
+    /** Get a compiled regex that matches output of this trace format */
+    public Pattern getRegex() {
+        return (Pattern) getAttribute(REGEX);
+    }
+
+    /** Set a compiled regex that matches output of this trace format */
+    public void setRegex(Pattern regex) {
+        setAttribute(REGEX, regex);
+    }
+
+    /**
+     * Get all parameters found in this trace format. The parameters were converted to camel case
+     * and match the group names in the regex.
+     */
+    public List<String> getParameters() {
+        return (List<String>) getAttribute(PARAMS);
+    }
+
+    /**
+     * Set all parameters found in this trace format. The parameters were converted to camel case
+     * and match the group names in the regex.
+     */
+    public void setParameters(List<String> parameters) {
+        setAttribute(PARAMS, parameters);
+    }
+
+    /**
+     * Get numeric parameters found in this trace format. The parameters were converted to camel
+     * case and match the group names in the regex.
+     */
+    public List<String> getNumericParameters() {
+        return (List<String>) getAttribute(NUM_PARAMS);
+    }
+
+    /**
+     * Set numeric parameters found in this trace format. The parameters were converted to camel
+     * case and match the group names in the regex.
+     */
+    public void setNumericParameters(List<String> numericParameters) {
+        setAttribute(NUM_PARAMS, numericParameters);
+    }
+
+    /**
+     * Get hexadecimal parameters found in this trace format. The parameters were converted to camel
+     * case and match the group names in the regex.
+     */
+    public List<String> getHexParameters() {
+        return (List<String>) getAttribute(HEX_PARAMS);
+    }
+
+    /**
+     * Set hexadecimal parameters found in this trace format. The parameters were converted to camel
+     * case and match the group names in the regex.
+     */
+    public void setHexParameters(List<String> hexParameters) {
+        setAttribute(HEX_PARAMS, hexParameters);
+    }
+
+    /**
+     * Get string parameters found in this trace format. The parameters were converted to camel case
+     * and match the group names in the regex.
+     */
+    public List<String> getStringParameters() {
+        return (List<String>) getAttribute(STR_PARAMS);
+    }
+
+    /**
+     * Set string parameters found in this trace format. The parameters were converted to camel case
+     * and match the group names in the regex.
+     */
+    public void setStringParameters(List<String> stringParameters) {
+        setAttribute(STR_PARAMS, stringParameters);
+    }
+}

src/com/android/loganalysis/parser/TraceFormatParser.java

+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.parser;
+
+import com.android.loganalysis.item.TraceFormatItem;
+
+import com.google.common.base.CaseFormat;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Read trace format and generate a regex that matches output of such format.
+ *
+ * <p>Traces under /d/tracing specify the output format with a printf string. This parser reads such
+ * string, finds all parameters, and generates a regex that matches output of such format. Each
+ * parameter corresponds to a named-capturing group in the regex. The parameter names are converted
+ * to camel case because Java regex group name must contain only letters and numbers.
+ *
+ * <p>An end-to-end example:
+ *
+ * <pre>{@code
+ * List<String> formatLine = Arrays.asList("print fmt: \"foo=%llu, bar:%s\", REC->foo, REC->bar");
+ * TraceFormatItem parsedFormat = new TraceFormatParser.parse(formatLine);
+ * parsedFormat.getParameters(); // "foo", "bar"
+ * parsedFormat.getNumericParameters(); // "foo"
+ * Matcher matcher = parsedFormat.getRegex.matcher("foo=123, bar:enabled");
+ * matcher.matches();
+ * matcher.group("foo") // 123
+ * matcher.group("bar") // "enabled"
+ * }</pre>
+ *
+ * <p>The initial implementation supports some commonly used specifiers: signed and unsigned integer
+ * (with or without long or long long modifier), floating point number (with or without precision),
+ * hexadecimal number (with or without 0's padding), and string (contains only [a-zA-Z_0-9]). It is
+ * assumed no characters found in the format line need to be escaped.
+ *
+ * <p>Some examples of trace format line:
+ *
+ * <p>(thermal/tsens_read)
+ *
+ * <p>print fmt: "temp=%lu sensor=tsens_tz_sensor%u", REC->temp, REC->sensor
+ *
+ * <p>(sched/sched_cpu_hotplug)
+ *
+ * <p>print fmt: "cpu %d %s error=%d", REC->affected_cpu, REC->status ? "online" : "offline",
+ * REC->error
+ *
+ * <p>(mmc/mmc_blk_erase_start)
+ *
+ * <p>print fmt: "cmd=%u,addr=0x%08x,size=0x%08x", REC->cmd, REC->addr, REC->size
+ */
+public class TraceFormatParser implements IParser {
+    // split the raw format line
+    private static final Pattern SPLIT_FORMAT_LINE =
+            Pattern.compile(".*?\"(?<printf>.*?)\"(?<params>.*)");
+    // match parameter names
+    private static final Pattern SPLIT_PARAMS = Pattern.compile("->(?<param>\\w+)");
+    // match and categorize common printf specifiers
+    // use ?: to flag all non-capturing group so any group captured correspond to a specifier
+    private static final Pattern PRINTF_SPECIFIERS =
+            Pattern.compile(
+                    "(?<num>%(?:llu|lu|u|lld|ld|d|(?:.\\d*)?f))|(?<hex>%\\d*(?:x|X))|(?<str>%s)");
+
+    // regex building blocks to match simple numeric/hex/string parameters, exposed for unit testing
+    static final String MATCH_NUM = "-?\\\\d+(?:\\\\.\\\\d+)?";
+    static final String MATCH_HEX = "[\\\\da-fA-F]+";
+    static final String MATCH_STR = "[\\\\w]*";
+
+    /** Parse a trace format line and return an {@link TraceFormatItem} */
+    @Override
+    public TraceFormatItem parse(List<String> lines) {
+        // sanity check
+        if (lines == null || lines.size() != 1) {
+            throw new RuntimeException("Cannot parse format line: expect one-line trace format");
+        }
+
+        // split the raw format line
+        Matcher formatLineMatcher = SPLIT_FORMAT_LINE.matcher(lines.get(0));
+        if (!formatLineMatcher.matches()) {
+            throw new RuntimeException("Cannot parse format line: unexpected format");
+        }
+        String printfString = formatLineMatcher.group("printf");
+        String paramsString = formatLineMatcher.group("params");
+
+        // list of parameters, to be populated soon
+        List<String> allParams = new ArrayList<>();
+        List<String> numParams = new ArrayList<>();
+        List<String> hexParams = new ArrayList<>();
+        List<String> strParams = new ArrayList<>();
+
+        // find all parameters and convert them to camel case
+        Matcher paramsMatcher = SPLIT_PARAMS.matcher(paramsString);
+        while (paramsMatcher.find()) {
+            String camelCasedParam =
+                    CaseFormat.LOWER_UNDERSCORE.to(
+                            CaseFormat.LOWER_CAMEL, paramsMatcher.group("param"));
+            allParams.add(camelCasedParam);
+        }
+
+        // scan the printf string, categorizing parameters and generating a matching regex
+        StringBuffer regexBuilder = new StringBuffer();
+        int paramIndex = 0;
+        String currentParam;
+
+        Matcher printfMatcher = PRINTF_SPECIFIERS.matcher(printfString);
+        while (printfMatcher.find()) {
+            // parameter corresponds to the found specifier
+            currentParam = allParams.get(paramIndex++);
+            if (printfMatcher.group("num") != null) {
+                printfMatcher.appendReplacement(
+                        regexBuilder, createNamedRegexGroup(MATCH_NUM, currentParam));
+                numParams.add(currentParam);
+            } else if (printfMatcher.group("hex") != null) {
+                printfMatcher.appendReplacement(
+                        regexBuilder, createNamedRegexGroup(MATCH_HEX, currentParam));
+                hexParams.add(currentParam);
+            } else if (printfMatcher.group("str") != null) {
+                printfMatcher.appendReplacement(
+                        regexBuilder, createNamedRegexGroup(MATCH_STR, currentParam));
+                strParams.add(currentParam);
+            } else {
+                throw new RuntimeException("Unrecognized specifier: " + printfMatcher.group());
+            }
+        }
+        printfMatcher.appendTail(regexBuilder);
+        Pattern generatedRegex = Pattern.compile(regexBuilder.toString());
+
+        // assemble and return a TraceFormatItem
+        TraceFormatItem item = new TraceFormatItem();
+        item.setRegex(generatedRegex);
+        item.setParameters(allParams);
+        item.setNumericParameters(numParams);
+        item.setHexParameters(hexParams);
+        item.setStringParameters(strParams);
+        return item;
+    }
+
+    /** Helper function to create a regex group with given name. */
+    private static String createNamedRegexGroup(String base, String name) {
+        return String.format("(?<%s>%s)", name, base);
+    }
+}

tests/src/com/android/loganalysis/parser/TraceFormatParserTest.java

+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.loganalysis.parser;
+
+import static org.junit.Assert.fail;
+
+import com.android.loganalysis.item.TraceFormatItem;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+
+/** Test for {@link TraceFormatParser}. */
+@RunWith(JUnit4.class)
+public class TraceFormatParserTest {
+    private TraceFormatParser mParser;
+
+    // "unwrap" the regex strings so that we can compare with the generated regex
+    private static final String MATCH_NUM_UNESCAPED =
+            TraceFormatParser.MATCH_NUM.replaceAll("\\\\\\\\", "\\\\");
+    private static final String MATCH_HEX_UNESCAPED =
+            TraceFormatParser.MATCH_HEX.replaceAll("\\\\\\\\", "\\\\");
+    private static final String MATCH_STR_UNESCAPED =
+            TraceFormatParser.MATCH_STR.replaceAll("\\\\\\\\", "\\\\");
+
+    @Before
+    public void setUp() {
+        mParser = new TraceFormatParser();
+    }
+
+    @Test
+    public void testParseFormatLine() {
+        List<String> formatLine =
+                Arrays.asList("print fmt: \"foo=%llu, bar=%s\", REC->foo, REC->bar");
+        String expectedRegex =
+                String.format(
+                        "foo=(?<foo>%s), bar=(?<bar>%s)", MATCH_NUM_UNESCAPED, MATCH_STR_UNESCAPED);
+        List<String> expectedParameters = Arrays.asList("foo", "bar");
+        List<String> expectedNumericParameters = Arrays.asList("foo");
+        List<String> expectedHexParameters = Arrays.asList();
+        List<String> expectedStringParameters = Arrays.asList("bar");
+        String shouldMatch = "foo=123, bar=enabled";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+        Assert.assertEquals(expectedNumericParameters, parsedItem.getNumericParameters());
+        Assert.assertEquals(expectedHexParameters, parsedItem.getHexParameters());
+        Assert.assertEquals(expectedStringParameters, parsedItem.getStringParameters());
+        Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "123");
+        Assert.assertEquals(m.group("bar"), "enabled");
+    }
+
+    @Test
+    public void testNoParameters() {
+        List<String> formatLine = Arrays.asList("print fmt: \"foo\"");
+        String expectedRegex = "foo";
+        List<String> expectedParameters = Arrays.asList();
+        String shouldMatch = "foo";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+        Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+    }
+
+    @Test
+    public void testNullInput() {
+        try {
+            mParser.parse(null);
+            fail("Expected an exception thrown by TraceFormatParser");
+        } catch (RuntimeException e) {
+            // expected
+        }
+    }
+
+    @Test
+    public void testEmptyInput() {
+        List<String> formatLine = Arrays.asList("");
+        try {
+            mParser.parse(formatLine);
+            fail("Expected an exception thrown by TraceFormatParser");
+        } catch (RuntimeException e) {
+            // expected
+        }
+    }
+
+    @Test
+    public void testMultiLineInput() {
+        List<String> formatLine = Arrays.asList("foo", "bar");
+        try {
+            mParser.parse(formatLine);
+            fail("Expected an exception thrown by TraceFormatParser");
+        } catch (RuntimeException e) {
+            // expected
+        }
+    }
+
+    @Test
+    public void testOneLineInvalidInput() {
+        List<String> formatLine = Arrays.asList("foo bar");
+        try {
+            mParser.parse(formatLine);
+            fail("Expected an exception thrown by TraceFormatParser");
+        } catch (RuntimeException e) {
+            // expected
+        }
+    }
+
+    @Test
+    public void testQuoteInParams() {
+        List<String> formatLine =
+                Arrays.asList("print fmt: \"foo %s\", REC->foo ? \"online\" : \"offline\"");
+        String expectedRegex = String.format("foo (?<foo>%s)", MATCH_STR_UNESCAPED);
+        String shouldMatch = "foo online";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Assert.assertEquals(expectedRegex, parsedItem.getRegex().toString());
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "online");
+    }
+
+    @Test
+    public void testCategorizeParameters() {
+        List<String> formatLine =
+                Arrays.asList(
+                        "print fmt: \"num1=%lu, num2=%f, hex=%08x, str=%s\", REC->num1, REC->num2, REC->hex, REC->str");
+        List<String> expectedNumericParameters = Arrays.asList("num1", "num2");
+        List<String> expectedHexParameters = Arrays.asList("hex");
+        List<String> expectedStringParameters = Arrays.asList("str");
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Assert.assertEquals(expectedNumericParameters, parsedItem.getNumericParameters());
+        Assert.assertEquals(expectedHexParameters, parsedItem.getHexParameters());
+        Assert.assertEquals(expectedStringParameters, parsedItem.getStringParameters());
+    }
+
+    @Test
+    public void testCaseConvertParameterName() {
+        List<String> formatLine = Arrays.asList("print fmt: \"foo_bar=%llu\", REC->foo_bar");
+        List<String> expectedParameters = Arrays.asList("fooBar");
+        String shouldMatch = "foo_bar=123";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Assert.assertEquals(expectedParameters, parsedItem.getParameters());
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("fooBar"), "123");
+    }
+
+    @Test
+    public void testMatchInt() {
+        List<String> formatLine =
+                Arrays.asList("print fmt: \"foo=%d, bar=%lu\", REC->foo, REC->bar");
+        String shouldMatch = "foo=-123, bar=456";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "-123");
+        Assert.assertEquals(m.group("bar"), "456");
+    }
+
+    @Test
+    public void testMatchFloat() {
+        List<String> formatLine =
+                Arrays.asList("print fmt: \"foo=%f, bar=%.2f\", REC->foo, REC->bar");
+        String shouldMatch = "foo=123.4567, bar=456.78";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "123.4567");
+        Assert.assertEquals(m.group("bar"), "456.78");
+    }
+
+    @Test
+    public void testMatchHex() {
+        List<String> formatLine =
+                Arrays.asList(
+                        "print fmt: \"foo=0x%04x, bar=0x%08X, baz=%x\", REC->foo, REC->bar, REC->baz");
+        String shouldMatch = "foo=0x007b, bar=0x000001C8, baz=7b";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "007b");
+        Assert.assertEquals(m.group("bar"), "000001C8");
+        Assert.assertEquals(m.group("baz"), "7b");
+    }
+
+    @Test
+    public void testMatchString() {
+        List<String> formatLine =
+                Arrays.asList("print fmt: \"foo=%s, bar=%s\", REC->foo, REC->bar");
+        String shouldMatch = "foo=oof, bar=123";
+
+        TraceFormatItem parsedItem = mParser.parse(formatLine);
+        Matcher m = parsedItem.getRegex().matcher(shouldMatch);
+        Assert.assertTrue(m.matches());
+        Assert.assertEquals(m.group("foo"), "oof");
+        Assert.assertEquals(m.group("bar"), "123");
+    }
+}