Improve data separation test coverage

Two areas need better coverage:
1. Tests are not verifying that files in /data/vendor do not have the
core_data_file_type attribute.
2. No error is thrown if a type lives in both /data/vendor
/data/<not vendor>.

Bug: 72998741
Test: build all selinux policies on master (assert build time tests)
Test: build and boot Marlin and Taimen, verify no selinux denials and
    everything works as expected.

Change-Id: I133a068123139a599b9b81ddcc254616894621eb
(cherry picked from commit 55d5e284)

tests/Android.bp

@@ -33,9 +33,10 @@ python_defaults {
 python_binary_host {
     name: "treble_sepolicy_tests",
     srcs: [
-        "treble_sepolicy_tests.py",
+        "FcSort.py",
         "mini_parser.py",
         "policy.py",
+        "treble_sepolicy_tests.py",
     ],
     required: ["libsepolwrap"],
     defaults: ["py2_only"],
@@ -44,8 +45,9 @@ python_binary_host {
 python_binary_host {
     name: "sepolicy_tests",
     srcs: [
-        "sepolicy_tests.py",
+        "FcSort.py",
         "policy.py",
+        "sepolicy_tests.py",
     ],
     required: ["libsepolwrap"],
     defaults: ["py2_only"],

tests/FcSort.py

+#!/usr/bin/env python
+import sys
+import os
+
+class FileContextsNode:
+    path = None
+    fileType = None
+    context = None
+    Type = None
+    meta = None
+    stemLen = None
+    strLen = None
+    Type = None
+    def __init__(self, path, fileType, context, meta, stemLen, strLen):
+        self.path = path
+        self.fileType = fileType
+        self.context = context
+        self.meta = meta
+        self.stemLen = stemLen
+        self.strlen = strLen
+        self.Type = context.split(":")[2]
+
+metaChars = frozenset(['.', '^', '$', '?', '*', '+', '|', '[', '(', '{'])
+escapedMetaChars = frozenset(['\.', '\^', '\$', '\?', '\*', '\+', '\|', '\[', '\(', '\{'])
+
+def getStemLen(path):
+    global metaChars
+    stemLen = 0
+    i = 0
+    while i < len(path):
+        if path[i] == "\\":
+            i += 1
+        elif path[i] in metaChars:
+            break
+        stemLen += 1
+        i += 1
+    return stemLen
+
+
+def getIsMeta(path):
+    global metaChars
+    global escapedMetaChars
+    metaCharsCount = 0
+    escapedMetaCharsCount = 0
+    for c in metaChars:
+        if c in path:
+            metaCharsCount += 1
+    for c in escapedMetaChars:
+        if c in path:
+            escapedMetaCharsCount += 1
+    return metaCharsCount > escapedMetaCharsCount
+
+def CreateNode(line):
+    global metaChars
+    if (len(line) == 0) or (line[0] == '#'):
+        return None
+
+    split = line.split()
+    path = split[0].strip()
+    context = split[-1].strip()
+    fileType = None
+    if len(split) == 3:
+        fileType = split[1].strip()
+    meta = getIsMeta(path)
+    stemLen = getStemLen(path)
+    strLen = len(path.replace("\\", ""))
+
+    return FileContextsNode(path, fileType, context, meta, stemLen, strLen)
+
+def ReadFileContexts(files):
+    fc = []
+    for f in files:
+        fd = open(f)
+        for line in fd:
+            node = CreateNode(line.strip())
+            if node != None:
+                fc.append(node)
+    return fc
+
+# Comparator function for list.sort() based off of fc_sort.c
+# Compares two FileContextNodes a and b and returns 1 if a is more
+# specific or -1 if b is more specific.
+def compare(a, b):
+    # The regex without metachars is more specific
+    if a.meta and not b.meta:
+        return -1
+    if b.meta and not a.meta:
+        return 1
+
+    # The regex with longer stemlen (regex before any meta characters) is more specific.
+    if a.stemLen < b.stemLen:
+        return -1
+    if b.stemLen < a.stemLen:
+        return 1
+
+    # The regex with longer string length is more specific
+    if a.strLen < b.strLen:
+        return -1
+    if b.strLen < a.strLen:
+        return 1
+
+    # A regex with a fileType defined (e.g. file, dir) is more specific.
+    if a.fileType is None and b.fileType is not None:
+        return -1
+    if b.fileType is None and a.fileType is not None:
+        return 1
+
+    # Regexes are equally specific.
+    return 0
+
+def FcSort(files):
+    for f in files:
+        if not os.path.exists(f):
+            sys.exit("Error: File_contexts file " + f + " does not exist\n")
+
+    Fc = ReadFileContexts(files)
+    Fc.sort(cmp=compare)
+
+    return Fc
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        sys.exit("Usage: fc_sort.py <file_contexts 1> <file_contexts 2> <file_contexts 3>")
+
+    FcSorted = FcSort(sys.argv[1:])

tests/policy.py

@@ -3,6 +3,7 @@ import re
 import os
 import sys
 import platform
+import FcSort
 
 ###
 # Check whether the regex will match a file path starting with the provided
@@ -45,10 +46,26 @@ class Policy:
     __ExpandedRules = set()
     __Rules = set()
     __FcDict = None
+    __FcSorted = None
     __libsepolwrap = None
     __policydbP = None
     __BUFSIZE = 2048
 
+    def AssertPathTypesDoNotHaveAttr(self, MatchPrefix, DoNotMatchPrefix, Attr):
+        # Query policy for the types associated with Attr
+        TypesPol = self.QueryTypeAttribute(Attr, True)
+        # Search file_contexts to find types associated with input paths.
+        TypesFc = self.__GetTypesByFilePathPrefix(MatchPrefix, DoNotMatchPrefix)
+        violators = TypesFc.intersection(TypesPol)
+        ret = ""
+        if len(violators) > 0:
+            ret += "The following types on "
+            ret += " ".join(str(x) for x in sorted(MatchPrefix))
+            ret += " must not be associated with the "
+            ret += "\"" + Attr + "\" attribute: "
+            ret += " ".join(str(x) for x in sorted(violators)) + "\n"
+        return ret
+
     # Check that path prefixes that match MatchPrefix, and do not Match
     # DoNotMatchPrefix have the attribute Attr.
     # For example assert that all types in /sys, and not in /sys/kernel/debugfs
@@ -198,18 +215,51 @@ class Policy:
         self.__libsepolwrap.destroy_type_iter(TypeIterP)
         return AllTypes
 
+    def __ExactMatchPathPrefix(self, pathregex, prefix):
+        pattern = re.compile('^' + pathregex + "$")
+        if pattern.match(prefix):
+            return True
+        return False
+
+    # Return a tuple (prefix, i) where i is the index of the most specific
+    # match of prefix in the sorted file_contexts. This is useful for limiting a
+    # file_contexts search to matches that are more specific and omitting less
+    # specific matches. For example, finding all matches to prefix /data/vendor
+    # should not include /data(/.*)? if /data/vendor(/.*)? is also specified.
+    def __FcSortedIndex(self, prefix):
+        index = 0
+        for i in range(0, len(self.__FcSorted)):
+            if self.__ExactMatchPathPrefix(self.__FcSorted[i].path, prefix):
+                index = i
+        return prefix, index
+
+    # Return a tuple of (path, Type) for all matching paths. Use the sorted
+    # file_contexts and index returned from __FcSortedIndex() to limit results
+    # to results that are more specific than the prefix.
+    def __MatchPathPrefixTypes(self, prefix, index):
+        PathType = []
+        for i in range(index, len(self.__FcSorted)):
+            if MatchPathPrefix(self.__FcSorted[i].path, prefix):
+                PathType.append((self.__FcSorted[i].path, self.__FcSorted[i].Type))
+        return PathType
+
+    # Return types that match MatchPrefixes but do not match
+    # DoNotMatchPrefixes
     def __GetTypesByFilePathPrefix(self, MatchPrefixes, DoNotMatchPrefixes):
         Types = set()
-        for Type in self.__FcDict:
-            for pathregex in self.__FcDict[Type]:
-                if not MatchPathPrefixes(pathregex, MatchPrefixes):
-                    continue
-                if MatchPathPrefixes(pathregex, DoNotMatchPrefixes):
+
+        MatchPrefixesWithIndex = []
+        for MatchPrefix in MatchPrefixes:
+            MatchPrefixesWithIndex.append(self.__FcSortedIndex(MatchPrefix))
+
+        for MatchPrefixWithIndex in MatchPrefixesWithIndex:
+            PathTypes = self.__MatchPathPrefixTypes(*MatchPrefixWithIndex)
+            for PathType in PathTypes:
+                if MatchPathPrefixes(PathType[0], DoNotMatchPrefixes):
                     continue
-                Types.add(Type)
+                Types.add(PathType[1])
         return Types
 
-
     def __GetTERules(self, policydbP, avtabIterP, Rules):
         if Rules is None:
             Rules = set()
@@ -313,6 +363,7 @@ class Policy:
                     self.__FcDict[t] = [rec[0]]
             except:
                 pass
+        self.__FcSorted = FcSort.FcSort(FcPaths)
 
     # load policy
     def __InitPolicy(self, PolicyPath):

tests/sepolicy_tests.py

@@ -24,8 +24,8 @@ def TestVendorTypeViolations(pol):
     return pol.AssertPathTypesHaveAttr(["/vendor/"], [], "vendor_file_type")
 
 def TestCoreDataTypeViolations(pol):
-    return pol.AssertPathTypesHaveAttr(["/data/"], ["/data/vendor/",
-            "/data/vendor_ce/", "/data/vendor_de/"], "core_data_file_type")
+    return pol.AssertPathTypesHaveAttr(["/data/"], ["/data/vendor",
+            "/data/vendor_ce", "/data/vendor_de"], "core_data_file_type")
 
 ###
 # extend OptionParser to allow the same option flag to be used multiple times.

tests/treble_sepolicy_tests.py

@@ -71,6 +71,7 @@ alldomains = {}
 coredomains = set()
 appdomains = set()
 vendordomains = set()
+pol = None
 
 # compat vars
 alltypes = set()
@@ -287,6 +288,12 @@ def TestViolatorAttributes():
     ret += TestViolatorAttribute("vendor_executes_system_violators")
     return ret
 
+# TODO move this to sepolicy_tests
+def TestCoreDataTypeViolations():
+    global pol
+    return pol.AssertPathTypesDoNotHaveAttr(["/data/vendor/", "/data/vendor_ce/",
+        "/data/vendor_de/"], [], "core_data_file_type")
+
 ###
 # extend OptionParser to allow the same option flag to be used multiple times.
 # This is used to allow multiple file_contexts files and tests to be
@@ -305,6 +312,7 @@ class MultipleOption(Option):
             Option.take_action(self, action, dest, opt, value, values, parser)
 
 Tests = {"CoredomainViolations": TestCoredomainViolations,
+         "CoreDatatypeViolations": TestCoreDataTypeViolations,
          "TrebleCompatMapping": TestTrebleCompatMapping,
          "ViolatorAttributes": TestViolatorAttributes}