Merge pull request #16 from p-l-/feature-pcapng

Initial Pcap-NG support

scapy/utils.py

@@ -527,44 +527,87 @@ endianness: "<" or ">", force endianness"""
 
 @conf.commands.register
 def rdpcap(filename, count=-1):
+    """Read a pcap or pcapng file and return a packet list
+
+count: read only <count> packets
+
+    """
+    try:
+        return _rdpcap(filename, count=count)
+    except Scapy_Exception:
+        pass
+    try:
+        return _rdpcapng(filename, count=count)
+    except Scapy_Exception:
+        raise Scapy_Exception("Not a valid pcap or pcapng file")
+
+
+def _rdpcap(filename, count=-1):
     """Read a pcap file and return a packet list
-count: read only <count> packets"""
+
+count: read only <count> packets
+
+    """
     with PcapReader(filename) as fdesc:
         return fdesc.read_all(count=count)
 
+def _rdpcapng(filename, count=-1):
+    """Read a pcapng file and return a packet list
+
+count: read only <count> packets
+
+    """
+    with PcapNgReader(filename) as fdesc:
+        return fdesc.read_all(count=count)
 
 
 class RawPcapReader:
     """A stateful pcap reader. Each packet is returned as a string"""
 
     def __init__(self, filename):
-        self.filename = filename
-        try:
-            self.f = gzip.open(filename,"rb")
-            magic = self.f.read(4)
-        except IOError:
-            self.f = open(filename,"rb")
-            magic = self.f.read(4)
+        magic = self.open(filename)
         if magic == "\xa1\xb2\xc3\xd4": #big endian
             self.endian = ">"
-        elif  magic == "\xd4\xc3\xb2\xa1": #little endian
+        elif magic == "\xd4\xc3\xb2\xa1": #little endian
             self.endian = "<"
         else:
-            raise Scapy_Exception("Not a pcap capture file (bad magic)")
+            try:
+                self.f.seek(-4, 1)
+            except:
+                pass
+            raise Scapy_Exception(
+                "Not a pcap capture file (bad magic: %r)" % magic
+            )
         hdr = self.f.read(20)
         if len(hdr)<20:
             raise Scapy_Exception("Invalid pcap file (too short)")
-        vermaj,vermin,tz,sig,snaplen,linktype = struct.unpack(self.endian+"HHIIII",hdr)
-
+        vermaj, vermin, tz, sig, snaplen, linktype = struct.unpack(
+            self.endian + "HHIIII", hdr
+        )
         self.linktype = linktype
 
-
+    def open(self, filename):
+        """Open (if necessary) filename"""
+        if isinstance(filename, basestring):
+            self.filename = filename
+            try:
+                self.f = gzip.open(filename,"rb")
+                return self.f.read(4)
+            except IOError:
+                self.f = open(filename,"rb")
+                return self.f.read(4)
+        else:
+            self.f = filename
+            self.filename = (filename.name
+                             if hasattr(filename, "name") else
+                             "No name")
+            return self.f.read(4)
 
     def __iter__(self):
         return self
 
     def next(self):
-        """impliment the iterator protocol on a set of packets in a pcap file"""
+        """implement the iterator protocol on a set of packets in a pcap file"""
         pkt = self.read_packet()
         if pkt == None:
             raise StopIteration
@@ -654,7 +697,102 @@ class PcapReader(RawPcapReader):
         return plist.PacketList(res,name = os.path.basename(self.filename))
     def recv(self, size=MTU):
         return self.read_packet(size=size)
-        
+
+
+class RawPcapNgReader(RawPcapReader):
+    """A stateful pcapng reader. Each packet is returned as a
+    string.
+
+    """
+
+    def __init__(self, filename):
+        # A list of (linktype, snaplen); will be populated by IDBs.
+        self.interfaces = []
+        self.blocktypes = {
+            1: self.read_block_idb,
+            6: self.read_block_epb,
+        }
+        magic = self.open(filename)
+        if magic != "\x0a\x0d\x0d\x0a": # PcapNg:
+            try:
+                self.f.seek(-4, 1)
+            except:
+                pass
+            raise Scapy_Exception(
+                "Not a pcapng capture file (bad magic: %r)" % magic
+            )
+        # see https://github.com/pcapng/pcapng
+        blocklen, magic = self.f.read(4), self.f.read(4)
+        if magic == "\x1a\x2b\x3c\x4d":
+            self.endian = ">"
+        elif magic == "\x4d\x3c\x2b\x1a":
+            self.endian = "<"
+        else:
+            raise Scapy_Exception("Not a pcapng capture file (bad magic)")
+        self.f.seek(0)
+
+    def read_packet(self, size=MTU):
+        """Read blocks until it reaches either EOF or a packet, and
+        returns None or (packet, (linktype, sec, usec, wirelen)),
+        where packet is a string.
+
+        """
+        while True:
+            try:
+                blocktype, blocklen = struct.unpack(self.endian + "2I",
+                                                    self.f.read(8))
+            except struct.error:
+                return None
+            block = self.f.read(blocklen - 12)
+            try:
+                if (blocklen,) != struct.unpack(self.endian + 'I',
+                                                self.f.read(4)):
+                    raise Scapy_Exception(
+                        "Invalid pcapng block (bad blocklen)"
+                    )
+            except struct.error:
+                return None
+            res = self.blocktypes.get(blocktype,
+                                      lambda block, size: None)(block, size)
+            if res is not None:
+                return res
+
+    def read_block_idb(self, block, _):
+        """Interface Description Block"""
+        self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8]))
+
+    def read_block_epb(self, block, size):
+        """Enhanced Packet Block"""
+        intid, sec, usec, caplen, wirelen = struct.unpack(self.endian + "5I",
+                                                          block[:20])
+        return (block[20:20 + caplen][:size],
+                (self.interfaces[intid][0], sec, usec, wirelen))
+
+
+class PcapNgReader(RawPcapNgReader):
+    def __init__(self, filename):
+        RawPcapNgReader.__init__(self, filename)
+    def read_packet(self, size=MTU):
+        rp = RawPcapNgReader.read_packet(self, size=size)
+        if rp is None:
+            return None
+        s, (linktype, sec, usec, wirelen) = rp
+        try:
+            p = conf.l2types[linktype](s)
+        except KeyboardInterrupt:
+            raise
+        except:
+            if conf.debug_dissector:
+                raise
+            p = conf.raw_layer(s)
+        p.time = sec+0.000001*usec
+        return p
+    def read_all(self,count=-1):
+        res = RawPcapNgReader.read_all(self, count)
+        import plist
+        return plist.PacketList(res, name=os.path.basename(self.filename))
+    def recv(self, size=MTU):
+        return self.read_packet()
 
 
 class RawPcapWriter:

test/regression.uts

@@ -2306,8 +2306,8 @@ c = ICMPv6NIQueryIPv6(flags="C")
 l = ICMPv6NIQueryIPv6(flags="L")
 s = ICMPv6NIQueryIPv6(flags="S")
 g = ICMPv6NIQueryIPv6(flags="G")
-all = ICMPv6NIQueryIPv6(flags="TALCLSG")
-t.flags == 1 and a.flags == 2 and c.flags == 4 and l.flags == 8 and s.flags == 16 and g.flags == 32 and all.flags == 63
+allflags = ICMPv6NIQueryIPv6(flags="TALCLSG")
+t.flags == 1 and a.flags == 2 and c.flags == 4 and l.flags == 8 and s.flags == 16 and g.flags == 32 and allflags.flags == 63
 
 
 = ICMPv6NIQuery* - flags handling (Test 2)
@@ -2317,8 +2317,8 @@ c = str(ICMPv6NIQueryNOOP(flags="C", nonce="A"*8))[6:8]
 l = str(ICMPv6NIQueryNOOP(flags="L", nonce="A"*8))[6:8]
 s = str(ICMPv6NIQueryNOOP(flags="S", nonce="A"*8))[6:8]
 g = str(ICMPv6NIQueryNOOP(flags="G", nonce="A"*8))[6:8]
-all = str(ICMPv6NIQueryNOOP(flags="TALCLSG", nonce="A"*8))[6:8]
-t == '\x00\x01' and a == '\x00\x02' and c == '\x00\x04' and l == '\x00\x08' and s == '\x00\x10' and g == '\x00\x20' and all == '\x00\x3F'
+allflags = str(ICMPv6NIQueryNOOP(flags="TALCLSG", nonce="A"*8))[6:8]
+t == '\x00\x01' and a == '\x00\x02' and c == '\x00\x04' and l == '\x00\x08' and s == '\x00\x10' and g == '\x00\x20' and allflags == '\x00\x3F'
 
 
 = ICMPv6NIReply* - flags handling (Test 1)
@@ -2328,8 +2328,8 @@ c = ICMPv6NIReplyIPv6(flags="C")
 l = ICMPv6NIReplyIPv6(flags="L")
 s = ICMPv6NIReplyIPv6(flags="S")
 g = ICMPv6NIReplyIPv6(flags="G")
-all = ICMPv6NIReplyIPv6(flags="TALCLSG")
-t.flags == 1 and a.flags == 2 and c.flags == 4 and l.flags == 8 and s.flags == 16 and g.flags == 32 and all.flags == 63
+allflags = ICMPv6NIReplyIPv6(flags="TALCLSG")
+t.flags == 1 and a.flags == 2 and c.flags == 4 and l.flags == 8 and s.flags == 16 and g.flags == 32 and allflags.flags == 63
 
 
 = ICMPv6NIReply* - flags handling (Test 2)
@@ -2339,8 +2339,8 @@ c = str(ICMPv6NIReplyNOOP(flags="C", nonce="A"*8))[6:8]
 l = str(ICMPv6NIReplyNOOP(flags="L", nonce="A"*8))[6:8]
 s = str(ICMPv6NIReplyNOOP(flags="S", nonce="A"*8))[6:8]
 g = str(ICMPv6NIReplyNOOP(flags="G", nonce="A"*8))[6:8]
-all = str(ICMPv6NIReplyNOOP(flags="TALCLSG", nonce="A"*8))[6:8]
-t == '\x00\x01' and a == '\x00\x02' and c == '\x00\x04' and l == '\x00\x08' and s == '\x00\x10' and g == '\x00\x20' and all == '\x00\x3F'
+allflags = str(ICMPv6NIReplyNOOP(flags="TALCLSG", nonce="A"*8))[6:8]
+t == '\x00\x01' and a == '\x00\x02' and c == '\x00\x04' and l == '\x00\x08' and s == '\x00\x10' and g == '\x00\x20' and allflags == '\x00\x3F'
 
 
 = ICMPv6NIQuery* - Flags Default values
@@ -4382,3 +4382,30 @@ str(NetflowHeader()/NetflowHeaderV5(count=1)/NetflowRecordV5(dst="192.168.0.1"))
 
 nf5 = NetflowHeader('\x00\x05\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00')
 nf5.version == 5 and nf5[NetflowHeaderV5].count == 2 and isinstance(nf5[NetflowRecordV5].payload, NetflowRecordV5)
+
+
+############
+############
++ pcap / pcapng format support
+
+= Variable creations
+import cStringIO
+pcapfile = cStringIO.StringIO('\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00')
+pcapngfile = cStringIO.StringIO('\n\r\r\n\\\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00,\x00File created by merging: \nFile1: test.pcap \n\x04\x00\x08\x00mergecap\x00\x00\x00\x00\\\x00\x00\x00\x01\x00\x00\x00\\\x00\x00\x00e\x00\x00\x00\xff\xff\x00\x00\x02\x006\x00Unknown/not available in original file format(libpcap)\x00\x00\t\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\\\x00\x00\x00\x06\x00\x00\x00H\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00/\xfc[\xcd(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00H\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\x1f\xff[\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r<\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\xb9\x02\\\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00<\x00\x00\x00')
+
+= Read a pcap file
+pktpcap = utils._rdpcap(pcapfile)
+
+= Read a pcapng file
+pktpcapng = utils._rdpcapng(pcapngfile)
+
+= Check both packet lists are the same
+assert list(pktpcap) == list(pktpcapng)
+
+= Check packets from pcap file
+assert all(IP in pkt for pkt in pktpcap)
+assert all(any(proto in pkt for pkt in pktpcap) for proto in [ICMP, UDP, TCP])
+
+= Check packets from pcap file
+assert all(IP in pkt for pkt in pktpcapng)
+assert all(any(proto in pkt for pkt in pktpcapng) for proto in [ICMP, UDP, TCP])