Update parse_dns.py with new datastructure that captures all dns requests per device...

diff --git a/parse_dns.py b/parse_dns.py
index 3bb7d0bf506c53ac96434ab523f120eeb314f55d..e81fd95e3477519f42599846987a1ed4a0e9e7ca 100644 (file)
--- a/parse_dns.py
+++ b/parse_dns.py
@@ -9,6 +9,9 @@ IP addresses that is associated with that hostname.
 import sys
 import json
 from collections import defaultdict
 import sys
 import json
 from collections import defaultdict

+from decimal import *
+
+ROUTER_MAC = "b0:b9:8a:73:69:8e"

 
 JSON_KEY_SOURCE = "_source"
 JSON_KEY_LAYERS = "layers"
 
 JSON_KEY_SOURCE = "_source"
 JSON_KEY_LAYERS = "layers"


@@ -19,32 +22,40 @@ JSON_KEY_DNS_RESP_TYPE = "dns.resp.type"
 JSON_KEY_DNS_A = "dns.a" # Key for retrieving IP. 'a' for type A DNS record.
 JSON_KEY_DNS_RESP_NAME = "dns.resp.name"
 JSON_KEY_DNS_CNAME = "dns.cname"
 JSON_KEY_DNS_A = "dns.a" # Key for retrieving IP. 'a' for type A DNS record.
 JSON_KEY_DNS_RESP_NAME = "dns.resp.name"
 JSON_KEY_DNS_CNAME = "dns.cname"

+JSON_KEY_ETH = "eth"
+JSON_KEY_ETH_DST = "eth.dst"
+JSON_KEY_FRAME = "frame"
+JSON_KEY_FRAME_TIME_EPOCH = "frame.time_epoch"

 
 def main():
        if len(sys.argv) < 2:
                print "Usage: python", sys.argv[0], "input_file"
                return
 
 def main():
        if len(sys.argv) < 2:
                print "Usage: python", sys.argv[0], "input_file"
                return

-       maps_tuple = parse_json_dns(sys.argv[1])
+       mac_to_ddm = parse_json_dns(sys.argv[1])
+       for mac in mac_to_ddm:
+               ddm = mac_to_ddm[mac]
+               ddm.print_mappings()
+       # maps_tuple = parse_json_dns(sys.argv[1])

        
        

-       # print hostname to ip map
-       hn_ip_map = maps_tuple[0]
-       for hn in hn_ip_map.keys():
-               print "====================================================================="
-               print hn, "maps to:"
-               for ip in hn_ip_map[hn]:
-                       print "    -", ip
-       print "====================================================================="
+       # # print hostname to ip map
+       # hn_ip_map = maps_tuple[0]
+       # for hn in hn_ip_map.keys():
+       #       print "====================================================================="
+       #       print hn, "maps to:"
+       #       for ip in hn_ip_map[hn]:
+       #               print "    -", ip
+       # print "====================================================================="

        
        

-       print " "
+       # print " "

 
 

-       # print ip to hostname map
-       ip_hn_map = maps_tuple[1]
-       for ip in ip_hn_map.keys():
-               print "====================================================================="
-               print ip, "maps to:"
-               for hn in ip_hn_map[ip]:
-                       print "    -", hn
-       print "====================================================================="
+       # # print ip to hostname map
+       # ip_hn_map = maps_tuple[1]
+       # for ip in ip_hn_map.keys():
+       #       print "====================================================================="
+       #       print ip, "maps to:"
+       #       for hn in ip_hn_map[ip]:
+       #               print "    -", hn
+       # print "====================================================================="

 
 class DeviceDNSMap:
        def __init__(self, mac_address):
 
 class DeviceDNSMap:
        def __init__(self, mac_address):


@@ -74,26 +85,32 @@ class DeviceDNSMap:
                return best_fit
 
        def add_mapping(self, ip, timestamp_hostname_tuple):
                return best_fit
 
        def add_mapping(self, ip, timestamp_hostname_tuple):

-               self.ip_mappings[ip].add(timestamp_hostname_tuple)
+               self.ip_mappings[ip].append(timestamp_hostname_tuple)
+
+       def print_mappings(self):
+               count = 0
+               print "### Mappings for MAC = ", self.mac, "###"
+               for ip in self.ip_mappings:
+                       print "--- IP ", ip, " maps to: ---"
+                       for t in self.ip_mappings[ip]:
+                               print t[1], "at epoch time =", t[0]
+                               count += 1
+               print "### Total of", count, "mappings for", self.mac, "###"

 
        # --------------------------------------------------------------------------
        # Define eq and hash such that instances of the class can be used as keys in dictionaries.
        # Equality is based on MAC as a MAC uniquely identifies the device.
        def __eq__(self, another):
 
        # --------------------------------------------------------------------------
        # Define eq and hash such that instances of the class can be used as keys in dictionaries.
        # Equality is based on MAC as a MAC uniquely identifies the device.
        def __eq__(self, another):

-       return hasattr(another, 'mac') and self.mac == another.mac
+               return hasattr(another, 'mac') and self.mac == another.mac

        def __hash__(self):
                return hash(self.data)
        # --------------------------------------------------------------------------
 
 
        def __hash__(self):
                return hash(self.data)
        # --------------------------------------------------------------------------
 
 

-# Convert JSON file containing DNS traffic to a tuple with two maps.
-# Index 0 of the tuple is a map in which a hostname points to its set of associated IPs.
-# Index 1 of the tuple is a map in which an ip points to its set of associated hostnames.

 def parse_json_dns(file_path):
 def parse_json_dns(file_path):

-       # Maps hostnames to IPs
-       host_ip_mappings = defaultdict(set)
-       # Maps ips to hostnames
-       ip_host_mappings = defaultdict(set)
+       # Our end output: dictionary of MAC addresses with DeviceDNSMaps as values.
+       # Each DeviceDNSMap contains DNS lookups performed by the device with the corresponding MAC.
+       result = defaultdict()

        with open(file_path) as jf:
                # Read JSON.
         # data becomes reference to root JSON object (or in our case json array)
        with open(file_path) as jf:
                # Read JSON.
         # data becomes reference to root JSON object (or in our case json array)


@@ -118,6 +135,20 @@ def parse_json_dns(file_path):
                        if len(queries.keys()) > 1:
                                # Unclear if script will behave correctly for DNS lookups with multiple queries
                                print "[ WARNING: Multi query DNS lookup ]"
                        if len(queries.keys()) > 1:
                                # Unclear if script will behave correctly for DNS lookups with multiple queries
                                print "[ WARNING: Multi query DNS lookup ]"

+                       # Get ethernet information for identifying the device performing the DNS lookup.
+                       eth = layers.get(JSON_KEY_ETH, None)
+                       if eth is None:
+                               print "[ WARNING: eth data not found ]"
+                               continue
+                       # As this is a response to a DNS query, the IoT device is the destination.
+                       # Get the device MAC of that device.
+                       device_mac = eth.get(JSON_KEY_ETH_DST, None)
+                       if device_mac is None:
+                               print "[ WARNING: eth.dst data not found ]"
+                               continue
+                       # Get the router's timestamp for this packet
+                       # so that we can mark when the DNS mapping occurred
+                       timestamp = Decimal(layers[JSON_KEY_FRAME][JSON_KEY_FRAME_TIME_EPOCH])

                        for ak in answers.keys():
                                a = answers[ak]
                                # We are looking for type A records as these are the ones that contain the IP.
                        for ak in answers.keys():
                                a = answers[ak]
                                # We are looking for type A records as these are the ones that contain the IP.


@@ -128,11 +159,19 @@ def parse_json_dns(file_path):
                                        # The answer may be the canonical name.
                                        # Now trace back the answer stack, looking for any higher level aliases.
                                        hostname = find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])
                                        # The answer may be the canonical name.
                                        # Now trace back the answer stack, looking for any higher level aliases.
                                        hostname = find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])

-                                       # Add mapping of hostname to ip to our data structure
-                                       host_ip_mappings[hostname].add(ip)
-                                       # Add mapping of ip to hostname to our data structure
-                                       ip_host_mappings[ip].add(hostname)
-       return (host_ip_mappings, ip_host_mappings)
+                                       # Create the tuple that indicates WHEN the ip to hostname mapping occurred
+                                       timestamp_hostname_tuple = (timestamp,hostname)
+                                       if device_mac in result:
+                                               # If we already have DNS data for the device with this MAC:
+                                               # Add the mapping to the DeviceDNSMap that is already present in the dict.
+                                               result[device_mac].add_mapping(ip, timestamp_hostname_tuple)
+                                       else:
+                                               # No DNS data for this device yet:
+                                               # Create a new DeviceDNSMap, add the mapping, and at it to the dict.
+                                               ddm = DeviceDNSMap(device_mac)
+                                               ddm.add_mapping(ip, timestamp_hostname_tuple)
+                                               result[device_mac] = ddm
+       return result

 
 # Recursively traverse set of answers trying to find the top most alias for a canonical name
 def find_alias_hostname(answers, hostname):
 
 # Recursively traverse set of answers trying to find the top most alias for a canonical name
 def find_alias_hostname(answers, hostname):