4 Script that takes a file (output by wireshark/tshark, in JSON format) with DNS traffic
5 and constructs a map (dictionary) in which a hostname points to a set that contains the
6 IP addresses that is associated with that hostname.
11 from collections import defaultdict
13 JSON_KEY_SOURCE = "_source"
14 JSON_KEY_LAYERS = "layers"
16 JSON_KEY_QUERIES = "Queries"
17 JSON_KEY_ANSWERS = "Answers"
18 JSON_KEY_DNS_RESP_TYPE = "dns.resp.type"
19 JSON_KEY_DNS_A = "dns.a" # Key for retrieving IP. 'a' for type A DNS record.
20 JSON_KEY_DNS_RESP_NAME = "dns.resp.name"
21 JSON_KEY_DNS_CNAME = "dns.cname"
25 print "Usage: python", sys.argv[0], "input_file"
27 maps_tuple = parse_json_dns(sys.argv[1])
29 # print hostname to ip map
30 hn_ip_map = maps_tuple[0]
31 for hn in hn_ip_map.keys():
32 print "====================================================================="
34 for ip in hn_ip_map[hn]:
36 print "====================================================================="
40 # print ip to hostname map
41 ip_hn_map = maps_tuple[1]
42 for ip in ip_hn_map.keys():
43 print "====================================================================="
45 for hn in ip_hn_map[ip]:
47 print "====================================================================="
50 def __init__(self, mac_address):
51 # MAC address of device
52 self.mac = mac_address
53 # Maps an external IP to a list of (timestamp,hostname) tuples.
54 # Entries in the list should be interpreted as follows:
55 # the timestamp indicates WHEN this device mapped the given ip (key in dict) to the hostname.
56 self.ip_mappings = defaultdict(list)
58 def hostname_for_ip_at_time(self, ip, timestamp):
59 # Does device have a mapping for the given IP?
60 if not ip in self.ip_mappings:
62 if not self.ip_mappings[ip]:
63 # If list of (timestamp,hostname) tuples is empty, there is no mapping to report.
65 # Best fit mapping: the mapping immediately BEFORE timestamp parameter.
66 # Start with random pick (element 0).
67 best_fit = self.ip_mappings[ip][0]
68 for t in self.ip_mappings[ip]:
69 # t is a (timestamp,hostname) tuple
70 if t[0] < timestamp and t[0] > best_fit[0]:
71 # t is a better fit if it happened BEFORE the input timestamp
72 # and is LATER than the current best_fit
76 def add_mapping(self, ip, timestamp_hostname_tuple):
77 self.ip_mappings[ip].add(timestamp_hostname_tuple)
79 # --------------------------------------------------------------------------
80 # Define eq and hash such that instances of the class can be used as keys in dictionaries.
81 # Equality is based on MAC as a MAC uniquely identifies the device.
82 def __eq__(self, another):
83 return hasattr(another, 'mac') and self.mac == another.mac
85 return hash(self.data)
86 # --------------------------------------------------------------------------
89 # Convert JSON file containing DNS traffic to a tuple with two maps.
90 # Index 0 of the tuple is a map in which a hostname points to its set of associated IPs.
91 # Index 1 of the tuple is a map in which an ip points to its set of associated hostnames.
92 def parse_json_dns(file_path):
93 # Maps hostnames to IPs
94 host_ip_mappings = defaultdict(set)
95 # Maps ips to hostnames
96 ip_host_mappings = defaultdict(set)
97 with open(file_path) as jf:
99 # data becomes reference to root JSON object (or in our case json array)
101 # Loop through json objects in data
102 # Each entry is a pcap entry (request/response (packet) and associated metadata)
104 # p is a JSON object, not an index
105 # Drill down to DNS part: _source->layers->dns
106 layers = p[JSON_KEY_SOURCE][JSON_KEY_LAYERS]
107 dns = layers.get(JSON_KEY_DNS, None)
108 # Skip any non DNS traffic
110 print "[ WARNING: Non DNS traffic ]"
112 # We only care about DNS responses as these also contain a copy of the query that they answer
113 answers = dns.get(JSON_KEY_ANSWERS, None)
116 ## Now that we know that it is an answer, the queries should also be available.
117 queries = dns.get(JSON_KEY_QUERIES)
118 if len(queries.keys()) > 1:
119 # Unclear if script will behave correctly for DNS lookups with multiple queries
120 print "[ WARNING: Multi query DNS lookup ]"
121 for ak in answers.keys():
123 # We are looking for type A records as these are the ones that contain the IP.
125 if a[JSON_KEY_DNS_RESP_TYPE] == "1":
127 ip = a[JSON_KEY_DNS_A]
128 # The answer may be the canonical name.
129 # Now trace back the answer stack, looking for any higher level aliases.
130 hostname = find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])
131 # Add mapping of hostname to ip to our data structure
132 host_ip_mappings[hostname].add(ip)
133 # Add mapping of ip to hostname to our data structure
134 ip_host_mappings[ip].add(hostname)
135 return (host_ip_mappings, ip_host_mappings)
137 # Recursively traverse set of answers trying to find the top most alias for a canonical name
138 def find_alias_hostname(answers, hostname):
139 for ak in answers.keys():
141 cname = a.get(JSON_KEY_DNS_CNAME, None)
142 # We only care about type=CNAME records
145 if cname == hostname:
146 # Located the right answer, perform recursive search for higher level aliases.
147 return find_alias_hostname(answers, a[JSON_KEY_DNS_RESP_NAME])
150 if __name__ == '__main__':
153 # ================================================================================================
154 # Notes/brainstorming how to do ip to host mappings.
156 # Maps IPs to hostnames. Uses a dictionary of dictionaries.
157 # IP lookup in the outer dictionary returns a dictionary that has hostnames as keys.
158 # Looking up a hostname in the inner dictionary returns a set of timestamps.
159 # Each timestamp indicate the time at which the IP<->hostname mapping was determined by a DNS query.
160 # Note that the keyset of the inner dictionary will be of size 1 in most cases.
161 # When this is the case, the value (the set of timestamps) can be ignored.
162 # The values are only relevant when one IP maps to more than 1 hostname.
163 # When this the case, the timestamps must be considered to find the most recent mapping.
164 # ip_host_mappings = defaultdict(defaultdict(set))
166 # ================================================================================================