From: rtrimana Date: Wed, 25 Jul 2018 23:10:52 +0000 (-0700) Subject: Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=af8c8476359986b7d5851777805aeaf5073605c9;hp=f3d73543514759adfe8600697f86218f3623f61f Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic --- diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/DnsMap.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/DnsMap.java index d88f0cf..0db01f8 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/DnsMap.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/DnsMap.java @@ -103,4 +103,9 @@ public class DnsMap implements PacketListener { public boolean isRelatedToCloudServer(String address, String hostname) { return ipToHostnameMap.getOrDefault(address, EMPTY_SET).contains(hostname); } + + public Set getHostnamesForIp(String ip) { + Set hostnames = ipToHostnameMap.get(ip); + return hostnames != null ? Collections.unmodifiableSet(hostnames) : null; + } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index e0a51a3..4a05501 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -11,10 +11,7 @@ import org.pcap4j.packet.namednumber.DataLinkType; import java.io.EOFException; import java.net.UnknownHostException; import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; +import java.util.*; import java.util.concurrent.TimeoutException; /** @@ -118,12 +115,14 @@ public class Main { // -------- 07-19-2018 -------- TriggerTimesFileReader ttfr = new TriggerTimesFileReader(); - List triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false); -// triggerTimes.stream().forEach(i -> System.out.println(i.atZone(TriggerTimesFileReader.ZONE_ID_LOS_ANGELES).toString())); - String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap"; +// List triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false); + List triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-june-14-2018-timestamps.txt", false); +// String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap"; + String pcapFile = "/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink.wlan1.local.pcap"; String tpLinkPlugIp = "192.168.1.159"; TriggerTrafficExtractor tte = new TriggerTrafficExtractor(pcapFile, triggerTimes, tpLinkPlugIp); - final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap"); +// final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap"); + final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-filtered.pcap"); DnsMap dnsMap = new DnsMap(); TcpReassembler tcpReassembler = new TcpReassembler(); tte.performExtraction(pkt -> { @@ -136,6 +135,7 @@ public class Main { outputter.flush(); outputter.close(); + /* int packets = 0; for (Conversation c : tcpReassembler.getTcpConversations()) { packets += c.getPackets().size(); @@ -147,22 +147,30 @@ public class Main { // Applying filter: "(tcp and not tcp.len == 0 and not tcp.analysis.retransmission and not tcp.analysis.fast_retransmission) or (tcp.flags.syn == 1) or (tcp.flags.fin == 1)" // to the file gives 295 packets, but there are 24 TCP-Out-Of-Order SYN/SYNACKs which are filtered as retransmissions in Conversation, so the numbers seem to match. System.out.println("number of packets: " + packets); + */ List> pairs = new ArrayList<>(); for (Conversation c : tcpReassembler.getTcpConversations()) { pairs.add(TcpConversationUtils.extractPacketPairs(c)); } + /* // Sort pairs according to timestamp of first packet of conversation for (debugging) convenience. Collections.sort(pairs, (l1, l2) -> { if (l1.get(0).getFirst().getTimestamp().isBefore(l2.get(0).getFirst().getTimestamp())) return -1; else if (l2.get(0).getFirst().getTimestamp().isBefore(l1.get(0).getFirst().getTimestamp())) return 1; else return 0; }); + */ System.out.println("list of pairs produced"); List eventstplinkraPairs = new ArrayList<>(); List> otherPairs = new ArrayList<>(); String hostname = "events.tplinkra.com"; + int emptyLists = 0; for (List lppp : pairs) { + if (lppp.size() < 1) { + emptyLists++; + continue; + } IpV4Packet ipPacket = lppp.get(0).getFirst().get(IpV4Packet.class); // If packets are associated with the hostname if (dnsMap.isRelatedToCloudServer(ipPacket.getHeader().getSrcAddr().getHostAddress(), hostname) || @@ -173,6 +181,7 @@ public class Main { otherPairs.add(lppp); } } + System.out.println("number of empty list of packet pairs: " + emptyLists); HashMap pairCount = new HashMap<>(); for (PcapPacketPair ppp : eventstplinkraPairs) { if (pairCount.containsKey(ppp.toString())) { @@ -182,6 +191,37 @@ public class Main { } } System.out.println("pairCount map built"); + + // Build map containing frequencies of packet lengths exchanged with events.tplinkra.com as well as a map with + // the frequencies of specific sequences of packet lengths for the same hostname + HashMap eventstplinkraPacketLengthFreqMap = new HashMap<>(); + HashMap eventstplinkraPacketSequenceFreqMap = new HashMap<>(); + for (Conversation c : tcpReassembler.getTcpConversations()) { + if (c.getPackets().size() == 0) { + continue; + } + PcapPacket firstPacket = c.getPackets().get(0); + IpV4Packet firstPacketIp = firstPacket.get(IpV4Packet.class); + if (!dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getSrcAddr().getHostAddress(), hostname) && + !dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getDstAddr().getHostAddress(), hostname)) { + continue; + } + // Update the packet length freq map + for (PcapPacket pp : c.getPackets()) { + eventstplinkraPacketLengthFreqMap.merge(pp.length(), 1, (i1, i2) -> i1 + i2); + } + // Update the packet sequence freq map + StringBuilder sb = new StringBuilder(); + for (PcapPacket pp : c.getPackets()) { + sb.append(pp.length() + " "); + } + eventstplinkraPacketSequenceFreqMap.merge(sb.toString(), 1, (i1, i2) -> i1+i2); + } + System.out.println("packet length frequency map created"); + + Map> hostnameConversationMap = + TcpConversationUtils.groupConversationsByHostname(tcpReassembler.getTcpConversations(), dnsMap); + System.out.println("hostnameConversationMap created"); // ---------------------------- } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java index 6cce343..d54e124 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java @@ -33,9 +33,8 @@ public class TcpReassembler implements PacketListener { /** * Holds terminated {@link Conversation}s. - * TODO: Should turn this into a list to avoid unintentional overwrite of a Conversation in case ephemeral port number is reused at a later stage...? */ - private final Map mTerminatedConversations = new HashMap<>(); + private final List mTerminatedConversations = new ArrayList<>(); @Override public void gotPacket(PcapPacket pcapPacket) { @@ -54,7 +53,7 @@ public class TcpReassembler implements PacketListener { */ public List getTcpConversations() { ArrayList combined = new ArrayList<>(); - combined.addAll(mTerminatedConversations.values()); + combined.addAll(mTerminatedConversations); combined.addAll(mOpenConversations.values()); return combined; } @@ -116,7 +115,7 @@ public class TcpReassembler implements PacketListener { // to establish a new conversation with the same four tuple as ongoingConv. // Mark existing connection as terminated. // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? - mTerminatedConversations.put(ongoingConv, ongoingConv); + mTerminatedConversations.add(ongoingConv); mOpenConversations.remove(ongoingConv); } } @@ -157,7 +156,7 @@ public class TcpReassembler implements PacketListener { private void processRstPacket(PcapPacket rstPacket) { Conversation conv = getOngoingConversationOrCreateNew(rstPacket); // Move conversation to set of terminated conversations. - mTerminatedConversations.put(conv, conv); + mTerminatedConversations.add(conv); mOpenConversations.remove(conv, conv); } @@ -176,7 +175,7 @@ public class TcpReassembler implements PacketListener { conv.attemptAcknowledgementOfFin(ackPacket); if (conv.isGracefullyShutdown()) { // Move conversation to set of terminated conversations. - mTerminatedConversations.put(conv, conv); + mTerminatedConversations.add(conv); mOpenConversations.remove(conv); } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java index a598f81..2f5f415 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java @@ -1,13 +1,13 @@ package edu.uci.iotproject.analysis; import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.DnsMap; import edu.uci.iotproject.util.PcapPacketUtils; import org.pcap4j.core.PcapPacket; import org.pcap4j.packet.IpV4Packet; import org.pcap4j.packet.TcpPacket; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** * TODO add class documentation. @@ -48,4 +48,53 @@ public class TcpConversationUtils { // TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases? } + + public static Map> groupConversationsByHostname(List tcpConversations, DnsMap ipHostnameMappings) { + HashMap> result = new HashMap<>(); + for (Conversation c : tcpConversations) { + if (c.getPackets().size() == 0) { + String warningStr = String.format("Detected a %s [%s] with no payload packets.", + c.getClass().getSimpleName(), c.toString()); + System.err.println(warningStr); + continue; + } + IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class); + String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress(); + String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress(); + // Check if src or dst IP is associated with one or more hostnames. + Set hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc); + if (hostnames == null) { + // No luck with src ip (possibly because it's a client->srv packet), try dst ip. + hostnames = ipHostnameMappings.getHostnamesForIp(ipDst); + } + if (hostnames != null) { + // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to. + for (String hostname : hostnames) { + List newValue = new ArrayList<>(); + newValue.add(c); + result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; }); + } + if (hostnames.size() > 1) { + // Print notice of IP mapping to multiple hostnames (debugging) + System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames", + TcpConversationUtils.class.getSimpleName(), hostnames.size())); + } + } else { + // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs. + // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP + // goes first. + String delimiter = "_"; + // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of + // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is + // communicating with itself!) + String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc; + List newValue = new ArrayList<>(); + newValue.add(c); + result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; }); + } + } + return result; + } + + } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java index 2c387f3..e01f712 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java @@ -43,12 +43,13 @@ public class PcapHandleReader { * @throws TimeoutException if packets are being read from a live capture and the timeout expired. */ public void readFromHandle() throws PcapNativeException, NotOpenException, TimeoutException { + int outOfOrderPackets = 0; try { PcapPacket prevPacket = null; PcapPacket packet; while ((packet = mHandle.getNextPacketEx()) != null) { if (prevPacket != null && packet.getTimestamp().isBefore(prevPacket.getTimestamp())) { - System.out.println("Out-of-order (in terms of timestamp) packet detected"); + outOfOrderPackets++; /* // Fail early if assumption doesn't hold. mHandle.close(); @@ -67,6 +68,11 @@ public class PcapHandleReader { // Reached end of file. All good. System.out.println(String.format("%s: finished reading pcap file", getClass().getSimpleName())); } + if (outOfOrderPackets > 0) { + System.err.println( + String.format("[[[ %s: %d packets appeared out of order (with regards to their timestamps) ]]]", + getClass().getSimpleName(), outOfOrderPackets)); + } mHandle.close(); }