Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic
authorrtrimana <rtrimana@uci.edu>
Wed, 25 Jul 2018 23:10:52 +0000 (16:10 -0700)
committerrtrimana <rtrimana@uci.edu>
Wed, 25 Jul 2018 23:10:52 +0000 (16:10 -0700)
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/DnsMap.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java

index d88f0cf..0db01f8 100644 (file)
@@ -103,4 +103,9 @@ public class DnsMap implements PacketListener {
     public boolean isRelatedToCloudServer(String address, String hostname) {
         return ipToHostnameMap.getOrDefault(address, EMPTY_SET).contains(hostname);
     }
+
+    public Set<String> getHostnamesForIp(String ip) {
+        Set<String> hostnames = ipToHostnameMap.get(ip);
+        return hostnames != null ? Collections.unmodifiableSet(hostnames) : null;
+    }
 }
index e0a51a3..4a05501 100644 (file)
@@ -11,10 +11,7 @@ import org.pcap4j.packet.namednumber.DataLinkType;
 import java.io.EOFException;
 import java.net.UnknownHostException;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.TimeoutException;
 
 /**
@@ -118,12 +115,14 @@ public class Main {
 
         // -------- 07-19-2018 --------
         TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
-        List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false);
-//        triggerTimes.stream().forEach(i -> System.out.println(i.atZone(TriggerTimesFileReader.ZONE_ID_LOS_ANGELES).toString()));
-        String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap";
+//        List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false);
+        List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-june-14-2018-timestamps.txt", false);
+//        String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap";
+        String pcapFile = "/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink.wlan1.local.pcap";
         String tpLinkPlugIp = "192.168.1.159";
         TriggerTrafficExtractor tte = new TriggerTrafficExtractor(pcapFile, triggerTimes, tpLinkPlugIp);
-        final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap");
+//        final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap");
+        final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-filtered.pcap");
         DnsMap dnsMap = new DnsMap();
         TcpReassembler tcpReassembler = new TcpReassembler();
         tte.performExtraction(pkt -> {
@@ -136,6 +135,7 @@ public class Main {
         outputter.flush();
         outputter.close();
 
+        /*
         int packets = 0;
         for (Conversation c : tcpReassembler.getTcpConversations()) {
             packets += c.getPackets().size();
@@ -147,22 +147,30 @@ public class Main {
         // Applying filter: "(tcp and not tcp.len == 0 and not tcp.analysis.retransmission and not tcp.analysis.fast_retransmission)  or (tcp.flags.syn == 1) or (tcp.flags.fin == 1)"
         // to the file gives 295 packets, but there are 24 TCP-Out-Of-Order SYN/SYNACKs which are filtered as retransmissions in Conversation, so the numbers seem to match.
         System.out.println("number of packets: " + packets);
+        */
 
         List<List<PcapPacketPair>> pairs = new ArrayList<>();
         for (Conversation c : tcpReassembler.getTcpConversations()) {
             pairs.add(TcpConversationUtils.extractPacketPairs(c));
         }
+        /*
         // Sort pairs according to timestamp of first packet of conversation for (debugging) convenience.
         Collections.sort(pairs, (l1, l2) -> {
             if (l1.get(0).getFirst().getTimestamp().isBefore(l2.get(0).getFirst().getTimestamp())) return -1;
             else if (l2.get(0).getFirst().getTimestamp().isBefore(l1.get(0).getFirst().getTimestamp())) return 1;
             else return 0;
         });
+        */
         System.out.println("list of pairs produced");
         List<PcapPacketPair> eventstplinkraPairs = new ArrayList<>();
         List<List<PcapPacketPair>> otherPairs = new ArrayList<>();
         String hostname = "events.tplinkra.com";
+        int emptyLists = 0;
         for (List<PcapPacketPair> lppp : pairs) {
+            if (lppp.size() < 1) {
+                emptyLists++;
+                continue;
+            }
             IpV4Packet ipPacket = lppp.get(0).getFirst().get(IpV4Packet.class);
             // If packets are associated with the hostname
             if (dnsMap.isRelatedToCloudServer(ipPacket.getHeader().getSrcAddr().getHostAddress(), hostname) ||
@@ -173,6 +181,7 @@ public class Main {
                 otherPairs.add(lppp);
             }
         }
+        System.out.println("number of empty list of packet pairs: " + emptyLists);
         HashMap<String, Integer> pairCount = new HashMap<>();
         for (PcapPacketPair ppp : eventstplinkraPairs) {
             if (pairCount.containsKey(ppp.toString())) {
@@ -182,6 +191,37 @@ public class Main {
             }
         }
         System.out.println("pairCount map built");
+
+        // Build map containing frequencies of packet lengths exchanged with events.tplinkra.com as well as a map with
+        // the frequencies of specific sequences of packet lengths for the same hostname
+        HashMap<Integer, Integer> eventstplinkraPacketLengthFreqMap = new HashMap<>();
+        HashMap<String, Integer> eventstplinkraPacketSequenceFreqMap = new HashMap<>();
+        for (Conversation c : tcpReassembler.getTcpConversations()) {
+            if (c.getPackets().size() == 0) {
+                continue;
+            }
+            PcapPacket firstPacket = c.getPackets().get(0);
+            IpV4Packet firstPacketIp = firstPacket.get(IpV4Packet.class);
+            if (!dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getSrcAddr().getHostAddress(), hostname) &&
+                    !dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getDstAddr().getHostAddress(), hostname)) {
+                continue;
+            }
+            // Update the packet length freq map
+            for (PcapPacket pp : c.getPackets()) {
+                eventstplinkraPacketLengthFreqMap.merge(pp.length(), 1, (i1, i2) -> i1 + i2);
+            }
+            // Update the packet sequence freq map
+            StringBuilder sb = new StringBuilder();
+            for (PcapPacket pp : c.getPackets()) {
+                sb.append(pp.length() + " ");
+            }
+            eventstplinkraPacketSequenceFreqMap.merge(sb.toString(), 1, (i1, i2) -> i1+i2);
+        }
+        System.out.println("packet length frequency map created");
+
+        Map<String, List<Conversation>> hostnameConversationMap =
+                TcpConversationUtils.groupConversationsByHostname(tcpReassembler.getTcpConversations(), dnsMap);
+        System.out.println("hostnameConversationMap created");
         // ----------------------------
     }
 
index 6cce343..d54e124 100644 (file)
@@ -33,9 +33,8 @@ public class TcpReassembler implements PacketListener {
 
     /**
      * Holds <em>terminated</em> {@link Conversation}s.
-     * TODO: Should turn this into a list to avoid unintentional overwrite of a Conversation in case ephemeral port number is reused at a later stage...?
      */
-    private final Map<Conversation, Conversation> mTerminatedConversations = new HashMap<>();
+    private final List<Conversation> mTerminatedConversations = new ArrayList<>();
 
     @Override
     public void gotPacket(PcapPacket pcapPacket) {
@@ -54,7 +53,7 @@ public class TcpReassembler implements PacketListener {
      */
     public List<Conversation> getTcpConversations() {
         ArrayList<Conversation> combined = new ArrayList<>();
-        combined.addAll(mTerminatedConversations.values());
+        combined.addAll(mTerminatedConversations);
         combined.addAll(mOpenConversations.values());
         return combined;
     }
@@ -116,7 +115,7 @@ public class TcpReassembler implements PacketListener {
                 // to establish a new conversation with the same four tuple as ongoingConv.
                 // Mark existing connection as terminated.
                 // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
-                mTerminatedConversations.put(ongoingConv, ongoingConv);
+                mTerminatedConversations.add(ongoingConv);
                 mOpenConversations.remove(ongoingConv);
             }
         }
@@ -157,7 +156,7 @@ public class TcpReassembler implements PacketListener {
     private void processRstPacket(PcapPacket rstPacket) {
         Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
         // Move conversation to set of terminated conversations.
-        mTerminatedConversations.put(conv, conv);
+        mTerminatedConversations.add(conv);
         mOpenConversations.remove(conv, conv);
     }
 
@@ -176,7 +175,7 @@ public class TcpReassembler implements PacketListener {
             conv.attemptAcknowledgementOfFin(ackPacket);
             if (conv.isGracefullyShutdown()) {
                 // Move conversation to set of terminated conversations.
-                mTerminatedConversations.put(conv, conv);
+                mTerminatedConversations.add(conv);
                 mOpenConversations.remove(conv);
             }
         }
index a598f81..2f5f415 100644 (file)
@@ -1,13 +1,13 @@
 package edu.uci.iotproject.analysis;
 
 import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.DnsMap;
 import edu.uci.iotproject.util.PcapPacketUtils;
 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 /**
  * TODO add class documentation.
@@ -48,4 +48,53 @@ public class TcpConversationUtils {
         // TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases?
     }
 
+
+    public static Map<String, List<Conversation>> groupConversationsByHostname(List<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
+        HashMap<String, List<Conversation>> result = new HashMap<>();
+        for (Conversation c : tcpConversations) {
+            if (c.getPackets().size() == 0) {
+                String warningStr = String.format("Detected a %s [%s] with no payload packets.",
+                        c.getClass().getSimpleName(), c.toString());
+                System.err.println(warningStr);
+                continue;
+            }
+            IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class);
+            String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress();
+            String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress();
+            // Check if src or dst IP is associated with one or more hostnames.
+            Set<String> hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc);
+            if (hostnames == null) {
+                // No luck with src ip (possibly because it's a client->srv packet), try dst ip.
+                hostnames = ipHostnameMappings.getHostnamesForIp(ipDst);
+            }
+            if (hostnames != null) {
+                // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to.
+                for (String hostname : hostnames) {
+                    List<Conversation> newValue = new ArrayList<>();
+                    newValue.add(c);
+                    result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+                }
+                if (hostnames.size() > 1) {
+                    // Print notice of IP mapping to multiple hostnames (debugging)
+                    System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames",
+                            TcpConversationUtils.class.getSimpleName(), hostnames.size()));
+                }
+            } else {
+                // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs.
+                // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP
+                // goes first.
+                String delimiter = "_";
+                // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of
+                // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is
+                // communicating with itself!)
+                String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc;
+                List<Conversation> newValue = new ArrayList<>();
+                newValue.add(c);
+                result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+            }
+        }
+        return result;
+    }
+
+
 }
index 2c387f3..e01f712 100644 (file)
@@ -43,12 +43,13 @@ public class PcapHandleReader {
      * @throws TimeoutException if packets are being read from a live capture and the timeout expired.
      */
     public void readFromHandle() throws PcapNativeException, NotOpenException, TimeoutException {
+        int outOfOrderPackets = 0;
         try {
             PcapPacket prevPacket = null;
             PcapPacket packet;
             while ((packet = mHandle.getNextPacketEx()) != null) {
                 if (prevPacket != null && packet.getTimestamp().isBefore(prevPacket.getTimestamp())) {
-                    System.out.println("Out-of-order (in terms of timestamp) packet detected");
+                    outOfOrderPackets++;
                     /*
                     // Fail early if assumption doesn't hold.
                     mHandle.close();
@@ -67,6 +68,11 @@ public class PcapHandleReader {
             // Reached end of file. All good.
             System.out.println(String.format("%s: finished reading pcap file", getClass().getSimpleName()));
         }
+        if (outOfOrderPackets > 0) {
+            System.err.println(
+                    String.format("[[[ %s: %d packets appeared out of order (with regards to their timestamps) ]]]",
+                            getClass().getSimpleName(), outOfOrderPackets));
+        }
         mHandle.close();
     }