Adding the basic signature harvesting (i.e., also saving to and reading back from...

diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
index 740525ed894d42070ccebfad2ab6d5fbce38e0a3..6569157f2b61f2e5f49a8d091d8decb801657ff9 100644 (file)
--- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
+++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
@@ -5,7 +5,6 @@
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/main">
       <sourceFolder url="file://$MODULE_DIR$/../../src/main/java" isTestSource="false" />
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/main">
       <sourceFolder url="file://$MODULE_DIR$/../../src/main/java" isTestSource="false" />

-      <sourceFolder url="file://$MODULE_DIR$/../../src/main/resources" type="java-resource" />

     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
index 8d4ae84e7bb669546c9cd219c3748bd37f5964ba..616bd5262c42114ee35fe7eb572645777c0545fc 100644 (file)
--- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
+++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
@@ -5,7 +5,6 @@
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/test">
       <sourceFolder url="file://$MODULE_DIR$/../../src/test/java" isTestSource="true" />
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/test">
       <sourceFolder url="file://$MODULE_DIR$/../../src/test/java" isTestSource="true" />

-      <sourceFolder url="file://$MODULE_DIR$/../../src/test/resources" type="java-test-resource" />

     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

diff --git a/Code/Projects/SmartPlugDetector/build.gradle b/Code/Projects/SmartPlugDetector/build.gradle
index 72c6b713e36eb4f85f9d2c85110b5f4c7bb052fa..bb07bce70462406025f3eab479c8483334f9bba6 100644 (file)
--- a/Code/Projects/SmartPlugDetector/build.gradle
+++ b/Code/Projects/SmartPlugDetector/build.gradle
@@ -26,4 +26,4 @@ dependencies {
 
     // Apache Commons Math for clustering
     compile 'org.apache.commons:commons-math3:3.6.1'
 
     // Apache Commons Math for clustering
     compile 'org.apache.commons:commons-math3:3.6.1'

-}
+}
\ No newline at end of file

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
index 0f837b29f41ad193b827bee13768a84c610bf43b..78676e8c828fc0080d11ed68e77e9effab5de58d 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
@@ -7,6 +7,7 @@ import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence;
 import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment;
 import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction;
 import edu.uci.iotproject.io.TriggerTimesFileReader;
 import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment;
 import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction;
 import edu.uci.iotproject.io.TriggerTimesFileReader;

+import edu.uci.iotproject.util.PcapPacketUtils;

 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.stat.clustering.Cluster;
 import org.apache.commons.math3.stat.clustering.DBSCANClusterer;
 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.stat.clustering.Cluster;
 import org.apache.commons.math3.stat.clustering.DBSCANClusterer;


@@ -41,17 +42,17 @@ public class Main {
         // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
         // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for
         // which traffic is to be extracted:
         // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
         // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for
         // which traffic is to be extracted:

-//        String path = "/scratch/July-2018"; // Rahmadi
-        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        String path = "/scratch/July-2018"; // Rahmadi
+//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus

         boolean verbose = true;
         final String onPairsPath = "/scratch/July-2018/on.txt";
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
         // 1) D-Link July 26 experiment
         boolean verbose = true;
         final String onPairsPath = "/scratch/July-2018/on.txt";
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
         // 1) D-Link July 26 experiment

-//        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
-//        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
-//        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
+        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
+        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
+        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?

 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";


@@ -135,10 +136,10 @@ public class Main {
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment

-        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
-        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
-        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
-        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
+//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren

 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";


@@ -268,27 +269,39 @@ public class Main {
         // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
         Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
         // Perform clustering on conversation logged as part of all ON events.
         // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
         Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
         // Perform clustering on conversation logged as part of all ON events.

-        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 5);
+        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);

         List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
         // Perform clustering on conversation logged as part of all OFF events.
         List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
         // Perform clustering on conversation logged as part of all OFF events.

-        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 5);
+        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);

         List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
         // Output clusters
         System.out.println("========================================");
         System.out.println("       Clustering results for ON        ");
         System.out.println("       Number of clusters: " + onClusters.size());
         int count = 0;
         List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
         // Output clusters
         System.out.println("========================================");
         System.out.println("       Clustering results for ON        ");
         System.out.println("       Number of clusters: " + onClusters.size());
         int count = 0;

+        List<List<PcapPacket>> ppListOfListReadOn = null;

         for (Cluster<PcapPacketPair> c : onClusters) {
             System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
         for (Cluster<PcapPacketPair> c : onClusters) {
             System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));

+            // Print to file
+            List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+            PrintUtils.serializeClustersIntoFile("./onSignature" + count + ".sig", ppListOfList);
+            ppListOfListReadOn =
+                    PrintUtils.serializeClustersFromFile("./onSignature" + count + ".sig");

         }
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");
         System.out.println("       Number of clusters: " + offClusters.size());
         count = 0;
         }
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");
         System.out.println("       Number of clusters: " + offClusters.size());
         count = 0;

+        List<List<PcapPacket>> ppListOfListReadOff = null;

         for (Cluster<PcapPacketPair> c : offClusters) {
             System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
         for (Cluster<PcapPacketPair> c : offClusters) {
             System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));

+            // Print to file
+            List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+            PrintUtils.serializeClustersIntoFile("./offSignature" + count + ".sig", ppListOfList);
+            ppListOfListReadOff =
+                    PrintUtils.serializeClustersFromFile("./offSignature" + count + ".sig");

         }
         System.out.println("========================================");
         // ============================================================================================================
         }
         System.out.println("========================================");
         // ============================================================================================================

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
index 9e25d1dd4159dc378fa6c930f55ac83d8327a33f..0db25e32737718a1d863f960040fe267ff2cd20a 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
@@ -62,6 +62,11 @@ public class TcpConversationUtils {
     // Helper method for implementing the public API of similarly named methods.
     private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
         List<PcapPacketPair> pairs = new ArrayList<>();
     // Helper method for implementing the public API of similarly named methods.
     private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
         List<PcapPacketPair> pairs = new ArrayList<>();

+        for(PcapPacket pp : packets) {
+            System.out.print(pp.length() + " ");
+        }
+        System.out.println();
+

         int i = 0;
         while (i < packets.size()) {
             PcapPacket p1 = packets.get(i);
         int i = 0;
         while (i < packets.size()) {
             PcapPacket p1 = packets.get(i);


@@ -79,6 +84,7 @@ public class TcpConversationUtils {
                     pairs.add(new PcapPacketPair(p1, p2));
                     // Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
                     i += 2;
                     pairs.add(new PcapPacketPair(p1, p2));
                     // Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
                     i += 2;

+                    //i++;

                 }
             } else {
                 // Last packet of conversation => one item pair
                 }
             } else {
                 // Last packet of conversation => one item pair

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
index 8f998eea3c8d3878c222fe63decd7c9e85d93491..167f0544df60b322e89b6c20ab5a5bad6ce32802 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
@@ -1,9 +1,13 @@
 package edu.uci.iotproject.util;
 
 package edu.uci.iotproject.util;
 

+import edu.uci.iotproject.analysis.PcapPacketPair;
+import org.apache.commons.math3.stat.clustering.Cluster;

 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
 
 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
 

+import java.util.ArrayList;
+import java.util.List;

 import java.util.Objects;
 
 /**
 import java.util.Objects;
 
 /**


@@ -79,4 +83,24 @@ public final class PcapPacketUtils {
         return tcp != null && tcp.getHeader().getAck();
     }
 
         return tcp != null && tcp.getHeader().getAck();
     }
 

+    /**
+     * Transform a {@code Cluster} of {@code PcapPacketPair} objects into a {@code List} of {@code List} of
+     * {@code PcapPacket} objects.
+     * @param cluster A {@link Cluster} of {@link PcapPacketPair} objects that needs to be transformed.
+     * @return A {@link List} of {@link List} of {@link PcapPacket} objects as the result of the transformation.
+     */
+    public static List<List<PcapPacket>> clusterToListOfPcapPackets(Cluster<PcapPacketPair> cluster) {
+        List<List<PcapPacket>> ppListOfList = new ArrayList<>();
+        for (PcapPacketPair ppp: cluster.getPoints()) {
+            // Create a list of PcapPacket objects (list of two members)
+            List<PcapPacket> ppList = new ArrayList<>();
+            ppList.add(ppp.getFirst());
+            ppList.add(ppp.getSecond().get());
+            // Create a list of list of PcapPacket objects
+            ppListOfList.add(ppList);
+        }
+
+        return ppListOfList;
+    }
+

 }
 }

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
index c685d7d7577f6301a630520cac7b33bc2b9f6333..34412d04adc6760ee13d1fabbc34d29bfdf08a56 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
@@ -4,10 +4,17 @@ import edu.uci.iotproject.DnsMap;
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import org.apache.commons.math3.stat.clustering.Cluster;
 
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import org.apache.commons.math3.stat.clustering.Cluster;
 

+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.List;

 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 

+import org.pcap4j.core.PcapPacket;
+

 /**
  * Utility methods for generating (output) strings.
  *
 /**
  * Utility methods for generating (output) strings.
  *


@@ -16,8 +23,73 @@ import java.util.stream.Collectors;
  */
 public class PrintUtils {
 
  */
 public class PrintUtils {
 

+    /**
+     * This is the path for writing the list of list of packet pairs {@code List<List<PcapPacket>>} into a file.
+     * The packet pairs are the pairs in one cluster, so the list represents a cluster that has been derived through
+     * the DBSCAN algorithm.
+     *
+     * E.g., this file could contain a list like the following:
+     *
+     * [[1109, 613],[1111, 613],[1115, 613],...]
+     *
+     * This list has lists of PcapPacket pairs as its members. We do not maintain the pairs in the form of
+     * {@code Cluster<PcapPacketPair>} objects because there might be a situation where we could combine multiple
+     * PcapPacketPair objects into a longer signature, i.e., a string of PcapPacket objects and not just a pair.
+     */
+    private static final String SERIALIZABLE_FILE_PATH = "./signature.sig";
+

     private PrintUtils() { /* private constructor to prevent instantiation */ }
 
     private PrintUtils() { /* private constructor to prevent instantiation */ }
 

+    /**
+     * Write the list of list of packet pairs {@code List<List<PcapPacket>>} into a file.
+     *
+     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
+     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
+     * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
+     * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
+     * PcapPacket objects and not just a pair.
+     *
+     * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the
+     *                 default file name {@code SERIALIZABLE_FILE_PATH}.
+     * @param clusterPackets The {@link Cluster} objects in the form of list of {@code PcapPacket} objects.
+     */
+    public static void serializeClustersIntoFile(String fileName, List<List<PcapPacket>> clusterPackets) {
+        if (fileName == null)
+            fileName = SERIALIZABLE_FILE_PATH;
+        try (ObjectOutputStream oos =
+                new ObjectOutputStream(new FileOutputStream(fileName))) {
+            oos.writeObject(clusterPackets);
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    /**
+     * Read the list of list of packet pairs {@code List<List<PcapPacket>>} from a file.
+     *
+     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
+     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
+     * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
+     * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
+     * PcapPacket objects and not just a pair.
+     *
+     * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the
+     *                 default file name {@code SERIALIZABLE_FILE_PATH}.
+     * @return The list of list of {@link Cluster} objects ({@code List<List<PcapPacket>>}) that is read from file.
+     */
+    public static List<List<PcapPacket>> serializeClustersFromFile(String fileName) {
+        if (fileName == null)
+            fileName = SERIALIZABLE_FILE_PATH;
+        List<List<PcapPacket>> ppListOfList = null;
+        try (ObjectInputStream ois =
+                     new ObjectInputStream(new FileInputStream(fileName))) {
+            ppListOfList = (List<List<PcapPacket>>) ois.readObject();
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
+
+        return ppListOfList;
+    }

 
     /**
      * Converts a {@code PcapPacketPair} into a CSV string containing the packet lengths of the two packets in the pair.
 
     /**
      * Converts a {@code PcapPacketPair} into a CSV string containing the packet lengths of the two packets in the pair.