From: rtrimana <rtrimana@uci.edu>
Date: Wed, 19 Sep 2018 16:54:30 +0000 (-0700)
Subject: Adding the basic signature harvesting (i.e., also saving to and reading back from... 
X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=f2d68ea45966e9905b3d0eb5c6958dd730f6c4f1

Adding the basic signature harvesting (i.e., also saving to and reading back from signature files).
---

diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
index 740525e..6569157 100644
--- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
+++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml
@@ -5,7 +5,6 @@
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/main">
       <sourceFolder url="file://$MODULE_DIR$/../../src/main/java" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/../../src/main/resources" type="java-resource" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
index 8d4ae84..616bd52 100644
--- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
+++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml
@@ -5,7 +5,6 @@
     <exclude-output />
     <content url="file://$MODULE_DIR$/../../src/test">
       <sourceFolder url="file://$MODULE_DIR$/../../src/test/java" isTestSource="true" />
-      <sourceFolder url="file://$MODULE_DIR$/../../src/test/resources" type="java-test-resource" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
diff --git a/Code/Projects/SmartPlugDetector/build.gradle b/Code/Projects/SmartPlugDetector/build.gradle
index 72c6b71..bb07bce 100644
--- a/Code/Projects/SmartPlugDetector/build.gradle
+++ b/Code/Projects/SmartPlugDetector/build.gradle
@@ -26,4 +26,4 @@ dependencies {
 
     // Apache Commons Math for clustering
     compile 'org.apache.commons:commons-math3:3.6.1'
-}
+}
\ No newline at end of file
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
index 0f837b2..78676e8 100644
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
@@ -7,6 +7,7 @@ import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence;
 import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment;
 import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction;
 import edu.uci.iotproject.io.TriggerTimesFileReader;
+import edu.uci.iotproject.util.PcapPacketUtils;
 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.stat.clustering.Cluster;
 import org.apache.commons.math3.stat.clustering.DBSCANClusterer;
@@ -41,17 +42,17 @@ public class Main {
         // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
         // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for
         // which traffic is to be extracted:
-//        String path = "/scratch/July-2018"; // Rahmadi
-        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        String path = "/scratch/July-2018"; // Rahmadi
+//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
         boolean verbose = true;
         final String onPairsPath = "/scratch/July-2018/on.txt";
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
         // 1) D-Link July 26 experiment
-//        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
-//        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
-//        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
+        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
+        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
+        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
@@ -135,10 +136,10 @@ public class Main {
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment
-        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
-        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
-        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
-        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
+//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
@@ -268,27 +269,39 @@ public class Main {
         // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
         Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
         // Perform clustering on conversation logged as part of all ON events.
-        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 5);
+        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
         List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
         // Perform clustering on conversation logged as part of all OFF events.
-        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 5);
+        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
         List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
         // Output clusters
         System.out.println("========================================");
         System.out.println("       Clustering results for ON        ");
         System.out.println("       Number of clusters: " + onClusters.size());
         int count = 0;
+        List<List<PcapPacket>> ppListOfListReadOn = null;
         for (Cluster<PcapPacketPair> c : onClusters) {
             System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
+            // Print to file
+            List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+            PrintUtils.serializeClustersIntoFile("./onSignature" + count + ".sig", ppListOfList);
+            ppListOfListReadOn =
+                    PrintUtils.serializeClustersFromFile("./onSignature" + count + ".sig");
         }
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");
         System.out.println("       Number of clusters: " + offClusters.size());
         count = 0;
+        List<List<PcapPacket>> ppListOfListReadOff = null;
         for (Cluster<PcapPacketPair> c : offClusters) {
             System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
+            // Print to file
+            List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+            PrintUtils.serializeClustersIntoFile("./offSignature" + count + ".sig", ppListOfList);
+            ppListOfListReadOff =
+                    PrintUtils.serializeClustersFromFile("./offSignature" + count + ".sig");
         }
         System.out.println("========================================");
         // ============================================================================================================
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
index 9e25d1d..0db25e3 100644
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
@@ -62,6 +62,11 @@ public class TcpConversationUtils {
     // Helper method for implementing the public API of similarly named methods.
     private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
         List<PcapPacketPair> pairs = new ArrayList<>();
+        for(PcapPacket pp : packets) {
+            System.out.print(pp.length() + " ");
+        }
+        System.out.println();
+
         int i = 0;
         while (i < packets.size()) {
             PcapPacket p1 = packets.get(i);
@@ -79,6 +84,7 @@ public class TcpConversationUtils {
                     pairs.add(new PcapPacketPair(p1, p2));
                     // Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
                     i += 2;
+                    //i++;
                 }
             } else {
                 // Last packet of conversation => one item pair
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
index 8f998ee..167f054 100644
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
@@ -1,9 +1,13 @@
 package edu.uci.iotproject.util;
 
+import edu.uci.iotproject.analysis.PcapPacketPair;
+import org.apache.commons.math3.stat.clustering.Cluster;
 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 
 /**
@@ -79,4 +83,24 @@ public final class PcapPacketUtils {
         return tcp != null && tcp.getHeader().getAck();
     }
 
+    /**
+     * Transform a {@code Cluster} of {@code PcapPacketPair} objects into a {@code List} of {@code List} of
+     * {@code PcapPacket} objects.
+     * @param cluster A {@link Cluster} of {@link PcapPacketPair} objects that needs to be transformed.
+     * @return A {@link List} of {@link List} of {@link PcapPacket} objects as the result of the transformation.
+     */
+    public static List<List<PcapPacket>> clusterToListOfPcapPackets(Cluster<PcapPacketPair> cluster) {
+        List<List<PcapPacket>> ppListOfList = new ArrayList<>();
+        for (PcapPacketPair ppp: cluster.getPoints()) {
+            // Create a list of PcapPacket objects (list of two members)
+            List<PcapPacket> ppList = new ArrayList<>();
+            ppList.add(ppp.getFirst());
+            ppList.add(ppp.getSecond().get());
+            // Create a list of list of PcapPacket objects
+            ppListOfList.add(ppList);
+        }
+
+        return ppListOfList;
+    }
+
 }
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
index c685d7d..34412d0 100644
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PrintUtils.java
@@ -4,10 +4,17 @@ import edu.uci.iotproject.DnsMap;
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import org.apache.commons.math3.stat.clustering.Cluster;
 
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.pcap4j.core.PcapPacket;
+
 /**
  * Utility methods for generating (output) strings.
  *
@@ -16,8 +23,73 @@ import java.util.stream.Collectors;
  */
 public class PrintUtils {
 
+    /**
+     * This is the path for writing the list of list of packet pairs {@code List<List<PcapPacket>>} into a file.
+     * The packet pairs are the pairs in one cluster, so the list represents a cluster that has been derived through
+     * the DBSCAN algorithm.
+     *
+     * E.g., this file could contain a list like the following:
+     *
+     * [[1109, 613],[1111, 613],[1115, 613],...]
+     *
+     * This list has lists of PcapPacket pairs as its members. We do not maintain the pairs in the form of
+     * {@code Cluster<PcapPacketPair>} objects because there might be a situation where we could combine multiple
+     * PcapPacketPair objects into a longer signature, i.e., a string of PcapPacket objects and not just a pair.
+     */
+    private static final String SERIALIZABLE_FILE_PATH = "./signature.sig";
+
     private PrintUtils() { /* private constructor to prevent instantiation */ }
 
+    /**
+     * Write the list of list of packet pairs {@code List<List<PcapPacket>>} into a file.
+     *
+     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
+     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
+     * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
+     * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
+     * PcapPacket objects and not just a pair.
+     *
+     * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the
+     *                 default file name {@code SERIALIZABLE_FILE_PATH}.
+     * @param clusterPackets The {@link Cluster} objects in the form of list of {@code PcapPacket} objects.
+     */
+    public static void serializeClustersIntoFile(String fileName, List<List<PcapPacket>> clusterPackets) {
+        if (fileName == null)
+            fileName = SERIALIZABLE_FILE_PATH;
+        try (ObjectOutputStream oos =
+                new ObjectOutputStream(new FileOutputStream(fileName))) {
+            oos.writeObject(clusterPackets);
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    /**
+     * Read the list of list of packet pairs {@code List<List<PcapPacket>>} from a file.
+     *
+     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
+     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
+     * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
+     * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
+     * PcapPacket objects and not just a pair.
+     *
+     * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the
+     *                 default file name {@code SERIALIZABLE_FILE_PATH}.
+     * @return The list of list of {@link Cluster} objects ({@code List<List<PcapPacket>>}) that is read from file.
+     */
+    public static List<List<PcapPacket>> serializeClustersFromFile(String fileName) {
+        if (fileName == null)
+            fileName = SERIALIZABLE_FILE_PATH;
+        List<List<PcapPacket>> ppListOfList = null;
+        try (ObjectInputStream ois =
+                     new ObjectInputStream(new FileInputStream(fileName))) {
+            ppListOfList = (List<List<PcapPacket>>) ois.readObject();
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
+
+        return ppListOfList;
+    }
 
     /**
      * Converts a {@code PcapPacketPair} into a CSV string containing the packet lengths of the two packets in the pair.