Adding range-based detection (improved the results for Nest Thermostat and Arlo Camera.
authorrtrimana <rtrimana@uci.edu>
Fri, 8 Mar 2019 00:29:38 +0000 (16:29 -0800)
committerrtrimana <rtrimana@uci.edu>
Fri, 8 Mar 2019 00:29:38 +0000 (16:29 -0800)
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PrintUtils.java

index 4ecba3f..d342508 100644 (file)
@@ -46,7 +46,7 @@ public class SignatureGenerator {
         final String onPairsPath = "/scratch/July-2018/on.txt";
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
-        // 1) D-Link July 26 experiment
+        // 1) TODO: D-LINK PLUG July 26 experiment
 //        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
@@ -60,10 +60,10 @@ public class SignatureGenerator {
 //        final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug-processed.pcap";
 //        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
-//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
-////        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
+////        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
 
-        // 2) TP-Link July 25 experiment
+        // 2) TODO: TP-LINK PLUG July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.timestamps";
@@ -88,7 +88,7 @@ public class SignatureGenerator {
 //        final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.truncated.timestamps";
 //        final String deviceIp = "192.168.1.159";
 
-        // 3) SmartThings Plug July 25 experiment
+        // 3) TODO: SMARTTHINGS PLUG July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/stplug/stplug.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-07/stplug/stplug-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-07/stplug/smartthings-july-25-2018.timestamps";
@@ -107,7 +107,7 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
 
-        // 4) Wemo July 30 experiment
+        // 4) TODO: WEMO PLUG July 30 experiment
 //        final String inputPcapFile = path + "/2018-07/wemo/wemo.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-07/wemo/wemo-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-07/wemo/wemo-july-30-2018.timestamps";
@@ -119,19 +119,19 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .145 == WeMo
 
-        // 5) Wemo Insight July 31 experiment
+        // 5) TODO: WEMO INSIGHT July 31 experiment
 //        final String inputPcapFile = path + "/2018-07/wemoinsight/wemoinsight.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-07/wemoinsight/wemoinsight-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-07/wemoinsight/wemo-insight-july-31-2018.timestamps";
 //        final String deviceIp = "192.168.1.135";
         // TODO: EXPERIMENT - November 21, 2018
-//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
-//        final String outputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug-processed.pcap";
-//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
-////        final String deviceIp = "192.168.1.145"; // .246 == phone; .135 == WeMo Insight
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .135 == WeMo Insight
+        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+        final String outputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug-processed.pcap";
+        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
+//        final String deviceIp = "192.168.1.145"; // .246 == phone; .135 == WeMo Insight
+        final String deviceIp = "192.168.1.246"; // .246 == phone; .135 == WeMo Insight
 
-        // 6) TP-Link Bulb August 1 experiment
+        // 6) TODO: TP-LINK BULB August 1 experiment
 //        final String inputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/tplink-bulb/tplink-bulb-aug-3-2018.timestamps";
@@ -145,7 +145,7 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == TP-Link bulb
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb
 
-        // 7) Kwikset Doorlock August 6 experiment
+        // 7) TODO: KWIKSET DOORLOCK August 6 experiment
 //        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
 ////        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-processed.pcap";
@@ -168,7 +168,7 @@ public class SignatureGenerator {
 //        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps";
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
 
-        // 8) Hue Bulb August 7 experiment
+        // 8) TODO: HUE BULB August 7 experiment
 //        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/hue-bulb/hue-bulb-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
@@ -185,7 +185,7 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
 
-        // 9) Lifx Bulb August 8 experiment
+        // 9) TODO: LIFX BULB August 8 experiment
 //        final String inputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/lifx-bulb/lifx-bulb-aug-8-2018.timestamps";
@@ -201,13 +201,13 @@ public class SignatureGenerator {
 //        final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-nov-1-2018.timestamps";
 //        final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx
 
-        // 10) Amcrest Camera August 9 experiment
+        // 10) TODO: AMCREST CAMERA August 9 experiment
 //        final String inputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/amcrest-camera/amcrest-camera-aug-9-2018.timestamps";
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .235 == camera
 
-        // 11) Arlo Camera August 10 experiment
+        // 11) TODO: ARLO CAMERA August 10 experiment
 //        final String inputPcapFile = path + "/2018-08/arlo-camera/arlo-camera.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/arlo-camera/arlo-camera-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/arlo-camera/arlo-camera-aug-10-2018.timestamps";
@@ -221,7 +221,7 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
 
-        // 12) Blossom sprinkler August 13 experiment
+        // 12) TODO: BLOSSOM SPRINKLER August 13 experiment
 //        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/blossom/blossom-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/blossom/blossom-aug-13-2018.timestamps";
@@ -231,7 +231,7 @@ public class SignatureGenerator {
 //        final String outputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-nov-2-2018.timestamps";
 //        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
-        // January 9, 11, 13, 14
+        // TODO: EXPERIMENT - January 9, 11, 13, 14
 //        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap";
 //        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
@@ -239,7 +239,7 @@ public class SignatureGenerator {
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 ////        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
 
-//        // 13) DLink siren August 14 experiment
+//        // 13) TODO: D-LINK SIREN August 14 experiment
 //        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
 //        //final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
 //        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
@@ -254,19 +254,19 @@ public class SignatureGenerator {
 ////        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
 
-        // 14) Nest thermostat August 15 experiment
+        // 14) TODO: NEST THERMOSTAT August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/nest/nest-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/nest/nest-aug-15-2018.timestamps";
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
 //        // TODO: EXPERIMENT - November 14, 2018
-        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap";
-//        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat-processed.pcap";
-        final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
-//        final String deviceIp = "192.168.1.127"; // .246 == phone; .127 == Nest thermostat
-        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
+////        final String deviceIp = "192.168.1.127"; // .246 == phone; .127 == Nest thermostat
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
 
         // 15) Alexa August 16 experiment
 //        final String inputPcapFile = path + "/2018-08/alexa/alexa.wlan1.local.pcap";
@@ -669,14 +669,12 @@ public class SignatureGenerator {
         // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
         Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
         // Perform clustering on conversation logged as part of all ON events.
-        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
-//        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(2, 2);
-        //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
+        double eps = 10.0;
+        int minPts = 45;
+        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(eps, minPts);
         List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
         // Perform clustering on conversation logged as part of all OFF events.
-        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
-//        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10, 2);
-        //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
+        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(eps, minPts);
         List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
         // Sort the conversations as reference
         List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
@@ -687,52 +685,51 @@ public class SignatureGenerator {
         int count = 0;
         List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
         List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+        List<List<List<PcapPacket>>> corePointRangeSignatureOn = new ArrayList<>();
         for (Cluster<PcapPacketPair> c : onClusters) {
             System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
             if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-            //if(c.getPoints().size() > 25) {
                 // Print to file
                 List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-                //TODO: DO EPSILON ANALYSIS HERE!!!
-
+                // Check for overlaps and decide whether to do range-based or conservative checking
+                corePointRangeSignatureOn.add(PcapPacketUtils.extractRangeCorePoints(ppListOfList, eps, minPts));
                 ppListOfListListOn.add(ppListOfList);
             }
         }
-        // TODO: Merging test
-        ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
-        // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
-//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
-        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
-//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
-        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
-        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
-        // TODO: This sequence actually belongs to the local communication between the plug and the phone
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
-        PcapPacketUtils.printSignatures(ppListOfListListOn);
-
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");
         System.out.println("       Number of clusters: " + offClusters.size());
         count = 0;
         List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
         List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+        List<List<List<PcapPacket>>> corePointRangeSignatureOff = new ArrayList<>();
         for (Cluster<PcapPacketPair> c : offClusters) {
             System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
             System.out.print(PrintUtils.toSummaryString(c));
             if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-            //if(c.getPoints().size() > 25) {
                 // Print to file
                 List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-                //TODO: DO EPSILON ANALYSIS HERE!!!
-
+                // Check for overlaps and decide whether to do range-based or conservative checking
+                corePointRangeSignatureOff.add(PcapPacketUtils.extractRangeCorePoints(ppListOfList, eps, minPts));
                 ppListOfListListOff.add(ppListOfList);
             }
         }
+        // TODO: Merging test
+        ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
+        // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
+//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
+        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+        // TODO: This sequence actually belongs to the local communication between the plug and the phone
+//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+        ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+
         // TODO: Merging test
         ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
         // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
@@ -745,19 +742,26 @@ public class SignatureGenerator {
         //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
         // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
         // TODO: This sequence actually belongs to the local communication between the plug and the phone
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+//        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
         ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
-        PcapPacketUtils.printSignatures(ppListOfListListOff);
 
+        // Write the signatures into the screen
+        PcapPacketUtils.printSignatures(ppListOfListListOn);
+        PcapPacketUtils.printSignatures(ppListOfListListOff);
 
         // Printing signatures into files
-        PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
-        ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
-        PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
-        ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
+        PrintUtils.serializeIntoFile("./onSignature.sig", ppListOfListListOn);
+        PrintUtils.serializeIntoFile("./offSignature.sig", ppListOfListListOff);
+        //ppListOfListReadOn = PrintUtils.deserializeFromFile("./onSignature.sig");
+        //ppListOfListReadOff = PrintUtils.deserializeFromFile("./offSignature.sig");
+
+        // Printing cluster analyses into files
+        PrintUtils.serializeIntoFile("./onClusters.cls", corePointRangeSignatureOn);
+        PrintUtils.serializeIntoFile("./offClusters.cls", corePointRangeSignatureOff);
         System.out.println("========================================");
+
         // ============================================================================================================
-//
+
 //        // TODO: This part is just for DBSCAN sensitivity experiment
 //        // TODO: This part is just for DBSCAN sensitivity experiment
 //        // TODO: This part is just for DBSCAN sensitivity experiment
@@ -898,22 +902,28 @@ public class SignatureGenerator {
     /**
      * Check if there is any overlap between the signature stored in this class and another signature.
      * Conditions:
-     * 1) If both signatures do not have any range, then we need to do conservative checking (return true).
-     * 2) If both signatures have the same number of packets/packet lengths, then we check the range; if the
+     * 1) If the signatures do not have any range, then we need to do conservative checking (return true).
+     * 2) If the signatures have the same number of packets/packet lengths, then we check the range; if the
      *    numbers of packets/packet lengths are different then we assume that there is no overlap.
      * 3) If there is any range in the signatures, then we need to check for overlap.
      * 4) If there is overlap for every packet/packet length, then we return false (range-based checking); otherwise,
      *    true (conservative checking).
      *
-     * @param otherSignature A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
-     *                       for overlaps with the signature stored in this class.
+     * @param signatures Multiple {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
+     *                       for overlaps.
      * @return A boolean that is true if there is an overlap; false otherwise.
      */
-//    public boolean isConservativeChecking(List<List<List<PcapPacket>>> otherSignature) {
-//
-//        // Get the ranges of the two signatures
-//        List<List<List<PcapPacket>>> signatureRanges = getSequenceRanges(mSignature);
-//        List<List<List<PcapPacket>>> otherSignatureRanges = getSequenceRanges(otherSignature);
+    public boolean isConservativeChecking(List<List<List<PcapPacket>>>...signatures) {
+
+        // If none of the pairs/sequences is range-based then we go conservative
+        boolean isRange = false;
+        for(List<List<List<PcapPacket>>> signature : signatures) {
+            if (isRangeBased(signature)) {
+                isRange = true;
+            }
+        }
+
+        return isRange;
 //        if (!isRangeBased(signatureRanges) && !isRangeBased(otherSignatureRanges)) {
 //            // Conservative checking when there is no range
 //            return true;
@@ -924,7 +934,7 @@ public class SignatureGenerator {
 //            // There is range; check if there is overlap
 //            return checkOverlap(signatureRanges, otherSignatureRanges);
 //        }
-//    }
+    }
 
     /*
      * Check for overlap since we have range in at least one of the signatures.
@@ -959,30 +969,30 @@ public class SignatureGenerator {
 //
 //        return true;
 //    }
-//
-//    /*
-//     * Check and see if there is any range in the signatures
-//     */
-//    private boolean isRangeBased(List<List<List<PcapPacket>>> signatureRanges) {
-//
-//        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
-//            // Lower bound of the range is in index 0
-//            // Upper bound of the range is in index 1
-//            List<PcapPacket> minSequence = listListPcapPacket.get(0);
-//            List<PcapPacket> maxSequence = listListPcapPacket.get(1);
-//            for(PcapPacket pcapPacket : minSequence) {
-//                int index = minSequence.indexOf(pcapPacket);
-//                if (pcapPacket.length() != maxSequence.get(index).length()) {
-//                    // If there is any packet length that differs in the minSequence
-//                    // and maxSequence, then it is range-based
-//                    return true;
-//                }
-//            }
-//        }
-//
-//        return false;
-//    }
-//
+
+    /*
+     * Check and see if there is any range in the signatures
+     */
+    private boolean isRangeBased(List<List<List<PcapPacket>>> signatureRanges) {
+
+        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
+            // Lower bound of the range is in index 0
+            // Upper bound of the range is in index 1
+            List<PcapPacket> minSequence = listListPcapPacket.get(0);
+            List<PcapPacket> maxSequence = listListPcapPacket.get(1);
+            for(PcapPacket pcapPacket : minSequence) {
+                int index = minSequence.indexOf(pcapPacket);
+                if (pcapPacket.length() != maxSequence.get(index).length()) {
+                    // If there is any packet length that differs in the minSequence
+                    // and maxSequence, then it is range-based
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
 //    /* Find the sequence with the minimum packet lengths.
 //     * The second-layer list should contain the minimum sequence for element 0 and maximum sequence for element 1.
 //     */
index 45c6a55..a01d1ba 100644 (file)
@@ -26,7 +26,7 @@ abstract public class AbstractClusterMatcher {
      */
     protected final List<ClusterMatcherObserver> mObservers;
 
-    protected AbstractClusterMatcher(List<List<PcapPacket>> cluster) {
+    protected AbstractClusterMatcher(List<List<PcapPacket>> cluster, boolean isRangeBased) {
         // ===================== PRECONDITION SECTION =====================
         cluster = Objects.requireNonNull(cluster, "cluster cannot be null");
         if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) {
@@ -38,8 +38,12 @@ abstract public class AbstractClusterMatcher {
             }
         }
         // ================================================================
-        // Let the subclass prune the provided cluster
-        mCluster = pruneCluster(cluster);
+        // Let the subclass prune the provided cluster---only if it is not range-based
+        if (!isRangeBased) {
+            mCluster = pruneCluster(cluster);
+        } else {
+            mCluster = cluster;
+        }
         mObservers = new ArrayList<>();
     }
 
index 5021c31..88cb64e 100644 (file)
@@ -36,7 +36,7 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye
      */
     public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
         // Consider all flows if no flow filter specified.
-        this(cluster, flow -> true);
+        this(cluster, flow -> true, false);
     }
 
     /**
@@ -48,9 +48,11 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye
      *                   namely when the {@link Layer2FlowReassembler} notifies the {@link Layer2ClusterMatcher} about
      *                   the new flow. This functionality may for example come in handy when one only wants to search
      *                   for matches in the subset of flows that involves a specific (range of) MAC(s).
+     * @param isRangeBased The boolean that decides if it is range-based vs. strict matching.
      */
-    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster, Function<Layer2Flow, Boolean> flowFilter) {
-        super(cluster);
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster, Function<Layer2Flow, Boolean> flowFilter,
+                                boolean isRangeBased) {
+        super(cluster, isRangeBased);
         mFlowFilter = flowFilter;
     }
 
index a721914..f5a314f 100644 (file)
@@ -109,13 +109,18 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         resultsWriter.flush();
 
+        // TODO: IMPLEMENT THE RANGE-BASED DETECTION HERE
+        boolean isRangeBased = true;
+
         // Create signature detectors and add observers that output their detected events.
-        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
-        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile);
         Layer2SignatureDetector onDetector = onSignatureMacFilters == null ?
-                new Layer2SignatureDetector(onSignature) : new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration);
+                new Layer2SignatureDetector(onSignature) :
+                new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBased);
         Layer2SignatureDetector offDetector = offSignatureMacFilters == null ?
-                new Layer2SignatureDetector(offSignature) : new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration);
+                new Layer2SignatureDetector(offSignature) :
+                new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBased);
         onDetector.addObserver((signature, match) -> {
             UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp());
             PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
@@ -175,10 +180,11 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
     private int mInclusionTimeMillis;
 
     public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature) {
-        this(searchedSignature, null, 0);
+        this(searchedSignature, null, 0, false);
     }
 
-    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, List<Function<Layer2Flow, Boolean>> flowFilters, int inclusionTimeMillis) {
+    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, List<Function<Layer2Flow,
+            Boolean>> flowFilters, int inclusionTimeMillis, boolean isRangeBased) {
         if (flowFilters != null && flowFilters.size() != searchedSignature.size()) {
             throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature.");
         }
@@ -187,7 +193,7 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         for (int i = 0; i < mSignature.size(); i++) {
             List<List<PcapPacket>> cluster = mSignature.get(i);
             Layer2ClusterMatcher clusterMatcher = flowFilters == null ?
-                    new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i));
+                    new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased);
             clusterMatcher.addObserver(this);
             clusterMatchers.add(clusterMatcher);
         }
index 472e3be..a4ad857 100644 (file)
@@ -66,19 +66,30 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
      */
     private final String mRouterWanIp;
 
+    /**
+     * Range-based vs. strict matching.
+     */
+    private final boolean mRangeBased;
+
+    /**
+     * Epsilon value used by the DBSCAN algorithm; it is used again for range-based matching here.
+     */
+    private final double mEps;
+
     /**
      * Create a {@link Layer3ClusterMatcher}.
      * @param cluster The cluster that traffic is matched against.
      * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for
      *                    determining the direction of packets).
+     * @param isRangeBased The boolean that decides if it is range-based vs. strict matching.
      * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that
      *                          (a subset of) the examined traffic is similar to the traffic that makes up
      *                          {@code cluster}, i.e., when the examined traffic is classified as pertaining to
      *                          {@code cluster}.
      */
-    public Layer3ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp, List<List<List<List<PcapPacket>>>> otherSignatures,
+    public Layer3ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp, boolean isRangeBased, double eps,
                                 ClusterMatcherObserver... detectionObservers) {
-        super(cluster);
+        super(cluster, isRangeBased);
         Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null");
         for (ClusterMatcherObserver obs : detectionObservers) {
             addObserver(obs);
@@ -92,16 +103,18 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
          * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted
          * in order to ensure correctness, especially during the development/debugging phase.
          */
-        if (mCluster.stream().
-                anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) {
-            throw new IllegalArgumentException(
-                    "cluster members must contain the same number of packets and exhibit the same packet direction " +
-                            "pattern"
-            );
+        mRangeBased = isRangeBased;
+        if (!mRangeBased) {    // Only when it is not range-based
+            if (mCluster.stream().
+                    anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) {
+                throw new IllegalArgumentException(
+                        "cluster members must contain the same number of packets and exhibit the same packet direction " +
+                                "pattern"
+                );
+            }
         }
+        mEps = eps;
         mRouterWanIp = routerWanIp;
-
-        checkOverlaps(otherSignatures);
     }
 
     @Override
@@ -110,18 +123,6 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
         mTcpReassembler.gotPacket(packet);
     }
 
-    // TODO: UNDER CONSTRUCTION NOW!
-    private void checkOverlaps(List<List<List<List<PcapPacket>>>> otherSignatures) {
-        // Unpack the list
-        for(List<List<List<PcapPacket>>> listListListPcapPacket : otherSignatures) {
-            for(List<List<PcapPacket>> listListPcapPacket : listListListPcapPacket) {
-                for(List<PcapPacket> listPcapPacket : listListPcapPacket) {
-
-                }
-            }
-        }
-    }
-
     /**
      * Get the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
      * @return the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
@@ -130,7 +131,42 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
         return mCluster;
     }
 
-    public void performDetection() {
+    public void performDetectionRangeBased() {
+        /*
+         * Let's start out simple by building a version that only works for signatures that do not span across multiple
+         * TCP conversations...
+         */
+        for (Conversation c : mTcpReassembler.getTcpConversations()) {
+            if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) {
+                // Skip empty conversations.
+                continue;
+            }
+            List<PcapPacket> lowerBound = mCluster.get(0);
+            List<PcapPacket> upperBound = mCluster.get(1);
+            if (isTlsSequence(lowerBound) != c.isTls() || isTlsSequence(upperBound) != c.isTls()) {
+                // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+                continue;
+            }
+            // Fetch set of packets to examine based on TLS or not.
+            List<PcapPacket> cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets();
+            Optional<List<PcapPacket>> match;
+            while ((match = findSubsequenceInSequence(lowerBound, upperBound, cPkts, mClusterMemberDirections, null)).
+                    isPresent()) {
+                List<PcapPacket> matchSeq = match.get();
+                // Notify observers about the match.
+                mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
+                /*
+                 * Get the index in cPkts of the last packet in the sequence of packets that matches the searched
+                 * signature sequence.
+                 */
+                int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size() - 1));
+                // We restart the search for the signature sequence immediately after that index, so truncate cPkts.
+                cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList());
+            }
+        }
+    }
+
+    public void performDetectionConservative() {
         /*
          * Let's start out simple by building a version that only works for signatures that do not span across multiple
          * TCP conversations...
@@ -165,11 +201,12 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
                      * Get the index in cPkts of the last packet in the sequence of packets that matches the searched
                      * signature sequence.
                      */
-                    int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1));
+                    int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size() - 1));
                     // We restart the search for the signature sequence immediately after that index, so truncate cPkts.
                     cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList());
                 }
             }
+
             /*
              * TODO:
              * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did
@@ -285,6 +322,95 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
         return Optional.empty();
     }
 
+    /**
+     * Overloading the method {@code findSubsequenceInSequence} for range-based matching. Instead of a sequence,
+     * we have sequences of lower and upper bounds.
+     *
+     * @param lowerBound The lower bound of the sequence we search for.
+     * @param upperBound The upper bound of the sequence we search for.
+     * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i},
+     *                              {@code subsequenceDirections[i]} is the direction of the packet returned by
+     *                              {@code subsequence.get(i)}. May be set to {@code null}, in which this call will
+     *                              internally compute the packet directions.
+     * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i},
+     *                           {@code sequenceDirections[i]} is the direction of the packet returned by
+     *                           {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally
+     *                           compute the packet directions.
+     *
+     * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty
+     *         {@link Optional} if no part of {@code sequence} matches {@code subsequence}.
+     */
+    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> lowerBound,
+                                                                 List<PcapPacket> upperBound,
+                                                                 List<PcapPacket> sequence,
+                                                                 Conversation.Direction[] subsequenceDirections,
+                                                                 Conversation.Direction[] sequenceDirections) {
+        // Just do the checks for either lower or upper bound!
+        // TODO: For now we use just the lower bound
+        if (sequence.size() < lowerBound.size()) {
+            // If subsequence is longer, it cannot be contained in sequence.
+            return Optional.empty();
+        }
+        if (isTlsSequence(lowerBound) != isTlsSequence(sequence)) {
+            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+            return Optional.empty();
+        }
+        // If packet directions have not been precomputed by calling code, we need to construct them.
+        if (subsequenceDirections == null) {
+            subsequenceDirections = getPacketDirections(lowerBound, mRouterWanIp);
+        }
+        if (sequenceDirections == null) {
+            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+        }
+        int subseqIdx = 0;
+        int seqIdx = 0;
+        while (seqIdx < sequence.size()) {
+            PcapPacket lowBndPkt = lowerBound.get(subseqIdx);
+            PcapPacket upBndPkt = upperBound.get(subseqIdx);
+            PcapPacket seqPkt = sequence.get(seqIdx);
+            // We only have a match if packet lengths and directions match.
+            // The packet lengths have to be in the range of [lowerBound - eps, upperBound+eps]
+            // TODO: Maybe we could do better here for the double to integer conversion?
+            int epsLowerBound = lowBndPkt.length() - (int) mEps;
+            int epsUpperBound = upBndPkt.length() + (int) mEps;
+            if (epsLowerBound <= seqPkt.getOriginalLength() &&
+                    seqPkt.getOriginalLength() <= epsUpperBound &&
+                    subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+                subseqIdx++;
+                seqIdx++;
+                if (subseqIdx == lowerBound.size()) {
+                    // We managed to match the entire subsequence in sequence.
+                    // Return the sublist of sequence that matches subsequence.
+                    /*
+                     * TODO:
+                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+                     * for live traces!
+                     */
+                    return Optional.of(sequence.subList(seqIdx - lowerBound.size(), seqIdx));
+                }
+            } else {
+                // Mismatch.
+                if (subseqIdx > 0) {
+                    /*
+                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+                     * leave seqIdx untouched.
+                     */
+                    subseqIdx = 0;
+                } else {
+                    /*
+                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+                     * sequence.
+                     */
+                    seqIdx++;
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
     /**
      * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
      * members. Two cluster members are considered identical if their packets lengths and packet directions are
index 340310d..6e5b87c 100644 (file)
@@ -5,6 +5,7 @@ import edu.uci.iotproject.analysis.UserAction;
 import edu.uci.iotproject.detection.AbstractClusterMatcher;
 import edu.uci.iotproject.detection.ClusterMatcherObserver;
 import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PcapPacketUtils;
 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.distribution.AbstractRealDistribution;
 import org.apache.commons.math3.distribution.NormalDistribution;
@@ -118,7 +119,7 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 
         // TODO: The following are tests for signatures against training data
 
-        // D-Link Plug experiment
+        // TODO: D-LINK PLUG experiment
 //        final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
         // D-Link Plug DEVICE signatures
 //        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
@@ -135,12 +136,16 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        // D-Link Plug DEVICE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-device-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-device-side.cls";
         // D-Link Plug PHONE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-phone-side.cls";
 
         // TODO: EXPERIMENT - November 9, 2018
-        // D-Link Siren experiment
+        // TODO: D-LINK SIREN experiment
         //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
         //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap";
 //        final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap";
@@ -153,8 +158,10 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
 //        final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-offClusters-phone-side.cls";
 
-        // TP-Link Plug experiment
+        // TODO: TP-LINK PLUG experiment
 ////        final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
 ////        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap";
 //        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap";
@@ -166,33 +173,18 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
 ////        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap";
 ////        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap";
-//        //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap";
-//        // TP-Link Plug DEVICE signatures
+//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap";
+////        // TP-Link Plug DEVICE signatures
 ////        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
 ////        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
 //        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
-        // TP-Link Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
-
-        // Arlo camera experiment
-//        final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-////        // TP-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
-        // TODO: EXPERIMENT - November 13, 2018
-        // Arlo Camera experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap";
-////        final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
-//        // Arlo Camera PHONE signatures
-////        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
-////        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
-//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete";
-//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete";
+//        // TP-Link Plug PHONE signatures
+////        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
+//        // TP-Link Plug cluster analyses
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-onClusters.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-offClusters.cls";
 
         // Amazon Alexa experiment
 //        final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap";
@@ -200,7 +192,25 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig";
 //        final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig";
 
-        // SmartThings Plug experiment
+        // TODO: KWIKSET DOORLOCK Sep 12 experiment
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
+//        // Kwikset Doorlock PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig";
+        // TODO: EXPERIMENT - November 10, 2018
+        // Kwikset Door lock experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap";
+//        // Kwikset Door lock PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
+////        final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-offClusters-phone-side.cls";
+
+        // TODO: SMARTTHINGS PLUG experiment
 //        final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap";
 //        // SmartThings Plug DEVICE signatures
 //        //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig";
@@ -213,24 +223,78 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
 //        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap";
 //        //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap";
-        final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap";
-//        // SmartThings Plug PHONE signatures
-        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
-        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap";
+////        // SmartThings Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
 //        final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-offClusters-phone-side.cls";
+
+        // TODO: ARLO CAMERA experiment
+//        final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+////        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 13, 2018
+        // Arlo Camera experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap";
+        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+        // Arlo Camera PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete";
+        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete";
+        final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls";
+        final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls";
+
+        // TODO: NEST THERMOSTAT experiment
+//        final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        // Nest Thermostat DEVICE signatures
+////        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig";
+////        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig";
+//        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+//        // TODO: EXPERIMENT - November 15, 2018
+        // Nest Thermostat experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap";
+////        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-offClusters-phone-side.cls";
 
         // TODO: EXPERIMENT - January 9, 2018
+        // TODO: BLOSSOM SPRINKLER experiment
         // Blossom Sprinkler experiment
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap";
+//        // Blossom Sprinkler DEVICE signatures
+//        final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+
 ////        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
 //        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
 ////        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
 //        // Blossom Sprinkler DEVICE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+////        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+////        final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls";
+////        final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls";
 //        // Blossom Sprinkler PHONE signatures
-////        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
-////        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls";
 
         // LiFX Bulb experiment
 //        final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap";
@@ -241,32 +305,6 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig";
 
-        // Blossom Sprinkler experiment
-//        //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
-//        //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap";
-//        // Blossom Sprinkler DEVICE signatures
-//        final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
-
-        // Nest Thermostat experiment
-//        final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-//        // Nest Thermostat DEVICE signatures
-////        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig";
-////        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig";
-//        // Nest Thermostat PHONE signatures
-//        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
-//        // TODO: EXPERIMENT - November 15, 2018
-        // Nest Thermostat experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap";
-////        // Nest Thermostat PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
-
         /*
         // Hue Bulb experiment
         final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
@@ -275,9 +313,7 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
         */
 
-
-
-        // TP-Link Bulb experiment
+        // TODO: TP-LINK BULB experiment
 //        final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
 //        // TP-Link Bulb PHONE signatures
 //        final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
@@ -291,16 +327,11 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        // TP-Link Bulb PHONE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-offClusters-phone-side.cls";
 
-        /*
-        // WeMo Plug experiment
-        final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
-        // WeMo Plug PHONE signatures
-        final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig";
-        final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig";
-        */
         // TODO: EXPERIMENT - November 20, 2018
-        // WeMo Plug experiment
+        // TODO: WEMO PLUG experiment
 //        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
 //        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap";
         // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE
@@ -310,16 +341,11 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        // WeMo Plug PHONE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-offClusters-phone-side.cls";
 
-        /*
-        // WeMo Insight Plug experiment
-        final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
-        // WeMo Insight Plug PHONE signatures
-        final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig";
-        final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig";
-        */
         // TODO: EXPERIMENT - November 21, 2018
-        // WeMo Insight Plug experiment
+        // TODO: WEMO INSIGHT Plug experiment
 //        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
 //        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap";
         // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG)
@@ -328,25 +354,22 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         // WeMo Insight Plug PHONE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
+//        final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-onClusters-phone-side.cls";
+//        final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-offClusters-phone-side.cls";
 
 
-        // Kwikset Doorlock Sep 12 experiment
-//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
-//        // Kwikset Doorlock PHONE signatures
-//        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig";
-//        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig";
-        // TODO: EXPERIMENT - November 10, 2018
-        // Kwikset Door lock experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
-        //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap";
-//        // Kwikset Door lock PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
-////        final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig";
-////        final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig";
-
-
+        /*
+        // WeMo Plug experiment
+        final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+        // WeMo Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig";
+        // WeMo Insight Plug experiment
+        final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+        // WeMo Insight Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig";
+        */
 
         // D-Link Siren experiment
 //        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
@@ -363,27 +386,36 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         System.out.println("OFF signature file in use is " + offSignatureFile);
         System.out.println("PCAP file that is the target of detection is " + inputPcapFile);
 
-        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
-        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+        // Specify epsilon
+        // TODO: This would be specified through command line option
+        double eps = 10.0;
+        // Load signatures
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile);
+        // Load signature analyses
+        List<List<List<PcapPacket>>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile);
+        List<List<List<PcapPacket>>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile);
+
+        // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS
+        // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR
+        // TODO: STRICT MATCHING
+        // Check if we should use range-based matching
+        boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature);
+        boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature);
+        // Update the signature with ranges if it is range-based
+        if (isRangeBasedForOn && isRangeBasedForOff) {
+            onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis);
+            offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis);
+        }
 
         // LAN
 //        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
 //        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
-
-        // TODO: We need the array that contains other signatures so that we can check for overlap and decide
-        // TODO: whether we use conservative or range-based matching
-        // Right now we have ON signature as other signature for OFF and OFF signature as other signature for ON
-        // In the future, we might have more other signatures
-        List<List<List<List<PcapPacket>>>> otherSignaturesOutsideOn = new ArrayList<>();
-        otherSignaturesOutsideOn.add(offSignature);
-        List<List<List<List<PcapPacket>>>> otherSignaturesOutsideOff = new ArrayList<>();
-        otherSignaturesOutsideOff.add(onSignature);
-
         // WAN
-        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", 0,
-                otherSignaturesOutsideOn);
-        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", 0,
-                otherSignaturesOutsideOff);
+        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105",
+                0, isRangeBasedForOn, eps);
+        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105",
+                0, isRangeBasedForOff, eps);
 
         final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).
                 withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles"));
@@ -428,16 +460,14 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector);
         reader.readFromHandle();
 
-        //if (onDetector.isConservativeChecking(offSignature)) {
-        // System.out.println("Do conservative checking!");
-        //} else {
-        // TODO: WORK ON THIS RANGE-BASED CHECKING
-        // System.out.println("Do range-based checking!");
-        //}
-
         // TODO: need a better way of triggering detection than this...
-        onDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
-        offDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
+        if (isRangeBasedForOn && isRangeBasedForOff) {
+            onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased());
+            offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased());
+        } else {
+            onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative());
+            offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative());
+        }
 
         // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger
         // times file.
@@ -451,11 +481,9 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
                 detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count());
 
-
         // TODO: Temporary clean up until we clean the pipeline
 //      List<UserAction> cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents);
 //      cleanedDetectedEvents.forEach(outputter);
-
     }
 
     /**
@@ -511,136 +539,14 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         return listUserActionClean;
     }
 
-//    /**
-//     * Check if there is any overlap between the signature stored in this class and another signature.
-//     * Conditions:
-//     * 1) If both signatures do not have any range, then we need to do conservative checking (return true).
-//     * 2) If both signatures have the same number of packets/packet lengths, then we check the range; if the
-//     *    numbers of packets/packet lengths are different then we assume that there is no overlap.
-//     * 3) If there is any range in the signatures, then we need to check for overlap.
-//     * 4) If there is overlap for every packet/packet length, then we return false (range-based checking); otherwise,
-//     *    true (conservative checking).
-//     *
-//     * @param otherSignature A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
-//     *                       for overlaps with the signature stored in this class.
-//     * @return A boolean that is true if there is an overlap; false otherwise.
-//     */
-//    public boolean isConservativeChecking(List<List<List<PcapPacket>>> otherSignature) {
-//
-//        // Get the ranges of the two signatures
-//        List<List<List<PcapPacket>>> signatureRanges = getSequenceRanges(mSignature);
-//        List<List<List<PcapPacket>>> otherSignatureRanges = getSequenceRanges(otherSignature);
-//        if (!isRangeBased(signatureRanges) && !isRangeBased(otherSignatureRanges)) {
-//            // Conservative checking when there is no range
-//            return true;
-//        } else if(signatureRanges.size() != otherSignatureRanges.size()) {
-//            // The two signatures have different numbers of packets/packet lengths
-//            return false;
-//        } else {
-//            // There is range; check if there is overlap
-//            return checkOverlap(signatureRanges, otherSignatureRanges);
-//        }
-//    }
-//
-//    /*
-//     * Check for overlap since we have range in at least one of the signatures.
-//     * Overlap is only true when all ranges overlap. We need to check in order.
-//     */
-//    private boolean checkOverlap(List<List<List<PcapPacket>>> signatureRanges,
-//                                 List<List<List<PcapPacket>>> otherSignatureRanges) {
-//
-//        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
-//            // Lower bound of the range is in index 0
-//            // Upper bound of the range is in index 1
-//            int sequenceSetIndex = signatureRanges.indexOf(listListPcapPacket);
-//            List<PcapPacket> minSequenceSignature = listListPcapPacket.get(0);
-//            List<PcapPacket> maxSequenceSignature = listListPcapPacket.get(1);
-//            for(PcapPacket pcapPacket : minSequenceSignature) {
-//                // Get the lower and upper bounds of the current signature
-//                int packetIndex = minSequenceSignature.indexOf(pcapPacket);
-//                int lowerBound = pcapPacket.length();
-//                int upperBound = maxSequenceSignature.get(packetIndex).length();
-//                // Check for range overlap in the other signature!
-//                // Check the packet/packet length at the same position
-//                List<PcapPacket> minSequenceSignatureOther = otherSignatureRanges.get(sequenceSetIndex).get(0);
-//                List<PcapPacket> maxSequenceSignatureOther = otherSignatureRanges.get(sequenceSetIndex).get(1);
-//                int lowerBoundOther = minSequenceSignatureOther.get(packetIndex).length();
-//                int upperBoundOther = maxSequenceSignatureOther.get(packetIndex).length();
-//                if (!(lowerBoundOther <= lowerBound && lowerBound <= upperBoundOther) &&
-//                    !(lowerBoundOther <= upperBound && upperBound <= upperBoundOther)) {
-//                    return false;
-//                }
-//            }
-//        }
-//
-//        return true;
-//    }
-//
-//    /*
-//     * Check and see if there is any range in the signatures
-//     */
-//    private boolean isRangeBased(List<List<List<PcapPacket>>> signatureRanges) {
-//
-//        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
-//            // Lower bound of the range is in index 0
-//            // Upper bound of the range is in index 1
-//            List<PcapPacket> minSequence = listListPcapPacket.get(0);
-//            List<PcapPacket> maxSequence = listListPcapPacket.get(1);
-//            for(PcapPacket pcapPacket : minSequence) {
-//                int index = minSequence.indexOf(pcapPacket);
-//                if (pcapPacket.length() != maxSequence.get(index).length()) {
-//                    // If there is any packet length that differs in the minSequence
-//                    // and maxSequence, then it is range-based
-//                    return true;
-//                }
-//            }
-//        }
-//
-//        return false;
-//    }
-
-    /* Find the sequence with the minimum packet lengths.
-     * The second-layer list should contain the minimum sequence for element 0 and maximum sequence for element 1.
-     */
-    private List<List<List<PcapPacket>>> getSequenceRanges(List<List<List<PcapPacket>>> signature) {
-
-        // Start from the first index
-        List<List<List<PcapPacket>>> rangeBasedSequence = new ArrayList<>();
-        for(List<List<PcapPacket>> listListPcapPacket : signature) {
-            List<List<PcapPacket>> minMaxSequence = new ArrayList<>();
-            // Both searches start from index 0
-            List<PcapPacket> minSequence = new ArrayList<>(listListPcapPacket.get(0));
-            List<PcapPacket> maxSequence = new ArrayList<>(listListPcapPacket.get(0));
-            for(List<PcapPacket> listPcapPacket : listListPcapPacket) {
-                for(PcapPacket pcapPacket : listPcapPacket) {
-                    int index = listPcapPacket.indexOf(pcapPacket);
-                    // Set the new minimum if length at the index is minimum
-                    if (pcapPacket.length() < minSequence.get(index).length()) {
-                        minSequence.set(index, pcapPacket);
-                    }
-                    // Set the new maximum if length at the index is maximum
-                    if (pcapPacket.length() > maxSequence.get(index).length()) {
-                        maxSequence.set(index, pcapPacket);
-                    }
-                }
-            }
-            // minSequence as element 0 and maxSequence as element 1
-            minMaxSequence.add(minSequence);
-            minMaxSequence.add(maxSequence);
-            rangeBasedSequence.add(minMaxSequence);
-        }
-
-        return rangeBasedSequence;
-    }
-
-    public SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String routerWanIp, int inclusionTimeMillis,
-                             List<List<List<List<PcapPacket>>>> otherSignatures) {
+    public SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String routerWanIp,
+                             int inclusionTimeMillis, boolean isRangeBased, double eps) {
         // note: doesn't protect inner lists from changes :'(
         mSignature = Collections.unmodifiableList(searchedSignature);
         // Generate corresponding/appropriate ClusterMatchers based on the provided signature
         List<Layer3ClusterMatcher> clusterMatchers = new ArrayList<>();
         for (List<List<PcapPacket>> cluster : mSignature) {
-            clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, otherSignatures, this));
+            clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, isRangeBased, eps, this));
         }
         mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
 
index 060387a..28e9805 100644 (file)
@@ -109,7 +109,7 @@ public class SanitySignatureGenerator {
                 cluster.add(sequence);
                 signature.add(cluster);
                 // Output the signature to a file.
-                PrintUtils.serializeSignatureIntoFile(mSignatureOutputPath, signature);
+                PrintUtils.serializeIntoFile(mSignatureOutputPath, signature);
             }
         }
 
index 067af93..a69ac13 100644 (file)
@@ -468,6 +468,339 @@ public final class PcapPacketUtils {
         }
     }
 
+    /**
+     * Extract core point range in the form of {@code List} of {@code List} of {@code PcapPacket} objects.
+     *
+     * @param pairs The pairs for core points extraction.
+     * @param eps Epsilon value for the DBSCAN algorithm.
+     * @param minPts minPts value for the DBSCAN algorithm.
+     * @return A {@link List} of {@link List} of {@code PcapPacket} objects that contains core points range
+     *          in the first and second element.
+     */
+    public static List<List<PcapPacket>> extractRangeCorePoints(List<List<PcapPacket>> pairs, double eps, int minPts) {
+
+        // Initialize min and max value
+        PcapPacket minFirstElement = null;
+        PcapPacket maxFirstElement = null;
+        PcapPacket minSecondElement = null;
+        PcapPacket maxSecondElement = null;
+
+        // Iterate over pairs
+        for(List<PcapPacket> pair : pairs) {
+            if (isCorePoint(pair, pairs, eps, minPts)) {
+                // Record the first element
+                if (minFirstElement == null || pair.get(0).length() < minFirstElement.length()) {
+                    minFirstElement = pair.get(0);
+                }
+                if (maxFirstElement == null || pair.get(0).length() > maxFirstElement.length()) {
+                    maxFirstElement = pair.get(0);
+                }
+                // Record the second element
+                if (minSecondElement == null || pair.get(1).length() < minSecondElement.length()) {
+                    minSecondElement = pair.get(1);
+                }
+                if (maxSecondElement == null || pair.get(1).length() > maxSecondElement.length()) {
+                    maxSecondElement = pair.get(1);
+                }
+            }
+        }
+        List<PcapPacket> corePointLowerBound = new ArrayList<>();
+        corePointLowerBound.add(0, minFirstElement);
+        corePointLowerBound.add(1, minSecondElement);
+        List<PcapPacket> corePointUpperBound = new ArrayList<>();
+        corePointUpperBound.add(0, maxFirstElement);
+        corePointUpperBound.add(1, maxSecondElement);
+        // Combine lower and upper bounds
+        List<List<PcapPacket>> listRangeCorePoints = new ArrayList<>();
+        listRangeCorePoints.add(corePointLowerBound);
+        listRangeCorePoints.add(corePointUpperBound);
+
+        return listRangeCorePoints;
+    }
+
+    /**
+     * Test whether the {@code List} of {@code PcapPacket} objects is a core point.
+     *
+     * @param pair The pair to be tested.
+     * @param pairs All of the pairs.
+     * @param eps Epsilon value for the DBSCAN algorithm.
+     * @param minPts minPts value for the DBSCAN algorithm.
+     * @return True if the pair is a core point.
+     */
+    private static boolean isCorePoint(List<PcapPacket> pair, List<List<PcapPacket>> pairs, double eps, int minPts) {
+
+        int corePoints = 0;
+        int x1 = pair.get(0) == null ? 0 : pair.get(0).length();
+        int y1 = pair.get(1) == null ? 0 : pair.get(1).length();
+        // Check if we have enough core points
+        for(List<PcapPacket> pairInPairs : pairs) {
+            int x2 = pairInPairs.get(0) == null ? 0 : pairInPairs.get(0).length();
+            int y2 = pairInPairs.get(1) == null ? 0 : pairInPairs.get(1).length();
+            // Measure distance between x and y
+            double distance = Math.sqrt(Math.pow((double)(x2 - x1), 2) + Math.pow((double)(y2 - y1), 2));
+            // Increment core points count if this point is within eps
+            if (distance <= eps) {
+                corePoints++;
+            }
+        }
+        // Return true if the number of core points >= minPts
+        if (corePoints >= minPts) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Test the conservativeness of the signatures (basically whether we want strict or range-based matching).
+     * We go for a conservative approach (strict matching) when there is no range or there are ranges but the
+     * ranges overlap across multiple signatures, e.g., ON and OFF signatures.
+     *
+     * @param signature The signature we want to check and overwrite if needed.
+     * @param eps Epsilon value for the DBSCAN algorithm.
+     * @param otherSignatures Other signatures we want to check against this signature.
+     * @return A boolean that is True when range-based matching is used.
+     */
+    public static boolean isRangeBasedMatching(List<List<List<PcapPacket>>> signature, double eps,
+                                                List<List<List<PcapPacket>>> ...otherSignatures) {
+        // Check against multiple signatures
+        // TODO: Per March 2019 we only support ON and OFF signatures though
+        for(List<List<List<PcapPacket>>> otherSig : otherSignatures) {
+            // Do conservative strict matching if there is any overlap
+            if (isConservativeChecking(signature, otherSig, eps)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Test the conservativeness of the signatures (basically whether we want strict or range-based matching).
+     * We go for a conservative approach (strict matching) when there is no range or there are ranges but the
+     * ranges overlap across multiple signatures, e.g., ON and OFF signatures.
+     *
+     * @param signature The signature we want to check and overwrite if needed.
+     * @param corePointRange The core points range of this signature.
+     * @return A boolean that is True when range-based matching is used.
+     */
+    public static List<List<List<PcapPacket>>> useRangeBasedMatching(List<List<List<PcapPacket>>> signature,
+                                                                     List<List<List<PcapPacket>>> corePointRange) {
+        // Do range-based checking instead if there is no overlap
+        // Transform our signature into a range-based format
+        List<List<List<PcapPacket>>> rangeBasedSignature = getSequenceRanges(signature);
+        // We have to iterate sequence by sequence in the signature that has already gone through concatenation/merging
+        // And compare the packet lengths against the ones in corePointRange that are still in pairs/points
+        List<List<List<PcapPacket>>> finalSignature = new ArrayList<>();
+
+        // Construct the range-based signature
+        for(List<List<PcapPacket>> listOfSequences : rangeBasedSignature) {
+            List<PcapPacket> sequenceLowerBound = listOfSequences.get(0);
+            List<PcapPacket> sequenceUpperBound = listOfSequences.get(1);
+            List<List<PcapPacket>> newList = new ArrayList<>();
+            List<PcapPacket> newListLowerBound = new ArrayList<>();
+            List<PcapPacket> newListUpperBound = new ArrayList<>();
+            // Iterate over the packets
+            for(PcapPacket lowerBound : sequenceLowerBound) {
+                // Look for the lower and upper bounds from the signature
+                PcapPacket upperBound = sequenceUpperBound.get(sequenceLowerBound.indexOf(lowerBound));
+                // Look for the lower and upper bounds from the cluster analysis (core point range)
+                List<PcapPacket> bounds = getCorePointBounds(corePointRange, lowerBound, upperBound);
+                // Add into list
+                // The first element is the lower bound and the second element is the upper bound
+                newListLowerBound.add(bounds.get(0));
+                newListUpperBound.add(bounds.get(1));
+            }
+            newList.add(0, newListLowerBound);
+            newList.add(1, newListUpperBound);
+            finalSignature.add(newList);
+        }
+
+        return finalSignature;
+    }
+
+    /*
+     * Get the corresponding PcapPacket object for lower and upper bounds.
+     */
+    private static List<PcapPacket> getCorePointBounds(List<List<List<PcapPacket>>> corePointRange,
+                                                       PcapPacket lowerBound, PcapPacket upperBound) {
+
+        List<PcapPacket> listBounds = new ArrayList<>();
+        // Iterate over PcapPacket one by one
+        for(List<List<PcapPacket>> listOfListPcapPacket : corePointRange) {
+            List<PcapPacket> listCorePointLowerBound = listOfListPcapPacket.get(0);
+            List<PcapPacket> listCorePointUpperBound = listOfListPcapPacket.get(1);
+            for(PcapPacket corePointLowerBound : listCorePointLowerBound) {
+                PcapPacket corePointUpperBound =
+                        listCorePointUpperBound.get(listCorePointLowerBound.indexOf(corePointLowerBound));
+                // Return if the match for the core point bounds is found
+                // Basically the core point range has to be within the signature range
+                if (lowerBound.length() <= corePointLowerBound.length() &&
+                    corePointUpperBound.length() <= upperBound.length()) {
+                    listBounds.add(0, corePointLowerBound);
+                    listBounds.add(1, corePointUpperBound);
+                    return listBounds;
+                }
+                // Just skip the null elements
+                if (lowerBound == null && upperBound == null) {
+                    continue;
+                }
+            }
+        }
+        // Return null if not found
+        return null;
+    }
+
+    /*
+     * Get the corresponding PcapPacket object for upper bound.
+     */
+//    private static PcapPacket getUpperBound(List<List<List<PcapPacket>>> corePointRange, PcapPacket pcapPacket) {
+//
+//        // Iterate over PcapPacket one by one
+//        int counter = 0;
+//        for(List<List<PcapPacket>> listOfListPcapPacket : corePointRange) {
+//            List<PcapPacket> listUpperBound = listOfListPcapPacket.get(1);
+//            for(PcapPacket upperBound : listUpperBound) {
+//                // Return the counter matches
+//                if (counter == pcapPacketIndex) {
+//                    return upperBound;
+//                }
+//                if (upperBound == null) {
+//                    continue;
+//                }
+//                counter++;
+//            }
+//        }
+//
+//        return null;
+//    }
+
+    /**
+     * Check if there is any overlap between the signature stored in this class and another signature.
+     * Conditions:
+     * 1) If both signatures do not have any range, then we need to do conservative checking (return true).
+     * 2) If both signatures have the same number of packets/packet lengths, then we check the range; if the
+     *    numbers of packets/packet lengths are different then we assume that there is no overlap.
+     * 3) If there is any range in the signatures, then we need to check for overlap.
+     * 4) If there is overlap for EVERY packet/packet length, then we return true (conservative checking);
+     *    otherwise false (range-based checking).
+     *
+     * @param signature A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
+     *                  for overlaps with the other signature.
+     * @param otherSignature A {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
+     *                       for overlaps with the signature.
+     * @param eps Epsilon value for the DBSCAN algorithm.
+     * @return A boolean that is true if there is an overlap; false otherwise.
+     */
+    public static boolean isConservativeChecking(List<List<List<PcapPacket>>> signature,
+                                                 List<List<List<PcapPacket>>> otherSignature,
+                                                 double eps) {
+
+        // Get the ranges of the two signatures
+        List<List<List<PcapPacket>>> signatureRanges = getSequenceRanges(signature);
+        List<List<List<PcapPacket>>> otherSignatureRanges = getSequenceRanges(otherSignature);
+        if (!isRangeBased(signatureRanges) && !isRangeBased(otherSignatureRanges)) {
+            // Conservative checking when there is no range
+            return true;
+        } else if(signatureRanges.size() != otherSignatureRanges.size()) {
+            // The two signatures have different numbers of packets/packet lengths
+            return false;
+        } else {
+            // There is range; check if there is overlap
+            return checkOverlap(signatureRanges, otherSignatureRanges, eps);
+        }
+    }
+
+    /* Find the sequence with the minimum packet lengths.
+     * The second-layer list should contain the minimum sequence for element 0 and maximum sequence for element 1.
+     */
+    private static List<List<List<PcapPacket>>> getSequenceRanges(List<List<List<PcapPacket>>> signature) {
+
+        // Start from the first index
+        List<List<List<PcapPacket>>> rangeBasedSequence = new ArrayList<>();
+        for(List<List<PcapPacket>> listListPcapPacket : signature) {
+            List<List<PcapPacket>> minMaxSequence = new ArrayList<>();
+            // Both searches start from index 0
+            List<PcapPacket> minSequence = new ArrayList<>(listListPcapPacket.get(0));
+            List<PcapPacket> maxSequence = new ArrayList<>(listListPcapPacket.get(0));
+            for(List<PcapPacket> listPcapPacket : listListPcapPacket) {
+                for(PcapPacket pcapPacket : listPcapPacket) {
+                    int index = listPcapPacket.indexOf(pcapPacket);
+                    // Set the new minimum if length at the index is minimum
+                    if (pcapPacket.length() < minSequence.get(index).length()) {
+                        minSequence.set(index, pcapPacket);
+                    }
+                    // Set the new maximum if length at the index is maximum
+                    if (pcapPacket.length() > maxSequence.get(index).length()) {
+                        maxSequence.set(index, pcapPacket);
+                    }
+                }
+            }
+            // minSequence as element 0 and maxSequence as element 1
+            minMaxSequence.add(minSequence);
+            minMaxSequence.add(maxSequence);
+            rangeBasedSequence.add(minMaxSequence);
+        }
+
+        return rangeBasedSequence;
+    }
+
+    /*
+     * Check for overlap since we have range in at least one of the signatures.
+     * Overlap is only true when all ranges overlap. We need to check in order.
+     */
+    private static boolean checkOverlap(List<List<List<PcapPacket>>> signatureRanges,
+                                 List<List<List<PcapPacket>>> otherSignatureRanges, double eps) {
+
+        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
+            // Lower bound of the range is in index 0
+            // Upper bound of the range is in index 1
+            int sequenceSetIndex = signatureRanges.indexOf(listListPcapPacket);
+            List<PcapPacket> minSequenceSignature = listListPcapPacket.get(0);
+            List<PcapPacket> maxSequenceSignature = listListPcapPacket.get(1);
+            for(PcapPacket pcapPacket : minSequenceSignature) {
+                // Get the lower and upper bounds of the current signature
+                int packetIndex = minSequenceSignature.indexOf(pcapPacket);
+                int lowerBound = pcapPacket.length();
+                int upperBound = maxSequenceSignature.get(packetIndex).length();
+                // Check for range overlap in the other signature!
+                // Check the packet/packet length at the same position
+                List<PcapPacket> minSequenceSignatureOther = otherSignatureRanges.get(sequenceSetIndex).get(0);
+                List<PcapPacket> maxSequenceSignatureOther = otherSignatureRanges.get(sequenceSetIndex).get(1);
+                int lowerBoundOther = minSequenceSignatureOther.get(packetIndex).length();
+                int upperBoundOther = maxSequenceSignatureOther.get(packetIndex).length();
+                if (!(lowerBoundOther-(int)eps <= lowerBound && lowerBound <= upperBoundOther+(int)eps) &&
+                    !(lowerBoundOther-(int)eps <= upperBound && upperBound <= upperBoundOther+(int)eps)) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /*
+     * Check and see if there is any range in the signatures
+     */
+    private static boolean isRangeBased(List<List<List<PcapPacket>>> signatureRanges) {
+
+        for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
+            // Lower bound of the range is in index 0
+            // Upper bound of the range is in index 1
+            List<PcapPacket> minSequence = listListPcapPacket.get(0);
+            List<PcapPacket> maxSequence = listListPcapPacket.get(1);
+            for(PcapPacket pcapPacket : minSequence) {
+                int index = minSequence.indexOf(pcapPacket);
+                if (pcapPacket.length() != maxSequence.get(index).length()) {
+                    // If there is any packet length that differs in the minSequence
+                    // and maxSequence, then it is range-based
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
     /**
      * Remove a sequence in a signature object.
      *
index 3d3a3be..3d25041 100644 (file)
@@ -65,10 +65,11 @@ public class PrintUtils {
     }
 
     /**
-     * Write the signature {@code List<List<List<PcapPacket>>>} into a file.
+     * Write the signature and cluster analysis {@code List<List<List<PcapPacket>>>} into a file.
      *
-     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
-     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
+     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature and cluster analysis
+     * in the form of list of packet pairs. We harvest the pairs and transform them back into a list of PcapPacket
+     * objects.
      * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
      * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
      * PcapPacket objects and not just a pair.
@@ -77,7 +78,7 @@ public class PrintUtils {
      *                 default file name {@code SERIALIZABLE_FILE_PATH}.
      * @param signature The {@link Cluster} objects in the form of list of {@code PcapPacket} objects.
      */
-    public static void serializeSignatureIntoFile(String fileName, List<List<List<PcapPacket>>> signature) {
+    public static void serializeIntoFile(String fileName, List<List<List<PcapPacket>>> signature) {
         if (fileName == null)
             fileName = SERIALIZABLE_FILE_PATH;
         try (ObjectOutputStream oos =
@@ -118,18 +119,12 @@ public class PrintUtils {
     /**
      * Read the list of list of packet pairs {@code List<List<List<PcapPacket>>>} from a file.
      *
-     * After the DBSCAN algorithm derives the clusters from pairs, we save the signature in the form of list of
-     * packet pairs. We harvest the pairs and transform them back into a list of PcapPacket objects.
-     * We do not maintain the pairs in the form of {@code Cluster<PcapPacketPair>} objects because there might be
-     * a situation where we could combine multiple PcapPacketPair objects into a longer signature, i.e., a string of
-     * PcapPacket objects and not just a pair.
-     *
      * @param fileName The path of the file in {@link String}. We could leave this one {@code null} if we wanted the
      *                 default file name {@code SERIALIZABLE_FILE_PATH}.
      * @return The list of list of list of {@link Cluster} objects ({@code List<List<List<PcapPacket>>>})
      *         that is read from file.
      */
-    public static List<List<List<PcapPacket>>> deserializeSignatureFromFile(String fileName) {
+    public static List<List<List<PcapPacket>>> deserializeFromFile(String fileName) {
         if (fileName == null)
             fileName = SERIALIZABLE_FILE_PATH;
         List<List<List<PcapPacket>>> ppListOfListOfList = null;