Adding a script to compare 2 timestamps files to validate the detection process.

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
index 00b0b9b748c43c303a853a25cf7e4140cef3f67f..f22bf2b0373c8464b850c4115db8e91a63953d0f 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
@@ -49,10 +49,10 @@ public class Main {
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
         // 1) D-Link July 26 experiment
         final String offPairsPath = "/scratch/July-2018/off.txt";
 
         // 1) D-Link July 26 experiment

-        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
-        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
-        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
-        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+//        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
+//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?

 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";


@@ -99,11 +99,11 @@ public class Main {
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
 
         // September 12, 2018 - includes both wlan1 and eth1 interfaces
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
 
         // September 12, 2018 - includes both wlan1 and eth1 interfaces

-        //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap";
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap";

 //        final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps";
 //        final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps";

-//        final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)

 
         // 8) Hue Bulb August 7 experiment
 //        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
 
         // 8) Hue Bulb August 7 experiment
 //        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";


@@ -136,10 +136,10 @@ public class Main {
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment

-//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
-//        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
-//        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
+        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
+        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
+        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren

 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";


@@ -163,6 +163,17 @@ public class Main {
 //        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps";
 //        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub; .199 == dlink plug; .183 == siren
 //        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps";
 //        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub; .199 == dlink plug; .183 == siren

+        // September 26 - D-Link noise
+//        final String inputPcapFile = path + "/2018-08/noise/noise.dlink.wlan1.pcap";
+//        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/noise/dlink-noise-sept-26-2018.timestamps";
+//        final String deviceIp = "192.168.1.183"; //  .199 == dlink plug; .183 == siren
+        // September 27 - Kwikset noise
+//        final String inputPcapFile = path + "/2018-08/noise/noise.kwikset.eth1.pcap";
+//        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps";
+//        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub;
+

 
         TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
         List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
 
         TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
         List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);


@@ -298,10 +309,12 @@ public class Main {
         PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
         PcapPacketUtils.sortSignatures(ppListOfListListOn);
         count = 0;
         PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
         PcapPacketUtils.sortSignatures(ppListOfListListOn);
         count = 0;

-        for (List<List<PcapPacket>> ll : ppListOfListListOn) {
+        /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {

             PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
             ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
             PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
             ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));

-        }
+        }*/
+        PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
+        ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");

 
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");
 
         System.out.println("========================================");
         System.out.println("       Clustering results for OFF       ");


@@ -322,10 +335,12 @@ public class Main {
         PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
         PcapPacketUtils.sortSignatures(ppListOfListListOff);
         count = 0;
         PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
         PcapPacketUtils.sortSignatures(ppListOfListListOff);
         count = 0;

-        for (List<List<PcapPacket>> ll : ppListOfListListOff) {
+        /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {

             PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
             ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
             PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
             ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));

-        }
+        }*/
+        PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
+        ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");

         System.out.println("========================================");
         // ============================================================================================================
 
         System.out.println("========================================");
         // ============================================================================================================
 

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java
index 6d1f54d3480b1d570ad0fe801373431bd7049e81..7c9bfdd3a2b185f7d9442b42ad648ca79e87d0e4 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java
@@ -27,8 +27,8 @@ public class SignatureDetector implements PacketListener, ClusterMatcher.Cluster
 
     // Test client
     public static void main(String[] args) throws PcapNativeException, NotOpenException {
 
     // Test client
     public static void main(String[] args) throws PcapNativeException, NotOpenException {

-        //        String path = "/scratch/July-2018"; // Rahmadi
-        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        String path = "/scratch/July-2018"; // Rahmadi
+        //String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus

 
         // D-Link Plug experiment
         final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap";
 
         // D-Link Plug experiment
         final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap";


@@ -91,8 +91,10 @@ public class SignatureDetector implements PacketListener, ClusterMatcher.Cluster
                 default:
                     throw new AssertionError("unhandled event type");
             }
                 default:
                     throw new AssertionError("unhandled event type");
             }

-            String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]",
-                    eventDescription, dateTimeFormatter.format(ua.getTimestamp()));
+            //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]",
+            //      eventDescription, dateTimeFormatter.format(ua.getTimestamp()));
+            String output = String.format("%s",
+                    dateTimeFormatter.format(ua.getTimestamp()));

             System.out.println(output);
         };
 
             System.out.println(output);
         };
 

diff --git a/python_ml/validate-detection.py b/python_ml/validate-detection.py
new file mode 100644 (file)
index 0000000..b22e0a3
--- /dev/null
+++ b/python_ml/validate-detection.py
@@ -0,0 +1,50 @@
+from datetime import datetime
+
+
+path = "/scratch/July-2018/evaluation/"
+device = "dlink"
+fileExperiment = "dlink-plug-8hr-data-oct-8-2018.timestamps"
+fileDetection = "dlink-plug.detection.timestamps"
+TIME_WINDOW = 15 # detection/signature window of 15 seconds
+#NEG_TIME_WINDOW = -15 # detection/signature window of 15 seconds
+
+# Open training timestamps file and store into a list
+with open(path + device + "/" + fileExperiment, "r") as experiment:
+       tsExperimentList = []
+       for line in experiment:
+               # Format "%m/%d/%Y %I:%M:%S %p"
+               tsE = datetime.strptime(line, "%m/%d/%Y %I:%M:%S %p\n")
+               tsExperimentList.append(tsE)
+
+# Open detection timestamps file and store into a list
+with open(path + device + "/" + fileDetection, "r") as detection:
+       tsDetectionList = []
+       for line in detection:
+               # Format "%b %d, %Y %I:%M:%S %p"
+               tsD = datetime.strptime(line, "%b %d, %Y %I:%M:%S %p\n")
+               tsDetectionList.append(tsD)
+               
+if (len(tsExperimentList) > len(tsDetectionList)):
+       maxTimestamps = len(tsExperimentList)
+else:
+       maxTimestamps = len(tsDetectionList)
+
+i = 0
+j = 0
+while i < maxTimestamps:
+       tsE = tsExperimentList[i]
+       tsD = tsDetectionList[j]
+       # Detection is always a bit later than training trigger
+       delta = tsD - tsE
+       # The following happens when we could detect less triggers than the experiment
+       if (delta.seconds > TIME_WINDOW):
+               print("Missing trigger at line: " + str(i) + ", t_experiment: " + str(tsE) + " and t_detection: " + str(tsD))
+               i = i + 1
+       # The following should not happen (we have more detected triggers than the experiment)
+       #elif (delta.seconds < NEG_TIME_WINDOW):
+       #       print("Mismatch at t_experiment: " + str(tsE) + " and t_detection: " + str(tsD))
+       #       j = j + 1
+       i = i + 1
+       j = j + 1
+
+print("Done parsing: " + str(i) + " lines")