Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
index a7e9a6c7b64e9f13e8fdeca43c6b51f18773d05c..2896ea740b68604d47510a13306649bc29d6c7d6 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
@@ -15,6 +15,7 @@ import org.pcap4j.packet.namednumber.DataLinkType;
 
 import java.io.EOFException;
 import java.net.UnknownHostException;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.TimeoutException;
@@ -59,8 +60,8 @@ public class Main {
 //        final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug-processed.pcap";
 //        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
-//        //final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
+//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+////        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
 
         // 2) TP-Link July 25 experiment
 //        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
@@ -71,7 +72,7 @@ public class Main {
 //        final String inputPcapFile = path + "/2018-10/tplink-plug/tplink-plug.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-10/tplink-plug/tplink-plug-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-10/tplink-plug/tplink-plug-oct-17-2018.timestamps";
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .159 == tplink plug
+//        final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug
         // TODO: EXPERIMENT - November 8, 2018
 //        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug-processed.pcap";
@@ -181,7 +182,7 @@ public class Main {
 //        final String inputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb-processed.pcap";
 //        final String triggerTimesFile = path + "/experimental_result/standalone/hue-bulb/timestamps/hue-bulb-nov-19-2018.timestamps";
-//        final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub
+////        final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
 
         // 9) Lifx Bulb August 8 experiment
@@ -212,13 +213,13 @@ public class Main {
 //        final String triggerTimesFile = path + "/2018-08/arlo-camera/arlo-camera-aug-10-2018.timestamps";
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
         // TODO: EXPERIMENT - November 13, 2018
-//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-//        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera-processed.pcap";
-////        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth1.local.pcap";
-////        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera-processed.pcap";
-//        final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
-////        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera
-//        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
+        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera-processed.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera-processed.pcap";
+        final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
+//        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera
+        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
 
         // 12) Blossom sprinkler August 13 experiment
 //        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
@@ -230,11 +231,13 @@ public class Main {
 //        final String outputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-nov-2-2018.timestamps";
 //        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
-        // January 9
-        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-        final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap";
-        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-9-2018.timestamps";
-        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
+        // January 9, 11, 13, 14
+//        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
+////        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-11-2019.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
+////        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
 
 //        // 13) DLink siren August 14 experiment
 //        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
@@ -248,15 +251,15 @@ public class Main {
 //        final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren-processed.pcap";
 //        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
-//        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
-////        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
+////        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
 
         // 14) Nest thermostat August 15 experiment
 //        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/2018-08/nest/nest-processed.pcap";
 //        final String triggerTimesFile = path + "/2018-08/nest/nest-aug-15-2018.timestamps";
 //        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
-        // TODO: EXPERIMENT - November 14, 2018
+//        // TODO: EXPERIMENT - November 14, 2018
 //        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
 //        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap";
 ////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap";
@@ -292,6 +295,200 @@ public class Main {
 //        final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps";
 //        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub;
 
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // D-Link plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
+////        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // TP-Link plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps";
+////        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
+        // D-Link siren
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
+        // Kwikset door lock
+//        final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
+        // SmartThings plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+        // Arlo Q
+//        final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        // Nest thermostat
+//        final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+        // Blossom sprinkler
+//        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+        // TP-Link bulb
+//        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+        // Philips hue
+//        final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
+//        final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+        // WeMo plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
+        // WeMo Insight plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
+
+
+//        TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
+//        List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
+//
+//        System.out.println("ON signature file in use is " + onSignatureFile);
+//        System.out.println("OFF signature file in use is " + offSignatureFile);
+//
+//        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+//        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+//
+//        List<Instant> signatureTimestamps = new ArrayList<>();
+//        // Load ON signature last packet's timestamp
+//        // Get the last only
+//        List<List<PcapPacket>> lastListOn = onSignature.get(onSignature.size()-1);
+//        for (List<PcapPacket> list : lastListOn) {
+//            // Get timestamp Instant from the last packet
+//            int lastPacketIndex = list.size()-1;
+//            signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+//        }
+//        // Load OFF signature last packet's timestamp
+//        // Get the last only
+//        List<List<PcapPacket>> lastListOff = offSignature.get(offSignature.size()-1);
+//        for (List<PcapPacket> list : lastListOff) {
+//            // Get timestamp Instant from the last packet
+//            int lastPacketIndex = list.size()-1;
+//            signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+//        }
+//        // Sort the timestamps
+//        signatureTimestamps.sort((p1, p2) -> {
+//            return p1.compareTo(p2);
+//        });
+//
+//        Iterator<Instant> iterTrig = triggerTimes.iterator();
+//        Iterator<Instant> iterSign = signatureTimestamps.iterator();
+//        System.out.println("Trigger to Last Packet:");
+//        while (iterTrig.hasNext() && iterSign.hasNext()) {
+//            Instant trigInst = (Instant) iterTrig.next();
+//            Instant signInst = (Instant) iterSign.next();
+//            Duration dur = Duration.between(trigInst, signInst);
+//            long duration = dur.toMillis();
+//            // Check duration --- should be below 15 seconds
+//            if (duration >= 0 && duration <= 15000) {
+//                System.out.println(dur.toMillis());
+//            } else if (duration > 15000) {
+//                while (duration > 15000) { // that means we have to move to the next trigger
+//                    trigInst = (Instant) iterTrig.next();
+//                    dur = Duration.between(trigInst, signInst);
+//                    duration = dur.toMillis();
+//                }
+//                System.out.println(dur.toMillis());
+//            } else { // below 0 / negative --- that means we have to move to the next signature
+//                while (duration < 0) { // that means we have to move to the next trigger
+//                    signInst = (Instant) iterSign.next();
+//                    dur = Duration.between(trigInst, signInst);
+//                    duration = dur.toMillis();
+//                }
+//                System.out.println(dur.toMillis());
+//            }
+//        }
+//
+//
+//        // ==========================================================================
+//        List<Instant> firstSignatureTimestamps = new ArrayList<>();
+//        List<Instant> lastSignatureTimestamps = new ArrayList<>();
+//        List<List<PcapPacket>> firstListOnSign = onSignature.get(0);
+//        List<List<PcapPacket>> lastListOnSign = onSignature.get(onSignature.size()-1);
+//        // Load ON signature first and last packet's timestamps
+//        for (List<PcapPacket> list : firstListOnSign) {
+//            // Get timestamp Instant from the last packet
+//            firstSignatureTimestamps.add(list.get(0).getTimestamp());
+//        }
+//        for (List<PcapPacket> list : lastListOnSign) {
+//            // Get timestamp Instant from the last packet
+//            int lastPacketIndex = list.size()-1;
+//            lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+//        }
+//
+//        List<List<PcapPacket>> firstListOffSign = offSignature.get(0);
+//        List<List<PcapPacket>> lastListOffSign = offSignature.get(offSignature.size()-1);
+//        // Load OFF signature first and last packet's timestamps
+//        for (List<PcapPacket> list : firstListOffSign) {
+//            // Get timestamp Instant from the last packet
+//            firstSignatureTimestamps.add(list.get(0).getTimestamp());
+//        }
+//        for (List<PcapPacket> list : lastListOffSign) {
+//            // Get timestamp Instant from the last packet
+//            int lastPacketIndex = list.size()-1;
+//            lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+//        }
+//        // Sort the timestamps
+//        firstSignatureTimestamps.sort((p1, p2) -> {
+//            return p1.compareTo(p2);
+//        });
+//        // Sort the timestamps
+//        lastSignatureTimestamps.sort((p1, p2) -> {
+//            return p1.compareTo(p2);
+//        });
+//
+//        Iterator<Instant> iterFirst = firstSignatureTimestamps.iterator();
+//        Iterator<Instant> iterLast = lastSignatureTimestamps.iterator();
+//        System.out.println("First to Last Packet:");
+//        while (iterFirst.hasNext() && iterLast.hasNext()) {
+//            Instant firstInst = (Instant) iterFirst.next();
+//            Instant lastInst = (Instant) iterLast.next();
+//            Duration dur = Duration.between(firstInst, lastInst);
+//            long duration = dur.toMillis();
+//            // Check duration --- should be below 15 seconds
+//            if (duration >= 0 && duration <= 15000) {
+//                System.out.println(dur.toMillis());
+//            } else if (duration > 15000) {
+//                while (duration > 15000) { // that means we have to move to the next trigger
+//                    firstInst = (Instant) iterFirst.next();
+//                    dur = Duration.between(firstInst, lastInst);
+//                    duration = dur.toMillis();
+//                }
+//                System.out.println(dur.toMillis());
+//            } else { // below 0 / negative --- that means we have to move to the next signature
+//                while (duration < 0) { // that means we have to move to the next trigger
+//                    lastInst = (Instant) iterLast.next();
+//                    dur = Duration.between(firstInst, lastInst);
+//                    duration = dur.toMillis();
+//                }
+//                System.out.println(dur.toMillis());
+//            }
+//            if (duration > 8000) {
+//                break;
+//            }
+//        }
+
+        // TODO: The above part is just for 15-second time sensitivity experiment
+        // TODO: The above part is just for 15-second time sensitivity experiment
+        // TODO: The above part is just for 15-second time sensitivity experiment
+
+
+
+
         TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
         List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
         // Tag each trigger with "ON" or "OFF", assuming that the first trigger is an "ON" and that they alternate.
@@ -444,6 +641,128 @@ public class Main {
         // ================================================ CLUSTERING ================================================
         // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List<Conversation> map
         // when don't care about hostnames and sequences (see comment earlier).
+//        List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+//                filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+//                map(e -> e.getValue()). // no longer interested in the UserActions
+//                flatMap(List::stream). // flatten List<List<T>> to a List<T>
+//                collect(Collectors.toList());
+//        List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+//                filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+//                map(e -> e.getValue()).
+//                flatMap(List::stream).
+//                collect(Collectors.toList());
+//        //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+//
+//        List<PcapPacketPair> onPairs = onConversations.stream().
+//                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+//                        TcpConversationUtils.extractPacketPairs(c)).
+//                flatMap(List::stream). // flatten List<List<>> to List<>
+//                collect(Collectors.toList());
+//        List<PcapPacketPair> offPairs = offConversations.stream().
+//                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+//                        TcpConversationUtils.extractPacketPairs(c)).
+//                flatMap(List::stream). // flatten List<List<>> to List<>
+//                collect(Collectors.toList());
+//        // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+//        Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+//        // Perform clustering on conversation logged as part of all ON events.
+////        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
+//        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(2, 2);
+//        //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
+//        List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+//        // Perform clustering on conversation logged as part of all OFF events.
+////        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
+//        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(2, 2);
+//        //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
+//        List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+//        // Sort the conversations as reference
+//        List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+//        // Output clusters
+//        System.out.println("========================================");
+//        System.out.println("       Clustering results for ON        ");
+//        System.out.println("       Number of clusters: " + onClusters.size());
+//        int count = 0;
+//        List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
+//        List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+//        for (Cluster<PcapPacketPair> c : onClusters) {
+//            System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+//            System.out.print(PrintUtils.toSummaryString(c));
+//            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+//            //if(c.getPoints().size() > 25) {
+//                // Print to file
+//                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+//                ppListOfListListOn.add(ppListOfList);
+//            }
+//        }
+//        // TODO: Merging test
+//        ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
+//        // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
+////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
+//        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+//        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+//        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+//        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+//        // TODO: This sequence actually belongs to the local communication between the plug and the phone
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+//        ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+//        PcapPacketUtils.printSignatures(ppListOfListListOn);
+//        //count = 0;
+//        /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
+//            PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
+//            ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
+//        }*/
+//        PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
+//        ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
+//
+//        System.out.println("========================================");
+//        System.out.println("       Clustering results for OFF       ");
+//        System.out.println("       Number of clusters: " + offClusters.size());
+//        count = 0;
+//        List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
+//        List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+//        for (Cluster<PcapPacketPair> c : offClusters) {
+//            System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+//            System.out.print(PrintUtils.toSummaryString(c));
+//            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+//            //if(c.getPoints().size() > 25) {
+//                // Print to file
+//                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+//                ppListOfListListOff.add(ppListOfList);
+//            }
+//        }
+//        // TODO: Merging test
+//        ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
+//        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3);
+//        // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+//        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+//        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
+//        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+//        // TODO: This sequence actually belongs to the local communication between the plug and the phone
+//        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+//        ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
+//        PcapPacketUtils.printSignatures(ppListOfListListOff);
+//        //count = 0;
+//        /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
+//            PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
+//            ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
+//        }*/
+//        PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
+//        ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
+//        System.out.println("========================================");
+        // ============================================================================================================
+
+        // TODO: This part is just for DBSCAN sensitivity experiment
+        // TODO: This part is just for DBSCAN sensitivity experiment
+        // TODO: This part is just for DBSCAN sensitivity experiment
+        // TODO: This part is just for DBSCAN sensitivity experiment
+        // TODO: This part is just for DBSCAN sensitivity experiment
         List<Conversation> onConversations = userActionToConversations.entrySet().stream().
                 filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
                 map(e -> e.getValue()). // no longer interested in the UserActions
@@ -468,101 +787,58 @@ public class Main {
                 collect(Collectors.toList());
         // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
         Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
-        // Perform clustering on conversation logged as part of all ON events.
-        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
-        //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(7.0, 45);
-        //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
-        List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
-        // Perform clustering on conversation logged as part of all OFF events.
-        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
-        //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(7.0, 45);
-        //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
-        List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
-        // Sort the conversations as reference
-        List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
-        // Output clusters
-        System.out.println("========================================");
-        System.out.println("       Clustering results for ON        ");
-        System.out.println("       Number of clusters: " + onClusters.size());
-        int count = 0;
-        List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
-        List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
-        for (Cluster<PcapPacketPair> c : onClusters) {
-            System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
-            System.out.print(PrintUtils.toSummaryString(c));
-            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-            //if(c.getPoints().size() > 25) {
-                // Print to file
-                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-                ppListOfListListOn.add(ppListOfList);
-            }
-        }
-        // TODO: Merging test
-        ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
-        ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
-        // TODO: Need to remove sequence numbers 0,1 for Blossom phone side since it is not a good signature!
-        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
-        // TODO: Need to remove sequence numbers 0 for Blossom device side since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
-        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
-        // TODO: This sequence actually belongs to the local communication between the plug and the phone
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
-        PcapPacketUtils.printSignatures(ppListOfListListOn);
-        //count = 0;
-        /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
-            PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
-            ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
-        }*/
-        PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
-        ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
-
-        System.out.println("========================================");
-        System.out.println("       Clustering results for OFF       ");
-        System.out.println("       Number of clusters: " + offClusters.size());
-        count = 0;
-        List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
-        List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
-        for (Cluster<PcapPacketPair> c : offClusters) {
-            System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
-            System.out.print(PrintUtils.toSummaryString(c));
-            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-            //if(c.getPoints().size() > 25) {
-                // Print to file
-                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-                ppListOfListListOff.add(ppListOfList);
+
+        double eps = 10; // loop from eps 1-10
+        int minPts = 50; // loop from minPts 30-50
+        for(int epsCount = 7; epsCount <= eps; epsCount++) {
+            for(int minPtsCount = 30; minPtsCount <= minPts; minPtsCount++) {
+                System.out.println("Eps: " + epsCount + " --- minPts: " + minPtsCount);
+                DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
+                DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
+                List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+                List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+                // Sort the conversations as reference
+                List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+                // Output clusters
+                System.out.println("========================================");
+                System.out.println("       Clustering results for ON        ");
+                System.out.println("       Number of clusters: " + onClusters.size());
+                int count = 0;
+                List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+                for (Cluster<PcapPacketPair> c : onClusters) {
+                    System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+//                    System.out.print(PrintUtils.toSummaryString(c));
+                    if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+//                        if(c.getPoints().size() > 25) {
+                        // Print to file
+                        List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+                        ppListOfListListOn.add(ppListOfList);
+                    }
+                }
+                PcapPacketUtils.printSignatures(ppListOfListListOn);
+
+                System.out.println("========================================");
+                System.out.println("       Clustering results for OFF       ");
+                System.out.println("       Number of clusters: " + offClusters.size());
+                count = 0;
+                List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+                for (Cluster<PcapPacketPair> c : offClusters) {
+                    System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+//                    System.out.print(PrintUtils.toSummaryString(c));
+                    if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+                        //if(c.getPoints().size() > 25) {
+                        // Print to file
+                        List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+                        ppListOfListListOff.add(ppListOfList);
+                    }
+                }
+                PcapPacketUtils.printSignatures(ppListOfListListOff);
+                System.out.println();
+                System.out.println();
+                System.out.println();
+                // ============================================================================================================
             }
         }
-        // TODO: Merging test
-        ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
-        //ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
-        // TODO: Need to remove sequence numbers 0,2 for Blossom device side since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
-        // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
-        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
-        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
-        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
-        // TODO: This sequence actually belongs to the local communication between the plug and the phone
-        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
-        PcapPacketUtils.printSignatures(ppListOfListListOff);
-        //count = 0;
-        /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
-            PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
-            ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
-        }*/
-        PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
-        ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
-        System.out.println("========================================");
-        // ============================================================================================================
 
 
 //        // ================================================================================================

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
index f1c264bf4dba24d454ececa0b68d50e85e83b7dd..a4217ccbb821421e86c4a75789c96fe93eaeb056 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
@@ -71,10 +71,10 @@ public class TcpConversationUtils {
     // Helper method for implementing the public API of similarly named methods.
     private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
         List<PcapPacketPair> pairs = new ArrayList<>();
-        for(PcapPacket pp : packets) {
-            System.out.print(pp.length() + " ");
-        }
-        System.out.println();
+//        for(PcapPacket pp : packets) {
+//            System.out.print(pp.length() + " ");
+//        }
+//        System.out.println();
 
         int i = 0;
         while (i < packets.size()) {

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
index 81002bd0e0b77ba13c776fbc93d55c33e0a654c6..b9584ff365f325885df7cb055521b0d9b30d4966 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
@@ -325,7 +325,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
             } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) {
                 directions[i] = Conversation.Direction.SERVER_TO_CLIENT;
             } else {
-                throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction");
+                //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction");
             }
         }
         return directions;

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
index 4095074594b6dca8232c0040e0a7b0a0774e39d1..52e2bdec47c4c4e498858978e24e5094fc13ba47 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
@@ -85,7 +85,7 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 
         // TODO: The following are negative tests against the PCAP file from UNSW
 //        final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload!
-//        final String inputPcapFile = path + "/UNSW/16-10-12.pcap";
+//          final String inputPcapFile = path + "/UNSW/16-10-12.pcap";
 
 //        final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload!
 //        final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken!
@@ -112,7 +112,7 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
 //        final String inputPcapFile = path + "/UNSW/16-10-01.pcap";
 //        final String inputPcapFile = path + "/UNSW/16-10-06.pcap";
         // Negative test: dataset from UNB
-//        final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap";
+//        final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap";
 
         // TODO: The following are tests for signatures against training data
 
@@ -219,8 +219,8 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         // TODO: EXPERIMENT - January 9, 2018
         // Blossom Sprinkler experiment
 //        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
-        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
+        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
         // Blossom Sprinkler DEVICE signatures
 //        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
 //        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
@@ -363,11 +363,11 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver
         List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
 
         // LAN
-        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
-        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
+//        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
+//        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
         // WAN
-//        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105");
-//        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105");
+        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105");
+        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105");
 
         final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).
                 withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles"));

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
index f07072abe8d9daf8ddd9a9bd2ce40e022ed5ee79..067af93edd6acf435ab154480b0a1434d2a9fa02 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java
@@ -28,6 +28,11 @@ public final class PcapPacketUtils {
      */
     private static final int SIGNATURE_MERGE_THRESHOLD = 5;
 
+    /**
+     * This is an overlap counter (we consider overlaps between signatures if it happens more than once)
+     */
+    private static int mOverlapCounter = 0;
+
 
     /**
      * Gets the source address of the Ethernet part of {@code packet}.
@@ -113,7 +118,7 @@ public final class PcapPacketUtils {
      * @param port The port to look for in the tcp.port field of {@code packet}.
      * @return {@code true} if the given ip+port match the corresponding fields in {@code packet}.
      */
-    public static boolean isSource(PcapPacket packet, String ip, int port) {
+    public static boolean isSource(PcapPacket packet,         String ip, int port) {
         IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class));
         // For now we only support TCP flows.
         TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class));
@@ -174,7 +179,7 @@ public final class PcapPacketUtils {
     }
 
     /**
-     * Checks if {@code packet} wraps a TCP packet that has the ACK flag set.
+     * Checks if {@code packet} wraps a TCP packet th        at has the ACK flag set.
      * @param packet A {@link PcapPacket} that is suspected to contain a {@link TcpPacket} for which the ACK flag is set.
      * @return {@code true} <em>iff</em> {@code packet} contains a {@code TcpPacket} for which the ACK flag is set,
      *         {@code false} otherwise.
@@ -204,7 +209,7 @@ public final class PcapPacketUtils {
             ppListOfList.add(ppList);
         }
         // Sort the list of lists based on the first packet's timestamp!
-        Collections.sort(ppListOfList, (p1, p2) -> p1.get(0).getTimestamp().compareTo(p2.get(0).getTimestamp()));
+        Collections.sort(ppListOfList, (p1, p2) -> p1.        get(0).getTimestamp().compareTo(p2.get(0).getTimestamp()));
         return ppListOfList;
     }
 
@@ -318,9 +323,11 @@ public final class PcapPacketUtils {
     public static List<List<List<PcapPacket>>> sortSignatures(List<List<List<PcapPacket>>> signatures) {
         // TODO: This is the simplest solution!!! Might not cover all corner cases.
         // TODO: Sort the list of lists based on the first packet's timestamps!
-        //Collections.sort(signatures, (p1, p2) -> {
-        //    return p1.get(0).get(0).getTimestamp().compareTo(p2.get(0).get(0).getTimestamp());
-        //});
+//        Collections.sort(signatures, (p1, p2) -> {
+//            //return p1.get(0).get(0).getTimestamp().compareTo(p2.get(0).get(0).getTimestamp());
+//            int compare = p1.get(0).get(0).getTimestamp().compareTo(p2.get(0).get(0).getTimestamp());
+//            return compare;
+//        });
         // TODO: The following is a more complete solution that covers corner cases.
         // Sort the list of lists based on one-to-one comparison between timestamps of signatures on both lists.
         // This also takes into account the fact that the number of signatures in the two lists could be different.
@@ -339,14 +346,17 @@ public final class PcapPacketUtils {
                 if (Math.abs(timestamp1 - timestamp2) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) {
                     // If these two are within INCLUSION_WINDOW_MILLIS window then compare!
                     compare = p1.get(count1).get(0).getTimestamp().compareTo(p2.get(count2).get(0).getTimestamp());
-                    if (comparePrev != 0) { // First time since it is 0
-                        if (Integer.signum(compare) != Integer.signum(comparePrev)) {
-                            // Throw an exception if the order of the two signatures is not consistent,
-                            // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other.
-//                            throw new Error("For some reason, the order of signatures are not always consistent!" +
-//                                    "Returning the original data structure of signatures...");
-                        }
-                    }
+//                    if (comparePrev != 0) { // First time since it is 0
+//                        if (Integer.signum(compare) != Integer.signum(comparePrev)) {
+//                            // Throw an exception if the order of the two signatures is not consistent,
+//                            // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other.
+//                            throw new Error("OVERLAP WARNING: " + "" +
+//                                    "Please remove one of the sequences: " +
+//                                    p1.get(0).get(0).length() + "... OR " +
+//                                    p2.get(0).get(0).length() + "...");
+//                        }
+//                    }
+                    overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2));
                     comparePrev = compare;
                     count1++;
                     count2++;
@@ -364,6 +374,46 @@ public final class PcapPacketUtils {
         return signatures;
     }
 
+    /**
+     * Checks for overlapping between two packet sequences.
+     * @param compare Current comparison value between packet sequences p1 and p2
+     * @param comparePrev Previous comparison value between packet sequences p1 and p2
+     * @param sequence1 The packet sequence ({@link List} of {@link PcapPacket} objects).
+     * @param sequence2 The packet sequence ({@link List} of {@link PcapPacket} objects).
+     */
+    private static void overlapChecking(int compare, int comparePrev, List<PcapPacket> sequence1, List<PcapPacket> sequence2) {
+
+        // Check if p1 occurs before p2 but both have same overlap
+        if (comparePrev != 0) { // First time since it is 0
+            if (Integer.signum(compare) != Integer.signum(comparePrev)) {
+                // Throw an exception if the order of the two signatures is not consistent,
+                // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other.
+                throw new Error("OVERLAP WARNING: " + "" +
+                        "Two sequences have some overlap. Please remove one of the sequences: " +
+                        sequence1.get(0).length() + "... OR " +
+                        sequence2.get(0).length() + "...");
+            }
+        }
+        // Check if p1 is longer than p2 and p2 occurs during the occurrence of p1
+        int lastIndexOfSequence1 = sequence1.size() - 1;
+        int lastIndexOfSequence2 = sequence2.size() - 1;
+        int compareLast =
+                sequence1.get(lastIndexOfSequence1).getTimestamp().compareTo(sequence2.get(lastIndexOfSequence2).getTimestamp());
+        // Check the signs of compare and compareLast
+        if ((compare <= 0 && compareLast > 0) ||
+            (compareLast <= 0 && compare > 0)) {
+            mOverlapCounter++;
+            // TODO: Probably not the best approach but we consider overlap if it happens more than once
+            if (mOverlapCounter > 1) {
+                throw new Error("OVERLAP WARNING: " + "" +
+                        "One sequence is in the other. Please remove one of the sequences: " +
+                        sequence1.get(0).length() + "... OR " +
+                        sequence2.get(0).length() + "...");
+            }
+        }
+
+    }
+
     /**
      * Gets the {@link IpV4Packet} contained in {@code packet}, or throws a {@link NullPointerException} if
      * {@code packet} does not contain an {@link IpV4Packet}.
@@ -398,7 +448,7 @@ public final class PcapPacketUtils {
         int sequenceCounter = 0;
         for(List<List<PcapPacket>> listListPcapPacket : signatures) {
             // Iterate over every member of a cluster/sequence
-            System.out.print("====== SEQUENCE " + sequenceCounter++);
+            System.out.print("====== SEQUENCE " + ++sequenceCounter);
             System.out.println(" - " + listListPcapPacket.size() + " MEMBERS ======");
             for(List<PcapPacket> listPcapPacket : listListPcapPacket) {
                 // Print out packet lengths in a sequence

diff --git a/python_ml/validate-detection.py b/python_ml/validate-detection.py
index 9ddde6c797d36bb41dad3ac49e480673e9a3939d..2d19ed4a23e01ac5c1187677eb2ed2b53c2efdd4 100644 (file)
--- a/python_ml/validate-detection.py
+++ b/python_ml/validate-detection.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 
+
 path = "/scratch/July-2018/experimental_result/smarthome/"
 #path = "/scratch/July-2018/experimental_result/standalone/"
 # D-Link plug
@@ -93,14 +94,20 @@ path = "/scratch/July-2018/experimental_result/smarthome/"
 #fileDetection = "wemo-insight-plug-smarthome-nov-22-2018.phone.wlan1.detections"
 
 # Blossom sprinkler
-device = "blossom-sprinkler/self-test"
-fileExperiment = "blossom-sprinkler-standalone-jan-9-2019.timestamps"
+#device = "blossom-sprinkler/self-test"
+#fileExperiment = "blossom-sprinkler-standalone-jan-9-2019.timestamps"
+#fileExperiment = "blossom-sprinkler-standalone-jan-11-2019.timestamps"
+#fileExperiment = "blossom-sprinkler-standalone-jan-14-2019.timestamps"
 #fileDetection = "phone-side-detection"
 #fileDetection = "device-side-detection"
 device = "blossom-sprinkler/timestamps"
-fileExperiment = "blossom-sprinkler-smarthome-jan-10-2019.timestamps"
-fileDetection = "blossom-sprinkler-smarthome-jan-10-2019.device.wlan1.detections"
-#fileDetection = "blossom-sprinkler-smarthome-jan-10-2019.phone.wlan1.detections"
+#fileExperiment = "blossom-sprinkler-smarthome-jan-10-2019.timestamps"
+#fileExperiment = "blossom-sprinkler-smarthome-jan-11-2019.timestamps"
+fileExperiment = "blossom-sprinkler-smarthome-jan-14-2019.timestamps"
+#fileDetection = "blossom-sprinkler-smarthome-jan-10-2019.device.wlan1.detections"
+#fileDetection = "blossom-sprinkler-smarthome-jan-11-2019.phone.wlan1.detections"
+#fileDetection = "blossom-sprinkler-smarthome-jan-14-2019.device.eth0.detections"
+fileDetection = "blossom-sprinkler-smarthome-jan-14-2019.phone.eth0.detections"
 
 TIME_WINDOW = 15 # detection/signature window of 15 seconds
 #NEG_TIME_WINDOW = -15 # detection/signature window of 15 seconds