Adding PacketLevelSignatureExtractor.
authorrtrimana <rtrimana@uci.edu>
Tue, 5 Feb 2019 01:41:22 +0000 (17:41 -0800)
committerrtrimana <rtrimana@uci.edu>
Tue, 5 Feb 2019 01:41:22 +0000 (17:41 -0800)
73 files changed:
Code/Projects/PacketLevelSignatureExtractor/.gitignore [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/.name [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/build.gradle [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/gradlew [new file with mode: 0755]
Code/Projects/PacketLevelSignatureExtractor/gradlew.bat [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/settings.gradle [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/LiveCapture.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PcapHandleReader.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/TriggerTimesFileReader.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPattern.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/maclayer/MacLayerFlowPatternFinder.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PrintUtils.java [new file with mode: 0644]
Code/Projects/PacketLevelSignatureExtractor/src/test/java/edu/uci/iotproject/test/SequenceAlignmentTest.java [new file with mode: 0644]

diff --git a/Code/Projects/PacketLevelSignatureExtractor/.gitignore b/Code/Projects/PacketLevelSignatureExtractor/.gitignore
new file mode 100644 (file)
index 0000000..b0ff774
--- /dev/null
@@ -0,0 +1,65 @@
+# Borrowed from https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+
+# Sensitive or high-churn files:
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+
+# Gradle: (combination of the JetBrains gitiginre and Gradle gitignore at )
+.idea/**/gradle.xml
+.idea/**/libraries
+.gradle
+/build/
+# Ignore Gradle GUI config
+gradle-app.setting
+# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
+!gradle-wrapper.jar
+# Cache of project
+.gradletasknamecache
+# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
+# gradle/wrapper/gradle-wrapper.properties
+
+# CMake
+cmake-build-debug/
+
+# Mongo Explorer plugin:
+.idea/**/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# ignore misc as it changes a lot depending on local settings -- however may need to be included later on.
+.idea/misc.xml
+
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/.name b/Code/Projects/PacketLevelSignatureExtractor/.idea/.name
new file mode 100644 (file)
index 0000000..297868a
--- /dev/null
@@ -0,0 +1 @@
+PacketLevelSignatureExtractor
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/compiler.xml
new file mode 100644 (file)
index 0000000..4ac14e9
--- /dev/null
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <bytecodeTargetLevel>
+      <module name="PacketLevelSignatureExtractor_main" target="1.8" />
+      <module name="PacketLevelSignatureExtractor_test" target="1.8" />
+    </bytecodeTargetLevel>
+  </component>
+</project>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/copyright/profiles_settings.xml
new file mode 100644 (file)
index 0000000..e7bedf3
--- /dev/null
@@ -0,0 +1,3 @@
+<component name="CopyrightManager">
+  <settings default="" />
+</component>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/encodings.xml
new file mode 100644 (file)
index 0000000..15a15b2
--- /dev/null
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with NO BOM" />
+</project>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules.xml
new file mode 100644 (file)
index 0000000..0b5d8f6
--- /dev/null
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor.iml" filepath="$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor_main.iml" filepath="$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor_main.iml" group="PacketLevelSignatureExtractor" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor_test.iml" filepath="$PROJECT_DIR$/.idea/modules/PacketLevelSignatureExtractor_test.iml" group="PacketLevelSignatureExtractor" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor.iml
new file mode 100644 (file)
index 0000000..21e02c3
--- /dev/null
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="PacketLevelSignatureExtractor" external.linked.project.path="$MODULE_DIR$/../.." external.root.project.path="$MODULE_DIR$/../.." external.system.id="GRADLE" external.system.module.group="edu.uci.iotproject" external.system.module.version="1.0-SNAPSHOT" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$/../..">
+      <excludeFolder url="file://$MODULE_DIR$/../../.gradle" />
+      <excludeFolder url="file://$MODULE_DIR$/../../build" />
+      <excludeFolder url="file://$MODULE_DIR$/../../out" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_main.iml
new file mode 100644 (file)
index 0000000..e1b3b2d
--- /dev/null
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="PacketLevelSignatureExtractor:main" external.linked.project.path="$MODULE_DIR$/../.." external.root.project.path="$MODULE_DIR$/../.." external.system.id="GRADLE" external.system.module.group="edu.uci.iotproject" external.system.module.type="sourceSet" external.system.module.version="1.0-SNAPSHOT" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/../../out/production/classes" />
+    <exclude-output />
+    <content url="file://$MODULE_DIR$/../../src/main">
+      <sourceFolder url="file://$MODULE_DIR$/../../src/main/java" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="Gradle: org.pcap4j:pcap4j-packetfactory-static:2.0.0-alpha" level="project" />
+    <orderEntry type="library" name="Gradle: org.pcap4j:pcap4j-core:2.0.0-alpha" level="project" />
+    <orderEntry type="library" name="Gradle: org.slf4j:slf4j-jdk14:1.8.0-beta2" level="project" />
+    <orderEntry type="library" name="Gradle: org.apache.commons:commons-math3:3.6.1" level="project" />
+    <orderEntry type="library" name="Gradle: org.jgrapht:jgrapht-core:1.2.0" level="project" />
+    <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.8.0-beta2" level="project" />
+    <orderEntry type="library" name="Gradle: net.java.dev.jna:jna:4.2.1" level="project" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml b/Code/Projects/PacketLevelSignatureExtractor/.idea/modules/PacketLevelSignatureExtractor_test.iml
new file mode 100644 (file)
index 0000000..c8b68de
--- /dev/null
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="PacketLevelSignatureExtractor:test" external.linked.project.path="$MODULE_DIR$/../.." external.root.project.path="$MODULE_DIR$/../.." external.system.id="GRADLE" external.system.module.group="edu.uci.iotproject" external.system.module.type="sourceSet" external.system.module.version="1.0-SNAPSHOT" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output-test url="file://$MODULE_DIR$/../../out/test/classes" />
+    <exclude-output />
+    <content url="file://$MODULE_DIR$/../../src/test">
+      <sourceFolder url="file://$MODULE_DIR$/../../src/test/java" isTestSource="true" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="PacketLevelSignatureExtractor_main" />
+    <orderEntry type="library" name="Gradle: org.pcap4j:pcap4j-packetfactory-static:2.0.0-alpha" level="project" />
+    <orderEntry type="library" name="Gradle: org.pcap4j:pcap4j-core:2.0.0-alpha" level="project" />
+    <orderEntry type="library" name="Gradle: org.slf4j:slf4j-jdk14:1.8.0-beta2" level="project" />
+    <orderEntry type="library" name="Gradle: org.apache.commons:commons-math3:3.6.1" level="project" />
+    <orderEntry type="library" name="Gradle: org.jgrapht:jgrapht-core:1.2.0" level="project" />
+    <orderEntry type="library" name="Gradle: junit:junit:4.11" level="project" />
+    <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.8.0-beta2" level="project" />
+    <orderEntry type="library" name="Gradle: net.java.dev.jna:jna:4.2.1" level="project" />
+    <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" />
+  </component>
+  <component name="TestModuleProperties" production-module="PacketLevelSignatureExtractor_main" />
+</module>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml b/Code/Projects/PacketLevelSignatureExtractor/.idea/vcs.xml
new file mode 100644 (file)
index 0000000..c2365ab
--- /dev/null
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/../../.." vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/build.gradle b/Code/Projects/PacketLevelSignatureExtractor/build.gradle
new file mode 100644 (file)
index 0000000..2027ed3
--- /dev/null
@@ -0,0 +1,40 @@
+group 'edu.uci.iotproject'
+version '1.0-SNAPSHOT'
+
+apply plugin: 'java'
+apply plugin: 'application'
+
+// Increase max memory
+applicationDefaultJvmArgs = ["-Xmx300g"]
+
+sourceCompatibility = 1.8
+
+//mainClassName = "edu.uci.iotproject.Main"
+//mainClassName = "edu.uci.iotproject.detection.SignatureDetector"
+//mainClassName = "edu.uci.iotproject.detection.layer2.Layer2SignatureDetector"
+//mainClassName = "edu.uci.iotproject.evaluation.DetectionResultsAnalyzer"
+mainClassName = System.getProperty("mainClass")
+
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    testCompile group: 'junit', name: 'junit', version: '4.11'
+
+    // pcap4j
+    // Updated to v2 alpha as the stable release does not include packet timestamps
+    // v2 should add support for TCP session reassembly as well, although it does not appear to be part of the lib yet.
+    compile 'org.pcap4j:pcap4j-core:2.0.0-alpha'
+    compile 'org.pcap4j:pcap4j-packetfactory-static:2.0.0-alpha'
+
+    // pcap4j logging dependency
+    compile 'org.slf4j:slf4j-jdk14:1.8.0-beta2'
+
+    // Apache Commons Math for clustering
+    compile 'org.apache.commons:commons-math3:3.6.1'
+
+    // JGraphT: Java Graph library
+    compile 'org.jgrapht:jgrapht-core:1.2.0'
+}
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection.sh
new file mode 100755 (executable)
index 0000000..1cf9087
--- /dev/null
@@ -0,0 +1,241 @@
+#!/bin/bash
+
+#set -x # echo invoked commands to std out
+
+# Base dir should point to the experimental_result folder which contains the subfolders:
+# - 'smarthome' which contains the traces collected while other devices are idle
+# - 'standalone' which contains signatures and the traces used to generate the signatures.
+BASE_DIR=$1
+readonly BASE_DIR
+
+OUTPUT_DIR=$2
+readonly OUTPUT_DIR
+
+PCAPS_BASE_DIR="$BASE_DIR/smarthome"
+readonly PCAPS_BASE_DIR
+
+SIGNATURES_BASE_DIR="$BASE_DIR/standalone"
+readonly SIGNATURES_BASE_DIR
+
+# ==================================================== ARLO CAMERA =====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE (TODO: may possibly be the .incomplete signatures)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="213"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= BLOSSOM SPRINKLER ==================================================
+PCAP_FILE="$PCAPS_BASE_DIR/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"
+
+# DEVICE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="9274"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="3670"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK PLUG =====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"
+
+# DEVICE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="8866"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="193"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK SIREN ====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="71"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== HUE BULB =======================================================
+PCAP_FILE="$PCAPS_BASE_DIR/hue-bulb/wlan1/hue-bulb.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="27"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= KWIKSET DOORLOCK ===================================================
+PCAP_FILE="$PCAPS_BASE_DIR/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="3161"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="1179"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ====================================================== ST PLUG =======================================================
+PCAP_FILE="$PCAPS_BASE_DIR/st-plug/wlan1/st-plug.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="2445"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK BULB ====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="162"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK PLUG ====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"
+
+# DEVICE SIDE (both the 112, 115 and 556, 1293 sequences)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="3660"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
+SIGNATURE_DURATION="224"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs.
+# ======================================================================================================================
+
+
+
+# ================================================== WEMO INSIGHT PLUG =================================================
+PCAP_FILE="$PCAPS_BASE_DIR/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="106"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== WEMO PLUG ======================================================
+PCAP_FILE="$PCAPS_BASE_DIR/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="147"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh
new file mode 100755 (executable)
index 0000000..7d0689d
--- /dev/null
@@ -0,0 +1,187 @@
+#!/bin/bash
+
+# Base directory where the smarthome evaluation traces and timestamp files are stored,
+# (i.e., /some/arbitrary/local/path/experimental_result/smarthome)
+TIMESTAMPS_BASE_DIR=$1
+readonly TIMESTAMPS_BASE_DIR
+
+# Base directory for the detection results files for the smarthome experiment
+RESULTS_BASE_DIR=$2
+readonly RESULTS_BASE_DIR
+
+
+
+# ==================================================== ARLO CAMERA =====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps"
+RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
+# Put the analysis results in the same folder as the detection results.
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+
+
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= BLOSSOM SPRINKLER ==================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps"
+
+# DEVICE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK PLUG =====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps"
+
+# DEVICE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK SIREN ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps"
+
+#PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== HUE BULB =======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= KWIKSET DOORLOCK ===================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ====================================================== ST PLUG =======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK BULB ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK PLUG ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps"
+
+# DEVICE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# DEVICE SIDE OUTBOUND
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================== WEMO INSIGHT PLUG =================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== WEMO PLUG ======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest.sh
new file mode 100755 (executable)
index 0000000..e32d6b9
--- /dev/null
@@ -0,0 +1,241 @@
+#!/bin/bash
+
+#set -x # echo invoked commands to std out
+
+# Base dir should point to the experimental_result folder which contains the subfolders:
+# - 'smarthome' which contains the traces collected while other devices are idle
+# - 'standalone' which contains signatures and the traces used to generate the signatures.
+BASE_DIR=$1
+readonly BASE_DIR
+
+OUTPUT_DIR=$2
+readonly OUTPUT_DIR
+
+PCAPS_BASE_DIR="$BASE_DIR/smarthome"
+readonly PCAPS_BASE_DIR
+
+SIGNATURES_BASE_DIR="$BASE_DIR/standalone"
+readonly SIGNATURES_BASE_DIR
+
+# # ==================================================== ARLO CAMERA =====================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE (TODO: may possibly be the .incomplete signatures)
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="213"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ================================================= BLOSSOM SPRINKLER ==================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"
+
+# # DEVICE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
+# SIGNATURE_DURATION="9274"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="3670"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== D-LINK PLUG =====================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"
+
+# # DEVICE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
+# SIGNATURE_DURATION="8866"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="193"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== D-LINK SIREN ====================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="71"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ===================================================== HUE BULB =======================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/hue-bulb/wlan1/hue-bulb.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="27"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ================================================= KWIKSET DOORLOCK ===================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="3161"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+PCAP_FILE="$PCAPS_BASE_DIR/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"
+
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="1179"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# # ====================================================== ST PLUG =======================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/st-plug/wlan1/st-plug.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="2445"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== TP-LINK BULB ====================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="162"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== TP-LINK PLUG ====================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"
+
+# # DEVICE SIDE (both the 112, 115 and 556, 1293 sequences)
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
+# SIGNATURE_DURATION="3660"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# # DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence)
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"
+# RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
+# SIGNATURE_DURATION="224"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# # Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs.
+# # ======================================================================================================================
+
+
+
+# # ================================================== WEMO INSIGHT PLUG =================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="106"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ===================================================== WEMO PLUG ======================================================
+# PCAP_FILE="$PCAPS_BASE_DIR/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"
+# OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"
+# RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# SIGNATURE_DURATION="147"
+
+# PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_nest_results_analysis.sh
new file mode 100644 (file)
index 0000000..61e029c
--- /dev/null
@@ -0,0 +1,187 @@
+#!/bin/bash
+
+# Base directory where the smarthome evaluation traces and timestamp files are stored,
+# (i.e., /some/arbitrary/local/path/experimental_result/smarthome)
+TIMESTAMPS_BASE_DIR=$1
+readonly TIMESTAMPS_BASE_DIR
+
+# Base directory for the detection results files for the smarthome experiment
+RESULTS_BASE_DIR=$2
+readonly RESULTS_BASE_DIR
+
+
+
+# # ==================================================== ARLO CAMERA =====================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps"
+# RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
+# # Put the analysis results in the same folder as the detection results.
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+
+
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ================================================= BLOSSOM SPRINKLER ==================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps"
+
+# # DEVICE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== D-LINK PLUG =====================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps"
+
+# # DEVICE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== D-LINK SIREN ====================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps"
+
+# #PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ===================================================== HUE BULB =======================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ================================================= KWIKSET DOORLOCK ===================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# # ====================================================== ST PLUG =======================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== TP-LINK BULB ====================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ==================================================== TP-LINK PLUG ====================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps"
+
+# # DEVICE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# # DEVICE SIDE OUTBOUND
+# RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ================================================== WEMO INSIGHT PLUG =================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
+
+
+
+# # ===================================================== WEMO PLUG ======================================================
+# TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps"
+
+# # Has no device side signature.
+
+# # PHONE SIDE
+# RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
+# ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+# PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+# ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# # ======================================================================================================================
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unb_all_detection.sh
new file mode 100755 (executable)
index 0000000..0a44835
--- /dev/null
@@ -0,0 +1,215 @@
+#!/bin/bash
+
+#set -x # echo invoked commands to std out
+
+# Arg1 should point to the UNB trace (PCAP w/o any expected events).
+PCAP_FILE=$1
+
+readonly PCAP_FILE
+
+# Arg2 should point to the base directory  for signature files (i.e., /some/local/path/experimental_result/standalone)
+SIGNATURES_BASE_DIR=$2
+readonly SIGNATURES_BASE_DIR
+
+# Arg3 should point to folder where the detection results for the UNB trace are to be output.
+OUTPUT_DIR=$3
+readonly OUTPUT_DIR
+
+# ==================================================== ARLO CAMERA =====================================================
+# Has no device side signature.
+
+# PHONE SIDE (TODO: may possibly be the .incomplete signatures)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="213"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= BLOSSOM SPRINKLER ==================================================
+# DEVICE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="9274"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="3670"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK PLUG =====================================================
+# DEVICE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="8866"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="193"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK SIREN ====================================================
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="71"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== HUE BULB =======================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="27"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= KWIKSET DOORLOCK ===================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="3161"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="1179"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ====================================================== ST PLUG =======================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="2445"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK BULB ====================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="162"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK PLUG ====================================================
+# DEVICE SIDE (both the 112, 115 and 556, 1293 sequences)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
+SIGNATURE_DURATION="3660"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.*;50:c7:bf:.* -offmacfilters 50:c7:bf:.*;50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence)
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"
+RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
+SIGNATURE_DURATION="224"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' -onmacfilters 50:c7:bf:.* -offmacfilters 50:c7:bf:.*"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+
+# Phone side does not make sense as it is merely a subset of the device side and does not differentiate ONs from OFFs.
+# ======================================================================================================================
+
+
+
+# ================================================== WEMO INSIGHT PLUG =================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="106"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== WEMO PLUG ======================================================
+# Has no device side signature.
+
+# PHONE SIDE
+ON_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"
+OFF_SIGNATURE="$SIGNATURES_BASE_DIR/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"
+RESULTS_FILE="$OUTPUT_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
+SIGNATURE_DURATION="147"
+
+PROGRAM_ARGS="'$PCAP_FILE' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION'"
+./gradlew run -DmainClass=edu.uci.iotproject.detection.layer2.Layer2SignatureDetector --args="$PROGRAM_ARGS"
+# ======================================================================================================================
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_unsw_all_detection.sh
new file mode 100755 (executable)
index 0000000..9ac4569
--- /dev/null
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Arg1 should point to the folder with UNSW traces (PCAP files w/o any expected events).
+UNSW_TRACES_DIR=$1
+
+# Arg2 should point to the base directory  for signature files (i.e., /some/local/path/experimental_result/standalone)
+SIGNATURES_BASE_DIR=$2
+readonly SIGNATURES_BASE_DIR
+
+# Arg3 should point to base directory where the detection results for the UNSW trace are to be output.
+# Subfolders will be created for each individual pcap file in UNSW_TRACES_DIR.
+OUTPUT_DIR=$3
+readonly OUTPUT_DIR
+
+#set -x # echo invoked commands to std out
+
+for PCAP_FILE in $UNSW_TRACES_DIR/*.pcap; do
+    # skip non pcap files
+    [ -e "$PCAP_FILE" ] || continue
+    # make an output sub dir in the base output dir that is the filename minus extension
+    OUTPUT_SUB_DIR=$(basename "$PCAP_FILE" .pcap)
+    ./execute_layer2_unb_all_detection.sh $PCAP_FILE $SIGNATURES_BASE_DIR $OUTPUT_DIR/$OUTPUT_SUB_DIR
+done
+
+
diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar
new file mode 100644 (file)
index 0000000..9411448
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties b/Code/Projects/PacketLevelSignatureExtractor/gradle/wrapper/gradle-wrapper.properties
new file mode 100644 (file)
index 0000000..a17f184
--- /dev/null
@@ -0,0 +1,6 @@
+#Tue Aug 21 11:14:11 PDT 2018
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip
diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradlew b/Code/Projects/PacketLevelSignatureExtractor/gradlew
new file mode 100755 (executable)
index 0000000..9d82f78
--- /dev/null
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
diff --git a/Code/Projects/PacketLevelSignatureExtractor/gradlew.bat b/Code/Projects/PacketLevelSignatureExtractor/gradlew.bat
new file mode 100644 (file)
index 0000000..aec9973
--- /dev/null
@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off\r
+@rem ##########################################################################\r
+@rem\r
+@rem  Gradle startup script for Windows\r
+@rem\r
+@rem ##########################################################################\r
+\r
+@rem Set local scope for the variables with windows NT shell\r
+if "%OS%"=="Windows_NT" setlocal\r
+\r
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\r
+set DEFAULT_JVM_OPTS=\r
+\r
+set DIRNAME=%~dp0\r
+if "%DIRNAME%" == "" set DIRNAME=.\r
+set APP_BASE_NAME=%~n0\r
+set APP_HOME=%DIRNAME%\r
+\r
+@rem Find java.exe\r
+if defined JAVA_HOME goto findJavaFromJavaHome\r
+\r
+set JAVA_EXE=java.exe\r
+%JAVA_EXE% -version >NUL 2>&1\r
+if "%ERRORLEVEL%" == "0" goto init\r
+\r
+echo.\r
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\r
+echo.\r
+echo Please set the JAVA_HOME variable in your environment to match the\r
+echo location of your Java installation.\r
+\r
+goto fail\r
+\r
+:findJavaFromJavaHome\r
+set JAVA_HOME=%JAVA_HOME:"=%\r
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe\r
+\r
+if exist "%JAVA_EXE%" goto init\r
+\r
+echo.\r
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\r
+echo.\r
+echo Please set the JAVA_HOME variable in your environment to match the\r
+echo location of your Java installation.\r
+\r
+goto fail\r
+\r
+:init\r
+@rem Get command-line arguments, handling Windowz variants\r
+\r
+if not "%OS%" == "Windows_NT" goto win9xME_args\r
+if "%@eval[2+2]" == "4" goto 4NT_args\r
+\r
+:win9xME_args\r
+@rem Slurp the command line arguments.\r
+set CMD_LINE_ARGS=\r
+set _SKIP=2\r
+\r
+:win9xME_args_slurp\r
+if "x%~1" == "x" goto execute\r
+\r
+set CMD_LINE_ARGS=%*\r
+goto execute\r
+\r
+:4NT_args\r
+@rem Get arguments from the 4NT Shell from JP Software\r
+set CMD_LINE_ARGS=%$\r
+\r
+:execute\r
+@rem Setup the command line\r
+\r
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar\r
+\r
+@rem Execute Gradle\r
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%\r
+\r
+:end\r
+@rem End local scope for the variables with windows NT shell\r
+if "%ERRORLEVEL%"=="0" goto mainEnd\r
+\r
+:fail\r
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\r
+rem the _cmd.exe /c_ return code!\r
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1\r
+exit /b 1\r
+\r
+:mainEnd\r
+if "%OS%"=="Windows_NT" endlocal\r
+\r
+:omega\r
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap
new file mode 100644 (file)
index 0000000..b30fad9
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_OFF.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap
new file mode 100644 (file)
index 0000000..a85b153
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap
new file mode 100644 (file)
index 0000000..209bfbf
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_LOCAL_ON_SUBSET.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap
new file mode 100644 (file)
index 0000000..73a92d4
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_CHARGING_ON.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap
new file mode 100644 (file)
index 0000000..d53625e
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/TP_LINK_REMOTE_ON.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap
new file mode 100644 (file)
index 0000000..8c945a0
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.dns.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap
new file mode 100644 (file)
index 0000000..7e96a6d
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.local.remote.dns.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap
new file mode 100644 (file)
index 0000000..8e99963
Binary files /dev/null and b/Code/Projects/PacketLevelSignatureExtractor/pcap/wlan1.remote.dns.pcap differ
diff --git a/Code/Projects/PacketLevelSignatureExtractor/settings.gradle b/Code/Projects/PacketLevelSignatureExtractor/settings.gradle
new file mode 100644 (file)
index 0000000..460cd63
--- /dev/null
@@ -0,0 +1,2 @@
+rootProject.name = 'PacketLevelSignatureExtractor'
+
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/ConversationPair.java
new file mode 100644 (file)
index 0000000..b864ee5
--- /dev/null
@@ -0,0 +1,149 @@
+package edu.uci.iotproject;
+
+import org.pcap4j.core.PcapHandle;
+import org.pcap4j.core.PcapPacket;
+
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a
+ * server).
+ * Holds a pair of packet lengths from {@link PcapPacket}s identified as pertaining to the flow.
+ * Here we consider pairs of packet lengths, e.g., from device to cloud and cloud to device.
+ * We collect these pairs of data points as signatures that we can plot on a graph.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class ConversationPair {
+
+    /* Begin instance properties */
+    /**
+     * The PrintWriter object that writes data points into file
+     */
+    private PrintWriter pw;
+
+    /**
+     * The direction of conversation
+     * true = device to server to device
+     */
+    private Direction direction;
+
+    /**
+     * If this is the first packet processed then the value is true (it is false otherwise).
+     */
+    private boolean firstPacket;
+
+    /**
+     * Count the frequencies of points
+     */
+    private Map<String, Integer> pointFreq;
+    private String dataPoint;
+
+    /**
+     * Four possible directions of conversations.
+     * E.g., DEVICE_TO_SERVER means the conversation is started from
+     * a device-server packet and then a server-device as a response.
+     * SERVER_TO_DEVICE means the conversation is started from a
+     * server-device packet and then a device-server packet as a response.
+     * The same pattern applies to PHONE_TO_SERVER and SERVER_TO_PHONE
+     * directions.
+     */
+    public enum Direction {
+        DEVICE_TO_SERVER,
+        SERVER_TO_DEVICE,
+        PHONE_TO_SERVER,
+        SERVER_TO_PHONE
+    }
+
+    /**
+     * Constructs a ConversationPair object.
+     * @param fileName The file name to write data points into.
+     * @param direction The direction of the first packet of the pair.
+     */
+    public ConversationPair(String fileName, Direction direction) {
+        try {
+            this.pw = new PrintWriter(fileName, "UTF-8");
+        } catch(UnsupportedEncodingException |
+                FileNotFoundException e) {
+            e.printStackTrace();
+        }
+        this.direction = direction;
+        this.firstPacket = true;
+        this.pointFreq = new HashMap<>();
+        this.dataPoint = null;
+    }
+
+    /**
+     * Writes conversation pair's packet lengths.
+     * @param packet The {@link PcapPacket} object that has packet information.
+     * @param fromClient If true then this packet comes from client, e.g., device.
+     * @param fromServer If true then this packet comes from server.
+     */
+    public void writeConversationPair(PcapPacket packet, boolean fromClient, boolean fromServer) {
+
+        // Write device data point first and then server
+        if (direction == Direction.DEVICE_TO_SERVER || direction == Direction.PHONE_TO_SERVER) {
+            if (fromClient && firstPacket) { // first packet
+                pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", ");
+                System.out.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", ");
+                dataPoint = Integer.toString(packet.getPayload().length()) + ", ";
+                firstPacket = false;
+            } else if (fromServer && !firstPacket) { // second packet
+                pw.println(packet.getPayload().length());
+                System.out.println(packet.getPayload().length());
+                dataPoint = dataPoint + Integer.toString(packet.getPayload().length());
+                countFrequency(dataPoint);
+                firstPacket = true;
+            }
+        // Write server data point first and then device
+        } else if (direction == Direction.SERVER_TO_DEVICE || direction == Direction.SERVER_TO_PHONE) {
+            if (fromServer && firstPacket) { // first packet
+                pw.print(packet.getTimestamp() + ", " + packet.getPayload().length() + ", ");
+                dataPoint = Integer.toString(packet.getPayload().length()) + ", ";
+                firstPacket = false;
+            } else if (fromClient && !firstPacket) { // second packet
+                pw.println(packet.getPayload().length());
+                dataPoint = dataPoint + Integer.toString(packet.getPayload().length());
+                countFrequency(dataPoint);
+                firstPacket = true;
+            }
+        }
+    }
+
+    /**
+     * Counts the frequencies of data points.
+     * @param dataPoint One data point for a conversation pair, e.g., 556, 1232.
+     */
+    private void countFrequency(String dataPoint) {
+
+        Integer freq = null;
+        if (pointFreq.containsKey(dataPoint)) {
+            freq = pointFreq.get(dataPoint);
+        } else {
+            freq = new Integer(0);
+        }
+        freq = freq + 1;
+        pointFreq.put(dataPoint, freq);
+    }
+
+    /**
+     * Prints the frequencies of data points from the Map.
+     */
+    public void printListFrequency() {
+        for(Map.Entry<String, Integer> entry : pointFreq.entrySet()) {
+            System.out.println(entry.getKey() + " - " + entry.getValue());
+        }
+    }
+
+    /**
+     * Close the PrintWriter object.
+     */
+    public void close() {
+        pw.close();
+    }
+}
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/DnsMap.java
new file mode 100644 (file)
index 0000000..0db01f8
--- /dev/null
@@ -0,0 +1,111 @@
+package edu.uci.iotproject;
+
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.Packet;
+import org.pcap4j.packet.DnsPacket;
+import org.pcap4j.packet.DnsResourceRecord;
+import org.pcap4j.packet.namednumber.DnsResourceRecordType;
+
+
+import java.net.Inet4Address;
+import java.net.UnknownHostException;
+import java.util.*;
+
+
+/**
+ * This is a class that does DNS mapping.
+ * Basically an IP address is mapped to its
+ * respective DNS hostnames.
+ *
+ * @author Rahmadi Trimananda (rtrimana@uci.edu)
+ * @version 0.1
+ */
+public class DnsMap implements PacketListener {
+
+    /* Class properties */
+    private Map<String, Set<String>> ipToHostnameMap;
+
+    /* Class constants */
+    private static final Set<String> EMPTY_SET = Collections.unmodifiableSet(new HashSet<>());
+
+    
+    /* Constructor */
+    public DnsMap() {
+        ipToHostnameMap = new HashMap<>();
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        try {
+            validateAndAddNewEntry(packet);
+        } catch (UnknownHostException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * Gets a packet and determine if this is a DNS packet
+     *
+     * @param   packet  Packet object
+     * @return          DnsPacket object or null
+     */
+    private DnsPacket getDnsPacket(Packet packet) {
+        DnsPacket dnsPacket = packet.get(DnsPacket.class);
+        return dnsPacket;
+    }
+
+    /**
+     * Checks DNS packet and build the map data structure that
+     * maps IP addresses to DNS hostnames
+     *
+     * @param   packet  PcapPacket object
+     */
+    public void validateAndAddNewEntry(PcapPacket packet) throws UnknownHostException {
+        // Make sure that this is a DNS packet
+        DnsPacket dnsPacket = getDnsPacket(packet);
+        if (dnsPacket != null) {
+            // We only care about DNS answers
+            if (dnsPacket.getHeader().getAnswers().size() != 0) {
+                String hostname = dnsPacket.getHeader().getQuestions().get(0).getQName().getName();
+                for(DnsResourceRecord answer : dnsPacket.getHeader().getAnswers()) {
+                    // We only care about type A records
+                    if (!answer.getDataType().equals(DnsResourceRecordType.A))
+                        continue;
+                    // Sanity check. For some reason the hostname appears to be the empty string in the answer .
+                    // We hence have to assume that all answers correspond to a single question that holds the hostname as part of its object tree.
+                    // Therefore, if there are more questions in one query-reply exchange, we are in trouble.
+                    if (!answer.getName().getName().equals("") && !answer.getName().getName().equals(hostname))
+                        throw new RuntimeException("[DNS parser] mismatch between hostname in question and hostname in answer");
+                    // The IP in byte representation.
+                    byte[] ipBytes = answer.getRData().getRawData();
+                    // Convert to string representation.
+                    String ip = Inet4Address.getByAddress(ipBytes).getHostAddress();
+                    Set<String> hostnameSet = new HashSet<>();
+                    hostnameSet.add(hostname);
+                    // Update or insert depending on presence of key:
+                    // Concat the existing set and the new set if ip already present as key,
+                    // otherwise add an entry for ip pointing to new set.
+                    ipToHostnameMap.merge(ip, hostnameSet, (v1, v2) -> { v1.addAll(v2); return v1; });
+                }
+            }
+        }
+    }
+    
+    
+    /**
+     * Checks DNS packet and build the map data structure that
+     * maps IP addresses to DNS hostnames
+     *
+     * @param   address     Address to check
+     * @param   hostname    Hostname to check
+     */
+    public boolean isRelatedToCloudServer(String address, String hostname) {
+        return ipToHostnameMap.getOrDefault(address, EMPTY_SET).contains(hostname);
+    }
+
+    public Set<String> getHostnamesForIp(String ip) {
+        Set<String> hostnames = ipToHostnameMap.get(ip);
+        return hostnames != null ? Collections.unmodifiableSet(hostnames) : null;
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPattern.java
new file mode 100644 (file)
index 0000000..aca63ab
--- /dev/null
@@ -0,0 +1,215 @@
+package edu.uci.iotproject;
+
+import org.pcap4j.core.*;
+import org.pcap4j.packet.*;
+import org.pcap4j.packet.DnsPacket;
+import org.pcap4j.packet.namednumber.DnsResourceRecordType;
+
+import java.io.EOFException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * TODO add class documentation.
+ * TODO: At this point, this class is still in transition to having multiple hostnames and lists of packets
+ *
+ * @author Janus Varmarken
+ */
+public class FlowPattern {
+
+    /**
+     * Class properties
+     */
+    private final String mPatternId;
+    private final String hostname;  // The hostname that this {@code FlowPattern} is associated with.
+
+    /**
+     * The order of packet lengths that defines this {@link FlowPattern}
+     * TODO: this is a simplified representation, we should also include information about direction of each packet.
+     */
+    private final List<Integer> flowPacketOrder;
+    private final Map<String, List<Integer>> mHostnameToPacketLengthsMap;
+    private final List<String> mHostnameList;
+    private final PcapHandle mPcap;
+
+    
+    /**
+     * Class constants
+     */
+     
+
+    /**
+     * Constructor #1
+     */
+    public FlowPattern(String mPatternId, String hostname, PcapHandle mPcap) {
+        this.mPatternId = mPatternId;
+        this.hostname = hostname;
+        this.mHostnameList = null;
+        this.mPcap = mPcap;
+        this.mHostnameToPacketLengthsMap = null;
+        this.flowPacketOrder = new ArrayList<Integer>();
+        processPcap();
+    }
+
+
+    /**
+     * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths
+     */
+    private void processPcap() {
+
+        PcapPacket packet;
+        try {
+            while ((packet = mPcap.getNextPacketEx()) != null) {
+                // For now, we only work support pattern search in TCP over IPv4.
+                IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+                TcpPacket tcpPacket = packet.get(TcpPacket.class);
+                if (ipPacket == null || tcpPacket == null)
+                    continue;
+                if (tcpPacket.getPayload() == null) // We skip non-payload control packets as these are less predictable
+                    continue; 
+                int packetLength = tcpPacket.getPayload().length();
+                flowPacketOrder.add(packetLength);
+            }
+        } catch (EOFException eofe) {
+            System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!");
+            System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(flowPacketOrder.toArray()));
+        } catch (PcapNativeException  |
+                 TimeoutException     |
+                 NotOpenException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+
+    /**
+     * Process the PcapHandle to strip off unnecessary packets.
+     * We then map list of hostnames to their respective arrays of packet lengths
+     */
+    private void processPcapToMap() {
+
+        PcapPacket packet;
+        try {
+            int hostIndex = -1;
+            Set<String> addressSet = new HashSet<>();
+            while ((packet = mPcap.getNextPacketEx()) != null) {
+                // For now, we only work support pattern search in TCP over IPv4.
+                IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+                TcpPacket tcpPacket = packet.get(TcpPacket.class);
+                if (ipPacket == null || tcpPacket == null) {
+                    continue;
+                }
+                if (tcpPacket.getPayload() == null) {
+                // We skip non-payload control packets as these are less predictable
+                    continue;
+                }
+                // We assume that if it is not a local address then it is a cloud server address
+                InetAddress srcAddress = ipPacket.getHeader().getSrcAddr();
+                InetAddress dstAddress = ipPacket.getHeader().getDstAddr();
+                boolean fromServer = !srcAddress.isSiteLocalAddress();
+                boolean fromClient = !dstAddress.isSiteLocalAddress();
+                if (!fromServer && !fromClient) {
+                    // Packet not related to pattern, skip it
+                    continue;
+                } else {
+                    // We relate and assume that this address is from our cloud server
+                    String cloudAddress = null;
+                    if (fromClient) {
+                        cloudAddress = dstAddress.getHostAddress();
+                    } else { // fromServer
+                        cloudAddress = srcAddress.getHostAddress();
+                    }
+                    //System.out.println("\nCloud address: " + cloudAddress);
+                    if (!addressSet.contains(cloudAddress)) {
+                        addressSet.add(cloudAddress);
+                        hostIndex++;
+                    }
+
+                    String hostname = mHostnameList.get(hostIndex);
+                    List<Integer> packetLengthsList = mHostnameToPacketLengthsMap.containsKey(hostname) ? 
+                        mHostnameToPacketLengthsMap.get(hostname) : new ArrayList<>();
+                    int packetLength = tcpPacket.getPayload().length();
+                    packetLengthsList.add(packetLength);
+                    mHostnameToPacketLengthsMap.put(hostname, packetLengthsList);
+                }
+            }
+        } catch (EOFException eofe) {
+            System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!");
+            System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(mHostnameToPacketLengthsMap.entrySet().toArray()));
+        } catch (PcapNativeException  |
+                 TimeoutException     |
+                 NotOpenException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    
+    /**
+     * Constructor #2
+     */
+    public FlowPattern(String mPatternId, List<String> mHostnameList, PcapHandle mPcap) {
+        this.mPatternId = mPatternId;
+        this.hostname = null;
+        this.mHostnameList = mHostnameList;
+        this.mPcap = mPcap;
+        this.flowPacketOrder = null;
+        this.mHostnameToPacketLengthsMap = new HashMap<>();
+        processPcapToMap();
+    }
+
+
+    public String getPatternId() {
+        return mPatternId;
+    }
+
+
+    public String getHostname() {
+        return hostname;
+    }
+
+
+    /**
+     * Get the sequence of packet lengths that defines this {@code FlowPattern}.
+     * @return the sequence of packet lengths that defines this {@code FlowPattern}.
+     */
+    public List<Integer> getPacketOrder() {
+        return flowPacketOrder;
+    }
+
+
+    /**
+     * Get the sequence of packet lengths based on input hostname.
+     * @return the sequence of packet lengths that defines this {@code FlowPattern}.
+     */
+    public List<Integer> getPacketOrder(String hostname) {
+        return mHostnameToPacketLengthsMap.get(hostname);
+    }
+
+
+    /**
+     * Get the list of associated hostnames.
+     * @return the associated hostnames that define this {@code FlowPattern}.
+     */
+    public List<String> getHostnameList() {
+        return mHostnameList;
+    }
+
+    
+    /**
+     * Get the length of the List of {@code FlowPattern}.
+     * @return the length of the List of {@code FlowPattern}.
+     */
+    public int getLength() {
+        return flowPacketOrder.size();
+    }  
+
+
+    /**
+     * Get the length of the List of {@code FlowPattern}.
+     * @return the length of the List of {@code FlowPattern}.
+     */
+    public int getLength(String hostname) {
+        return mHostnameToPacketLengthsMap.get(hostname).size();
+    } 
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
new file mode 100644 (file)
index 0000000..c384852
--- /dev/null
@@ -0,0 +1,357 @@
+package edu.uci.iotproject;
+
+import edu.uci.iotproject.comparison.ComparisonFunctions;
+import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult;
+import edu.uci.iotproject.comparison.PatternComparisonTask;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import org.pcap4j.core.NotOpenException;
+import org.pcap4j.core.PcapHandle;
+import org.pcap4j.core.PcapNativeException;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.DnsPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.TcpPacket;
+
+import java.io.*;
+import java.net.UnknownHostException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+
+
+/**
+ * <p>Provides functionality for searching for the presence of a {@link FlowPattern} in a PCAP trace.</p>
+ *
+ * <p>
+ * The (entire) PCAP trace is traversed and parsed on one thread (specifically, the thread that calls
+ * {@link #findFlowPattern()}). This thread builds a {@link DnsMap} using the DNS packets present in the trace and uses
+ * that {@code DnsMap} to reassemble {@link Conversation}s that <em>potentially</em> match the provided
+ * {@link FlowPattern} (in that one end/party of said conversations matches the hostname(s) specified by the given
+ * {@code FlowPattern}).
+ * These potential matches are then examined on background worker thread(s) to determine if they are indeed a (complete)
+ * match of the provided {@code FlowPattern}.
+ * </p>
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class FlowPatternFinder {
+
+    /* Begin class properties */
+    /**
+     * {@link ExecutorService} responsible for parallelizing pattern searches.
+     * Declared as static to allow for reuse of threads across different instances of {@code FlowPatternFinder} and to
+     * avoid the overhead of initializing a new thread pool for each {@code FlowPatternFinder} instance.
+     */
+    private static final ExecutorService EXECUTOR_SERVICE = Executors.newCachedThreadPool();
+    /* End class properties */
+
+    /* Begin instance properties */
+    /**
+     * Holds a set of {@link Conversation}s that <em>potentially</em> match {@link #mPattern} since each individual
+     * {@code Conversation} is communication with the hostname identified by {@code mPattern.getHostname()}.
+     * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method),
+     * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)}
+     * functionality.
+     *
+     * @see <a href="https://stackoverflow.com/questions/7283338/getting-an-element-from-a-set">this question on StackOverflow.com</a>
+     */
+    private final Map<Conversation, Conversation> mConversations;
+
+    /**
+     * Holds a list of trigger times.
+     */
+    private final List<Long> mTriggerTimes;
+    private static int triggerListCounter;
+
+    private final DnsMap mDnsMap;
+    private final PcapHandle mPcap;
+    private final FlowPattern mPattern;
+    private final ConversationPair mConvPair;
+    private final String FILE = "./devices/tplink_switch/datapoints.csv";
+    //private final String REF_FILE = "./devices/dlink_switch/dlink-june-26-2018.timestamps";
+    private final String REF_FILE = "./devices/tplink_switch/tplink-june-14-2018.timestamps";
+    //private final String REF_FILE = "./devices/tplink_switch/tplink-feb-13-2018.timestamps";
+    // Router time is in CET and we use PST for the trigger times
+    // Difference is 7 hours x 3600 x 1000ms = 25,200,000ms
+    private final long TIME_OFFSET = 25200000;
+
+    private final List<Future<CompleteMatchPatternComparisonResult>> mPendingComparisons = new ArrayList<>();
+    /* End instance properties */
+
+    /**
+     * Constructs a new {@code FlowPatternFinder}.
+     * @param pcap an <em>open</em> {@link PcapHandle} that provides access to the trace that is to be examined.
+     * @param pattern the {@link FlowPattern} to search for.
+     */
+    public FlowPatternFinder(PcapHandle pcap, FlowPattern pattern) {
+        this.mConversations = new HashMap<>();
+        this.mTriggerTimes = readTriggerTimes(REF_FILE);
+        triggerListCounter = 0;
+        this.mDnsMap = new DnsMap();
+        this.mPcap = Objects.requireNonNull(pcap,
+                String.format("Argument of type '%s' cannot be null", PcapHandle.class.getSimpleName()));
+        this.mPattern = Objects.requireNonNull(pattern,
+                String.format("Argument of type '%s' cannot be null", FlowPattern.class.getSimpleName()));
+        this.mConvPair = new ConversationPair(FILE, ConversationPair.Direction.DEVICE_TO_SERVER);
+    }
+
+
+    private List<Long> readTriggerTimes(String refFileName) {
+
+        List<Long> listTriggerTimes = new ArrayList<>();
+        try {
+            File file = new File(refFileName);
+            BufferedReader br = new BufferedReader(new FileReader(file));
+            String s;
+            while ((s = br.readLine()) != null) {
+                listTriggerTimes.add(timeToMillis(s, false));
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        System.out.println("List has: " + listTriggerTimes.size());
+
+        return listTriggerTimes;
+    }
+
+    /**
+     * Starts the pattern search.
+     */
+    public void start() {
+
+        //findFlowPattern();
+        findSignatureBasedOnTimestamp();
+    }
+
+    /**
+     * Find patterns based on the FlowPattern object (run by a thread)
+     */
+    private void findFlowPattern() {
+        try {
+            PcapPacket packet;
+//            TODO: The new comparison method is pending
+//            TODO: For now, just compare using one hostname and one list per FlowPattern
+//            List<String> hostnameList = mPattern.getHostnameList();
+//            int hostIndex = 0;
+            while ((packet = mPcap.getNextPacketEx()) != null) {
+                // Let DnsMap handle DNS packets.
+                if (packet.get(DnsPacket.class) != null) {
+                    // Check if this is a valid DNS packet
+                    mDnsMap.validateAndAddNewEntry(packet);
+                    continue;
+                }
+                // For now, we only work support pattern search in TCP over IPv4.
+                final IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+                final TcpPacket tcpPacket = packet.get(TcpPacket.class);
+                if (ipPacket == null || tcpPacket == null) {
+                    continue;
+                }
+
+                String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress();
+                String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress();
+                int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
+                int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
+                // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server?
+                boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname());
+                boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname());
+//                String currentHostname = hostnameList.get(hostIndex);
+//                boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname);
+//                boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname);
+                if (!fromServer && !fromClient) {
+                    // Packet not related to pattern, skip it.
+                    continue;
+                }
+
+                // Conversations (connections/sessions) are identified by the four-tuple
+                // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc).
+                // Create "dummy" conversation for looking up an existing entry.
+                Conversation conversation = fromClient ? new Conversation(srcAddress, srcPort, dstAddress, dstPort) :
+                        new Conversation(dstAddress, dstPort, srcAddress, srcPort);
+                // Add the packet so that the "dummy" conversation can be immediately added to the map if no entry
+                // exists for the conversation that the current packet belongs to.
+                if (tcpPacket.getHeader().getFin()) {
+                    // Record FIN packets.
+                    conversation.addFinPacket(packet);
+                }
+                if (tcpPacket.getPayload() != null) {
+                    // Record regular payload packets.
+                    conversation.addPacket(packet, true);
+                }
+                // Note: does not make sense to call attemptAcknowledgementOfFin here as the new packet has no FINs
+                // in its list, so if this packet is an ACK, it would not be added anyway.
+                // Need to retain a final reference to get access to the packet in the lambda below.
+                final PcapPacket finalPacket = packet;
+                // Add the new conversation to the map if an equal entry is not already present.
+                // If an existing entry is already present, the current packet is simply added to that conversation.
+                mConversations.merge(conversation, conversation, (existingEntry, toMerge) -> {
+                    // toMerge may not have any payload packets if the current packet is a FIN packet.
+                    if (toMerge.getPackets().size() > 0) {
+                        existingEntry.addPacket(toMerge.getPackets().get(0), true);
+                    }
+                    if (toMerge.getFinAckPairs().size() > 0) {
+                        // Add the FIN packet to the existing entry.
+                        existingEntry.addFinPacket(toMerge.getFinAckPairs().get(0).getFinPacket());
+                    }
+                    if (finalPacket.get(TcpPacket.class).getHeader().getAck()) {
+                        existingEntry.attemptAcknowledgementOfFin(finalPacket);
+                    }
+                    return existingEntry;
+                });
+                // Refresh reference to point to entry in map (in case packet was added to existing entry).
+                conversation = mConversations.get(conversation);
+                if (conversation.isGracefullyShutdown()) {
+                    // Conversation terminated gracefully, so we can now start analyzing it.
+                    // Remove the Conversation from the map and start the analysis.
+                    // Any future packets identified by the same four tuple will be tied to a new Conversation instance.
+                    mConversations.remove(conversation);
+                    // Create comparison task and send to executor service.
+                    PatternComparisonTask<CompleteMatchPatternComparisonResult> comparisonTask =
+                            new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.SUB_SEQUENCE_COMPLETE_MATCH);
+                    mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask));
+                    // Increment hostIndex to find the next
+
+                }
+            }
+        } catch (EOFException eofe) {
+            // TODO should check for leftover conversations in map here and fire tasks for those.
+            // TODO [cont'd] such tasks may be present if connections did not terminate gracefully or if there are longlived connections.
+            System.out.println("[ findFlowPattern ] Finished processing entire PCAP stream!");
+            System.out.println("[ findFlowPattern ] Now waiting for comparisons to finish...");
+            // Wait for all comparisons to finish, then output their results to std.out.
+            for(Future<CompleteMatchPatternComparisonResult> comparisonTask : mPendingComparisons) {
+                try {
+                    // Blocks until result is ready.
+                    CompleteMatchPatternComparisonResult comparisonResult = comparisonTask.get();
+                    if (comparisonResult.getResult()) {
+                        System.out.println(comparisonResult.getTextualDescription());
+                    }
+                } catch (InterruptedException|ExecutionException e) {
+                    e.printStackTrace();
+                }
+            }
+        } catch (UnknownHostException |
+                 PcapNativeException  |
+                 NotOpenException     |
+                 TimeoutException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    /**
+     * Find patterns based on the FlowPattern object (run by a thread)
+     */
+    private void findSignatureBasedOnTimestamp() {
+        try {
+            PcapPacket packet;
+//            TODO: The new comparison method is pending
+//            TODO: For now, just compare using one hostname and one list per FlowPattern
+            while ((packet = mPcap.getNextPacketEx()) != null) {
+                // Let DnsMap handle DNS packets.
+                if (packet.get(DnsPacket.class) != null) {
+                    // Check if this is a valid DNS packet
+                    mDnsMap.validateAndAddNewEntry(packet);
+                    continue;
+                }
+                // For now, we only work support pattern search in TCP over IPv4.
+                final IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+                final TcpPacket tcpPacket = packet.get(TcpPacket.class);
+                if (ipPacket == null || tcpPacket == null) {
+                    continue;
+                }
+
+                String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress();
+                String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress();
+                int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
+                int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
+                //System.out.println("Timestamp packet: " + packet.getTimestamp());
+                // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server?
+                boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname());
+                boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname());
+                if (!fromServer && !fromClient) {
+                    // Packet not related to pattern, skip it.
+                    continue;
+                }
+                // Record the conversation pairs
+                if (tcpPacket.getPayload() != null && checkTimeStamp(packet)) {
+                //if (tcpPacket.getPayload() != null) {
+                    mConvPair.writeConversationPair(packet, fromClient, fromServer);
+                }
+            }
+        } catch (EOFException eofe) {
+            triggerListCounter = 0;
+            mConvPair.close();
+            System.out.println("[ findFlowPattern ] ConversationPair writer closed!");
+            System.out.println("[ findFlowPattern ] Frequencies of data points:");
+            mConvPair.printListFrequency();
+        } catch (UnknownHostException |
+                PcapNativeException  |
+                NotOpenException     |
+                TimeoutException ex) {
+            ex.printStackTrace();
+        }
+    }
+
+    private boolean checkTimeStamp(PcapPacket packet) {
+
+        // Extract time from the packet's timestamp
+        String timeStamp = packet.getTimestamp().toString();
+        String timeString = timeStamp.substring(timeStamp.indexOf("T") + 1, timeStamp.indexOf("."));
+        // Timestamps are in CET (ahead of PST) so it should be deducted by TIME_OFFSET
+        long time = timeToMillis(timeString, true) - TIME_OFFSET;
+        //long time = timeToMillis(timeString, true);
+
+        //System.out.println("Gets here: " + time + " trigger time: " + mTriggerTimes.get(triggerListCounter));
+
+        // We accept packets that are at most 3 seconds away from the trigger time
+        if ((mTriggerTimes.get(triggerListCounter) <= time) &&
+                (time <= mTriggerTimes.get(triggerListCounter) + 3000)) {
+            //System.out.println("Gets here 1: " + timeString + " index: " + triggerListCounter);
+            return true;
+        } else {
+            // Handle the case that the timestamp is > 3000, but < next timestamp
+            // in the list. We ignore these packets.
+            if (time < mTriggerTimes.get(triggerListCounter)) {
+                // Timestamp is smaller than trigger, ignore!
+                //System.out.println("Gets here 2: " + timeString + " index: " + triggerListCounter);
+                return false;
+            } else { // Timestamp is greater than trigger, increment!
+                triggerListCounter = triggerListCounter + 1;
+                //System.out.println("Gets here 3: " + timeString + " index: " + triggerListCounter);
+                //return false;
+                return checkTimeStamp(packet);
+            }
+        }
+
+        //System.out.println("Timestamp: " + timeToMillis(time, true));
+        //String time2 = "21:38:08";
+        //System.out.println("Timestamp: " + timeToMillis(time2, true));
+    }
+
+    /**
+     * A private function that returns time in milliseconds.
+     * @param time The time in the form of String.
+     * @param is24Hr If true, then this is in 24-hour format.
+     */
+    private long timeToMillis(String time, boolean is24Hr) {
+
+        String format = null;
+        if (is24Hr) {
+            format = "hh:mm:ss";
+        } else { // 12 Hr format
+            format = "hh:mm:ss aa";
+        }
+        DateFormat sdf = new SimpleDateFormat(format);
+        Date date = null;
+        try {
+            date = sdf.parse(time);
+        } catch(Exception e) {
+            e.printStackTrace();
+        }
+        if (date == null)
+            return 0;
+        return date.getTime();
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/Main.java
new file mode 100644 (file)
index 0000000..f1056b0
--- /dev/null
@@ -0,0 +1,905 @@
+package edu.uci.iotproject;
+
+import static edu.uci.iotproject.analysis.UserAction.Type;
+
+import edu.uci.iotproject.analysis.*;
+import edu.uci.iotproject.io.TriggerTimesFileReader;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import edu.uci.iotproject.util.PrintUtils;
+import org.apache.commons.math3.stat.clustering.Cluster;
+import org.apache.commons.math3.stat.clustering.DBSCANClusterer;
+import org.pcap4j.core.*;
+import org.pcap4j.packet.namednumber.DataLinkType;
+
+import java.io.EOFException;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This is a system that reads PCAP files to compare
+ * patterns of DNS hostnames, packet sequences, and packet
+ * lengths with training data to determine certain events
+ * or actions for smart home devices.
+ *
+ * @author Janus Varmarken
+ * @author Rahmadi Trimananda (rtrimana@uci.edu)
+ * @version 0.1
+ */
+public class Main {
+
+
+    public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException {
+        // -------------------------------------------------------------------------------------------------------------
+        // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
+        // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for
+        // which traffic is to be extracted:
+        String path = "/scratch/July-2018"; // Rahmadi
+//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        boolean verbose = true;
+        final String onPairsPath = "/scratch/July-2018/on.txt";
+        final String offPairsPath = "/scratch/July-2018/off.txt";
+
+        // 1) D-Link July 26 experiment
+//        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
+//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+          // Actual training
+//        final String inputPcapFile = path + "/2018-10/dlink-plug/dlink-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/dlink-plug/dlink-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/dlink-plug/dlink-plug-oct-17-2018.timestamps";
+//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+        // TODO: EXPERIMENT - November 7, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
+//        final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
+////        final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
+
+        // 2) TP-Link July 25 experiment
+//        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.timestamps";
+//        final String deviceIp = "192.168.1.159";
+          // Actual training
+//        final String inputPcapFile = path + "/2018-10/tplink-plug/tplink-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/tplink-plug/tplink-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/tplink-plug/tplink-plug-oct-17-2018.timestamps";
+//        final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug
+        // TODO: EXPERIMENT - November 8, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps";
+//        final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug
+////        final String deviceIp = "192.168.1.246"; // .246 == phone; .159 == tplink plug
+
+        // 2b) TP-Link July 25 experiment TRUNCATED:
+        // Only contains "true local" events, i.e., before the behavior changes to remote-like behavior.
+        // Last included event is at July 25 10:38:11; file filtered to only include packets with arrival time <= 10:38:27.
+//        final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.truncated.pcap";
+//        final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.truncated.pcap";
+//        final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.truncated.timestamps";
+//        final String deviceIp = "192.168.1.159";
+
+        // 3) SmartThings Plug July 25 experiment
+//        final String inputPcapFile = path + "/2018-07/stplug/stplug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/stplug/stplug-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/stplug/smartthings-july-25-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+        // October 18
+//        final String inputPcapFile = path + "/2018-10/st-plug/st-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/st-plug/st-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/st-plug/st-plug-oct-18-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+        // TODO: EXPERIMENT - November 12, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps";
+////        final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+
+        // 4) Wemo July 30 experiment
+//        final String inputPcapFile = path + "/2018-07/wemo/wemo.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/wemo/wemo-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/wemo/wemo-july-30-2018.timestamps";
+//        final String deviceIp = "192.168.1.145";  // .246 == phone; .145 == WeMo
+        // TODO: EXPERIMENT - November 20, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps";
+////        final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .145 == WeMo
+
+        // 5) Wemo Insight July 31 experiment
+//        final String inputPcapFile = path + "/2018-07/wemoinsight/wemoinsight.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-07/wemoinsight/wemoinsight-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-07/wemoinsight/wemo-insight-july-31-2018.timestamps";
+//        final String deviceIp = "192.168.1.135";
+        // TODO: EXPERIMENT - November 21, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
+////        final String deviceIp = "192.168.1.145"; // .246 == phone; .135 == WeMo Insight
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .135 == WeMo Insight
+
+        // 6) TP-Link Bulb August 1 experiment
+//        final String inputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/tplink-bulb/tplink-bulb-aug-3-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb
+        // TODO: EXPERIMENT - November 16, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps";
+////        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == TP-Link bulb
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb
+
+        // 7) Kwikset Doorlock August 6 experiment
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
+////        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-processed.pcap";
+////        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-aug-6-2018.timestamps";
+//        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-8hr-data-oct-11-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+        // TODO: EXPERIMENT - November 10, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps";
+////        final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+
+        // September 12, 2018 - includes both wlan1 and eth1 interfaces
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
+
+        // 8) Hue Bulb August 7 experiment
+//        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/hue-bulb/hue-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
+//        final String deviceIp = "192.168.1.246";
+        // October 30 experiment
+//        final String inputPcapFile = path + "/2018-10/hue-bulb/hue-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/hue-bulb/hue-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/hue-bulb/hue-bulb-oct-30-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
+        // TODO: EXPERIMENT - November 19, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/hue-bulb/timestamps/hue-bulb-nov-19-2018.timestamps";
+////        final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
+
+        // 9) Lifx Bulb August 8 experiment
+//        final String inputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/lifx-bulb/lifx-bulb-aug-8-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .231 == Lifx
+        // October 18
+//        final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-oct-18-2018.timestamps";
+//        final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx
+        // November 1
+//        final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-nov-1-2018.timestamps";
+//        final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx
+
+        // 10) Amcrest Camera August 9 experiment
+//        final String inputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/amcrest-camera/amcrest-camera-aug-9-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .235 == camera
+
+        // 11) Arlo Camera August 10 experiment
+//        final String inputPcapFile = path + "/2018-08/arlo-camera/arlo-camera.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/arlo-camera/arlo-camera-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/arlo-camera/arlo-camera-aug-10-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
+        // TODO: EXPERIMENT - November 13, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
+////        final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
+
+        // 12) Blossom sprinkler August 13 experiment
+//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/blossom/blossom-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/blossom/blossom-aug-13-2018.timestamps";
+//        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
+//        // 2 November
+//        final String inputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-nov-2-2018.timestamps";
+//        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
+        // January 9, 11, 13, 14
+//        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
+////        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-11-2019.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
+////        final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
+
+//        // 13) DLink siren August 14 experiment
+//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
+//        final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-oct-12-2018.timestamps";
+//        //final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
+//        //final String triggerTimesFile = path + "/actual/timestamps/dlink-siren-8hr-data-oct-10-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
+        // TODO: EXPERIMENT - November 9, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
+////        final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
+
+        // 14) Nest thermostat August 15 experiment
+//        final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/nest/nest-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/nest/nest-aug-15-2018.timestamps";
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
+//        // TODO: EXPERIMENT - November 14, 2018
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap";
+////        final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat-processed.pcap";
+//        final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
+////        final String deviceIp = "192.168.1.127"; // .246 == phone; .127 == Nest thermostat
+//        final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
+
+        // 15) Alexa August 16 experiment
+//        final String inputPcapFile = path + "/2018-08/alexa/alexa.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/alexa/alexa-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-16-2018.timestamps";
+//        final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa
+        // August 17
+//        final String inputPcapFile = path + "/2018-08/alexa/alexa2.wlan1.local.pcap";
+//        final String outputPcapFile = path + "/2018-08/alexa/alexa2-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-17-2018.timestamps";
+//        final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa
+
+        // September 17
+//        final String inputPcapFile = path + "/2018-08/noise/noise.eth1.pcap";
+//        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps";
+//        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub; .199 == dlink plug; .183 == siren
+        // September 26 - D-Link noise
+//        final String inputPcapFile = path + "/2018-08/noise/noise.dlink.wlan1.pcap";
+//        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/noise/dlink-noise-sept-26-2018.timestamps";
+//        final String deviceIp = "192.168.1.183"; //  .199 == dlink plug; .183 == siren
+        // September 27 - Kwikset noise
+//        final String inputPcapFile = path + "/2018-08/noise/noise.kwikset.eth1.pcap";
+//        final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
+//        final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps";
+//        final String deviceIp = "192.168.1.142"; //  .142 == SmartThings Hub;
+
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // TODO: The below part is just for 15-second time sensitivity experiment
+        // D-Link plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
+////        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // TP-Link plug
+        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps";
+////        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
+////        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
+        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
+
+        // D-Link siren
+//        final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
+        // Kwikset door lock
+//        final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
+        // SmartThings plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+        // Arlo Q
+//        final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        // Nest thermostat
+//        final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+        // Blossom sprinkler
+//        final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+        // TP-Link bulb
+//        final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+        // Philips hue
+//        final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
+//        final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+        // WeMo plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
+        // WeMo Insight plug
+//        final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
+
+
+        TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
+        List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
+
+        System.out.println("ON signature file in use is " + onSignatureFile);
+        System.out.println("OFF signature file in use is " + offSignatureFile);
+
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+
+        List<Instant> signatureTimestamps = new ArrayList<>();
+        // Load ON signature last packet's timestamp
+        // Get the last only
+        List<List<PcapPacket>> lastListOn = onSignature.get(onSignature.size()-1);
+        for (List<PcapPacket> list : lastListOn) {
+            // Get timestamp Instant from the last packet
+            int lastPacketIndex = list.size()-1;
+            signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+        }
+        // Load OFF signature last packet's timestamp
+        // Get the last only
+        List<List<PcapPacket>> lastListOff = offSignature.get(offSignature.size()-1);
+        for (List<PcapPacket> list : lastListOff) {
+            // Get timestamp Instant from the last packet
+            int lastPacketIndex = list.size()-1;
+            signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+        }
+        // Sort the timestamps
+        signatureTimestamps.sort((p1, p2) -> {
+            return p1.compareTo(p2);
+        });
+
+        Iterator<Instant> iterTrig = triggerTimes.iterator();
+        Iterator<Instant> iterSign = signatureTimestamps.iterator();
+        System.out.println("Trigger to Last Packet:");
+        while (iterTrig.hasNext() && iterSign.hasNext()) {
+            Instant trigInst = (Instant) iterTrig.next();
+            Instant signInst = (Instant) iterSign.next();
+            Duration dur = Duration.between(trigInst, signInst);
+            long duration = dur.toMillis();
+            // Check duration --- should be below 15 seconds
+            if (duration >= 0 && duration <= 15000) {
+                System.out.println(dur.toMillis());
+            } else if (duration > 15000) {
+                while (duration > 15000) { // that means we have to move to the next trigger
+                    trigInst = (Instant) iterTrig.next();
+                    dur = Duration.between(trigInst, signInst);
+                    duration = dur.toMillis();
+                }
+                System.out.println(dur.toMillis());
+            } else { // below 0 / negative --- that means we have to move to the next signature
+                while (duration < 0) { // that means we have to move to the next trigger
+                    signInst = (Instant) iterSign.next();
+                    dur = Duration.between(trigInst, signInst);
+                    duration = dur.toMillis();
+                }
+                System.out.println(dur.toMillis());
+            }
+        }
+
+
+        // ==========================================================================
+        List<Instant> firstSignatureTimestamps = new ArrayList<>();
+        List<Instant> lastSignatureTimestamps = new ArrayList<>();
+        List<List<PcapPacket>> firstListOnSign = onSignature.get(0);
+        List<List<PcapPacket>> lastListOnSign = onSignature.get(onSignature.size()-1);
+        // Load ON signature first and last packet's timestamps
+        for (List<PcapPacket> list : firstListOnSign) {
+            // Get timestamp Instant from the last packet
+            firstSignatureTimestamps.add(list.get(0).getTimestamp());
+        }
+        for (List<PcapPacket> list : lastListOnSign) {
+            // Get timestamp Instant from the last packet
+            int lastPacketIndex = list.size()-1;
+            lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+        }
+
+        List<List<PcapPacket>> firstListOffSign = offSignature.get(0);
+        List<List<PcapPacket>> lastListOffSign = offSignature.get(offSignature.size()-1);
+        // Load OFF signature first and last packet's timestamps
+        for (List<PcapPacket> list : firstListOffSign) {
+            // Get timestamp Instant from the last packet
+            firstSignatureTimestamps.add(list.get(0).getTimestamp());
+        }
+        for (List<PcapPacket> list : lastListOffSign) {
+            // Get timestamp Instant from the last packet
+            int lastPacketIndex = list.size()-1;
+            lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+        }
+        // Sort the timestamps
+        firstSignatureTimestamps.sort((p1, p2) -> {
+            return p1.compareTo(p2);
+        });
+        // Sort the timestamps
+        lastSignatureTimestamps.sort((p1, p2) -> {
+            return p1.compareTo(p2);
+        });
+
+        Iterator<Instant> iterFirst = firstSignatureTimestamps.iterator();
+        Iterator<Instant> iterLast = lastSignatureTimestamps.iterator();
+        System.out.println("First to Last Packet:");
+        while (iterFirst.hasNext() && iterLast.hasNext()) {
+            Instant firstInst = (Instant) iterFirst.next();
+            Instant lastInst = (Instant) iterLast.next();
+            Duration dur = Duration.between(firstInst, lastInst);
+            long duration = dur.toMillis();
+            // Check duration --- should be below 15 seconds
+            if (duration >= 0 && duration <= 15000) {
+                System.out.println(dur.toMillis());
+            } else if (duration > 15000) {
+                while (duration > 15000) { // that means we have to move to the next trigger
+                    firstInst = (Instant) iterFirst.next();
+                    dur = Duration.between(firstInst, lastInst);
+                    duration = dur.toMillis();
+                }
+                System.out.println(dur.toMillis());
+            } else { // below 0 / negative --- that means we have to move to the next signature
+                while (duration < 0) { // that means we have to move to the next trigger
+                    lastInst = (Instant) iterLast.next();
+                    dur = Duration.between(firstInst, lastInst);
+                    duration = dur.toMillis();
+                }
+                System.out.println(dur.toMillis());
+            }
+            if (duration > 8000) {
+                break;
+            }
+        }
+
+        // TODO: The above part is just for 15-second time sensitivity experiment
+        // TODO: The above part is just for 15-second time sensitivity experiment
+        // TODO: The above part is just for 15-second time sensitivity experiment
+
+
+
+
+//        TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
+//        List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
+//        // Tag each trigger with "ON" or "OFF", assuming that the first trigger is an "ON" and that they alternate.
+//        List<UserAction> userActions = new ArrayList<>();
+//        for (int i = 0; i < triggerTimes.size(); i++) {
+//            userActions.add(new UserAction(i % 2 == 0 ? Type.TOGGLE_ON : Type.TOGGLE_OFF, triggerTimes.get(i)));
+//        }
+//        TriggerTrafficExtractor tte = new TriggerTrafficExtractor(inputPcapFile, triggerTimes, deviceIp);
+//        final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen(outputPcapFile);
+//        DnsMap dnsMap = new DnsMap();
+//        TcpReassembler tcpReassembler = new TcpReassembler();
+//        TrafficLabeler trafficLabeler = new TrafficLabeler(userActions);
+//        tte.performExtraction(pkt -> {
+//            try {
+//                outputter.dump(pkt);
+//            } catch (NotOpenException e) {
+//                e.printStackTrace();
+//            }
+//        }, dnsMap, tcpReassembler, trafficLabeler);
+//        outputter.flush();
+//        outputter.close();
+//
+//        if (tte.getPacketsIncludedCount() != trafficLabeler.getTotalPacketCount()) {
+//            // Sanity/debug check
+//            throw new AssertionError(String.format("mismatch between packet count in %s and %s",
+//                    TriggerTrafficExtractor.class.getSimpleName(), TrafficLabeler.class.getSimpleName()));
+//        }
+//
+//        // Extract all conversations present in the filtered trace.
+//        List<Conversation> allConversations = tcpReassembler.getTcpConversations();
+//        // Group conversations by hostname.
+//        Map<String, List<Conversation>> convsByHostname = TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap);
+//        System.out.println("Grouped conversations by hostname.");
+//        // For each hostname, count the frequencies of packet lengths exchanged with that hostname.
+//        final Map<String, Map<Integer, Integer>> pktLenFreqsByHostname = new HashMap<>();
+//        convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host, TcpConversationUtils.countPacketLengthFrequencies(convs)));
+//        System.out.println("Counted frequencies of packet lengths exchanged with each hostname.");
+//        // For each hostname, count the frequencies of packet sequences (i.e., count how many conversations exchange a
+//        // sequence of packets of some specific lengths).
+//        final Map<String, Map<String, Integer>> pktSeqFreqsByHostname = new HashMap<>();
+//        convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host, TcpConversationUtils.countPacketSequenceFrequencies(convs)));
+//        System.out.println("Counted frequencies of packet sequences exchanged with each hostname.");
+//        // For each hostname, count frequencies of packet pairs exchanged with that hostname across all conversations
+//        final Map<String, Map<String, Integer>> pktPairFreqsByHostname =
+//                TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap);
+//        System.out.println("Counted frequencies of packet pairs per hostname");
+//        // For each user action, reassemble the set of TCP connections occurring shortly after
+//        final Map<UserAction, List<Conversation>> userActionToConversations = trafficLabeler.getLabeledReassembledTcpTraffic();
+//        final Map<UserAction, Map<String, List<Conversation>>> userActionsToConvsByHostname = trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap);
+//        System.out.println("Reassembled TCP conversations occurring shortly after each user event");
+//
+//
+//
+//        /*
+//         * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and
+//         * sequence identifiers as we do not care about hostnames and sequences during clustering.
+//         * We can simply use the UserAction->List<Conversation> map to generate ON/OFF groupings of conversations.
+//         */
+//
+////        // Contains all ON events: hostname -> sequence identifier -> list of conversations with that sequence
+////        Map<String, Map<String, List<Conversation>>> ons = new HashMap<>();
+////        // Contains all OFF events: hostname -> sequence identifier -> list of conversations with that sequence
+////        Map<String, Map<String, List<Conversation>>> offs = new HashMap<>();
+////        userActionsToConvsByHostname.forEach((ua, hostnameToConvs) -> {
+////            Map<String, Map<String, List<Conversation>>> outer = ua.getType() == Type.TOGGLE_ON ? ons : offs;
+////            hostnameToConvs.forEach((host, convs) -> {
+////                Map<String, List<Conversation>> seqsToConvs = TcpConversationUtils.
+////                        groupConversationsByPacketSequence(convs, verbose);
+////                outer.merge(host, seqsToConvs, (oldMap, newMap) -> {
+////                    newMap.forEach((sequence, cs) -> oldMap.merge(sequence, cs, (list1, list2) -> {
+////                        list1.addAll(list2);
+////                        return list1;
+////                    }));
+////                    return oldMap;
+////                });
+////            });
+////        });
+////
+////        System.out.println("==== ON ====");
+////        // Print out all the pairs into a file for ON events
+////        File fileOnEvents = new File(onPairsPath);
+////        PrintWriter pwOn = null;
+////        try {
+////            pwOn = new PrintWriter(fileOnEvents);
+////        } catch(Exception ex) {
+////            ex.printStackTrace();
+////        }
+////        for(Map.Entry<String, Map<String, List<Conversation>>> entry : ons.entrySet()) {
+////            Map<String, List<Conversation>> seqsToConvs = entry.getValue();
+////            for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
+////                List<Conversation> listConv = entryConv.getValue();
+////                // Just get the first Conversation because all Conversations in this group
+////                // should have the same pairs of Application Data.
+////                for(Conversation conv : listConv) {
+////                    // Process only if it is a TLS packet
+////                    if (conv.isTls()) {
+////                        List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
+////                        for(PcapPacketPair pair: tlsAppDataList) {
+////                            System.out.println(PrintUtils.toCsv(pair, dnsMap));
+////                            pwOn.println(PrintUtils.toCsv(pair, dnsMap));
+////                        }
+////                    } else { // Non-TLS conversations
+////                        List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
+////                        for(PcapPacketPair pair: packetList) {
+////                            System.out.println(PrintUtils.toCsv(pair, dnsMap));
+////                            pwOn.println(PrintUtils.toCsv(pair, dnsMap));
+////                        }
+////                    }
+////                }
+////            }
+////        }
+////        pwOn.close();
+////
+////        System.out.println("==== OFF ====");
+////        // Print out all the pairs into a file for ON events
+////        File fileOffEvents = new File(offPairsPath);
+////        PrintWriter pwOff = null;
+////        try {
+////            pwOff = new PrintWriter(fileOffEvents);
+////        } catch(Exception ex) {
+////            ex.printStackTrace();
+////        }
+////        for(Map.Entry<String, Map<String, List<Conversation>>> entry : offs.entrySet()) {
+////            Map<String, List<Conversation>> seqsToConvs = entry.getValue();
+////            for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
+////                List<Conversation> listConv = entryConv.getValue();
+////                // Just get the first Conversation because all Conversations in this group
+////                // should have the same pairs of Application Data.
+////                for(Conversation conv : listConv) {
+////                    // Process only if it is a TLS packet
+////                    if (conv.isTls()) {
+////                        List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
+////                        for(PcapPacketPair pair: tlsAppDataList) {
+////                            System.out.println(PrintUtils.toCsv(pair, dnsMap));
+////                            pwOff.println(PrintUtils.toCsv(pair, dnsMap));
+////                        }
+////                    } else { // Non-TLS conversations
+////                        List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
+////                        for (PcapPacketPair pair : packetList) {
+////                            System.out.println(PrintUtils.toCsv(pair, dnsMap));
+////                            pwOff.println(PrintUtils.toCsv(pair, dnsMap));
+////                        }
+////                    }
+////                }
+////            }
+////        }
+////        pwOff.close();
+//
+//
+//        // ================================================ CLUSTERING ================================================
+//        // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List<Conversation> map
+//        // when don't care about hostnames and sequences (see comment earlier).
+////        List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+////                filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+////                map(e -> e.getValue()). // no longer interested in the UserActions
+////                flatMap(List::stream). // flatten List<List<T>> to a List<T>
+////                collect(Collectors.toList());
+////        List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+////                filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+////                map(e -> e.getValue()).
+////                flatMap(List::stream).
+////                collect(Collectors.toList());
+////        //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+////
+////        List<PcapPacketPair> onPairs = onConversations.stream().
+////                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+////                        TcpConversationUtils.extractPacketPairs(c)).
+////                flatMap(List::stream). // flatten List<List<>> to List<>
+////                collect(Collectors.toList());
+////        List<PcapPacketPair> offPairs = offConversations.stream().
+////                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+////                        TcpConversationUtils.extractPacketPairs(c)).
+////                flatMap(List::stream). // flatten List<List<>> to List<>
+////                collect(Collectors.toList());
+////        // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+////        Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+////        // Perform clustering on conversation logged as part of all ON events.
+//////        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
+////        DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(2, 2);
+////        //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
+////        List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+////        // Perform clustering on conversation logged as part of all OFF events.
+//////        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
+////        DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(2, 2);
+////        //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
+////        List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+////        // Sort the conversations as reference
+////        List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+////        // Output clusters
+////        System.out.println("========================================");
+////        System.out.println("       Clustering results for ON        ");
+////        System.out.println("       Number of clusters: " + onClusters.size());
+////        int count = 0;
+////        List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
+////        List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+////        for (Cluster<PcapPacketPair> c : onClusters) {
+////            System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+////            System.out.print(PrintUtils.toSummaryString(c));
+////            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+////            //if(c.getPoints().size() > 25) {
+////                // Print to file
+////                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+////                ppListOfListListOn.add(ppListOfList);
+////            }
+////        }
+////        // TODO: Merging test
+////        ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
+////        // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
+//////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
+////        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+//////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+////        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+////        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+////        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+////        // TODO: This sequence actually belongs to the local communication between the plug and the phone
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+////        ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+////        PcapPacketUtils.printSignatures(ppListOfListListOn);
+////        //count = 0;
+////        /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
+////            PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
+////            ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
+////        }*/
+////        PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
+////        ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
+////
+////        System.out.println("========================================");
+////        System.out.println("       Clustering results for OFF       ");
+////        System.out.println("       Number of clusters: " + offClusters.size());
+////        count = 0;
+////        List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
+////        List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+////        for (Cluster<PcapPacketPair> c : offClusters) {
+////            System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+////            System.out.print(PrintUtils.toSummaryString(c));
+////            if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+////            //if(c.getPoints().size() > 25) {
+////                // Print to file
+////                List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+////                ppListOfListListOff.add(ppListOfList);
+////            }
+////        }
+////        // TODO: Merging test
+////        ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
+////        // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+//////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3);
+////        // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+////        // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+//////        PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+////        // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
+////        // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+////        // TODO: This sequence actually belongs to the local communication between the plug and the phone
+////        //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+////        ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
+////        PcapPacketUtils.printSignatures(ppListOfListListOff);
+////        //count = 0;
+////        /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
+////            PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
+////            ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
+////        }*/
+////        PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
+////        ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
+////        System.out.println("========================================");
+//        // ============================================================================================================
+//
+//        // TODO: This part is just for DBSCAN sensitivity experiment
+//        // TODO: This part is just for DBSCAN sensitivity experiment
+//        // TODO: This part is just for DBSCAN sensitivity experiment
+//        // TODO: This part is just for DBSCAN sensitivity experiment
+//        // TODO: This part is just for DBSCAN sensitivity experiment
+//        List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+//                filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+//                map(e -> e.getValue()). // no longer interested in the UserActions
+//                flatMap(List::stream). // flatten List<List<T>> to a List<T>
+//                collect(Collectors.toList());
+//        List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+//                filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+//                map(e -> e.getValue()).
+//                flatMap(List::stream).
+//                collect(Collectors.toList());
+//        //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+//
+//        List<PcapPacketPair> onPairs = onConversations.stream().
+//                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+//                        TcpConversationUtils.extractPacketPairs(c)).
+//                flatMap(List::stream). // flatten List<List<>> to List<>
+//                collect(Collectors.toList());
+//        List<PcapPacketPair> offPairs = offConversations.stream().
+//                map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+//                        TcpConversationUtils.extractPacketPairs(c)).
+//                flatMap(List::stream). // flatten List<List<>> to List<>
+//                collect(Collectors.toList());
+//        // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+//        Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+//
+//        double eps = 10; // loop from eps 1-10
+//        int minPts = 50; // loop from minPts 30-50
+//        for(int epsCount = 7; epsCount <= eps; epsCount++) {
+//            for(int minPtsCount = 30; minPtsCount <= minPts; minPtsCount++) {
+//                System.out.println("Eps: " + epsCount + " --- minPts: " + minPtsCount);
+//                DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
+//                DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
+//                List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+//                List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+//                // Sort the conversations as reference
+//                List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+//                // Output clusters
+//                System.out.println("========================================");
+//                System.out.println("       Clustering results for ON        ");
+//                System.out.println("       Number of clusters: " + onClusters.size());
+//                int count = 0;
+//                List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+//                for (Cluster<PcapPacketPair> c : onClusters) {
+//                    System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+////                    System.out.print(PrintUtils.toSummaryString(c));
+//                    if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+////                        if(c.getPoints().size() > 25) {
+//                        // Print to file
+//                        List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+//                        ppListOfListListOn.add(ppListOfList);
+//                    }
+//                }
+//                PcapPacketUtils.printSignatures(ppListOfListListOn);
+//
+//                System.out.println("========================================");
+//                System.out.println("       Clustering results for OFF       ");
+//                System.out.println("       Number of clusters: " + offClusters.size());
+//                count = 0;
+//                List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+//                for (Cluster<PcapPacketPair> c : offClusters) {
+//                    System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+////                    System.out.print(PrintUtils.toSummaryString(c));
+//                    if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+//                        //if(c.getPoints().size() > 25) {
+//                        // Print to file
+//                        List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+//                        ppListOfListListOff.add(ppListOfList);
+//                    }
+//                }
+//                PcapPacketUtils.printSignatures(ppListOfListListOff);
+//                System.out.println();
+//                System.out.println();
+//                System.out.println();
+//                // ============================================================================================================
+//            }
+//        }
+
+
+//        // ================================================================================================
+//        // <<< Some work-in-progress/explorative code that extracts a "representative" sequence >>>
+//        //
+//        // Currently need to know relevant hostname in advance :(
+//        String hostname = "events.tplinkra.com";
+////        String hostname = "rfe-us-west-1.dch.dlink.com";
+//        // Conversations with 'hostname' for ON events.
+//        List<Conversation> onsForHostname = new ArrayList<>();
+//        // Conversations with 'hostname' for OFF events.
+//        List<Conversation> offsForHostname = new ArrayList<>();
+//        // "Unwrap" sequence groupings in ons/offs maps.
+//        ons.get(hostname).forEach((k,v) -> onsForHostname.addAll(v));
+//        offs.get(hostname).forEach((k,v) -> offsForHostname.addAll(v));
+//
+//
+//        Map<String, List<Conversation>> onsForHostnameGroupedByTlsAppDataSequence = TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(onsForHostname);
+//
+//
+//        // Extract representative sequence for ON and OFF by providing the list of conversations with
+//        // 'hostname' observed for each event type (the training data).
+//        SequenceExtraction seqExtraction = new SequenceExtraction();
+////        ExtractedSequence extractedSequenceForOn = seqExtraction.extract(onsForHostname);
+////        ExtractedSequence extractedSequenceForOff = seqExtraction.extract(offsForHostname);
+//
+//        ExtractedSequence extractedSequenceForOn = seqExtraction.extractByTlsAppData(onsForHostname);
+//        ExtractedSequence extractedSequenceForOff = seqExtraction.extractByTlsAppData(offsForHostname);
+//
+//        // Let's check how many ONs align with OFFs and vice versa (that is, how many times an event is incorrectly
+//        // labeled).
+//        int onsLabeledAsOff = 0;
+//        Integer[] representativeOnSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOn.getRepresentativeSequence());
+//        Integer[] representativeOffSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOff.getRepresentativeSequence());
+//        SequenceAlignment<Integer> seqAlg = seqExtraction.getAlignmentAlgorithm();
+//        for (Conversation c : onsForHostname) {
+//            Integer[] onSeq = TcpConversationUtils.getPacketLengthSequence(c);
+//            if (seqAlg.calculateAlignment(representativeOffSeq, onSeq) <= extractedSequenceForOff.getMaxAlignmentCost()) {
+//                onsLabeledAsOff++;
+//            }
+//        }
+//        int offsLabeledAsOn = 0;
+//        for (Conversation c : offsForHostname) {
+//            Integer[] offSeq = TcpConversationUtils.getPacketLengthSequence(c);
+//            if (seqAlg.calculateAlignment(representativeOnSeq, offSeq) <= extractedSequenceForOn.getMaxAlignmentCost()) {
+//                offsLabeledAsOn++;
+//            }
+//        }
+//        System.out.println("");
+//        // ================================================================================================
+//
+//
+//        // -------------------------------------------------------------------------------------------------------------
+//        // -------------------------------------------------------------------------------------------------------------
+    }
+
+}
+
+
+// TP-Link MAC 50:c7:bf:33:1f:09 and usually IP 192.168.1.159 (remember to verify per file)
+// frame.len >= 556 && frame.len <= 558 && ip.addr == 192.168.1.159
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketFilter.java
new file mode 100644 (file)
index 0000000..529faf4
--- /dev/null
@@ -0,0 +1,14 @@
+package edu.uci.iotproject.analysis;
+
+import org.pcap4j.core.PcapPacket;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public interface PcapPacketFilter {
+
+    boolean shouldIncludePacket(PcapPacket packet);
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/PcapPacketPair.java
new file mode 100644 (file)
index 0000000..2d6e9aa
--- /dev/null
@@ -0,0 +1,182 @@
+package edu.uci.iotproject.analysis;
+
+import edu.uci.iotproject.DnsMap;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.apache.commons.math3.stat.clustering.Clusterable;
+import org.pcap4j.core.PcapPacket;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static edu.uci.iotproject.util.PcapPacketUtils.getSourceIp;
+
+/**
+ * <p>
+ *     A simple wrapper for holding a pair of packets (e.g., a request and associated reply packet).
+ * </p>
+ *
+ * <b>Note:</b> we use the deprecated version
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class PcapPacketPair implements Clusterable<PcapPacketPair> {
+
+    /**
+     * If {@code true}, {@link #distanceFrom(PcapPacketPair)} will only consider if the sources of the two packets in
+     * the {@link PcapPacketPair}s being compared match in terms of whether the IP is a local or a remote IP. It will
+     * <em>not</em> check if the IPs/hostnames are actually the same. Set to {@code false} to make the comparison more
+     * strict, i.e., to enforce the requirement that the respective IPs (or hostnames) in the packets of the two
+     * {@link PcapPacketPair}s must be identical.
+     */
+    private static final boolean SIMPLIFIED_SOURCE_COMPARISON = true;
+
+    private final PcapPacket mFirst;
+
+    private final Optional<PcapPacket> mSecond;
+
+    /**
+     * IP to hostname mappings.
+     * Allows for grouping packets with different source IPs that map to the same hostname into one cluster.
+     */
+    private DnsMap mDnsMap; // TODO implement and invoke setter
+
+    public PcapPacketPair(PcapPacket first, PcapPacket second) {
+        mFirst = first;
+        mSecond = Optional.ofNullable(second);
+    }
+
+    public PcapPacket getFirst() { return mFirst; }
+
+    public boolean isFirstClient() {
+        String firstIp = PcapPacketUtils.getSourceIp(mFirst);
+        InetAddress ia = null;
+        try {
+            ia = InetAddress.getByName(firstIp);
+        } catch (UnknownHostException ex) {
+            ex.printStackTrace();
+        }
+        return ia.isSiteLocalAddress();
+    }
+
+    public Optional<PcapPacket> getSecond() { return mSecond; }
+
+    public boolean isSecondClient() {
+        // Return the value of the second source if it is not null
+        if (mSecond.isPresent()) {
+            String secondIp = PcapPacketUtils.getSourceIp(mSecond.get());
+            InetAddress ia = null;
+            try {
+                ia = InetAddress.getByName(secondIp);
+            } catch (UnknownHostException ex) {
+                ex.printStackTrace();
+            }
+            return ia.isSiteLocalAddress();
+        } else {
+            // When it is null, we always return the opposite of the first source's status
+            return !isFirstClient();
+        }
+    }
+
+    /**
+     * Get the {@link DnsMap} that is queried for hostnames mappings when performing IP/hostname-sensitive clustering.
+     * @return the {@link DnsMap} that is queried for hostnames mappings when performing IP/hostname-sensitive clustering.
+     */
+    public DnsMap getDnsMap() {
+        return mDnsMap;
+    }
+
+    /**
+     * Set the {@link DnsMap} to be queried for hostnames mappings when performing IP/hostname-sensitive clustering.
+     * @param dnsMap a {@code DnsMap} to be queried for hostnames mappings when performing IP/hostname-sensitive clustering.
+     */
+    public void setDnsMap(final DnsMap dnsMap) {
+        mDnsMap = dnsMap;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%d, %s",
+                getFirst().getOriginalLength(),
+                getSecond().map(pkt -> Integer.toString(pkt.getOriginalLength())).orElse("null"));
+    }
+
+    // =================================================================================================================
+    // Begin implementation of org.apache.commons.math3.stat.clustering.Clusterable interface
+    @Override
+    public double distanceFrom(PcapPacketPair that) {
+        if (SIMPLIFIED_SOURCE_COMPARISON) {
+            // Direction of packets in terms of client-to-server or server-to-client must match, but we don't care about
+            // IPs and hostnames
+            if (this.isFirstClient() != that.isFirstClient() || this.isSecondClient() != that.isSecondClient()) {
+                // Distance is maximal if mismatch in direction of packets
+                return Double.MAX_VALUE;
+            }
+        } else {
+            // Strict mode enabled: IPs/hostnames must match!
+            // Extract src ips of both packets of each pair.
+            String thisSrc1 = getSourceIp(this.getFirst());
+            String thisSrc2 = this.getSecond().map(pp -> getSourceIp(pp)).orElse("");
+            String thatSrc1 = getSourceIp(that.getFirst());
+            String thatSrc2 = that.getSecond().map(pp -> getSourceIp(pp)).orElse("");
+
+            // Replace IPs with hostnames if possible.
+            thisSrc1 = mapToHostname(thisSrc1);
+            thisSrc2 = mapToHostname(thisSrc2);
+            thatSrc1 = mapToHostname(thatSrc1);
+            thatSrc2 = mapToHostname(thatSrc2);
+
+            if(!thisSrc1.equals(thatSrc1) || !thisSrc2.equals(thatSrc2)) {
+                // Distance is maximal if sources differ.
+                return Double.MAX_VALUE;
+            }
+        }
+
+        // If the sources match, the distance is the Euclidean distance between each pair of packet lengths.
+        int thisLen1 = this.getFirst().getOriginalLength();
+        // TODO should discard pairs w/o second packet from clustering; replace below with getSecond().get() when done.
+        int thisLen2 = this.getSecond().map(pp -> pp.getOriginalLength()).orElse(0);
+        int thatLen1 = that.getFirst().getOriginalLength();
+        // TODO should discard pairs w/o second packet from clustering; replace below with getSecond().get() when done.
+        int thatLen2 = that.getSecond().map(pp -> pp.getOriginalLength()).orElse(0);
+        return Math.sqrt(
+                Math.pow(thisLen1 - thatLen1, 2) +
+                        Math.pow(thisLen2 - thatLen2, 2)
+        );
+    }
+
+    @Override
+    public PcapPacketPair centroidOf(Collection<PcapPacketPair> p) {
+        // No notion of centroid in DBSCAN
+        throw new UnsupportedOperationException("Not implemented; no notion of a centroid in DBSCAN.");
+    }
+    // End implementation of org.apache.commons.math3.stat.clustering.Clusterable interface
+    // =================================================================================================================
+
+    private String mapToHostname(String ip) {
+        Set<String> hostnames = mDnsMap.getHostnamesForIp(ip);
+        if (hostnames != null && hostnames.size() > 0) {
+            // append hostnames back-to-back separated by a delimiter if more than one item in set
+            // note: use sorted() to ensure that output remains consistent (as Set has no internal ordering of elements)
+            String result = hostnames.stream().sorted().collect(Collectors.joining(" "));
+            if (hostnames.size() > 1) {
+                // One IP can map to multiple hostnames, although that is rare. For now just raise a warning.
+                String warningStr = String.format(
+                        "%s.mapToHostname(): encountered an IP (%s) that maps to multiple hostnames (%s)",
+                        getClass().getSimpleName(), ip, result);
+                System.err.println(warningStr);
+            }
+            return result;
+        }
+        // If unable to map to a hostname, return ip for ease of use; caller can overwrite input value, defaulting to
+        // the original value if no mapping is found:
+        // String src = "<some-ip>";
+        // src = mapToHostname(src); // src is now either a hostname or the original ip.
+        return ip;
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
new file mode 100644 (file)
index 0000000..a4217cc
--- /dev/null
@@ -0,0 +1,464 @@
+package edu.uci.iotproject.analysis;
+
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.DnsMap;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.TcpPacket;
+
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static edu.uci.iotproject.util.PcapPacketUtils.*;
+
+/**
+ * Utility functions for analyzing and structuring (sets of) {@link Conversation}s.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class TcpConversationUtils {
+
+    /**
+     * Identifies the adjacency type of the signature for merging.
+     */
+    public enum SignaturePosition {
+        NOT_ADJACENT,
+        LEFT_ADJACENT,
+        RIGHT_ADJACENT
+    }
+
+    /**
+     * <p>
+     *      Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets.
+     *      <em>The extracted pairs are formed from the full set of payload-carrying TCP packets.</em>
+     * </p>
+     *
+     * <b>Note:</b> in the current implementation, if one endpoint sends multiple packets back-to-back with no
+     * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances
+     * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
+     *
+     * @param conv The {@code Conversation} for which packet pairs are to be extracted.
+     * @return The packet pairs extracted from {@code conv}.
+     */
+    public static List<PcapPacketPair> extractPacketPairs(Conversation conv) {
+        return extractPacketPairs(conv.getPackets());
+    }
+
+
+    /**
+     * <p>
+     *      Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets.
+     *      <em>The extracted pairs are formed from the full set of TLS Application Data packets.</em>
+     * </p>
+     *
+     * <b>Note:</b> in the current implementation, if one endpoint sends multiple packets back-to-back with no
+     * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances
+     * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
+     *
+     * @param conv The {@code Conversation} for which packet pairs are to be extracted.
+     * @return The packet pairs extracted from {@code conv}.
+     */
+    public static List<PcapPacketPair> extractTlsAppDataPacketPairs(Conversation conv) {
+        if (!conv.isTls()) {
+            throw new IllegalArgumentException(String.format("Provided %s argument is not a TLS session"));
+        }
+        return extractPacketPairs(conv.getTlsApplicationDataPackets());
+    }
+
+    // Helper method for implementing the public API of similarly named methods.
+    private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
+        List<PcapPacketPair> pairs = new ArrayList<>();
+//        for(PcapPacket pp : packets) {
+//            System.out.print(pp.length() + " ");
+//        }
+//        System.out.println();
+
+        int i = 0;
+        while (i < packets.size()) {
+            PcapPacket p1 = packets.get(i);
+            String p1SrcIp = p1.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
+            int p1SrcPort = p1.get(TcpPacket.class).getHeader().getSrcPort().valueAsInt();
+            if (i+1 < packets.size()) {
+                PcapPacket p2 = packets.get(i+1);
+                if (PcapPacketUtils.isSource(p2, p1SrcIp, p1SrcPort)) {
+                    // Two packets in a row going in the same direction -> create one item pair for p1
+                    pairs.add(new PcapPacketPair(p1, null));
+                    // Advance one packet as the following two packets may form a valid two-item pair.
+                    i++;
+                } else {
+                    // The two packets form a response-reply pair, create two-item pair.
+                    pairs.add(new PcapPacketPair(p1, p2));
+                    // Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
+                    i += 2;
+                    //i++;
+                }
+            } else {
+                // Last packet of conversation => one item pair
+                pairs.add(new PcapPacketPair(p1, null));
+                // Advance i to ensure termination.
+                i++;
+            }
+        }
+        return pairs;
+        // TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases?
+    }
+
+    /**
+     * Given a collection of TCP conversations and associated DNS mappings, groups the conversations by hostname.
+     * @param tcpConversations The collection of TCP conversations.
+     * @param ipHostnameMappings The associated DNS mappings.
+     * @return A map where each key is a hostname and its associated value is a list of conversations where one of the
+     *         two communicating hosts is that hostname (i.e. its IP maps to the hostname).
+     */
+    public static Map<String, List<Conversation>> groupConversationsByHostname(Collection<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
+        HashMap<String, List<Conversation>> result = new HashMap<>();
+        for (Conversation c : tcpConversations) {
+            if (c.getPackets().size() == 0) {
+                String warningStr = String.format("Detected a %s [%s] with no payload packets.",
+                        c.getClass().getSimpleName(), c.toString());
+                System.err.println(warningStr);
+                continue;
+            }
+            IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class);
+            String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress();
+            String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress();
+            // Check if src or dst IP is associated with one or more hostnames.
+            Set<String> hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc);
+            if (hostnames == null) {
+                // No luck with src ip (possibly because it's a client->srv packet), try dst ip.
+                hostnames = ipHostnameMappings.getHostnamesForIp(ipDst);
+            }
+            if (hostnames != null) {
+                // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to.
+                for (String hostname : hostnames) {
+                    List<Conversation> newValue = new ArrayList<>();
+                    newValue.add(c);
+                    result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+                }
+                if (hostnames.size() > 1) {
+                    // Print notice of IP mapping to multiple hostnames (debugging)
+                    System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames",
+                            TcpConversationUtils.class.getSimpleName(), hostnames.size()));
+                }
+            } else {
+                // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs.
+                // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP
+                // goes first.
+                String delimiter = "_";
+                // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of
+                // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is
+                // communicating with itself!)
+                String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc;
+                List<Conversation> newValue = new ArrayList<>();
+                newValue.add(c);
+                result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+            }
+        }
+        return result;
+    }
+
+    public static Map<String, Integer> countPacketSequenceFrequencies(Collection<Conversation> conversations) {
+        Map<String, Integer> result = new HashMap<>();
+        for (Conversation conv : conversations) {
+            if (conv.getPackets().size() == 0) {
+                // Skip conversations with no payload packets.
+                continue;
+            }
+            StringBuilder sb = new StringBuilder();
+            for (PcapPacket pp : conv.getPackets()) {
+                sb.append(pp.length() + " ");
+            }
+            result.merge(sb.toString(), 1, (i1, i2) -> i1+i2);
+        }
+        return result;
+    }
+
+    /**
+     * Given a {@link Collection} of {@link Conversation}s, builds a {@link Map} from {@link String} to {@link List}
+     * of {@link Conversation}s such that each key is the <em>concatenation of the packet lengths of all payload packets
+     * (i.e., the set of packets returned by {@link Conversation#getPackets()}) separated by a delimiter</em> of any
+     * {@link Conversation} pointed to by that key. In other words, what the {@link Conversation}s {@code cs} pointed to
+     * by the key {@code s} have in common is that they all contain exactly the same number of payload packets <em>and
+     * </em> these payload packets are identical across all {@code Conversation}s in {@code cs} in terms of packet
+     * length and packet order. For example, if the key is "152 440 550", this means that every individual
+     * {@code Conversation} in the list of {@code Conversation}s pointed to by that key contain exactly three payload
+     * packet of lengths 152, 440, and 550, and these three packets are ordered in the order prescribed by the key.
+     *
+     * @param conversations The collection of {@code Conversation}s to group by packet sequence.
+     * @param verbose If set to {@code true}, the grouping (and therefore the key) will also include SYN/SYNACK,
+     *                FIN/FINACK, RST packets, and each payload-carrying packet will have an indication of the direction
+     *                of the packet prepended.
+     * @return a {@link Map} from {@link String} to {@link List} of {@link Conversation}s such that each key is the
+     *         <em>concatenation of the packet lengths of all payload packets (i.e., the set of packets returned by
+     *         {@link Conversation#getPackets()}) separated by a delimiter</em> of any {@link Conversation} pointed to
+     *         by that key.
+     */
+    public static Map<String, List<Conversation>> groupConversationsByPacketSequence(Collection<Conversation> conversations, boolean verbose) {
+        return conversations.stream().collect(Collectors.groupingBy(c -> toSequenceString(c, verbose)));
+    }
+
+    public static Map<String, List<Conversation>> groupConversationsByTlsApplicationDataPacketSequence(Collection<Conversation> conversations) {
+        return conversations.stream().collect(Collectors.groupingBy(
+                c -> c.getTlsApplicationDataPackets().stream().map(p -> Integer.toString(p.getOriginalLength())).
+                        reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2))
+        );
+    }
+
+    /**
+     * Given a {@link Conversation}, counts the frequencies of each unique packet length seen as part of the
+     * {@code Conversation}.
+     * @param c The {@code Conversation} for which unique packet length frequencies are to be determined.
+     * @return A mapping from packet length to its frequency.
+     */
+    public static Map<Integer, Integer> countPacketLengthFrequencies(Conversation c) {
+        Map<Integer, Integer> result = new HashMap<>();
+        for (PcapPacket packet : c.getPackets()) {
+            result.merge(packet.length(), 1, (i1, i2) -> i1 + i2);
+        }
+        return result;
+    }
+
+    /**
+     * Like {@link #countPacketLengthFrequencies(Conversation)}, but counts packet length frequencies for a collection
+     * of {@code Conversation}s, i.e., the frequency of a packet length becomes the total number of packets with that
+     * length across <em>all</em> {@code Conversation}s in {@code conversations}.
+     * @param conversations The collection of {@code Conversation}s for which packet length frequencies are to be
+     *                      counted.
+     * @return A mapping from packet length to its frequency.
+     */
+    public static Map<Integer, Integer> countPacketLengthFrequencies(Collection<Conversation> conversations) {
+        Map<Integer, Integer> result = new HashMap<>();
+        for (Conversation c : conversations) {
+            Map<Integer, Integer> intermediateResult = countPacketLengthFrequencies(c);
+            for (Map.Entry<Integer, Integer> entry : intermediateResult.entrySet()) {
+                result.merge(entry.getKey(), entry.getValue(), (i1, i2) -> i1 + i2);
+            }
+        }
+        return result;
+    }
+
+    public static Map<String, Integer> countPacketPairFrequencies(Collection<PcapPacketPair> pairs) {
+        Map<String, Integer> result = new HashMap<>();
+        for (PcapPacketPair ppp : pairs) {
+            result.merge(ppp.toString(), 1, (i1, i2) -> i1 + i2);
+        }
+        return result;
+    }
+
+    public static Map<String, Map<String, Integer>> countPacketPairFrequenciesByHostname(Collection<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
+        Map<String, List<Conversation>> convsByHostname = groupConversationsByHostname(tcpConversations, ipHostnameMappings);
+        HashMap<String, Map<String, Integer>> result = new HashMap<>();
+        for (Map.Entry<String, List<Conversation>> entry : convsByHostname.entrySet()) {
+            // Merge all packet pairs exchanged during the course of all conversations with hostname into one list
+            List<PcapPacketPair> allPairsExchangedWithHostname = new ArrayList<>();
+            entry.getValue().forEach(conversation -> allPairsExchangedWithHostname.addAll(extractPacketPairs(conversation)));
+            // Then count the frequencies of packet pairs exchanged with the hostname, irrespective of individual
+            // conversations
+            result.put(entry.getKey(), countPacketPairFrequencies(allPairsExchangedWithHostname));
+        }
+        return result;
+    }
+
+    /**
+     * Given a {@link Conversation}, extract its packet length sequence.
+     * @param c The {@link Conversation} from which a packet length sequence is to be extracted.
+     * @return An {@code Integer[]} that holds the packet lengths of all payload-carrying packets in {@code c}. The
+     *         packet lengths in the returned array are ordered by packet timestamp.
+     */
+    public static Integer[] getPacketLengthSequence(Conversation c) {
+        return getPacketLengthSequence(c.getPackets());
+    }
+
+
+    /**
+     * Given a {@link Conversation}, extract its packet length sequence, but only include packet lengths of those
+     * packets that carry TLS Application Data.
+     * @param c The {@link Conversation} from which a TLS Application Data packet length sequence is to be extracted.
+     * @return An {@code Integer[]} that holds the packet lengths of all packets in {@code c} that carry TLS Application
+     *         Data. The packet lengths in the returned array are ordered by packet timestamp.
+     */
+    public static Integer[] getPacketLengthSequenceTlsAppDataOnly(Conversation c) {
+        if (!c.isTls()) {
+            throw new IllegalArgumentException("Provided " + c.getClass().getSimpleName() + " was not a TLS session");
+        }
+        return getPacketLengthSequence(c.getTlsApplicationDataPackets());
+    }
+
+    /**
+     * Given a list of packets, extract the packet lengths and wrap them in an array such that the packet lengths in the
+     * resulting array appear in the same order as their corresponding packets in the input list.
+     * @param packets The list of packets for which the packet lengths are to be extracted.
+     * @return An array containing the packet lengths in the same order as their corresponding packets in the input list.
+     */
+    private static Integer[] getPacketLengthSequence(List<PcapPacket> packets) {
+        return packets.stream().map(pkt -> pkt.getOriginalLength()).toArray(Integer[]::new);
+    }
+
+    /**
+     * Builds a string representation of the sequence of packets exchanged as part of {@code c}.
+     * @param c The {@link Conversation} for which a string representation of the packet sequence is to be constructed.
+     * @param verbose {@code true} if set to true, the returned sequence string will also include SYN/SYNACK,
+     *                FIN/FINACK, RST packets, as well as an indication of the direction of payload-carrying packets.
+     * @return a string representation of the sequence of packets exchanged as part of {@code c}.
+     */
+    private static String toSequenceString(Conversation c, boolean verbose) {
+        // Payload-parrying packets are always included, but only prepend direction if verbose output is chosen.
+        Stream<String> s = c.getPackets().stream().map(p -> verbose ? c.getDirection(p).toCompactString() + p.getOriginalLength() : Integer.toString(p.getOriginalLength()));
+        if (verbose) {
+            // In the verbose case, we also print SYN, FIN and RST packets.
+            // Convert the SYN packets to a string representation and prepend them in front of the payload packets.
+            s = Stream.concat(c.getSynPackets().stream().map(p -> isSyn(p) && isAck(p) ? "SYNACK" : "SYN"), s);
+            // Convert the FIN packets to a string representation and append them after the payload packets.
+            s = Stream.concat(s, c.getFinAckPairs().stream().map(f -> f.isAcknowledged() ? "FINACK" : "FIN"));
+            // Convert the RST packets to a string representation and append at the end.
+            s = Stream.concat(s, c.getRstPackets().stream().map(r -> "RST"));
+        }
+        /*
+         * Note: the collector internally uses a StringBuilder, which is more efficient than simply doing string
+         * concatenation as in the following example:
+         * s.reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2);
+         * (above code is O(N^2) where N is the number of characters)
+         */
+        return s.collect(Collectors.joining(" "));
+    }
+
+    /**
+     * Set of port numbers that we consider TLS traffic.
+     * Note: purposefully initialized as a {@link HashSet} to get O(1) {@code contains()} call.
+     */
+    private static final Set<Integer> TLS_PORTS = Stream.of(443, 8443, 41143).
+            collect(Collectors.toCollection(HashSet::new));
+
+    /**
+     * Check if a given port number is considered a TLS port.
+     * @param port The port number to check.
+     * @return {@code true} if the port number is considered a TLS port, {@code false} otherwise.
+     */
+    public static boolean isTlsPort(int port) {
+        return TLS_PORTS.contains(port);
+    }
+
+    /**
+     * Appends a space to {@code sb} <em>iff</em> {@code sb} already contains some content.
+     * @param sb A {@link StringBuilder} that should have a space appended <em>iff</em> it is not empty.
+     */
+    private static void appendSpaceIfNotEmpty(StringBuilder sb) {
+        if (sb.length() != 0) {
+            sb.append(" ");
+        }
+    }
+
+    /**
+     * Given a list of {@link Conversation} objects, sort them by timestamps.
+     * @param conversations The list of {@link Conversation} objects to be sorted.
+     * @return A sorted list of {@code Conversation} based on timestamps of the first
+     *          packet in the {@code Conversation}.
+     */
+    public static List<Conversation> sortConversationList(List<Conversation> conversations) {
+        // Get rid of Conversation objects with no packets.
+        conversations.removeIf(x -> x.getPackets().size() == 0);
+        // Sort the list based on the first packet's timestamp!
+        Collections.sort(conversations, (c1, c2) ->
+                c1.getPackets().get(0).getTimestamp().compareTo(c2.getPackets().get(0).getTimestamp()));
+        return conversations;
+    }
+
+    /**
+     * Given a {@code List} of {@link Conversation} objects, find one that has the given {@code List}
+     * of {@code PcapPacket}.
+     * @param conversations The {@code List} of {@link Conversation} objects as reference.
+     * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@code List} of {@link Conversation}.
+     * @return A {@code Conversation} that contains the given {@code List} of {@code PcapPacket}.
+     */
+    public static Conversation returnConversation(List<PcapPacket> ppList, List<Conversation> conversations) {
+        // TODO: This part of comparison takes into account that the list of conversations is not sorted
+        // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list
+        // TODO:    as a parameter
+        // Find a Conversation that ppList is part of
+        for (Conversation c : conversations) {
+            // Figure out if c is the Conversation that ppList is in
+            if (isPartOfConversation(ppList, c)) {
+                return c;
+            }
+        }
+        // Return null if not found
+        return null;
+    }
+
+    /**
+     * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it and return the
+     * adjacency label based on {@code SignaturePosition}.
+     * @param conversation The {@link Conversation} object as reference.
+     * @param ppListFirst The first {@code List} of {@code PcapPacket} objects in the {@link Conversation}.
+     * @param ppListSecond The second {@code List} of {@code PcapPacket} objects in the {@link Conversation} whose
+     *                     position will be observed in the {@link Conversation} with respect to ppListFirst.
+     * @return A {@code SignaturePosition} that represents the position of the signature against another signature
+     *          in a {@link Conversation}.
+     */
+    public static SignaturePosition isPartOfConversationAndAdjacent(List<PcapPacket> ppListFirst,
+                                                                    List<PcapPacket> ppListSecond,
+                                                                    Conversation conversation) {
+        // Take the first element in ppList and compare it
+        // The following elements in ppList are guaranteed to be in the same Conversation
+        // TODO: This part of comparison takes into account that the list of conversations is not sorted
+        // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list
+        // TODO:    as a parameter
+        if (isPartOfConversation(ppListSecond, conversation)) {
+            // Compare the first element of ppListSecond with the last element of ppListFirst to know
+            // whether ppListSecond is RIGHT_ADJACENT relative to ppListFirst.
+            PcapPacket lastElOfFirstList = ppListFirst.get(ppListFirst.size() - 1);
+            PcapPacket firstElOfSecondList = ppListSecond.get(0);
+            // If the positions of the two are in order, then they are adjacent.
+            int indexOfLastElOfFirstList = returnIndexInConversation(lastElOfFirstList, conversation);
+            int indexOfFirstElOfSecondList = returnIndexInConversation(firstElOfSecondList, conversation);
+            if(indexOfLastElOfFirstList + 1 == indexOfFirstElOfSecondList) {
+                return SignaturePosition.RIGHT_ADJACENT;
+            }
+            // NOT RIGHT_ADJACENT, so check for LEFT_ADJACENT.
+            // Compare the first element of ppListRight with the last element of ppListSecond to know
+            // whether ppListSecond is LEFT_ADJACENT relative to ppListFirst.
+            PcapPacket firstElOfFirstList = ppListFirst.get(0);
+            PcapPacket lastElOfSecondList = ppListSecond.get(ppListSecond.size() - 1);
+            // If the positions of the two are in order, then they are adjacent.
+            int indexOfFirstElOfFirstList = returnIndexInConversation(firstElOfFirstList, conversation);
+            int indexOfLastElOfSecondList = returnIndexInConversation(lastElOfSecondList, conversation);
+            if(indexOfLastElOfSecondList + 1 == indexOfFirstElOfFirstList) {
+                return SignaturePosition.LEFT_ADJACENT;
+            }
+        }
+        // Return NOT_ADJACENT if not found.
+        return SignaturePosition.NOT_ADJACENT;
+    }
+
+    /**
+     * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it.
+     * @param conversation The {@link Conversation} object as reference.
+     * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@link Conversation}.
+     * @return A {@code Boolean} value that represents the presence of the {@code List} of {@code PcapPacket} in
+     *         the {@link Conversation}.
+     */
+    private static boolean isPartOfConversation(List<PcapPacket> ppList, Conversation conversation) {
+        // Find the first element of ppList in conversation.
+        if (conversation.getPackets().contains(ppList.get(0)))
+            return true;
+        // Return false if not found.
+        return false;
+    }
+
+    /**
+     * Given a {@link Conversation} objects, check the index of a {@code PcapPacket} in it.
+     * @param conversation The {@link Conversation} object as reference.
+     * @param pp The {@code PcapPacket} object to search in the {@link Conversation}.
+     * @return An {@code Integer} value that gives the index of the {@code PcapPacket} in the {@link Conversation}.
+     */
+    private static int returnIndexInConversation(PcapPacket pp, Conversation conversation) {
+        // Find pp in conversation.
+        if (conversation.getPackets().contains(pp))
+            return conversation.getPackets().indexOf(pp);
+        // Return -1 if not found.
+        return -1;
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java
new file mode 100644 (file)
index 0000000..983de12
--- /dev/null
@@ -0,0 +1,146 @@
+package edu.uci.iotproject.analysis;
+
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.DnsMap;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.function.Function;
+
+/**
+ * A {@link PacketListener} that marks network traffic as (potentially) related to a user's actions by comparing the
+ * timestamp of each packet to the timestamps of the provided list of user actions.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class TrafficLabeler implements PacketListener {
+
+    private final Map<UserAction, List<PcapPacket>> mActionToTrafficMap;
+    private final List<UserAction> mActionsSorted;
+    /**
+     * The total number of packets labeled, i.e, the sum of the sizes of the values in {@link #mActionToTrafficMap}.
+     */
+    private long mPackets = 0;
+
+    public TrafficLabeler(List<UserAction> userActions) {
+        // Init map with empty lists (no packets have been mapped to UserActions at the onset).
+        mActionToTrafficMap = new HashMap<>();
+        userActions.forEach(ua -> mActionToTrafficMap.put(ua, new ArrayList<>()));
+        // Sort list of UserActions by timestamp in order to facilitate fast Packet-to-UserAction mapping.
+        // For safety reasons, we create an internal copy of the list to prevent external code from changing the list's
+        // contents as that would render our assumptions about order of elements invalid.
+        // In addition, this also ensures that we do not break assumptions made by external code as we avoid reordering
+        // the elements of the list passed from the external code.
+        // If performance is to be favored over safety, assign userActions to mActionsSorted directly.
+        mActionsSorted = new ArrayList<>();
+        mActionsSorted.addAll(userActions);
+        Collections.sort(mActionsSorted, (ua1, ua2) -> ua1.getTimestamp().compareTo(ua2.getTimestamp()));
+    }
+
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // Locate UserAction corresponding to packet, if any.
+        int index = Collections.binarySearch(mActionsSorted, new UserAction(null, packet.getTimestamp()), (listItem, key) -> {
+            // Start of inclusion interval is the time of the user action
+            Instant intervalStart = listItem.getTimestamp();
+            // End of inclusion interval is some arbitrary number of milliseconds after the user action.
+            Instant intervalEnd = intervalStart.plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS);
+            if (key.getTimestamp().isAfter(intervalStart) && key.getTimestamp().isBefore(intervalEnd)) {
+                // Packet lies within specified interval after the current UserAction, so we're done.
+                // Communicate termination to binarySearch by returning 0 which indicates equality.
+                return 0;
+            }
+            // If packet lies outside inclusion interval of current list item, continue search in lower or upper half of
+            // list depending on whether the timestamp of the current list item is smaller or greater than that of the
+            // packet.
+            return listItem.getTimestamp().compareTo(key.getTimestamp());
+        });
+        if (index >= 0) {
+            // Associate the packet to the its corresponding user action (located during the binary search above).
+            mActionToTrafficMap.get(mActionsSorted.get(index)).add(packet);
+            mPackets++;
+        }
+        // Ignore packet if it is not found to be in temporal proximity of a user action.
+    }
+
+    /**
+     * Get the total number of packets labeled by this {@code TrafficLabeler}.
+     *
+     * @return the total number of packets labeled by this {@code TrafficLabeler}.
+     */
+    public long getTotalPacketCount() {
+        return mPackets;
+    }
+
+    /**
+     * Get the labeled traffic.
+     *
+     * @return A {@link Map} in which a {@link UserAction} points to a {@link List} of {@link PcapPacket}s believed to
+     *         be related (occurring as a result of) that {@code UserAction}.
+     */
+    public Map<UserAction, List<PcapPacket>> getLabeledTraffic() {
+        return Collections.unmodifiableMap(mActionToTrafficMap);
+    }
+
+    /**
+     * Like {@link #getLabeledTraffic()}, but allows the caller to supply a mapping function that is applied to
+     * the traffic associated with each {@link UserAction} (the traffic label) before returning the labeled traffic.
+     * This may for example be useful for a caller who wishes to perform some postprocessing of labeled traffic, e.g.,
+     * in order to perform additional filtering or to transform the representation of labeled traffic.
+     * <p>
+     *     An example usecase is provided in {@link #getLabeledReassembledTcpTraffic()} which uses this function to
+     *     build a {@link Map} in which a {@link UserAction} points to the reassembled TCP connections believed to have
+     *     occurred as a result of that {@code UserAction}.
+     * </p>
+     *
+     * @param mappingFunction A mapping function that converts a {@link List} of {@link PcapPacket} into some other type
+     *                        {@code T}.
+     * @param <T> The return type of {@code mappingFunction}.
+     * @return A {@link Map} in which a {@link UserAction} points to the result of applying {@code mappingFunction} to
+     *         the set of packets believed to be related (occurring as a result of) that {@code UserAction}.
+     */
+    public <T> Map<UserAction, T> getLabeledTraffic(Function<List<PcapPacket>, T> mappingFunction) {
+        Map<UserAction, T> result = new HashMap<>();
+        mActionToTrafficMap.forEach((ua, packets) -> result.put(ua, mappingFunction.apply(packets)));
+        return result;
+    }
+
+
+    /**
+     * Get the labeled traffic reassembled as TCP connections (<b>note:</b> <em>discards</em> all non-TCP traffic).
+     *
+     * @return A {@link Map} in which a {@link UserAction} points to a {@link List} of {@link Conversation}s believed to
+     *         be related (occurring as a result of) that {@code UserAction}.
+     */
+    public Map<UserAction, List<Conversation>> getLabeledReassembledTcpTraffic() {
+        return getLabeledTraffic(packets -> {
+            TcpReassembler tcpReassembler = new TcpReassembler();
+            packets.forEach(p -> tcpReassembler.gotPacket(p));
+            return tcpReassembler.getTcpConversations();
+        });
+    }
+
+    /**
+     * Like {@link #getLabeledReassembledTcpTraffic()}, but uses the provided {@code ipHostnameMappings} to group
+     * {@link Conversation}s by hostname.
+     *
+     * @param ipHostnameMappings A {@link DnsMap} with IP to hostname mappings used for reverse DNS lookup.
+     * @return A {@link Map} in which a {@link UserAction} points to the set of {@link Conversation}s believed to be
+     *         related (occurring as a result of) that {@code UserAction}. More precisely, each {@code UserAction} in
+     *         the returned {@code Map} points to <em>another</em> {@code Map} in which a hostname points to the set of
+     *         {@code Conversation}s involving that hostname.
+     */
+    public Map<UserAction, Map<String, List<Conversation>>> getLabeledReassembledTcpTraffic(DnsMap ipHostnameMappings) {
+        return getLabeledTraffic(packets -> {
+            TcpReassembler tcpReassembler = new TcpReassembler();
+            packets.forEach(p -> tcpReassembler.gotPacket(p));
+            return TcpConversationUtils.groupConversationsByHostname(tcpReassembler.getTcpConversations(), ipHostnameMappings);
+        });
+    }
+
+}
\ No newline at end of file
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java
new file mode 100644 (file)
index 0000000..0a22d63
--- /dev/null
@@ -0,0 +1,114 @@
+package edu.uci.iotproject.analysis;
+
+import edu.uci.iotproject.io.PcapHandleReader;
+import org.pcap4j.core.*;
+
+import java.time.Instant;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class TriggerTrafficExtractor implements PcapPacketFilter {
+
+    private final String mPcapFilePath;
+    private final List<Instant> mTriggerTimes;
+    private final String mDeviceIp;
+
+    private int mTriggerIndex = 0;
+
+    /**
+     * The total number of packets marked for inclusion during one run of {@link #performExtraction(PacketListener...)}.
+     */
+    private long mIncludedPackets = 0;
+
+    public static final int INCLUSION_WINDOW_MILLIS = 15_000;
+
+    public TriggerTrafficExtractor(String pcapFilePath, List<Instant> triggerTimes, String deviceIp) throws PcapNativeException, NotOpenException {
+        mPcapFilePath = pcapFilePath;
+        // Ensure that trigger times are sorted in ascending as we rely on this fact in the logic that works out if a
+        // packet is related to a trigger.
+        Collections.sort(triggerTimes, (i1, i2) -> {
+            if (i1.isBefore(i2)) return -1;
+            else if (i2.isBefore(i1)) return 1;
+            else return 0;
+        });
+        mTriggerTimes = Collections.unmodifiableList(triggerTimes);
+        mDeviceIp = deviceIp;
+    }
+
+
+    public void performExtraction(PacketListener... extractedPacketsConsumers) throws PcapNativeException, NotOpenException, TimeoutException {
+        // Reset trigger index and packet counter in case client code chooses to rerun the extraction.
+        mTriggerIndex = 0;
+        mIncludedPackets = 0;
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(mPcapFilePath, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(mPcapFilePath);
+        }
+        // Use the native support for BPF to immediately filter irrelevant traffic.
+        handle.setFilter("ip host " + mDeviceIp, BpfProgram.BpfCompileMode.OPTIMIZE);
+        PcapHandleReader pcapReader = new PcapHandleReader(handle, this, extractedPacketsConsumers);
+        pcapReader.readFromHandle();
+
+    }
+
+    /**
+     * Return the number of extracted packets (i.e., packets selected for inclusion) as a result of the most recent call
+     * to {@link #performExtraction(PacketListener...)}.
+     *
+     * @return the number of extracted packets (i.e., packets selected for inclusion) as a result of the most recent
+     *         call to {@link #performExtraction(PacketListener...)}.
+     */
+    public long getPacketsIncludedCount() {
+        return mIncludedPackets;
+    }
+
+    @Override
+    public boolean shouldIncludePacket(PcapPacket packet) {
+        // New version. Simpler, but slower: the later a packet arrives, the more elements of mTriggerTimes will need to
+        // be traversed.
+        boolean include = mTriggerTimes.stream().anyMatch(
+                trigger -> trigger.isBefore(packet.getTimestamp()) &&
+                        packet.getTimestamp().isBefore(trigger.plusMillis(INCLUSION_WINDOW_MILLIS))
+        );
+        if (include) {
+            mIncludedPackets++;
+        }
+        return include;
+
+        /*
+        // Old version. Faster, but more complex - is it correct?
+        if (mTriggerIndex >= mTriggerTimes.size()) {
+            // Don't include packet if we've exhausted the list of trigger times.
+            return false;
+        }
+
+        // TODO hmm, is this correct?
+        Instant trigger = mTriggerTimes.get(mTriggerIndex);
+        if (trigger.isBefore(packet.getTimestamp()) &&
+                packet.getTimestamp().isBefore(trigger.plusMillis(INCLUSION_WINDOW_MILLIS))) {
+            // Packet lies within INCLUSION_WINDOW_MILLIS after currently considered trigger, include it.
+            return true;
+        } else {
+            if (!trigger.isBefore(packet.getTimestamp())) {
+                // Packet is before currently considered trigger, so it shouldn't be included
+                return false;
+            } else {
+                // Packet is >= INCLUSION_WINDOW_MILLIS after currently considered trigger.
+                // Proceed to next trigger to see if it lies in range of that.
+                // Note that there's an assumption here that no two trigger intervals don't overlap!
+                mTriggerIndex++;
+                return shouldIncludePacket(packet);
+            }
+        }
+        */
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/UserAction.java
new file mode 100644 (file)
index 0000000..408d66a
--- /dev/null
@@ -0,0 +1,108 @@
+package edu.uci.iotproject.analysis;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+/**
+ * Models a user's action, such as toggling the smart plug on/off at a given time.
+ *
+ * @author Janus Varmarken
+ */
+public class UserAction {
+
+    private static volatile DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ISO_ZONED_DATE_TIME.
+            withZone(ZoneId.of("America/Los_Angeles"));
+
+    /**
+     * Sets the {@link DateTimeFormatter} used when outputting a user action as a string and parsing a user action from
+     * a string.
+     * @param formatter The formatter to use for outputting and parsing.
+     */
+    public static void setTimestampFormatter(DateTimeFormatter formatter) {
+        TIMESTAMP_FORMATTER = formatter;
+    }
+
+    /**
+     * Instantiates a {@code UserAction} from a string that obeys the format used in {@link UserAction#toString()}.
+     * @param string The string that represents a {@code UserAction}
+     * @return A {@code UserAction} resulting from deserializing the string.
+     */
+    public static UserAction fromString(String string) {
+        String[] parts = string.split("@");
+        if (parts.length != 2) {
+            throw new IllegalArgumentException("Invalid string format");
+        }
+        // If any of these two parses fail, an exception is thrown -- no need to check return values.
+        UserAction.Type actionType = UserAction.Type.valueOf(parts[0].trim());
+        Instant timestamp = TIMESTAMP_FORMATTER.parse(parts[1].trim(), Instant::from);
+        return new UserAction(actionType, timestamp);
+    }
+
+
+    /**
+     * The specific type of action the user performed.
+     */
+    private final Type mType;
+
+    /**
+     * The time the action took place.
+     */
+    private final Instant mTimestamp;
+
+    public UserAction(Type typeOfAction, Instant timeOfAction) {
+        mType = typeOfAction;
+        mTimestamp = timeOfAction;
+    }
+
+    /**
+     * Get the specific type of action performed by the user.
+     * @return the specific type of action performed by the user.
+     */
+    public Type getType() {
+        return mType;
+    }
+
+    /**
+     * Get the time at which the user performed this action.
+     * @return the time at which the user performed this action.
+     */
+    public Instant getTimestamp() {
+        return mTimestamp;
+    }
+
+    /**
+     * Enum for indicating what type of action the user performed.
+     */
+    public enum Type {
+        TOGGLE_ON, TOGGLE_OFF
+    }
+
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof UserAction) {
+            UserAction that = (UserAction) obj;
+            return this.mType == that.mType && this.mTimestamp.equals(that.mTimestamp);
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int hashCode = 17;
+        hashCode = prime * hashCode + mType.hashCode();
+        hashCode = prime * hashCode + mTimestamp.hashCode();
+        return hashCode;
+    }
+
+    @Override
+    public String toString() {
+       return String.format("%s @ %s", mType.name(), TIMESTAMP_FORMATTER.format(mTimestamp));
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/AlignmentPricer.java
new file mode 100644 (file)
index 0000000..0552279
--- /dev/null
@@ -0,0 +1,70 @@
+package edu.uci.iotproject.comparison.seqalignment;
+
+import java.util.function.ToIntBiFunction;
+import java.util.function.ToIntFunction;
+
+/**
+ * Provides a generic implementation for the calculation of the cost of aligning two elements of a sequence as part of
+ * the sequence alignment algorithm (the algorithm is implemented in {@link SequenceAlignment}).
+ *
+ * @param <T> The type of the elements that are being aligned.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class AlignmentPricer<T> {
+
+    /**
+     * A function that provides the cost of aligning a {@link T} with a gap.
+     */
+    private final ToIntFunction<T> mGapCostFunction;
+
+    /**
+     * A function that provides the cost of aligning a {@link T} with some other {@link T}.
+     */
+    private final ToIntBiFunction<T,T> mAlignmentCostFunction;
+
+    /**
+     * Constructs a new {@link AlignmentPricer}.
+     *
+     * @param alignmentCostFunction A function that specifies the cost of aligning a {@link T} with some other {@link T}
+     *                              (e.g., based on the values of the properties of the two instances).
+     * @param gapCostFunction A function that specifies the cost of aligning a {@link T} with a gap. Note that the
+     *                        function is free to specify <em>different</em> gap costs for different {@link T}s.
+     */
+    public AlignmentPricer(ToIntBiFunction<T,T> alignmentCostFunction, ToIntFunction<T> gapCostFunction) {
+        mAlignmentCostFunction = alignmentCostFunction;
+        mGapCostFunction = gapCostFunction;
+    }
+
+    /**
+     * Calculate the cost of aligning {@code item1} with {@code item2}. If either of the two arguments is set to
+     * {@code null}, the cost of aligning the other argument with a gap will be returned. Note that both arguments
+     * cannot be {@code null} at the same time as that translates to aligning a gap with a gap, which is pointless.
+     *
+     * @param item1 The first of the two aligned objects. Set to {@code null} to calculate the cost of aligning
+     *              {@code item2} with a gap.
+     * @param item2 The second of the two aligned objects. Set to {@code null} to calculate the cost of aligning
+     *              {@code item2} with a gap.
+     * @return The cost of aligning {@code item1} with {@code item2}.
+     */
+    public int alignmentCost(T item1, T item2) {
+        // If both arguments are null, the caller is aligning a gap with a gap which is pointless might as well remove
+        // both gaps in that case!)
+        if (item1 == null && item2 == null) {
+            throw new IllegalArgumentException("Both arguments cannot be null: you are aligning a gap with a gap!");
+        }
+        // If one item is null, it means we're aligning an int with a gap.
+        // Invoke the provided gap cost function to get the gap cost.
+        if (item1 == null) {
+            return mGapCostFunction.applyAsInt(item2);
+        }
+        if (item2 == null) {
+            return mGapCostFunction.applyAsInt(item1);
+        }
+        // If both arguments are present, we simply delegate the task of calculating the cost of aligning the two items
+        // to the provided alignment cost function.
+        return mAlignmentCostFunction.applyAsInt(item1, item2);
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java
new file mode 100644 (file)
index 0000000..2d193a9
--- /dev/null
@@ -0,0 +1,41 @@
+package edu.uci.iotproject.comparison.seqalignment;
+
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class ExtractedSequence {
+
+    private final Conversation mRepresentativeSequence;
+
+    private final int mMaxAlignmentCost;
+
+    private final String mSequenceString;
+
+    public ExtractedSequence(Conversation sequence, int maxAlignmentCost, boolean tlsAppDataAlignment) {
+        mRepresentativeSequence = sequence;
+        mMaxAlignmentCost = maxAlignmentCost;
+        StringBuilder sb = new StringBuilder();
+        List<PcapPacket> pkts = tlsAppDataAlignment ? sequence.getTlsApplicationDataPackets() : sequence.getPackets();
+        pkts.forEach(p -> {
+            if (sb.length() != 0) sb.append(" ");
+            sb.append(p.getOriginalLength());
+        });
+        mSequenceString = sb.toString();
+    }
+
+    public Conversation getRepresentativeSequence() {
+        return mRepresentativeSequence;
+    }
+
+    public int getMaxAlignmentCost() {
+        return mMaxAlignmentCost;
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SampleIntegerAlignmentPricer.java
new file mode 100644 (file)
index 0000000..a09a10d
--- /dev/null
@@ -0,0 +1,23 @@
+package edu.uci.iotproject.comparison.seqalignment;
+
+/**
+ * A sample {@link AlignmentPricer} for computing the cost of aligning integer values. In this sample implementation,
+ * the cost of aligning two integers {@code i1} and {@code i2} is {@code Math.abs(i1 - i2)}, i.e., it is the absolute
+ * value of the difference between {@code i1} and {@code i2}. The cost of aligning an integer {@code i} with a gap is
+ * simply {@code i}, i.e., the gap is essentially treated as a zero.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class SampleIntegerAlignmentPricer extends AlignmentPricer<Integer> {
+
+    /**
+     * Constructs a new {@link SampleIntegerAlignmentPricer}.
+     */
+    public SampleIntegerAlignmentPricer() {
+        // Cost of aligning integers i1 and i2 is the absolute value of their difference.
+        // Cost of aligning integer i with a gap is i (as it was aligned with 0).
+        super((i1,i2) -> Math.abs(i1 - i2) , (i) -> i);
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceAlignment.java
new file mode 100644 (file)
index 0000000..005d7ff
--- /dev/null
@@ -0,0 +1,75 @@
+package edu.uci.iotproject.comparison.seqalignment;
+
+/**
+ * A generic implementation of the sequence alignment algorithm given in Kleinberg's and Tardos' "Algorithm Design".
+ * This implementation is the basic version. There is a more complex version which significantly reduces the space
+ * complexity at a slight cost to time complexity.
+ *
+ * @param <ALIGNMENT_UNIT> The <em>unit of the alignment</em>, or, in other words, the <em>granularity</em> of the
+ *                        alignment. For example, for 'classical' string alignment (as in sequence alignment where we
+ *                        try to align two strings character by character -- the example most often used in books on
+ *                        algorithms) this would be a {@link Character}. As a second example, by specifying
+ *                        {@link String}, one can decrease the granularity so as to align <em>blocks</em> of characters
+ *                        (e.g., if one wants to align to two string arrays).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class SequenceAlignment<ALIGNMENT_UNIT> {
+
+
+    /**
+     * Provides the cost of aligning two {@link ALIGNMENT_UNIT}s with one another as well as the cost of aligning an
+     * {@link ALIGNMENT_UNIT} with a gap.
+     */
+    private final AlignmentPricer<ALIGNMENT_UNIT> mAlignmentPricer;
+
+    /**
+     * Constructs a new {@link SequenceAlignment}. The new instance relies on the provided {@code alignmentPricer} to
+     * provide the cost of aligning two {@link ALIGNMENT_UNIT}s as well as the cost of aligning an
+     * {@link ALIGNMENT_UNIT} with a gap.
+     *
+     * @param alignmentPricer An {@link AlignmentPricer} that provides the cost of aligning two {@link ALIGNMENT_UNIT}s
+     *                        with one another as well as the cost of aligning an {@link ALIGNMENT_UNIT} with a gap.
+     */
+    public SequenceAlignment(AlignmentPricer<ALIGNMENT_UNIT> alignmentPricer) {
+        mAlignmentPricer = alignmentPricer;
+    }
+
+
+    /**
+     * Calculates the cost of aligning {@code sequence1} with {@code sequence2}.
+     *
+     * @param sequence1 A sequence that is to be aligned with {@code sequence2}.
+     * @param sequence2 A sequence that is to be aligned with {@code sequence1}.
+     *
+     * @return The cost of aligning {@code sequence1} with {@code sequence2}.
+     */
+    public int calculateAlignment(ALIGNMENT_UNIT[] sequence1, ALIGNMENT_UNIT[] sequence2) {
+        int[][] costs = new int[sequence1.length + 1][sequence2.length +1];
+        /*
+         * TODO:
+         * This is a homebrewn initialization; it is different from the one in the Kleinberg book - is it correct?
+         * It tries to add support for *different* gap costs depending on the input (e.g., such that one can say that
+         * matching a 'c' with a gap is more expensive than matching a 'b' with a gap).
+         */
+        for (int i = 1; i <= sequence1.length; i++) {
+            costs[i][0] = mAlignmentPricer.alignmentCost(sequence1[i-1], null) + costs[i-1][0];
+        }
+        for (int j = 1; j <= sequence2.length; j++) {
+            costs[0][j] = mAlignmentPricer.alignmentCost(sequence2[j-1], null) + costs[0][j-1];
+        }
+        for (int j = 1; j <= sequence2.length; j++) {
+            for (int i = 1; i <= sequence1.length; i++) {
+                // The cost when current items of both sequences are aligned.
+                int costAligned = mAlignmentPricer.alignmentCost(sequence2[j-1], sequence1[i-1]) + costs[i-1][j-1];
+                // The cost when current item from sequence1 is not aligned (it's matched with a gap)
+                int seq1ItemNotMached = mAlignmentPricer.alignmentCost(sequence1[i-1], null) + costs[i-1][j];
+                // The cost when current item from sequence2 is not aligned (it's matched with a gap)
+                int seq2ItemNotMached = mAlignmentPricer.alignmentCost(sequence2[j-1], null) + costs[i][j-1];
+                costs[i][j] = Math.min(costAligned, Math.min(seq1ItemNotMached, seq2ItemNotMached));
+            }
+        }
+        return costs[sequence1.length][sequence2.length];
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java
new file mode 100644 (file)
index 0000000..6aaa318
--- /dev/null
@@ -0,0 +1,152 @@
+package edu.uci.iotproject.comparison.seqalignment;
+
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.analysis.TcpConversationUtils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class SequenceExtraction {
+
+
+    private final SequenceAlignment<Integer> mAlignmentAlg;
+
+
+    public SequenceExtraction() {
+        mAlignmentAlg = new SequenceAlignment<>(new AlignmentPricer<>((i1,i2) -> Math.abs(i1-i2), i -> 10));
+    }
+
+
+    public SequenceExtraction(SequenceAlignment<Integer> alignmentAlgorithm) {
+        mAlignmentAlg = alignmentAlgorithm;
+    }
+
+    /**
+     * Gets the {@link SequenceAlignment} used to perform the sequence extraction.
+     * @return the {@link SequenceAlignment} used to perform the sequence extraction.
+     */
+    public SequenceAlignment<Integer> getAlignmentAlgorithm() {
+        return mAlignmentAlg;
+    }
+
+    // Initial
+//    /**
+//     *
+//     * @param convsForAction A set of {@link Conversation}s known to be associated with a single type of user action.
+//     */
+//    public void extract(List<Conversation> convsForAction) {
+//        int maxDifference = 0;
+//
+//        for (int i = 0; i < convsForAction.size(); i++) {
+//            for (int j = i+1; j < convsForAction.size(); i++) {
+//                Integer[] sequence1 = getPacketLengthSequence(convsForAction.get(i));
+//                Integer[] sequence2 = getPacketLengthSequence(convsForAction.get(j));
+//                int alignmentCost = mAlignmentAlg.calculateAlignment(sequence1, sequence2);
+//                if (alignmentCost > maxDifference) {
+//                    maxDifference = alignmentCost;
+//                }
+//            }
+//        }
+//
+//    }
+
+
+//    public void extract(Map<String, List<Conversation>> hostnameToConvs) {
+//        int maxDifference = 0;
+//
+//        for (int i = 0; i < convsForAction.size(); i++) {
+//            for (int j = i+1; j < convsForAction.size(); i++) {
+//                Integer[] sequence1 = getPacketLengthSequence(convsForAction.get(i));
+//                Integer[] sequence2 = getPacketLengthSequence(convsForAction.get(j));
+//                int alignmentCost = mAlignmentAlg.calculateAlignment(sequence1, sequence2);
+//                if (alignmentCost > maxDifference) {
+//                    maxDifference = alignmentCost;
+//                }
+//            }
+//        }
+//
+//    }
+
+    // Building signature from entire sequence
+    public ExtractedSequence extract(List<Conversation> convsForActionForHostname) {
+        // First group conversations by packet sequences.
+        // TODO: the introduction of SYN/SYNACK, FIN/FINACK and RST as part of the sequence ID may be undesirable here
+        // as it can potentially result in sequences that are equal in terms of payload packets to be considered
+        // different due to differences in how they are terminated.
+        Map<String, List<Conversation>> groupedBySequence =
+                TcpConversationUtils.groupConversationsByPacketSequence(convsForActionForHostname, false);
+
+        // Then get a hold of one of the conversations that gave rise to the most frequent sequence.
+        Conversation mostFrequentConv = null;
+        int maxFrequency = 0;
+        for (Map.Entry<String, List<Conversation>> seqMapEntry : groupedBySequence.entrySet()) {
+            if (seqMapEntry.getValue().size() > maxFrequency) {
+                // Found a more frequent sequence
+                maxFrequency = seqMapEntry.getValue().size();
+                // We just pick the first conversation as the representative conversation for this sequence type.
+                mostFrequentConv = seqMapEntry.getValue().get(0);
+            } else if (seqMapEntry.getValue().size() == maxFrequency) {
+                // This sequence has the same frequency as the max frequency seen so far.
+                // Break ties by choosing the longest sequence.
+                // First get an arbitrary representative of currently examined sequence; we just pick the first.
+                Conversation c = seqMapEntry.getValue().get(0);
+                mostFrequentConv = c.getPackets().size() > mostFrequentConv.getPackets().size() ? c : mostFrequentConv;
+            }
+        }
+        // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the
+        // each of the rest of the conversations also associated with this action and hostname.
+        int maxCost = 0;
+        final Integer[] mostFrequentConvSeq = TcpConversationUtils.getPacketLengthSequence(mostFrequentConv);
+        for (Conversation c : convsForActionForHostname) {
+            if (c == mostFrequentConv) {
+                // Don't compute distance to self.
+                continue;
+            }
+            Integer[] cSeq = TcpConversationUtils.getPacketLengthSequence(c);
+            int alignmentCost = mAlignmentAlg.calculateAlignment(mostFrequentConvSeq, cSeq);
+            if (alignmentCost > maxCost) {
+                maxCost = alignmentCost;
+            }
+        }
+        return new ExtractedSequence(mostFrequentConv, maxCost, false);
+    }
+
+    // Building signature from only TLS Application Data packets
+    public ExtractedSequence extractByTlsAppData(List<Conversation> convsForActionForHostname) {
+        // TODO: temporary hack to avoid 97-only conversations for dlink plug. We need some preprocessing/data cleaning.
+        convsForActionForHostname = convsForActionForHostname.stream().filter(c -> c.getTlsApplicationDataPackets().size() > 1).collect(Collectors.toList());
+
+        Map<String, List<Conversation>> groupedByTlsAppDataSequence =
+                TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(convsForActionForHostname);
+        // Get a Conversation representing the most frequent TLS application data sequence.
+        Conversation mostFrequentConv = groupedByTlsAppDataSequence.values().stream().max((l1, l2) -> {
+            // The frequency of a conversation with a specific packet sequence is the list size as that represents how
+            // many conversations exhibit that packet sequence.
+            // Hence, the difference between the list sizes can be used directly as the return value of the Comparator.
+            // Note: we break ties by choosing the one with the most TLS application data packets (i.e., the longest
+            // sequence) in case the frequencies are equal.
+            int diff = l1.size() - l2.size();
+            return diff != 0 ? diff : l1.get(0).getTlsApplicationDataPackets().size() - l2.get(0).getTlsApplicationDataPackets().size();
+        }).get().get(0); // Just pick the first as a representative of the most frequent sequence.
+        // Lengths of TLS Application Data packets in the most frequent (or most frequent and longest) conversation.
+        Integer[] mostFreqSeq = TcpConversationUtils.getPacketLengthSequenceTlsAppDataOnly(mostFrequentConv);
+        // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the
+        // each of the rest of the conversations also associated with this action and hostname.
+        int maxCost = 0;
+        for (Conversation c : convsForActionForHostname) {
+            if (c == mostFrequentConv) continue;
+            int cost = mAlignmentAlg.calculateAlignment(mostFreqSeq, TcpConversationUtils.getPacketLengthSequenceTlsAppDataOnly(c));
+            maxCost = cost > maxCost ? cost : maxCost;
+        }
+        return new ExtractedSequence(mostFrequentConv, maxCost, true);
+        // Now find the maximum cost of aligning the most frequent (or, alternatively longest) conversation with the
+        // each of the rest of the conversations also associated with this action and hostname.
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java
new file mode 100644 (file)
index 0000000..45c6a55
--- /dev/null
@@ -0,0 +1,72 @@
+package edu.uci.iotproject.detection;
+
+import org.pcap4j.core.PcapPacket;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Base class for classes that search a traffic trace for sequences of packets that "belong to" a given cluster (in
+ * other words, classes that attempt to classify traffic as pertaining to a given cluster).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+abstract public class AbstractClusterMatcher {
+
+    /**
+     * The cluster that describes the sequence of packets that this {@link AbstractClusterMatcher} is trying to detect
+     * in the observed traffic.
+     */
+    protected final List<List<PcapPacket>> mCluster;
+
+    /**
+     * Observers registered for callbacks from this {@link AbstractClusterMatcher}.
+     */
+    protected final List<ClusterMatcherObserver> mObservers;
+
+    protected AbstractClusterMatcher(List<List<PcapPacket>> cluster) {
+        // ===================== PRECONDITION SECTION =====================
+        cluster = Objects.requireNonNull(cluster, "cluster cannot be null");
+        if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) {
+            throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)");
+        }
+        for (List<PcapPacket> clusterMember : cluster) {
+            if (clusterMember.size() != cluster.get(0).size()) {
+                throw new IllegalArgumentException("All sequences in cluster must contain the same number of packets");
+            }
+        }
+        // ================================================================
+        // Let the subclass prune the provided cluster
+        mCluster = pruneCluster(cluster);
+        mObservers = new ArrayList<>();
+    }
+
+    /**
+     * Register for callbacks from this cluster matcher.
+     * @param observer The target of the callbacks.
+     */
+    public final void addObserver(ClusterMatcherObserver observer) {
+        mObservers.add(observer);
+    }
+
+    /**
+     * Deregister for callbacks from this cluster matcher.
+     * @param observer The callback target that is to be deregistered.
+     */
+    public final void removeObserver(ClusterMatcherObserver observer) {
+        mObservers.remove(observer);
+    }
+
+    /**
+     * Allows subclasses to specify how to prune the input cluster provided to the constructor.
+     * @param cluster The input cluster provided to the constructor.
+     * @return The pruned cluster to use in place of the input cluster.
+     */
+    abstract protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster);
+
+    // TODO: move Direction outside Conversation so that this is less confusing.
+//    abstract protected Conversation.Direction[] getPacketDirections(List<PcapPacket> packets);
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/AbstractSignatureDetector.java
new file mode 100644 (file)
index 0000000..b99116c
--- /dev/null
@@ -0,0 +1,77 @@
+package edu.uci.iotproject.detection;
+
+import org.pcap4j.core.PcapPacket;
+
+import java.util.*;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public abstract class AbstractSignatureDetector implements ClusterMatcherObserver {
+
+
+    /**
+     * The signature that this {@link AbstractSignatureDetector} is searching for.
+     */
+    private final List<List<List<PcapPacket>>> mSignature;
+
+    /**
+     * The {@link AbstractClusterMatcher}s in charge of detecting each individual sequence of packets that together make
+     * up the the signature.
+     */
+    private final List<AbstractClusterMatcher> mClusterMatchers;
+
+    /**
+     * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches
+     * found by the {@link AbstractClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed",
+     * i.e., have yet to be included in a signature detected by this {@link AbstractSignatureDetector} (a signature can
+     * be encompassed of multiple packet sequences occurring shortly after one another on multiple connections).
+     */
+    private final List<List<PcapPacket>>[] pendingMatches;
+
+    /**
+     * Maps an {@link AbstractClusterMatcher} to its corresponding index in {@link #pendingMatches}.
+     */
+    private final Map<AbstractClusterMatcher, Integer> mClusterMatcherIds;
+
+    public AbstractSignatureDetector(List<List<List<PcapPacket>>> searchedSignature) {
+        mSignature = Collections.unmodifiableList(searchedSignature);
+        List<AbstractClusterMatcher> clusterMatchers = new ArrayList<>();
+        for (List<List<PcapPacket>> cluster : mSignature) {
+            AbstractClusterMatcher clusterMatcher = constructClusterMatcher(cluster);
+            clusterMatcher.addObserver(this);
+            clusterMatchers.add(clusterMatcher);
+        }
+        mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
+        pendingMatches = new List[mClusterMatchers.size()];
+        for (int i = 0; i < pendingMatches.length; i++) {
+            pendingMatches[i] = new ArrayList<>();
+        }
+        Map<AbstractClusterMatcher, Integer> clusterMatcherIds = new HashMap<>();
+        for (int i = 0; i < mClusterMatchers.size(); i++) {
+            clusterMatcherIds.put(mClusterMatchers.get(i), i);
+        }
+        mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
+    }
+
+    abstract protected AbstractClusterMatcher constructClusterMatcher(List<List<PcapPacket>> cluster);
+
+    /**
+     * Encapsulates a {@code List<PcapPacket>} so as to allow the list to be used as a vertex in a graph while avoiding
+     * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph.
+     * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)}
+     * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not
+     * recognize two lists that contain the same items--from a value and not reference point of view--as the same
+     * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more
+     * appropriate.
+     */
+    private static class Vertex {
+        private final List<PcapPacket> sequence;
+        private Vertex(List<PcapPacket> wrappedSequence) {
+            sequence = wrappedSequence;
+        }
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java
new file mode 100644 (file)
index 0000000..d67c520
--- /dev/null
@@ -0,0 +1,26 @@
+package edu.uci.iotproject.detection;
+
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+
+/**
+ * Interface used by client code to register for receiving a notification whenever an {@link AbstractClusterMatcher}
+ * detects traffic that matches an element of its associated cluster.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public interface ClusterMatcherObserver {
+
+    /**
+     * Callback that is invoked by an {@link AbstractClusterMatcher} whenever it detects traffic that matches an element
+     * of its associated cluster.
+     *
+     * @param clusterMatcher The {@link AbstractClusterMatcher} that detected a match (i.e., classified traffic as
+     *                       pertaining to its associated cluster).
+     * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}.
+     */
+    void onMatch(AbstractClusterMatcher clusterMatcher, List<PcapPacket> match);
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/SignatureDetectorObserver.java
new file mode 100644 (file)
index 0000000..c2d8e09
--- /dev/null
@@ -0,0 +1,22 @@
+package edu.uci.iotproject.detection;
+
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+
+/**
+ * Used for registering for notifications from a signature detector.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public interface SignatureDetectorObserver {
+
+    /**
+     * Invoked when the signature detector has detected the presence of a signature in the traffic that it's examining.
+     * @param searchedSignature The signature that the signature detector reporting the match is searching for.
+     * @param matchingTraffic The actual traffic trace that matches the searched signature.
+     */
+    void onSignatureDetected(List<List<List<PcapPacket>>> searchedSignature, List<List<PcapPacket>> matchingTraffic);
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java
new file mode 100644 (file)
index 0000000..5021c31
--- /dev/null
@@ -0,0 +1,153 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver;
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowObserver;
+import org.pcap4j.core.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Attempts to detect members of a cluster (packet sequence mutations) in layer 2 flows.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver {
+
+    /**
+     * Maps from a flow to a table of {@link Layer2SequenceMatcher}s for that particular flow. The table {@code t} is
+     * structured such that {@code t[i][j]} is a {@link Layer2SequenceMatcher} that attempts to match member {@code i}
+     * of {@link #mCluster} and has so far matched {@code j} packets of that particular sequence.
+     */
+    private final Map<Layer2Flow, Layer2SequenceMatcher[][]> mPerFlowSeqMatchers = new HashMap<>();
+
+    private final Function<Layer2Flow, Boolean> mFlowFilter;
+
+    /**
+     * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members.
+     * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for.
+     */
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
+        // Consider all flows if no flow filter specified.
+        this(cluster, flow -> true);
+    }
+
+    /**
+     * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members.
+     * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for.
+     * @param flowFilter A filter that defines what {@link Layer2Flow}s the new {@link Layer2ClusterMatcher} should
+     *                   search for {@code cluster}'s members in. If {@code flowFilter} returns {@code true}, the flow
+     *                   will be included (searched). Note that {@code flowFilter} is only queried once for each flow,
+     *                   namely when the {@link Layer2FlowReassembler} notifies the {@link Layer2ClusterMatcher} about
+     *                   the new flow. This functionality may for example come in handy when one only wants to search
+     *                   for matches in the subset of flows that involves a specific (range of) MAC(s).
+     */
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster, Function<Layer2Flow, Boolean> flowFilter) {
+        super(cluster);
+        mFlowFilter = flowFilter;
+    }
+
+    @Override
+    public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) {
+        if (mPerFlowSeqMatchers.get(flow) == null) {
+            // If this is the first time we encounter this flow, we need to set up sequence matchers for it.
+            // All sequences of the cluster have the same length, so we only need to compute the length of the nested
+            // arrays once. We want to make room for a cluster matcher in each state, including the initial empty state
+            // but excluding the final "full match" state (as there is no point in keeping a terminated sequence matcher
+            // around), so the length of the inner array is simply the sequence length.
+            Layer2SequenceMatcher[][] matchers = new Layer2SequenceMatcher[mCluster.size()][mCluster.get(0).size()];
+            // Prepare a "state 0" sequence matcher for each sequence variation in the cluster.
+            for (int i = 0; i < matchers.length; i++) {
+                matchers[i][0] = new Layer2SequenceMatcher(mCluster.get(i));
+            }
+            // Associate the new sequence matcher table with the new flow
+            mPerFlowSeqMatchers.put(flow, matchers);
+        }
+        // Fetch table that contains sequence matchers for this flow.
+        Layer2SequenceMatcher[][] matchers = mPerFlowSeqMatchers.get(flow);
+        // Present the packet to all sequence matchers.
+        for (int i = 0; i < matchers.length; i++) {
+            // Present packet to the sequence matchers that has advanced the most first. This is to prevent discarding
+            // the sequence matchers that have advanced the most in the special case where the searched sequence
+            // contains two packets of the same length going in the same direction.
+            for (int j = matchers[i].length - 1; j >= 0 ; j--) {
+                Layer2SequenceMatcher sm = matchers[i][j];
+                if (sm == null) {
+                    // There is currently no sequence matcher that has managed to match j packets.
+                    continue;
+                }
+                boolean matched = sm.matchPacket(newPacket);
+                if (matched) {
+                    if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) {
+                        // Sequence matcher has a match. Report it to observers.
+                        mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets()));
+                        // Remove the now terminated sequence matcher.
+                        matchers[i][j] = null;
+                    } else {
+                        // Sequence matcher advanced one step, so move it to its corresponding new position iff the
+                        // packet that advanced it has a later timestamp than that of the last matched packet of the
+                        // sequence matcher at the new index, if any. In most traces, a small amount of the packets
+                        // appear out of order (with regards to their timestamp), which is why this check is required.
+                        // Obviously it would not be needed if packets where guaranteed to be processed in timestamp
+                        // order here.
+                        if (matchers[i][j+1] == null ||
+                                newPacket.getTimestamp().isAfter(matchers[i][j+1].getLastPacket().getTimestamp())) {
+                            matchers[i][j+1] = sm;
+                        }
+                    }
+                    // We always want to have a sequence matcher in state 0, regardless of if the one that advanced
+                    // from state zero completed its matching or if it replaced a different one in state 1 or not.
+                    if (sm.getMatchedPacketsCount() == 1) {
+                        matchers[i][j] = new Layer2SequenceMatcher(sm.getTargetSequence());
+                    }
+                }
+            }
+        }
+    }
+
+
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        // Note: we assume that all sequences in the input cluster are of the same length and that their packet
+        // directions are identical.
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> {
+                for (int i = 0; i < pcPkts.size(); i++) {
+                    if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+            if (!alreadyPresent) {
+                // Add the sequence if not already present in the pruned cluster.
+                prunedCluster.add(originalClusterSeq);
+            }
+        }
+        return prunedCluster;
+    }
+
+    private static final boolean DEBUG = false;
+
+    @Override
+    public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) {
+        // New flow detected. Check if we should consider it when searching for cluster member matches.
+        if (mFlowFilter.apply(newFlow)) {
+            if (DEBUG) {
+                System.out.println(">>> ACCEPTING FLOW: " + newFlow + " <<<");
+            }
+            // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed.
+            newFlow.addFlowObserver(this);
+        } else if (DEBUG) {
+            System.out.println(">>> IGNORING FLOW: " + newFlow + " <<<");
+        }
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java
new file mode 100644 (file)
index 0000000..672fb72
--- /dev/null
@@ -0,0 +1,173 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls
+ * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2SequenceMatcher {
+
+    /**
+     * The sequence this {@link Layer2SequenceMatcher} is searching for.
+     */
+    private final List<PcapPacket> mSequence;
+
+    /**
+     * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the
+     * searched sequence).
+     */
+    private final List<PcapPacket> mMatchedPackets = new ArrayList<>();
+
+    /**
+     * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented
+     * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a
+     * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses
+     * of devices in advance during matching.
+     */
+    private final boolean[] mPacketDirections;
+
+    /**
+     * Create a {@code Layer2SequenceMatcher}.
+     * @param sequence The sequence to match against (search for).
+     */
+    public Layer2SequenceMatcher(List<PcapPacket> sequence) {
+        mSequence = sequence;
+        // Compute packet directions for sequence.
+        mPacketDirections = new boolean[sequence.size()];
+        for (int i = 0; i < sequence.size(); i++) {
+            if (i == 0) {
+                // No previous packet; boolean parameter is ignored in this special case.
+                mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i));
+            } else {
+                // Base direction marker on direction of previous packet.
+                PcapPacket prevPkt = mSequence.get(i-1);
+                boolean prevPktDirection = mPacketDirections[i-1];
+                mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i));
+            }
+        }
+    }
+
+    /**
+     * Attempt to advance this {@code Layer2SequenceMatcher} by matching {@code packet} against the packet that this
+     * {@code Layer2SequenceMatcher} expects as the next packet of the sequence it is searching for.
+     * @param packet
+     * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of
+     *         matched packets, {@code false} otherwise.
+     */
+    public boolean matchPacket(PcapPacket packet) {
+        if (getMatchedPacketsCount() == getTargetSequencePacketCount()) {
+            // We already matched the entire sequence, so we can't match any more packets.
+            return false;
+        }
+
+        // Verify that new packet pertains to same flow as previously matched packets, if any.
+        if (getMatchedPacketsCount() > 0) {
+            MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet);
+            MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet);
+            MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0));
+            MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0));
+            if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) ||
+                    pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) {
+                return false;
+            }
+        }
+
+        // Get representative of the packet we expect to match next.
+        PcapPacket expected = mSequence.get(mMatchedPackets.size());
+        // First verify if the received packet has the length we're looking for.
+        if (packet.getOriginalLength() == expected.getOriginalLength()) {
+            // If this is the first packet, we only need to verify that its length is correct. Time constraints are
+            // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we
+            // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet
+            // must originate from this particular MAC".
+            if (getMatchedPacketsCount() == 0) {
+                // Store packet as matched and advance.
+                mMatchedPackets.add(packet);
+                return true;
+            }
+            // Check if direction of packet matches expected direction.
+            boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1),
+                    mPacketDirections[getMatchedPacketsCount()-1], packet);
+            boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()];
+            if (actualDirection != expectedDirection) {
+                return false;
+            }
+            // Next apply timing constraints:
+            // 1: to be a match, the packet must have a later timestamp than any other packet currently matched
+            // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded?
+            if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) {
+                return false;
+            }
+            if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp().
+                            plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) {
+                return false;
+            }
+            // If we made it here, it means that this packet has the expected length, direction, and obeys the timing
+            // constraints, so we store it and advance.
+            mMatchedPackets.add(packet);
+            if (mMatchedPackets.size() == mSequence.size()) {
+                // TODO report (to observers?) that we are done?
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public int getMatchedPacketsCount() {
+        return mMatchedPackets.size();
+    }
+
+    public int getTargetSequencePacketCount() {
+        return mSequence.size();
+    }
+
+    public List<PcapPacket> getTargetSequence() {
+        return mSequence;
+    }
+
+    public List<PcapPacket> getMatchedPackets() {
+        return mMatchedPackets;
+    }
+
+    /**
+     * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}.
+     * @return The last matched packet, or {@code null} if no packets have been matched yet.
+     */
+    public PcapPacket getLastPacket() {
+        return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null;
+    }
+
+    /**
+     * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction
+     * of {@code currPkt} is {@code true} by definition.
+     * @param prevPkt The previous packet, if any.
+     * @param prevPktDirection The computed direction of the previous packet
+     * @param currPkt The current packet for which the direction is to be determined.
+     * @return The direction of {@code currPkt}.
+     */
+    private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) {
+        if (prevPkt == null) {
+            // By definition, use true as direction marker for first packet
+            return true;
+        }
+        if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) {
+            // Current packet goes in same direction as previous packet.
+            return prevPktDirection;
+        } else {
+            // Current packet goes in opposite direction of previous packet.
+            return !prevPktDirection;
+        }
+    }
+
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java
new file mode 100644 (file)
index 0000000..a721914
--- /dev/null
@@ -0,0 +1,352 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import edu.uci.iotproject.analysis.UserAction;
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.detection.ClusterMatcherObserver;
+import edu.uci.iotproject.detection.SignatureDetectorObserver;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.io.PrintWriterUtils;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler;
+import edu.uci.iotproject.util.PrintUtils;
+import org.jgrapht.GraphPath;
+import org.jgrapht.alg.shortestpath.DijkstraShortestPath;
+import org.jgrapht.graph.DefaultWeightedEdge;
+import org.jgrapht.graph.SimpleDirectedWeightedGraph;
+import org.pcap4j.core.*;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.time.Duration;
+import java.util.*;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+
+/**
+ * Performs layer 2 signature detection.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2SignatureDetector implements PacketListener, ClusterMatcherObserver {
+
+    /**
+     * If set to {@code true}, output written to the results file is also dumped to standard out.
+     */
+    private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
+
+    private static List<Function<Layer2Flow, Boolean>> parseSignatureMacFilters(String filtersString) {
+        List<Function<Layer2Flow, Boolean>> filters = new ArrayList<>();
+        String[] filterRegexes = filtersString.split(";");
+        for (String filterRegex : filterRegexes) {
+            final Pattern regex = Pattern.compile(filterRegex);
+            // Create a filter that includes all flows where one of the two MAC addresses match the regex.
+            filters.add(flow -> regex.matcher(flow.getEndpoint1().toString()).matches() || regex.matcher(flow.getEndpoint2().toString()).matches());
+        }
+        return filters;
+    }
+
+    public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException {
+        // Parse required parameters.
+        if (args.length < 5) {
+            String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" +
+                            "\n  inputPcapFile: the target of the detection" +
+                            "\n  onSignatureFile: the file that contains the ON signature to search for" +
+                            "\n  offSignatureFile: the file that contains the OFF signature to search for" +
+                            "\n  resultsFile: where to write the results of the detection" +
+                            "\n  signatureDuration: the maximum duration of signature detection",
+                    Layer2SignatureDetector.class.getSimpleName());
+            System.out.println(errMsg);
+            String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " +
+                    "following options and associated positional arguments may be used:\n" +
+                    "  '-onmacfilters <regex>;<regex>;...;<regex>' which specifies that sequence matching should ONLY" +
+                    " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " +
+                    "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " +
+                    "aggressive filtering on parts of the signature (e.g., the communication that involves the " +
+                    "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " +
+                    "that matches the vendor's prefix).\n" +
+                    "  '-offmacfilters <regex>;<regex>;...;<regex>' works exactly the same as onmacfilters, but " +
+                    "applies to the OFF signature instead of the ON signature.\n" +
+                    "  '-sout <boolean literal>' true/false literal indicating if output should also be printed to std out; default is true.";
+            System.out.println(optParamsExplained);
+            return;
+        }
+        final String pcapFile = args[0];
+        final String onSignatureFile = args[1];
+        final String offSignatureFile = args[2];
+        final String resultsFile = args[3];
+        final int signatureDuration = Integer.parseInt(args[4]);
+
+        // Parse optional parameters.
+        List<Function<Layer2Flow, Boolean>> onSignatureMacFilters = null, offSignatureMacFilters = null;
+        final int optParamsStartIdx = 5;
+        if (args.length > optParamsStartIdx) {
+            for (int i = optParamsStartIdx; i < args.length; i++) {
+                if (args[i].equalsIgnoreCase("-onMacFilters")) {
+                    // Next argument is the cluster-wise MAC filters (separated by semicolons).
+                    onSignatureMacFilters = parseSignatureMacFilters(args[i+1]);
+                } else if (args[i].equalsIgnoreCase("-offMacFilters")) {
+                    // Next argument is the cluster-wise MAC filters (separated by semicolons).
+                    offSignatureMacFilters = parseSignatureMacFilters(args[i+1]);
+                } else if (args[i].equalsIgnoreCase("-sout")) {
+                    // Next argument is a boolean true/false literal.
+                    DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]);
+                }
+            }
+        }
+
+        // Prepare file outputter.
+        File outputFile = new File(resultsFile);
+        outputFile.getParentFile().mkdirs();
+        final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile));
+        // Include metadata as comments at the top
+        PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        resultsWriter.flush();
+
+        // Create signature detectors and add observers that output their detected events.
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+        Layer2SignatureDetector onDetector = onSignatureMacFilters == null ?
+                new Layer2SignatureDetector(onSignature) : new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration);
+        Layer2SignatureDetector offDetector = offSignatureMacFilters == null ?
+                new Layer2SignatureDetector(offSignature) : new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration);
+        onDetector.addObserver((signature, match) -> {
+            UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp());
+            PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        });
+        offDetector.addObserver((signature, match) -> {
+            UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, match.get(0).get(0).getTimestamp());
+            PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        });
+
+        // Load the PCAP file
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(pcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(pcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector);
+        // Parse the file
+        reader.readFromHandle();
+
+        // Flush output to results file and close it.
+        resultsWriter.flush();
+        resultsWriter.close();
+    }
+
+    /**
+     * The signature that this {@link Layer2SignatureDetector} is searching for.
+     */
+    private final List<List<List<PcapPacket>>> mSignature;
+
+    /**
+     * The {@link Layer2ClusterMatcher}s in charge of detecting each individual sequence of packets that together make
+     * up the the signature.
+     */
+    private final List<Layer2ClusterMatcher> mClusterMatchers;
+
+    /**
+     * For each {@code i} ({@code i >= 0 && i < mPendingMatches.length}), {@code mPendingMatches[i]} holds the matches
+     * found by the {@link Layer2ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed",
+     * i.e., have yet to be included in a signature detected by this {@link Layer2SignatureDetector} (a signature can
+     * be encompassed of multiple packet sequences occurring shortly after one another on multiple connections).
+     */
+    private final List<List<PcapPacket>>[] mPendingMatches;
+
+    /**
+     * Maps a {@link Layer2ClusterMatcher} to its corresponding index in {@link #mPendingMatches}.
+     */
+    private final Map<Layer2ClusterMatcher, Integer> mClusterMatcherIds;
+
+    /**
+     * In charge of reassembling layer 2 packet flows.
+     */
+    private final Layer2FlowReassembler mFlowReassembler = new Layer2FlowReassembler();
+
+    private final List<SignatureDetectorObserver> mObservers = new ArrayList<>();
+
+    private int mInclusionTimeMillis;
+
+    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature) {
+        this(searchedSignature, null, 0);
+    }
+
+    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, List<Function<Layer2Flow, Boolean>> flowFilters, int inclusionTimeMillis) {
+        if (flowFilters != null && flowFilters.size() != searchedSignature.size()) {
+            throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature.");
+        }
+        mSignature = Collections.unmodifiableList(searchedSignature);
+        List<Layer2ClusterMatcher> clusterMatchers = new ArrayList<>();
+        for (int i = 0; i < mSignature.size(); i++) {
+            List<List<PcapPacket>> cluster = mSignature.get(i);
+            Layer2ClusterMatcher clusterMatcher = flowFilters == null ?
+                    new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i));
+            clusterMatcher.addObserver(this);
+            clusterMatchers.add(clusterMatcher);
+        }
+        mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
+        mPendingMatches = new List[mClusterMatchers.size()];
+        for (int i = 0; i < mPendingMatches.length; i++) {
+            mPendingMatches[i] = new ArrayList<>();
+        }
+        Map<Layer2ClusterMatcher, Integer> clusterMatcherIds = new HashMap<>();
+        for (int i = 0; i < mClusterMatchers.size(); i++) {
+            clusterMatcherIds.put(mClusterMatchers.get(i), i);
+        }
+        mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
+        // Register all cluster matchers to receive a notification whenever a new flow is encountered.
+        mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm));
+        mInclusionTimeMillis =
+                inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // Forward packet processing to the flow reassembler that in turn notifies the cluster matchers as appropriate
+        mFlowReassembler.gotPacket(packet);
+    }
+
+    @Override
+    public void onMatch(AbstractClusterMatcher clusterMatcher, List<PcapPacket> match) {
+        // TODO: a cluster matcher found a match
+        if (clusterMatcher instanceof Layer2ClusterMatcher) {
+            // Add the match at the corresponding index
+            mPendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
+            checkSignatureMatch();
+        }
+    }
+
+    public void addObserver(SignatureDetectorObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public boolean removeObserver(SignatureDetectorObserver observer) {
+        return mObservers.remove(observer);
+    }
+
+
+    @SuppressWarnings("Duplicates")
+    private void checkSignatureMatch() {
+        // << Graph-based approach using Balint's idea. >>
+        // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp.
+
+        // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence.
+        if (Arrays.stream(mPendingMatches).noneMatch(l -> l.isEmpty())) {
+            // Construct the DAG
+            final SimpleDirectedWeightedGraph<Vertex, DefaultWeightedEdge> graph =
+                    new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class);
+            // Add a vertex for each match found by all cluster matchers.
+            // And maintain an array to keep track of what cluster matcher each vertex corresponds to
+            final List<Vertex>[] vertices = new List[mPendingMatches.length];
+            for (int i = 0; i < mPendingMatches.length; i++) {
+                vertices[i] = new ArrayList<>();
+                for (List<PcapPacket> sequence : mPendingMatches[i]) {
+                    Vertex v = new Vertex(sequence);
+                    vertices[i].add(v); // retain reference for later when we are to add edges
+                    graph.addVertex(v); // add to vertex to graph
+                }
+            }
+            // Add dummy source and sink vertices to facilitate search.
+            final Vertex source = new Vertex(null);
+            final Vertex sink = new Vertex(null);
+            graph.addVertex(source);
+            graph.addVertex(sink);
+            // The source is connected to all vertices that wrap the sequences detected by cluster matcher at index 0.
+            // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node.
+            for (Vertex v : vertices[0]) {
+                DefaultWeightedEdge edge = graph.addEdge(source, v);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Similarly, all vertices that wrap the sequences detected by the last cluster matcher of the signature
+            // are connected to the sink node.
+            for (Vertex v : vertices[vertices.length-1]) {
+                DefaultWeightedEdge edge = graph.addEdge(v, sink);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Now link sequences detected by the cluster matcher at index i to sequences detected by the cluster
+            // matcher at index i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than
+            // the former).
+            for (int i = 0; i < vertices.length; i++) {
+                int j = i + 1;
+                if (j < vertices.length) {
+                    for (Vertex iv : vertices[i]) {
+                        PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1);
+                        for (Vertex jv : vertices[j]) {
+                            PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1);
+                            if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) {
+                                DefaultWeightedEdge edge = graph.addEdge(iv, jv);
+                                // The weight is the duration of the i'th sequence plus the duration between the i'th
+                                // and i+1'th sequence.
+                                Duration d = Duration.
+                                        between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp());
+                                // Unfortunately weights are double values, so must convert from long to double.
+                                // TODO: need nano second precision? If so, use d.toNanos().
+                                // TODO: risk of overflow when converting from long to double..?
+                                graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue());
+                            }
+                            // Alternative version if we cannot assume that sequences are ordered by timestamp:
+//                            if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get()
+//                                    .getTimestamp().isBefore(jv.sequence.stream().min(
+//                                            Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) {
+//
+//                            }
+                        }
+                    }
+                }
+            }
+            // Graph construction complete, run shortest-path to find a (potential) signature match.
+            DijkstraShortestPath<Vertex, DefaultWeightedEdge> dijkstra = new DijkstraShortestPath<>(graph);
+            GraphPath<Vertex, DefaultWeightedEdge> shortestPath = dijkstra.getPath(source, sink);
+            if (shortestPath != null) {
+                // The total weight is the duration between the first packet of the first sequence and the last packet
+                // of the last sequence, so we simply have to compare the weight against the timeframe that we allow
+                // the signature to span. For now we just use the inclusion window we defined for training purposes.
+                // Note however, that we must convert back from double to long as the weight is stored as a double in
+                // JGraphT's API.
+                if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) {
+                    // There's a signature match!
+                    // Extract the match from the vertices
+                    List<List<PcapPacket>> signatureMatch = new ArrayList<>();
+                    for(Vertex v : shortestPath.getVertexList()) {
+                        if (v == source || v == sink) {
+                            // Skip the dummy source and sink nodes.
+                            continue;
+                        }
+                        signatureMatch.add(v.sequence);
+                        // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that
+                        // the sequence we've "consumed" for index i of the matched signature is also at index i in
+                        // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct
+                        // another signature match in a later call.
+                        mPendingMatches[signatureMatch.size()-1].remove(v.sequence);
+                    }
+                    // Declare success: notify observers
+                    mObservers.forEach(obs -> obs.onSignatureDetected(mSignature,
+                            Collections.unmodifiableList(signatureMatch)));
+                }
+            }
+        }
+    }
+
+    /**
+     * Encapsulates a {@code List<PcapPacket>} so as to allow the list to be used as a vertex in a graph while avoiding
+     * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph.
+     * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)}
+     * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not
+     * recognize two lists that contain the same items--from a value and not reference point of view--as the same
+     * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more
+     * appropriate.
+     */
+    private static class Vertex {
+        private final List<PcapPacket> sequence;
+        private Vertex(List<PcapPacket> wrappedSequence) {
+            sequence = wrappedSequence;
+        }
+    }
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
new file mode 100644 (file)
index 0000000..b9584ff
--- /dev/null
@@ -0,0 +1,334 @@
+package edu.uci.iotproject.detection.layer3;
+
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.detection.ClusterMatcherObserver;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
+import edu.uci.iotproject.analysis.TcpConversationUtils;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.pcap4j.core.*;
+
+import java.time.ZoneId;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static edu.uci.iotproject.util.PcapPacketUtils.*;
+
+/**
+ * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify
+ * traffic as pertaining to a given cluster).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer3ClusterMatcher extends AbstractClusterMatcher implements PacketListener {
+
+    // Test client
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+
+//        String path = "/scratch/July-2018"; // Rahmadi
+        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+        final String signatureFile = path + "/2018-07/dlink/offSignature1.sig";
+
+        List<List<PcapPacket>> signature = PrintUtils.deserializeClustersFromFile(signatureFile);
+        Layer3ClusterMatcher clusterMatcher = new Layer3ClusterMatcher(signature, null,
+                (sig, match) -> System.out.println(
+                        String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]",
+                                match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles")))
+                )
+        );
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher);
+        reader.readFromHandle();
+        clusterMatcher.performDetection();
+    }
+
+    /**
+     * The ordered directions of packets in the sequences that make up {@link #mCluster}.
+     */
+    private final Conversation.Direction[] mClusterMemberDirections;
+
+    /**
+     * For reassembling the observed traffic into TCP connections.
+     */
+    private final TcpReassembler mTcpReassembler = new TcpReassembler();
+
+    /**
+     * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
+     */
+    private final String mRouterWanIp;
+
+    /**
+     * Create a {@link Layer3ClusterMatcher}.
+     * @param cluster The cluster that traffic is matched against.
+     * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for
+     *                    determining the direction of packets).
+     * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that
+     *                          (a subset of) the examined traffic is similar to the traffic that makes up
+     *                          {@code cluster}, i.e., when the examined traffic is classified as pertaining to
+     *                          {@code cluster}.
+     */
+    public Layer3ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp,
+                                ClusterMatcherObserver... detectionObservers) {
+        super(cluster);
+        Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null");
+        for (ClusterMatcherObserver obs : detectionObservers) {
+            addObserver(obs);
+        }
+        // Build the cluster members' direction sequence.
+        // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null).
+        mClusterMemberDirections = getPacketDirections(cluster.get(0), null);
+        /*
+         * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and
+         * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later
+         * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted
+         * in order to ensure correctness, especially during the development/debugging phase.
+         */
+        if (mCluster.stream().
+                anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) {
+            throw new IllegalArgumentException(
+                    "cluster members must contain the same number of packets and exhibit the same packet direction " +
+                            "pattern"
+            );
+        }
+        mRouterWanIp = routerWanIp;
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet).
+        mTcpReassembler.gotPacket(packet);
+    }
+
+    /**
+     * Get the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
+     * @return the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
+     */
+    public List<List<PcapPacket>> getCluster() {
+        return mCluster;
+    }
+
+    public void performDetection() {
+        /*
+         * Let's start out simple by building a version that only works for signatures that do not span across multiple
+         * TCP conversations...
+         */
+        for (Conversation c : mTcpReassembler.getTcpConversations()) {
+            if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) {
+                // Skip empty conversations.
+                continue;
+            }
+            for (List<PcapPacket> signatureSequence : mCluster) {
+                if (isTlsSequence(signatureSequence) != c.isTls()) {
+                    // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+                    continue;
+                }
+                // Fetch set of packets to examine based on TLS or not.
+                List<PcapPacket> cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets();
+                /*
+                 * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases
+                 * where the same signature sequence appears multiple times in one Conversation.
+                 *
+                 * Note: since we expect all sequences that together make up the signature to exhibit the same direction
+                 * pattern, we can simply pass the precomputed direction array for the signature sequence so that it
+                 * won't have to be recomputed internally in each call to findSubsequenceInSequence().
+                 */
+                Optional<List<PcapPacket>> match;
+                while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)).
+                        isPresent()) {
+                    List<PcapPacket> matchSeq = match.get();
+                    // Notify observers about the match.
+                    mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
+                    /*
+                     * Get the index in cPkts of the last packet in the sequence of packets that matches the searched
+                     * signature sequence.
+                     */
+                    int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1));
+                    // We restart the search for the signature sequence immediately after that index, so truncate cPkts.
+                    cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList());
+                }
+            }
+            /*
+             * TODO:
+             * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did
+             * not manage to capture every single mutation of the sequence during training.
+             *
+             * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if
+             * distance between input conversation and cluster average/centroid is smaller than or equal to the computed
+             * variance?
+             */
+        }
+    }
+
+    /**
+     * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection
+     * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence}
+     * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and
+     * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src
+     * and which one is the dst, as packets in {@code sequence} may be in alternating directions).
+     * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or
+     *                 non-TLS packets.
+     * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise.
+     */
+    private boolean isTlsSequence(List<PcapPacket> sequence) {
+        // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection!
+        PcapPacket firstPkt = sequence.get(0);
+        int srcPort = getSourcePort(firstPkt);
+        int dstPort = getDestinationPort(firstPkt);
+        return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort);
+    }
+
+    /**
+     * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets
+     * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow
+     * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if
+     * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of  packet lengths
+     * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow
+     * interleaving packets, we need a modified version of
+     * <a href="https://stackoverflow.com/a/20545604/1214974">this</a>.
+     *
+     * @param subsequence The sequence to search for.
+     * @param sequence The sequence to search.
+     * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i},
+     *                              {@code subsequenceDirections[i]} is the direction of the packet returned by
+     *                              {@code subsequence.get(i)}. May be set to {@code null}, in which this call will
+     *                              internally compute the packet directions.
+     * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i},
+     *                           {@code sequenceDirections[i]} is the direction of the packet returned by
+     *                           {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally
+     *                           compute the packet directions.
+     *
+     * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty
+     *         {@link Optional} if no part of {@code sequence} matches {@code subsequence}.
+     */
+    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+                                                                 List<PcapPacket> sequence,
+                                                                 Conversation.Direction[] subsequenceDirections,
+                                                                 Conversation.Direction[] sequenceDirections) {
+        if (sequence.size() < subsequence.size()) {
+            // If subsequence is longer, it cannot be contained in sequence.
+            return Optional.empty();
+        }
+        if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
+            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+            return Optional.empty();
+        }
+        // If packet directions have not been precomputed by calling code, we need to construct them.
+        if (subsequenceDirections == null) {
+            subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
+        }
+        if (sequenceDirections == null) {
+            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+        }
+        int subseqIdx = 0;
+        int seqIdx = 0;
+        while (seqIdx < sequence.size()) {
+            PcapPacket subseqPkt = subsequence.get(subseqIdx);
+            PcapPacket seqPkt = sequence.get(seqIdx);
+            // We only have a match if packet lengths and directions match.
+            if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() &&
+                    subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+                subseqIdx++;
+                seqIdx++;
+                if (subseqIdx == subsequence.size()) {
+                    // We managed to match the entire subsequence in sequence.
+                    // Return the sublist of sequence that matches subsequence.
+                    /*
+                     * TODO:
+                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+                     * for live traces!
+                     */
+                    return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
+                }
+            } else {
+                // Mismatch.
+                if (subseqIdx > 0) {
+                    /*
+                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+                     * leave seqIdx untouched.
+                     */
+                    subseqIdx = 0;
+                } else {
+                    /*
+                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+                     * sequence.
+                     */
+                    seqIdx++;
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
+    /**
+     * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
+     * members. Two cluster members are considered identical if their packets lengths and packet directions are
+     * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the
+     * nested lists) in order to preserve its integrity when exposed to external code (e.g., through
+     * {@link #getCluster()}).
+     *
+     * @param cluster A cluster to prune.
+     * @return The resulting pruned cluster.
+     */
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = false;
+            for (List<PcapPacket> prunedClusterSeq : prunedCluster) {
+                Optional<List<PcapPacket>> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq,
+                        mClusterMemberDirections, mClusterMemberDirections);
+                if (duplicate.isPresent()) {
+                    alreadyPresent = true;
+                    break;
+                }
+            }
+            if (!alreadyPresent) {
+                prunedCluster.add(Collections.unmodifiableList(originalClusterSeq));
+            }
+        }
+        return Collections.unmodifiableList(prunedCluster);
+    }
+
+    /**
+     * Given a {@code List<PcapPacket>}, generate a {@code Conversation.Direction[]} such that each entry in the
+     * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding
+     * index in the input list.
+     * @param packets The list of packets for which to construct a corresponding array of packet directions.
+     * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when
+     *                    the traffic is captured just outside the local network (at the ISP side of the router). Set to
+     *                    {@code null} if {@code packets} stem from traffic captured within the local network.
+     * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the
+     *         corresponding index in {@code packets}.
+     */
+    private static Conversation.Direction[] getPacketDirections(List<PcapPacket> packets, String routerWanIp) {
+        Conversation.Direction[] directions = new Conversation.Direction[packets.size()];
+        for (int i = 0; i < packets.size(); i++) {
+            PcapPacket pkt = packets.get(i);
+            if (getSourceIp(pkt).equals(getDestinationIp(pkt))) {
+                // Sanity check: we shouldn't be processing loopback traffic
+                throw new AssertionError("loopback traffic detected");
+            }
+            if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) {
+                directions[i] = Conversation.Direction.CLIENT_TO_SERVER;
+            } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) {
+                directions[i] = Conversation.Direction.SERVER_TO_CLIENT;
+            } else {
+                //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction");
+            }
+        }
+        return directions;
+    }
+
+}
diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
new file mode 100644 (file)
index 0000000..0c4324d
--- /dev/null
@@ -0,0 +1,666 @@
+package edu.uci.iotproject.detection.layer3;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import edu.uci.iotproject.analysis.UserAction;
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.detection.ClusterMatcherObserver;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.jgrapht.GraphPath;
+import org.jgrapht.alg.shortestpath.DijkstraShortestPath;
+import org.jgrapht.graph.DefaultWeightedEdge;
+import org.jgrapht.graph.SimpleDirectedWeightedGraph;
+import org.pcap4j.core.*;
+
+import java.time.Duration;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.FormatStyle;
+import java.util.*;
+import java.util.function.Consumer;
+
+/**
+ * Detects an event signature that spans one or multiple TCP connections.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class SignatureDetector implements PacketListener, ClusterMatcherObserver {
+
+    // Test client
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+//        if (args.length < 3) {
+//            String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile",
+//                    SignatureDetector.class.getSimpleName());
+//            System.out.println(errMsg);
+//            return;
+//        }
+//        final String inputPcapFile = args[0];
+//        final String onSignatureFile = args[1];
+//        final String offSignatureFile = args[2];
+
+        String path = "/scratch/July-2018"; // Rahmadi
+//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+//        String path = "/home/jvarmark/iot_project/datasets"; // Hera (server)
+//        String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server)
+
+        // No activity test
+        //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap";
+
+        // D-Link Siren experiment
+//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
+//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap";
+        // D-Link Siren DEVICE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
+        // TODO: EXPERIMENT - November 19, 2018
+        // Hue Bulb experiment
+//        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
+        // Hue Bulb PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+
+        /*
+        // Kwikset Doorlock Sep 12 experiment
+//        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
+        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap";
+//        // Kwikset Doorlock PHONE signatures
+        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig";
+        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig";
+        */
+
+        // D-Link Plug experiment
+        //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap";
+//        final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap";
+
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig";
+//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig";
+//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig";
+
+        // TODO: The following are negative tests against the PCAP file from UNSW
+//        final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload!
+//          final String inputPcapFile = path + "/UNSW/16-10-12.pcap";
+
+//        final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken!
+        // TODO: The following one is very long!!! - Split into smaller files!
+//        final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap";
+
+//        final String inputPcapFile = path + "/UNSW/16-09-23.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-24.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-25.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-26.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-27.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-29.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-01.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-06.pcap";
+        // Negative test: dataset from UNB
+//        final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap";
+
+        // TODO: The following are tests for signatures against training data
+
+        // D-Link Plug experiment
+//        final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - November 7, 2018
+        // D-Link Plug experiment
+        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap";
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - November 9, 2018
+        // D-Link Siren experiment
+        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap";
+        // D-Link Siren DEVICE signatures
+        // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig";
+
+        // TP-Link Plug experiment
+////        final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap";
+//        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap";
+//        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
+        // TODO: EXPERIMENT - November 8, 2018
+        // TP-Link Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap";
+        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
+        // TP-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
+
+        // Arlo camera experiment
+//        final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+////        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 13, 2018
+        // Arlo Camera experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+        // Arlo Camera PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+
+        // Amazon Alexa experiment
+//        final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap";
+//        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig";
+
+        // SmartThings Plug experiment
+//        final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap";
+//        // SmartThings Plug DEVICE signatures
+//        //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig";
+//        //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig";
+//        // SmartThings Plug PHONE signatures
+//        final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 12, 2018
+        // SmartThings Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap";
+//        //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap";
+//        // SmartThings Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - January 9, 2018
+        // Blossom Sprinkler experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
+        // Blossom Sprinkler DEVICE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+        // Blossom Sprinkler PHONE signatures
+        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
+        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+
+        // LiFX Bulb experiment
+//        final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap";
+//        // LiFX Bulb DEVICE signatures
+//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig";
+        // LiFX Bulb PHONE signatures
+//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig";
+
+        // Blossom Sprinkler experiment
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap";
+//        // Blossom Sprinkler DEVICE signatures
+//        final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+
+        // Nest Thermostat experiment
+//        final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        // Nest Thermostat DEVICE signatures
+////        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig";
+////        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig";
+//        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 15, 2018
+        // Nest Thermostat experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap";
+////        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+
+        /*
+        // Hue Bulb experiment
+        final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
+        // Hue Bulb PHONE signatures
+        final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+        final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+        */
+
+
+
+        // TP-Link Bulb experiment
+//        final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
+//        // TP-Link Bulb PHONE signatures
+//        final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 16, 2018
+        // TP-Link Bulb experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap";
+//        // TP-Link Bulb PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+
+        /*
+        // WeMo Plug experiment
+        final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+        // WeMo Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig";
+        */
+        // TODO: EXPERIMENT - November 20, 2018
+        // WeMo Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap";
+        // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE
+        // TODO: THE ACTUAL TRIGGERS
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap";
+//        // WeMo Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
+
+        /*
+        // WeMo Insight Plug experiment
+        final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+        // WeMo Insight Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig";
+        */
+        // TODO: EXPERIMENT - November 21, 2018
+        // WeMo Insight Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap";
+        // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG)
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap";
+        // WeMo Insight Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
+
+
+        // Kwikset Doorlock Sep 12 experiment
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
+//        // Kwikset Doorlock PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig";
+        // TODO: EXPERIMENT - November 10, 2018
+        // Kwikset Door lock experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap";
+////        // Kwikset Door lock PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig";
+
+
+
+        // D-Link Siren experiment
+//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+        // D-Link Siren DEVICE signatures
+        //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
+        //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
+
+
+        // Output file names used (to make it easy to catch if one forgets to change them)
+        System.out.println("ON signature file in use is " + onSignatureFile);
+        System.out.println("OFF signature file in use is " + offSignatureFile);
+        System.out.println("PCAP file that is the target of detection is " + inputPcapFile);
+
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+
+        // LAN
+//        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
+//        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
+        // WAN
+        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", 0);
+        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", 0);
+
+        final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).
+                withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles"));
+
+        // Outputs information about a detected event to std.out
+        final Consumer<UserAction> outputter = ua -> {
+            String eventDescription;
+            switch (ua.getType()) {
+                case TOGGLE_ON:
+                    eventDescription = "ON";
+                    break;
+                case TOGGLE_OFF:
+                    eventDescription = "OFF";
+                    break;
+                default:
+                    throw new AssertionError("unhandled event type");
+            }
+            //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]",
+            //      eventDescription, dateTimeFormatter.format(ua.getTimestamp()));
+            String output = String.format("%s",
+                    dateTimeFormatter.format(ua.getTimestamp()));
+            System.out.println(output);
+        };
+
+        // Let's create observers that construct a UserAction representing the detected event.
+        final List<UserAction> detectedEvents = new ArrayList<>();
+        onDetector.addObserver((searched, match) -> {
+            PcapPacket firstPkt = match.get(0).get(0);
+            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp()));
+        });
+        offDetector.addObserver((searched, match) -> {
+            PcapPacket firstPkt = match.get(0).get(0);
+            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp()));
+        });
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector);
+        reader.readFromHandle();
+
+        // TODO: need a better way of triggering detection than this...
+        onDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
+        offDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
+
+        // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger
+        // times file.
+        Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp));
+
+        // Output the detected events
+        detectedEvents.forEach(outputter);
+
+        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count());
+        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count());
+
+
+        // TODO: Temporary clean up until we clean the pipeline
+//        List<UserAction> cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents);
+//        cleanedDetectedEvents.forEach(outputter);
+    }
+
+    /**
+     * The signature that this {@link SignatureDetector} is searching for.
+     */
+    private final List<List<List<PcapPacket>>> mSignature;
+
+    /**
+     * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the
+     * the signature.
+     */
+    private final List<Layer3ClusterMatcher> mClusterMatchers;
+
+    /**
+     * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches
+     * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e.,
+     * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed
+     * of multiple packet sequences occurring shortly after one another on multiple connections).
+     */
+    private final List<List<PcapPacket>>[] pendingMatches;
+
+    /**
+     * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}.
+     */
+    private final Map<Layer3ClusterMatcher, Integer> mClusterMatcherIds;
+
+    private final List<SignatureDetectionObserver> mObservers = new ArrayList<>();
+
+    private int mInclusionTimeMillis;
+
+    /**
+     * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions
+     * that appear multiple times.
+     * TODO: This static method is probably just for temporary and we could get rid of this after we clean up
+     * TODO:    the pipeline
+     *
+     * @param listUserAction A {@link List} of {@code UserAction}.
+     *
+     */
+    public static List<UserAction> removeDuplicates(List<UserAction> listUserAction) {
+
+        // Iterate and check for duplicates (check timestamps)
+        Set<Long> epochSecondSet = new HashSet<>();
+        // Create a target list for cleaned up list
+        List<UserAction> listUserActionClean = new ArrayList<>();
+        for(UserAction userAction : listUserAction) {
+            // Don't insert if any duplicate is found
+            if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) {
+                listUserActionClean.add(userAction);
+                epochSecondSet.add(userAction.getTimestamp().getEpochSecond());
+            }
+        }
+        return listUserActionClean;
+    }
+
+    public SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String routerWanIp, int inclusionTimeMillis) {
+        // note: doesn't protect inner lists from changes :'(
+        mSignature = Collections.unmodifiableList(searchedSignature);
+        // Generate corresponding/appropriate ClusterMatchers based on the provided signature
+        List<Layer3ClusterMatcher> clusterMatchers = new ArrayList<>();
+        for (List<List<PcapPacket>> cluster : mSignature) {
+            clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, this));
+        }
+        mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
+
+        // < exploratory >
+        pendingMatches = new List[mClusterMatchers.size()];
+        for (int i = 0; i < pendingMatches.length; i++) {
+            pendingMatches[i] = new ArrayList<>();
+        }
+        Map<Layer3ClusterMatcher, Integer> clusterMatcherIds = new HashMap<>();
+        for (int i = 0; i < mClusterMatchers.size(); i++) {
+            clusterMatcherIds.put(mClusterMatchers.get(i), i);
+        }
+        mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
+        mInclusionTimeMillis =
+                inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
+    }
+
+    public void addObserver(SignatureDetectionObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public boolean removeObserver(SignatureDetectionObserver observer) {
+        return mObservers.remove(observer);
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // simply delegate packet reception to all ClusterMatchers.
+        mClusterMatchers.forEach(cm -> cm.gotPacket(packet));
+    }
+
+    @Override
+    public void onMatch(AbstractClusterMatcher clusterMatcher, List<PcapPacket> match) {
+        // Add the match at the corresponding index
+        pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
+        checkSignatureMatch();
+    }
+
+    private void checkSignatureMatch() {
+        // << Graph-based approach using Balint's idea. >>
+        // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp.
+
+        // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence.
+        if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) {
+            // Construct the DAG
+            final SimpleDirectedWeightedGraph<Vertex, DefaultWeightedEdge> graph =
+                    new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class);
+            // Add a vertex for each match found by all ClusterMatchers
+            // And maintain an array to keep track of what cluster matcher each vertex corresponds to
+            final List<Vertex>[] vertices = new List[pendingMatches.length];
+            for (int i = 0; i < pendingMatches.length; i++) {
+                vertices[i] = new ArrayList<>();
+                for (List<PcapPacket> sequence : pendingMatches[i]) {
+                    Vertex v = new Vertex(sequence);
+                    vertices[i].add(v); // retain reference for later when we are to add edges
+                    graph.addVertex(v); // add to vertex to graph
+                }
+            }
+            // Add dummy source and sink vertices to facilitate search.
+            final Vertex source = new Vertex(null);
+            final Vertex sink = new Vertex(null);
+            graph.addVertex(source);
+            graph.addVertex(sink);
+            // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0.
+            // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node.
+            for (Vertex v : vertices[0]) {
+                DefaultWeightedEdge edge = graph.addEdge(source, v);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature
+            // are connected to the sink node.
+            for (Vertex v : vertices[vertices.length-1]) {
+                DefaultWeightedEdge edge = graph.addEdge(v, sink);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index
+            // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former).
+            for (int i = 0; i < vertices.length; i++) {
+                int j = i + 1;
+                if (j < vertices.length) {
+                    for (Vertex iv : vertices[i]) {
+                        PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1);
+                        for (Vertex jv : vertices[j]) {
+                            PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1);
+                            if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) {
+                                DefaultWeightedEdge edge = graph.addEdge(iv, jv);
+                                // The weight is the duration of the i'th sequence plus the duration between the i'th
+                                // and i+1'th sequence.
+                                Duration d = Duration.
+                                        between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp());
+                                // Unfortunately weights are double values, so must convert from long to double.
+                                // TODO: need nano second precision? If so, use d.toNanos().
+                                // TODO: risk of overflow when converting from long to double..?
+                                graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue());
+                            }
+                            // Alternative version if we cannot assume that sequences are ordered by timestamp:
+//                            if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get()
+//                                    .getTimestamp().isBefore(jv.sequence.stream().min(
+//                                            Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) {
+//
+//                            }
+                        }
+                    }
+                }
+            }
+            // Graph construction complete, run shortest-path to find a (potential) signature match.
+            DijkstraShortestPath<Vertex, DefaultWeightedEdge> dijkstra = new DijkstraShortestPath<>(graph);
+            GraphPath<Vertex, DefaultWeightedEdge> shortestPath = dijkstra.getPath(source, sink);
+            if (shortestPath != null) {
+                // The total weight is the duration between the first packet of the first sequence and the last packet
+                // of the last sequence, so we simply have to compare the weight against the timeframe that we allow
+                // the signature to span. For now we just use the inclusion window we defined for training purposes.
+                // Note however, that we must convert back from double to long as the weight is stored as a double in
+                // JGraphT's API.
+                if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) {
+                    // There's a signature match!
+                    // Extract the match from the vertices
+                    List<List<PcapPacket>> signatureMatch = new ArrayList<>();
+                    for(Vertex v : shortestPath.getVertexList()) {
+                        if (v == source || v == sink) {
+                            // Skip the dummy source and sink nodes.
+                            continue;
+                        }
+                        signatureMatch.add(v.sequence);
+                        // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that
+                        // the sequence we've "consumed" for index i of the matched signature is also at index i in
+                        // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct
+                        // another signature match in a later call.
+                        pendingMatches[signatureMatch.size()-1].remove(v.sequence);
+                    }
+                    // Declare success: notify observers
+                    mObservers.forEach(obs -> obs.onSignatureDetected(mSignature,
+                            Collections.unmodifiableList(signatureMatch)));
+                }
+            }
+        }
+    }
+
+    /**
+     * Used for registering for notifications of signatures detected by a {@link SignatureDetector}.
+     */
+    interface SignatureDetectionObserver {
+
+        /**
+         * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's
+         * examining.
+         * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching
+ &n