Adding flexibility to detection result analyzer: to not consider event type, e.g...

diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh
index 7d0689d5c3dc6b2d45d0added9820c20f4ecb380..fd6228e9b107baf7f7c3ff1c60a065af47583c6e 100755 (executable)
--- a/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh
+++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer2_smarthome_all_detection_results_analysis.sh
@@ -16,9 +16,10 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smartho
 RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.wlan1.detection.pcap___phone-side.detectionresults"
 # Put the analysis results in the same folder as the detection results.
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
 
 
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -30,13 +31,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprin
 # DEVICE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___device-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="false"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="false"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -48,13 +51,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome
 # DEVICE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___device-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -66,7 +71,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smartho
 #PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -80,8 +86,9 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
-./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+#./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
 
@@ -94,7 +101,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlo
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -108,7 +116,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -122,7 +131,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-1
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -136,7 +146,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smartho
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -148,13 +159,15 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smartho
 # DEVICE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 
 # DEVICE SIDE OUTBOUND
 RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.detection.pcap___device-side-outbound.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -168,7 +181,8 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
@@ -182,6 +196,7 @@ TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-n
 # PHONE SIDE
 RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.detection.pcap___phone-side.detectionresults"
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
-PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE'"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
 ./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
\ No newline at end of file

diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh
index 74a3810340a48a505941555ff391fdf4b24f6d5a..30f9cc86c52f42a6c69f75cb66f62b719983dbcc 100755 (executable)
--- a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh
+++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection.sh
@@ -57,7 +57,7 @@ ON_ANALYSIS="$SIGNATURES_BASE_DIR/blossom-sprinkler/analysis/blossom-sprinkler-o
 OFF_ANALYSIS="$SIGNATURES_BASE_DIR/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls"
 ON_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"
 OFF_SIGNATURE="$SIGNATURES_BASE_DIR/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"
-RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.wlan1.detection.pcap___phone-side.detectionresults"
+RESULTS_FILE="$OUTPUT_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___phone-side.detectionresults"
 SIGNATURE_DURATION="3670"
 EPSILON="10.0"
 
@@ -185,7 +185,7 @@ ON_ANALYSIS="$SIGNATURES_BASE_DIR/st-plug/analysis/st-plug-onClusters-phone-side
 OFF_ANALYSIS="$SIGNATURES_BASE_DIR/st-plug/analysis/st-plug-offClusters-phone-side.cls"
 ON_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-onSignature-phone-side.sig"
 OFF_SIGNATURE="$SIGNATURES_BASE_DIR/st-plug/signatures/st-plug-offSignature-phone-side.sig"
-RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.wlan1.detection.pcap___phone-side.detectionresults"
+RESULTS_FILE="$OUTPUT_DIR/st-plug/st-plug.eth0.detection.pcap___phone-side.detectionresults"
 SIGNATURE_DURATION="2445"
 EPSILON="10.0"
 
@@ -216,7 +216,7 @@ PROGRAM_ARGS="'$PCAP_FILE' '$ON_ANALYSIS' '$OFF_ANALYSIS' '$ON_SIGNATURE' '$OFF_
 
 
 # ==================================================== TP-LINK PLUG ====================================================
-PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"
+PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"
 
 # DEVICE SIDE (both the 112, 115 and 556, 1293 sequences)
 ON_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-onClusters.cls"
@@ -230,12 +230,14 @@ EPSILON="10.0"
 PROGRAM_ARGS="'$PCAP_FILE' '$ON_ANALYSIS' '$OFF_ANALYSIS' '$ON_SIGNATURE' '$OFF_SIGNATURE' '$RESULTS_FILE' '$SIGNATURE_DURATION' '$EPSILON'"
 ./gradlew run -DmainClass=edu.uci.iotproject.detection.layer3.Layer3SignatureDetector --args="$PROGRAM_ARGS"
 
+PCAP_FILE="$PCAPS_BASE_DIR/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"
+
 # DEVICE SIDE OUTBOUND (contains only those packets that go through the WAN port, i.e., only the 556, 1293 sequence)
 ON_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-onClusters.cls"
 OFF_ANALYSIS="$SIGNATURES_BASE_DIR/tplink-plug/analysis/tplink-plug-offClusters.cls"
 ON_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"
 OFF_SIGNATURE="$SIGNATURES_BASE_DIR/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"
-RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.wlan1.wan-detection.pcap___device-side-outbound.detectionresults"
+RESULTS_FILE="$OUTPUT_DIR/tplink-plug/tplink-plug.eth0.detection.pcap___device-side-outbound.detectionresults"
 SIGNATURE_DURATION="224"
 EPSILON="10.0"
 

diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh
new file mode 100755 (executable)
index 0000000..7eb0d3e
--- /dev/null
+++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh
@@ -0,0 +1,206 @@
+#!/bin/bash
+
+# Base directory where the smarthome evaluation traces and timestamp files are stored,
+# (i.e., /some/arbitrary/local/path/experimental_result/smarthome)
+TIMESTAMPS_BASE_DIR=$1
+readonly TIMESTAMPS_BASE_DIR
+
+# Base directory for the detection results files for the smarthome experiment
+RESULTS_BASE_DIR=$2
+readonly RESULTS_BASE_DIR
+
+
+
+# ==================================================== ARLO CAMERA =====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/arlo-camera/timestamps/arlo-camera-smarthome-nov-15-2018.timestamps"
+RESULTS_FILE="$RESULTS_BASE_DIR/arlo-camera/arlo-camera.eth0.detection.pcap___phone-side.detectionresults"
+# Put the analysis results in the same folder as the detection results.
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+
+
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= BLOSSOM SPRINKLER ==================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/blossom-sprinkler/timestamps/blossom-sprinkler-smarthome-jan-14-2019.timestamps"
+
+# DEVICE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="false"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/blossom-sprinkler/blossom-sprinkler.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="false"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK PLUG =====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-plug/timestamps/dlink-plug-smarthome-nov-8-2018.timestamps"
+
+# DEVICE SIDE
+# This one is going to generate 95 FPs because every event is counted twice (same signatures for ON and OFF).
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.eth0.detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-plug/dlink-plug.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== D-LINK SIREN ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/dlink-siren/timestamps/dlink-siren-smarthome-nov-10-2018.timestamps"
+
+#PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/dlink-siren/dlink-siren.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== HUE BULB =======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/hue-bulb/timestamps/hue-bulb-smarthome-nov-20-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/hue-bulb/hue-bulb.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+# Note: Hue bulb's signature was gone after September/October 2018, perhaps due to some firmware update.
+#./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= KWIKSET DOORLOCK ===================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/kwikset-doorlock/timestamps/kwikset-doorlock-smarthome-nov-10-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/kwikset-doorlock/kwikset-doorlock.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================= NEST THERMOSTAT ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/nest-thermostat/timestamps/nest-thermostat-smarthome-nov-16-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/nest-thermostat/nest-thermostat.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ====================================================== ST PLUG =======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/st-plug/timestamps/st-plug-smarthome-nov-13-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/st-plug/st-plug.eth0.detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK BULB ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-bulb/timestamps/tplink-bulb-smarthome-nov-19-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-bulb/tplink-bulb.wlan1.wan-detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ==================================================== TP-LINK PLUG ====================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/tplink-plug/timestamps/tplink-plug-smarthome-nov-9-2018.timestamps"
+
+# DEVICE SIDE
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.wlan1.wan-detection.pcap___device-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+
+# DEVICE SIDE OUTBOUND
+RESULTS_FILE="$RESULTS_BASE_DIR/tplink-plug/tplink-plug.eth0.detection.pcap___device-side-outbound.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ================================================== WEMO INSIGHT PLUG =================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-insight-plug/timestamps/wemo-insight-plug-smarthome-nov-22-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+# This one is going to generate >100 FPs because every event is counted twice (same signatures for ON and OFF).
+RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.wlan1.wan-detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
+
+
+
+# ===================================================== WEMO PLUG ======================================================
+TIMESTAMPS_FILE="$TIMESTAMPS_BASE_DIR/wemo-plug/timestamps/wemo-plug-smarthome-nov-21-2018.timestamps"
+
+# Has no device side signature.
+
+# PHONE SIDE
+# This one is going to generate >100 FPs because every event is counted twice (same signatures for ON and OFF).
+RESULTS_FILE="$RESULTS_BASE_DIR/wemo-plug/wemo-plug.wlan1.wan-detection.pcap___phone-side.detectionresults"
+ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
+EXACT_MATCH="true"
+PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
+./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+# ======================================================================================================================
\ No newline at end of file

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java
index 83b5b91383c932d0f3584c71ee4b5f3888b7c439..2be6de3c3116cfa0519408272d04afe46fdd9c3e 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java
@@ -87,12 +87,6 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         final int signatureDuration = Integer.parseInt(args[6]);
         final double eps = Double.parseDouble(args[7]);
 
-//        final String pcapFile = args[0];
-//        final String onSignatureFile = args[1];
-//        final String offSignatureFile = args[2];
-//        final String resultsFile = args[3];
-//        final int signatureDuration = Integer.parseInt(args[4]);
-
         // Parse optional parameters.
         List<Function<Layer2Flow, Boolean>> onSignatureMacFilters = null, offSignatureMacFilters = null;
         final int optParamsStartIdx = 7;
@@ -176,9 +170,9 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         // Parse the file
         reader.readFromHandle();
 
-        String resultOn = "Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+        String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
                 detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count();
-        String resultOff = "Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+        String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
                 detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count();
         PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java
index 4afa21612547a9bed682bfd1fb2de372cd047338..0fc43be5e8a665c15711b76c682a77b66b543db9 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java
@@ -177,9 +177,9 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb
         // Output the detected events
         detectedEvents.forEach(outputter);
 
-        String resultOn = "Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+        String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
                 detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count();
-        String resultOff = "Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+        String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
                 detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count();
         PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java
index 34063d5a7f5e6635d3ee8547a5f3830cc012fa52..8eaa78b58a14998b90b8c66e33c77bf91e8c34a3 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java
@@ -20,22 +20,28 @@ import java.util.Optional;
 public class DetectionResultsAnalyzer {
 
     private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
+    private static boolean DETECTED_EVENT_EXACT_MATCH;
 
     public static void main(String[] args) throws IOException {
-        if (args.length < 3) {
+        if (args.length < 4) {
             String errMsg = String.format("Usage: %s triggerTimesFile detectionOutputFile [stdOut]" +
                             "\n - triggerTimesFile: the file that contains the timestamps for the user actions" +
                             "\n - detectionOutputFile: the file that contains the detected events" +
                             "\n - analysisResultsFile: where to write the results of the detection analysis" +
-                            "\n - stdOut: optional true/false literal indicating if output should also be printed to std out; default is true",
+                            "\n - matchEventType: true/false literal indicating if a detected event should" +
+                                " have a matching type" +
+                            "\n - stdOut: optional true/false literal indicating if output should also be printed to" +
+                                " std out; default is true",
                     DetectionResultsAnalyzer.class.getSimpleName());
+            System.out.println(errMsg);
             return;
         }
         String triggerTimesFile = args[0];
         File detectionOutputFile = new File(args[1]);
         String analysisResultsFile = args[2];
-        if (args.length > 3) {
-            DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[3]);
+        DETECTED_EVENT_EXACT_MATCH = Boolean.parseBoolean(args[3]);
+        if (args.length > 4) {
+            DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[4]);
         }
 
         // -------------------------------------- Parse the input files --------------------------------------
@@ -68,20 +74,39 @@ public class DetectionResultsAnalyzer {
 
         // To contain all detected events that could be mapped to a trigger
         List<UserAction> truePositives = new ArrayList<>();
-        for (UserAction detectedEvent : detectedEvents) {
-            Optional<UserAction> matchingTrigger = triggers.stream()
-                    .filter(t -> t.getType() == detectedEvent.getType() &&
-                            t.getTimestamp().isBefore(detectedEvent.getTimestamp()) &&
-                            t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS).
-                                    isAfter(detectedEvent.getTimestamp())
-                    ).findFirst();
-            matchingTrigger.ifPresent(mt -> {
-                // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with
-                // another detected event.
-                triggers.remove(mt);
-                // The current detected event was a true positive as we could match it with a trigger.
-                truePositives.add(detectedEvent);
-            });
+        if (DETECTED_EVENT_EXACT_MATCH) {
+            for (UserAction detectedEvent : detectedEvents) {
+                Optional<UserAction> matchingTrigger = triggers.stream()
+                        .filter(t -> t.getType() == detectedEvent.getType() &&
+                                t.getTimestamp().isBefore(detectedEvent.getTimestamp()) &&
+                                t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS).
+                                        isAfter(detectedEvent.getTimestamp())
+                        ).findFirst();
+                matchingTrigger.ifPresent(mt -> {
+                    // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with
+                    // another detected event.
+                    triggers.remove(mt);
+                    // The current detected event was a true positive as we could match it with a trigger.
+                    truePositives.add(detectedEvent);
+                });
+            }
+            // TODO: Experimental
+        } else { // DETECTED_EVENT_EXACT_MATCH == false
+            for (UserAction detectedEvent : detectedEvents) {
+                Optional<UserAction> matchingTrigger = triggers.stream()
+                        .filter(t ->
+                                t.getTimestamp().isBefore(detectedEvent.getTimestamp()) &&
+                                t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS).
+                                        isAfter(detectedEvent.getTimestamp())
+                        ).findFirst();
+                matchingTrigger.ifPresent(mt -> {
+                    // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with
+                    // another detected event.
+                    triggers.remove(mt);
+                    // The current detected event was a true positive as we could match it with a trigger.
+                    truePositives.add(detectedEvent);
+                });
+            }
         }
         // Now the false positives are those elements in detectedEvents that are not in truePositives
         List<UserAction> falsePositives = new ArrayList<>();
@@ -90,7 +115,7 @@ public class DetectionResultsAnalyzer {
 
         // Output the results...
         PrintWriter outputter = new PrintWriter(new FileWriter(analysisResultsFile));
-        PrintWriterUtils.println("---------- False negatives (events that where not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("---------- False negatives (events that were not detected) ----------", outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
         for (UserAction missing : triggers) {
             PrintWriterUtils.println(missing, outputter, DUPLICATE_OUTPUT_TO_STD_OUT);
         }