X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=blobdiff_plain;f=Code%2FProjects%2FPacketLevelSignatureExtractor%2Fsrc%2Fmain%2Fjava%2Fedu%2Fuci%2Fiotproject%2Fdetection%2Flayer3%2FLayer3SignatureDetector.java;h=03e4bd1cb242531bd6a4858eaba01a2de4b054b5;hp=859c056a1508ded480d6824e21dbc1d0a00516c2;hb=987ea910fed24a1f3f51ded41b6aa98c4e2618ae;hpb=92a31b0967b3acff9e473ce62c136e84298b3aab diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java index 859c056..03e4bd1 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java @@ -5,6 +5,7 @@ import edu.uci.iotproject.analysis.UserAction; import edu.uci.iotproject.detection.AbstractClusterMatcher; import edu.uci.iotproject.detection.ClusterMatcherObserver; import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.apache.commons.math3.distribution.AbstractRealDistribution; @@ -15,6 +16,10 @@ import org.jgrapht.graph.DefaultWeightedEdge; import org.jgrapht.graph.SimpleDirectedWeightedGraph; import org.pcap4j.core.*; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.time.Duration; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -30,341 +35,61 @@ import java.util.function.Consumer; */ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherObserver { - // Test client - public static void main(String[] args) throws PcapNativeException, NotOpenException { -// if (args.length < 3) { -// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", -// SignatureDetector.class.getSimpleName()); -// System.out.println(errMsg); -// return; -// } -// final String inputPcapFile = args[0]; -// final String onSignatureFile = args[1]; -// final String offSignatureFile = args[2]; - - String path = "/scratch/July-2018"; // Rahmadi -// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus -// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) -// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) - - // No activity test - //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; - - // D-Link Siren experiment -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; - // D-Link Siren DEVICE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - // TODO: EXPERIMENT - November 19, 2018 - // Hue Bulb experiment -// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - - /* - // Kwikset Doorlock Sep 12 experiment -// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; - final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; -// // Kwikset Doorlock PHONE signatures - final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; - final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; - */ - - // D-Link Plug experiment - //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; - - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; - - // TODO: The following are negative tests against the PCAP file from UNSW -// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! - // TODO: The following one is very long!!! - Split into smaller files! -// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; - // Negative test: dataset from UNB -// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap"; - - // TODO: The following are tests for signatures against training data - - // TODO: D-LINK PLUG experiment -// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; - - // TODO: EXPERIMENT - November 7, 2018 - // D-Link Plug experiment -// //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; -// // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-device-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-device-side.cls"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 9, 2018 - // TODO: D-LINK SIREN experiment - //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; -// // D-Link Siren DEVICE signatures -// // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-offClusters-phone-side.cls"; - - // TODO: TP-LINK PLUG experiment -//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; -// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; - // TODO: EXPERIMENT - November 8, 2018 - // TP-Link Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; -//// // TP-Link Plug DEVICE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; -// // TP-Link Plug PHONE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; -// // TP-Link Plug cluster analyses -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-onClusters.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-offClusters.cls"; - - // Amazon Alexa experiment -// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; - - // TODO: KWIKSET DOORLOCK Sep 12 experiment -// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; -// // Kwikset Doorlock PHONE signatures -// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; -// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; - // TODO: EXPERIMENT - November 10, 2018 - // Kwikset Door lock experiment -// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; -// // Kwikset Door lock PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; -//// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; -//// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-offClusters-phone-side.cls"; - - // TODO: SMARTTHINGS PLUG experiment -// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// // SmartThings Plug DEVICE signatures -// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; -// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; -// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 12, 2018 - // SmartThings Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; -//// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-offClusters-phone-side.cls"; - - // TODO: ARLO CAMERA experiment -// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -//// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 13, 2018 - // Arlo Camera experiment -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; -// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; - // Arlo Camera PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete"; -// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls"; - - // TODO: NEST THERMOSTAT experiment -// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -// // Nest Thermostat DEVICE signatures -//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; -// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; -// // TODO: EXPERIMENT - November 15, 2018 - // Nest Thermostat experiment -// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; -//// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - January 9, 2018 - // TODO: BLOSSOM SPRINKLER experiment - // Blossom Sprinkler experiment -// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; -// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; -// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; -// // Blossom Sprinkler DEVICE signatures -// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; - -// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; - final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; - // Blossom Sprinkler DEVICE signatures - final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; - final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; - final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls"; - final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls"; - // Blossom Sprinkler PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls"; - - // LiFX Bulb experiment -// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; -// // LiFX Bulb DEVICE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; - // LiFX Bulb PHONE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; - - /* - // Hue Bulb experiment - final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures - final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; - final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - */ - - // TODO: TP-LINK BULB experiment -// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 16, 2018 - // TP-Link Bulb experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 20, 2018 - // TODO: WEMO PLUG experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; - // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE - // TODO: THE ACTUAL TRIGGERS -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; -// // WeMo Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 21, 2018 - // TODO: WEMO INSIGHT Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; - // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; - // WeMo Insight Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-offClusters-phone-side.cls"; - - // Output file names used (to make it easy to catch if one forgets to change them) - System.out.println("ON signature file in use is " + onSignatureFile); - System.out.println("OFF signature file in use is " + offSignatureFile); - System.out.println("PCAP file that is the target of detection is " + inputPcapFile); - - // Specify epsilon - // TODO: This would be specified through command line option - double eps = 10.0; + /** + * If set to {@code true}, output written to the results file is also dumped to standard out. + */ + private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + + /** + * Router's IP. + */ + private static String ROUTER_WAN_IP = "128.195.205.105"; + + public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { + if (args.length < 8) { + String errMsg = String.format("SPECTO version 1.0\n" + + "Copyright (C) 2018-2019 Janus Varmarken and Rahmadi Trimananda.\n" + + "University of California, Irvine.\n" + + "All rights reserved.\n\n" + + "Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile resultsFile" + + "\n inputPcapFile: the target of the detection" + + "\n onAnalysisFile: the file that contains the ON clusters analysis" + + "\n offAnalysisFile: the file that contains the OFF clusters analysis" + + "\n onSignatureFile: the file that contains the ON signature to search for" + + "\n offSignatureFile: the file that contains the OFF signature to search for" + + "\n resultsFile: where to write the results of the detection" + + "\n signatureDuration: the maximum duration of signature detection" + + "\n epsilon: the epsilon value for the DBSCAN algorithm", + Layer3SignatureDetector.class.getSimpleName()); + System.out.println(errMsg); + return; + } + final String pcapFile = args[0]; + final String onClusterAnalysisFile = args[1]; + final String offClusterAnalysisFile = args[2]; + final String onSignatureFile = args[3]; + final String offSignatureFile = args[4]; + final String resultsFile = args[5]; + // TODO: THIS IS TEMPORARILY SET TO DEFAULT SIGNATURE DURATION + // TODO: WE DO NOT WANT TO BE TOO STRICT AT THIS POINT SINCE LAYER 3 ALREADY APPLIES BACK-TO-BACK REQUIREMENT + // TODO: FOR PACKETS IN A SIGNATURE +// final int signatureDuration = Integer.parseInt(args[6]); + final int signatureDuration = TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS; + final double eps = Double.parseDouble(args[7]); + + // Prepare file outputter. + File outputFile = new File(resultsFile); + outputFile.getParentFile().mkdirs(); + final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile)); + // Include metadata as comments at the top + PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + resultsWriter.flush(); + // Load signatures List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); @@ -379,16 +104,17 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); // Update the signature with ranges if it is range-based - if (isRangeBasedForOn && isRangeBasedForOff) { + if (isRangeBasedForOn) { onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); + } + if (isRangeBasedForOff) { offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); } - // WAN - SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", - 0, isRangeBasedForOn, eps); - SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", - 0, isRangeBasedForOff, eps); + Layer3SignatureDetector onDetector = new Layer3SignatureDetector(onSignature, ROUTER_WAN_IP, + signatureDuration, isRangeBasedForOn, eps); + Layer3SignatureDetector offDetector = new Layer3SignatureDetector(offSignature, ROUTER_WAN_IP, + signatureDuration, isRangeBasedForOff, eps); final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); @@ -406,39 +132,45 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb default: throw new AssertionError("unhandled event type"); } - //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", - // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); - String output = String.format("%s", - dateTimeFormatter.format(ua.getTimestamp())); - System.out.println(output); + // TODO: Uncomment the following if we want the old style print-out messages + // String output = String.format("%s", + // dateTimeFormatter.format(ua.getTimestamp())); + // System.out.println(output); + PrintWriterUtils.println(ua, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); }; // Let's create observers that construct a UserAction representing the detected event. final List detectedEvents = new ArrayList<>(); onDetector.addObserver((searched, match) -> { PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); + UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp()); + detectedEvents.add(event); }); offDetector.addObserver((searched, match) -> { PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); + UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp()); + //PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + detectedEvents.add(event); }); PcapHandle handle; try { - handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + handle = Pcaps.openOffline(pcapFile, PcapHandle.TimestampPrecision.NANO); } catch (PcapNativeException pne) { - handle = Pcaps.openOffline(inputPcapFile); + handle = Pcaps.openOffline(pcapFile); } PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); reader.readFromHandle(); // TODO: need a better way of triggering detection than this... - if (isRangeBasedForOn && isRangeBasedForOff) { + if (isRangeBasedForOn) { onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); - offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); } else { onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); + } + if (isRangeBasedForOff) { + offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); + } else { offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); } @@ -449,18 +181,23 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb // Output the detected events detectedEvents.forEach(outputter); - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); + String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count(); + String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count(); + PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + // Flush output to results file and close it. + resultsWriter.flush(); + resultsWriter.close(); // TODO: Temporary clean up until we clean the pipeline // List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); // cleanedDetectedEvents.forEach(outputter); } /** - * The signature that this {@link SignatureDetector} is searching for. + * The signature that this {@link Layer3SignatureDetector} is searching for. */ private final List>> mSignature; @@ -473,7 +210,7 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb /** * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., - * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed + * have yet to be included in a signature detected by this {@link Layer3SignatureDetector} (a signature can be encompassed * of multiple packet sequences occurring shortly after one another on multiple connections). */ private final List>[] pendingMatches; @@ -519,7 +256,8 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb // Generate corresponding/appropriate ClusterMatchers based on the provided signature List clusterMatchers = new ArrayList<>(); for (List> cluster : mSignature) { - clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, isRangeBased, eps, this)); + clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, inclusionTimeMillis, + isRangeBased, eps, this)); } mClusterMatchers = Collections.unmodifiableList(clusterMatchers); @@ -659,14 +397,14 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb } /** - * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. + * Used for registering for notifications of signatures detected by a {@link Layer3SignatureDetector}. */ interface SignatureDetectionObserver { /** - * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's + * Invoked when the {@link Layer3SignatureDetector} detects the presence of a signature in the traffic that it's * examining. - * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching + * @param searchedSignature The signature that the {@link Layer3SignatureDetector} reporting the match is searching * for. * @param matchingTraffic The actual traffic trace that matches the searched signature. */