public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException {
// -------------------------------------------------------------------------------------------------------------
// ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------
- // Paths to input and output files (consider supplying these as arguments instead) and IP of the device for
- // which traffic is to be extracted:
- String path = "/scratch/July-2018"; // Rahmadi
-// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+ if (args.length < 11) {
+ String errMsg = String.format("Usage: %s inputPcapFile outputPcapFile triggerTimesFile deviceIp" +
+ " onSignatureFile offSignatureFile onClusterAnalysisFile offClusterAnalysisFile epsilon" +
+ " deletedSequencesOn deletedSequencesOff" +
+ "\n inputPcapFile: the target of the detection" +
+ "\n outputPcapFile: the processed PCAP file through 15-second window filtering" +
+ "\n triggerTimesFile: the trigger timestamps" +
+ "\n deviceIp: the IP address of the device we want to generate a signature for" +
+ "\n onSignatureFile: name of the ON signature file" +
+ "\n offSignatureFile: name of the OFF signature file" +
+ "\n onClusterAnalysisFile: name of the ON signature cluster analysis file" +
+ "\n offClusterAnalysisFile: name of the OFF signature cluster analysis file" +
+ "\n epsilon: epsilon value of the DBSCAN algorithm" +
+ "\n deletedSequencesOn: sequences to be deleted from the final ON signature" +
+ " (please separate with commas, e.g., 0,1,2, or put '-1' if not needed)" +
+ "\n deletedSequencesOff: sequences to be deleted from the final OFF signature" +
+ " (please separate with commas, e.g., 0,1,2, or put '-1' if not needed)",
+ SignatureGenerator.class.getSimpleName());
+ System.out.println(errMsg);
+ return;
+ }
boolean verbose = true;
- final String onPairsPath = "/scratch/July-2018/on.txt";
- final String offPairsPath = "/scratch/July-2018/off.txt";
-
- // 1) TODO: D-LINK PLUG July 26 experiment
-// final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-07/dlink/dlink-processed.pcap";
-// final String triggerTimesFile = path + "/2018-07/dlink/dlink-july-26-2018.timestamps";
-// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
- // Actual training
-// final String inputPcapFile = path + "/2018-10/dlink-plug/dlink-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/dlink-plug/dlink-plug-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/dlink-plug/dlink-plug-oct-17-2018.timestamps";
-// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
- // TODO: EXPERIMENT - November 7, 2018
- final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
- final String outputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug-processed.pcap";
- final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
-// final String deviceIp = "192.168.1.199"; // .246 == phone; .199 == dlink plug?
- final String deviceIp = "192.168.1.246"; // .246 == phone; .199 == dlink plug?
-
- // 2) TODO: TP-LINK PLUG July 25 experiment
-// final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.pcap";
-// final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.timestamps";
-// final String deviceIp = "192.168.1.159";
- // Actual training
-// final String inputPcapFile = path + "/2018-10/tplink-plug/tplink-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/tplink-plug/tplink-plug-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/tplink-plug/tplink-plug-oct-17-2018.timestamps";
-// final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug
- // TODO: EXPERIMENT - November 8, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps";
-// final String deviceIp = "192.168.1.159"; // .246 == phone; .159 == tplink plug
-//// final String deviceIp = "192.168.1.246"; // .246 == phone; .159 == tplink plug
-
- // 2b) TP-Link July 25 experiment TRUNCATED:
- // Only contains "true local" events, i.e., before the behavior changes to remote-like behavior.
- // Last included event is at July 25 10:38:11; file filtered to only include packets with arrival time <= 10:38:27.
-// final String inputPcapFile = path + "/2018-07/tplink/tplink.wlan1.local.truncated.pcap";
-// final String outputPcapFile = path + "/2018-07/tplink/tplink-processed.truncated.pcap";
-// final String triggerTimesFile = path + "/2018-07/tplink/tplink-july-25-2018.truncated.timestamps";
-// final String deviceIp = "192.168.1.159";
-
- // 3) TODO: SMARTTHINGS PLUG July 25 experiment
-// final String inputPcapFile = path + "/2018-07/stplug/stplug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-07/stplug/stplug-processed.pcap";
-// final String triggerTimesFile = path + "/2018-07/stplug/smartthings-july-25-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
- // October 18
-// final String inputPcapFile = path + "/2018-10/st-plug/st-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/st-plug/st-plug-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/st-plug/st-plug-oct-18-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
- // TODO: EXPERIMENT - November 12, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug-processed.pcap";
-//// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug.eth1.local.pcap";
-//// final String outputPcapFile = path + "/experimental_result/standalone/st-plug/eth1/st-plug-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps";
-//// final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
-
- // 4) TODO: WEMO PLUG July 30 experiment
-// final String inputPcapFile = path + "/2018-07/wemo/wemo.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-07/wemo/wemo-processed.pcap";
-// final String triggerTimesFile = path + "/2018-07/wemo/wemo-july-30-2018.timestamps";
-// final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo
- // TODO: EXPERIMENT - November 20, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps";
-//// final String deviceIp = "192.168.1.145"; // .246 == phone; .145 == WeMo
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .145 == WeMo
-
- // 5) TODO: WEMO INSIGHT July 31 experiment
-// final String inputPcapFile = path + "/2018-07/wemoinsight/wemoinsight.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-07/wemoinsight/wemoinsight-processed.pcap";
-// final String triggerTimesFile = path + "/2018-07/wemoinsight/wemo-insight-july-31-2018.timestamps";
-// final String deviceIp = "192.168.1.135";
- // TODO: EXPERIMENT - November 21, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
-//// final String deviceIp = "192.168.1.145"; // .246 == phone; .135 == WeMo Insight
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .135 == WeMo Insight
-
- // 6) TODO: TP-LINK BULB August 1 experiment
-// final String inputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/tplink-bulb/tplinkbulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/tplink-bulb/tplink-bulb-aug-3-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb
- // TODO: EXPERIMENT - November 16, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb-processed.pcap";
-//// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth1.local.pcap";
-//// final String outputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps";
-//// final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == TP-Link bulb
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == TP-Link bulb
-
- // 7) TODO: KWIKSET DOORLOCK August 6 experiment
-// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
-//// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-processed.pcap";
-//// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-aug-6-2018.timestamps";
-// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-8hr-data-oct-11-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
- // TODO: EXPERIMENT - November 10, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock-processed.pcap";
-//// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock.eth1.local.pcap";
-//// final String outputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/eth1/kwikset-doorlock-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps";
-//// final String deviceIp = "192.168.1.142"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
-
- // September 12, 2018 - includes both wlan1 and eth1 interfaces
-// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
-// //final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.eth1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/kwikset-doorlock/kwikset-doorlock-sept-12-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .142 == SmartThings Hub (note: use eth0 capture for this!)
-
- // 8) TODO: HUE BULB August 7 experiment
-// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/hue-bulb/hue-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
-// final String deviceIp = "192.168.1.246";
- // October 30 experiment
-// final String inputPcapFile = path + "/2018-10/hue-bulb/hue-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/hue-bulb/hue-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/hue-bulb/hue-bulb-oct-30-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
- // TODO: EXPERIMENT - November 19, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/hue-bulb/wlan1/hue-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/hue-bulb/timestamps/hue-bulb-nov-19-2018.timestamps";
-//// final String deviceIp = "192.168.1.100"; // .246 == phone; .100 == Hue hub
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .100 == Hue hub
-
- // 9) TODO: LIFX BULB August 8 experiment
-// final String inputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/lifx-bulb/lifx-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/lifx-bulb/lifx-bulb-aug-8-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .231 == Lifx
- // October 18
-// final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-oct-18-2018.timestamps";
-// final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx
- // November 1
-// final String inputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/lifx-bulb/lifx-bulb-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/lifx-bulb/lifx-bulb-nov-1-2018.timestamps";
-// final String deviceIp = "192.168.1.231"; // .246 == phone; .231 == Lifx
-
- // 10) TODO: AMCREST CAMERA August 9 experiment
-// final String inputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/amcrest-camera/amcrest-camera-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/amcrest-camera/amcrest-camera-aug-9-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .235 == camera
-
- // 11) TODO: ARLO CAMERA August 10 experiment
-// final String inputPcapFile = path + "/2018-08/arlo-camera/arlo-camera.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/arlo-camera/arlo-camera-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/arlo-camera/arlo-camera-aug-10-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
- // TODO: EXPERIMENT - November 13, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera-processed.pcap";
-//// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth1.local.pcap";
-//// final String outputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
-//// final String deviceIp = "192.168.1.140"; // .246 == phone; .140 == camera
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .140 == camera
-
- // 12) TODO: BLOSSOM SPRINKLER August 13 experiment
-// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/blossom/blossom-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/blossom/blossom-aug-13-2018.timestamps";
-// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
-// // 2 November
-// final String inputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-processed.pcap";
-// final String triggerTimesFile = path + "/2018-10/blossom-sprinkler/blossom-sprinkler-nov-2-2018.timestamps";
-// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
- // TODO: EXPERIMENT - January 9, 11, 13, 14
-// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
-//// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-11-2019.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .229 == sprinkler
-//// final String deviceIp = "192.168.1.229"; // .246 == phone; .229 == sprinkler
-
-// // 13) TODO: D-LINK SIREN August 14 experiment
-// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
-// //final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
-// final String outputPcapFile = path + "/2018-08/dlink-siren/dlink-siren-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-oct-12-2018.timestamps";
-// //final String triggerTimesFile = path + "/2018-08/dlink-siren/dlink-siren-aug-14-2018.timestamps";
-// //final String triggerTimesFile = path + "/actual/timestamps/dlink-siren-8hr-data-oct-10-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
- // TODO: EXPERIMENT - November 9, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
-//// final String deviceIp = "192.168.1.183"; // .246 == phone; .183 == siren
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .183 == siren
-
- // 14) TODO: NEST THERMOSTAT August 15 experiment
-// final String inputPcapFile = path + "/2018-08/nest/nest.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/nest/nest-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/nest/nest-aug-15-2018.timestamps";
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
-// // TODO: EXPERIMENT - November 14, 2018
-// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-// final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat-processed.pcap";
-//// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth1.local.pcap";
-//// final String outputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat-processed.pcap";
-// final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
-//// final String deviceIp = "192.168.1.127"; // .246 == phone; .127 == Nest thermostat
-// final String deviceIp = "192.168.1.246"; // .246 == phone; .127 == Nest thermostat
-
- // 15) Alexa August 16 experiment
-// final String inputPcapFile = path + "/2018-08/alexa/alexa.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/alexa/alexa-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-16-2018.timestamps";
-// final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa
- // August 17
-// final String inputPcapFile = path + "/2018-08/alexa/alexa2.wlan1.local.pcap";
-// final String outputPcapFile = path + "/2018-08/alexa/alexa2-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/alexa/alexa-aug-17-2018.timestamps";
-// final String deviceIp = "192.168.1.225"; // .246 == phone; .225 == Alexa
-
- // September 17
-// final String inputPcapFile = path + "/2018-08/noise/noise.eth1.pcap";
-// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/noise/noise-sept-17-2018.timestamps";
-// final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub; .199 == dlink plug; .183 == siren
- // September 26 - D-Link noise
-// final String inputPcapFile = path + "/2018-08/noise/noise.dlink.wlan1.pcap";
-// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/noise/dlink-noise-sept-26-2018.timestamps";
-// final String deviceIp = "192.168.1.183"; // .199 == dlink plug; .183 == siren
- // September 27 - Kwikset noise
-// final String inputPcapFile = path + "/2018-08/noise/noise.kwikset.eth1.pcap";
-// final String outputPcapFile = path + "/2018-08/noise/noise-processed.pcap";
-// final String triggerTimesFile = path + "/2018-08/noise/kwikset-doorlock-noise-sept-27-2018.timestamps";
-// final String deviceIp = "192.168.1.142"; // .142 == SmartThings Hub;
-
- // TODO: The below part is just for 15-second time sensitivity experiment
- // TODO: The below part is just for 15-second time sensitivity experiment
- // TODO: The below part is just for 15-second time sensitivity experiment
- // D-Link plug
-// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-plug/timestamps/dlink-plug-nov-7-2018.timestamps";
-//// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
-//// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
-// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
- // TP-Link plug
-// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-plug/timestamps/tplink-plug-nov-8-2018.timestamps";
-////// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
-////// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
-//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
-//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
-// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
-
- // D-Link siren
-// final String triggerTimesFile = path + "/experimental_result/standalone/dlink-siren/timestamps/dlink-siren-nov-9-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
- // Kwikset door lock
-// final String triggerTimesFile = path + "/experimental_result/standalone/kwikset-doorlock/timestamps/kwikset-doorlock-nov-10-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
- // SmartThings plug
-// final String triggerTimesFile = path + "/experimental_result/standalone/st-plug/timestamps/st-plug-nov-12-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
- // Arlo Q
-// final String triggerTimesFile = path + "/experimental_result/standalone/arlo-camera/timestamps/arlo-camera-nov-13-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.complete.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.complete.sig";
- // Nest thermostat
-// final String triggerTimesFile = path + "/experimental_result/standalone/nest-thermostat/timestamps/nest-thermostat-nov-15-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
- // Blossom sprinkler
-// final String triggerTimesFile = path + "/experimental_result/standalone/blossom-sprinkler/timestamps/blossom-sprinkler-standalone-jan-14-2019.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
-// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
- // TP-Link bulb
-// final String triggerTimesFile = path + "/experimental_result/standalone/tplink-bulb/timestamps/tplink-bulb-nov-16-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
- // Philips hue
-// final String triggerTimesFile = path + "/2018-08/hue-bulb/hue-bulb-aug-7-2018.timestamps";
-// final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
- // WeMo plug
-// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-plug/timestamps/wemo-plug-nov-20-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
- // WeMo Insight plug
-// final String triggerTimesFile = path + "/experimental_result/standalone/wemo-insight-plug/timestamps/wemo-insight-plug-nov-21-2018.timestamps";
-// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
-// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
-
-
-// TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
-// List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
-//
-// System.out.println("ON signature file in use is " + onSignatureFile);
-// System.out.println("OFF signature file in use is " + offSignatureFile);
-//
-// List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile);
-// List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile);
-//
-// List<Instant> signatureTimestamps = new ArrayList<>();
-// // Load ON signature last packet's timestamp
-// // Get the last only
-// List<List<PcapPacket>> lastListOn = onSignature.get(onSignature.size()-1);
-// for (List<PcapPacket> list : lastListOn) {
-// // Get timestamp Instant from the last packet
-// int lastPacketIndex = list.size()-1;
-// signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
-// }
-// // Load OFF signature last packet's timestamp
-// // Get the last only
-// List<List<PcapPacket>> lastListOff = offSignature.get(offSignature.size()-1);
-// for (List<PcapPacket> list : lastListOff) {
-// // Get timestamp Instant from the last packet
-// int lastPacketIndex = list.size()-1;
-// signatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
-// }
-// // Sort the timestamps
-// signatureTimestamps.sort((p1, p2) -> {
-// return p1.compareTo(p2);
-// });
-//
-// Iterator<Instant> iterTrig = triggerTimes.iterator();
-// Iterator<Instant> iterSign = signatureTimestamps.iterator();
-// System.out.println("Trigger to Last Packet:");
-// while (iterTrig.hasNext() && iterSign.hasNext()) {
-// Instant trigInst = (Instant) iterTrig.next();
-// Instant signInst = (Instant) iterSign.next();
-// Duration dur = Duration.between(trigInst, signInst);
-// long duration = dur.toMillis();
-// // Check duration --- should be below 15 seconds
-// if (duration >= 0 && duration <= 15000) {
-// System.out.println(dur.toMillis());
-// } else if (duration > 15000) {
-// while (duration > 15000) { // that means we have to move to the next trigger
-// trigInst = (Instant) iterTrig.next();
-// dur = Duration.between(trigInst, signInst);
-// duration = dur.toMillis();
-// }
-// System.out.println(dur.toMillis());
-// } else { // below 0 / negative --- that means we have to move to the next signature
-// while (duration < 0) { // that means we have to move to the next trigger
-// signInst = (Instant) iterSign.next();
-// dur = Duration.between(trigInst, signInst);
-// duration = dur.toMillis();
-// }
-// System.out.println(dur.toMillis());
-// }
-// }
-//
-//
-// // ==========================================================================
-// List<Instant> firstSignatureTimestamps = new ArrayList<>();
-// List<Instant> lastSignatureTimestamps = new ArrayList<>();
-// List<List<PcapPacket>> firstListOnSign = onSignature.get(0);
-// List<List<PcapPacket>> lastListOnSign = onSignature.get(onSignature.size()-1);
-// // Load ON signature first and last packet's timestamps
-// for (List<PcapPacket> list : firstListOnSign) {
-// // Get timestamp Instant from the last packet
-// firstSignatureTimestamps.add(list.get(0).getTimestamp());
-// }
-// for (List<PcapPacket> list : lastListOnSign) {
-// // Get timestamp Instant from the last packet
-// int lastPacketIndex = list.size()-1;
-// lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
-// }
-//
-// List<List<PcapPacket>> firstListOffSign = offSignature.get(0);
-// List<List<PcapPacket>> lastListOffSign = offSignature.get(offSignature.size()-1);
-// // Load OFF signature first and last packet's timestamps
-// for (List<PcapPacket> list : firstListOffSign) {
-// // Get timestamp Instant from the last packet
-// firstSignatureTimestamps.add(list.get(0).getTimestamp());
-// }
-// for (List<PcapPacket> list : lastListOffSign) {
-// // Get timestamp Instant from the last packet
-// int lastPacketIndex = list.size()-1;
-// lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
-// }
-// // Sort the timestamps
-// firstSignatureTimestamps.sort((p1, p2) -> {
-// return p1.compareTo(p2);
-// });
-// // Sort the timestamps
-// lastSignatureTimestamps.sort((p1, p2) -> {
-// return p1.compareTo(p2);
-// });
-//
-// Iterator<Instant> iterFirst = firstSignatureTimestamps.iterator();
-// Iterator<Instant> iterLast = lastSignatureTimestamps.iterator();
-// System.out.println("First to Last Packet:");
-// while (iterFirst.hasNext() && iterLast.hasNext()) {
-// Instant firstInst = (Instant) iterFirst.next();
-// Instant lastInst = (Instant) iterLast.next();
-// Duration dur = Duration.between(firstInst, lastInst);
-// long duration = dur.toMillis();
-// // Check duration --- should be below 15 seconds
-// if (duration >= 0 && duration <= 15000) {
-// System.out.println(dur.toMillis());
-// } else if (duration > 15000) {
-// while (duration > 15000) { // that means we have to move to the next trigger
-// firstInst = (Instant) iterFirst.next();
-// dur = Duration.between(firstInst, lastInst);
-// duration = dur.toMillis();
-// }
-// System.out.println(dur.toMillis());
-// } else { // below 0 / negative --- that means we have to move to the next signature
-// while (duration < 0) { // that means we have to move to the next trigger
-// lastInst = (Instant) iterLast.next();
-// dur = Duration.between(firstInst, lastInst);
-// duration = dur.toMillis();
-// }
-// System.out.println(dur.toMillis());
-// }
-// if (duration > 8000) {
-// break;
-// }
-// }
-
- // TODO: The above part is just for 15-second time sensitivity experiment
- // TODO: The above part is just for 15-second time sensitivity experiment
- // TODO: The above part is just for 15-second time sensitivity experiment
-
-
-
+ final String inputPcapFile = args[0];
+ final String outputPcapFile = args[1];
+ final String triggerTimesFile = args[2];
+ final String deviceIp = args[3];
+ final String onSignatureFile = args[4];
+ final String offSignatureFile = args[5];
+ final String onClusterAnalysisFile = args[6];
+ final String offClusterAnalysisFile = args[7];
+ final double eps = Double.parseDouble(args[8]);
+ final String deletedSequencesOn = args[9];
+ final String deletedSequencesOff = args[10];
+
+ // =========================================== TRAFFIC FILTERING ============================================
TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
List<Instant> triggerTimes = ttfr.readTriggerTimes(triggerTimesFile, false);
// Extract all conversations present in the filtered trace.
List<Conversation> allConversations = tcpReassembler.getTcpConversations();
// Group conversations by hostname.
- Map<String, List<Conversation>> convsByHostname = TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap);
+ Map<String, List<Conversation>> convsByHostname =
+ TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap);
System.out.println("Grouped conversations by hostname.");
// For each hostname, count the frequencies of packet lengths exchanged with that hostname.
final Map<String, Map<Integer, Integer>> pktLenFreqsByHostname = new HashMap<>();
- convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host, TcpConversationUtils.countPacketLengthFrequencies(convs)));
+ convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host,
+ TcpConversationUtils.countPacketLengthFrequencies(convs)));
System.out.println("Counted frequencies of packet lengths exchanged with each hostname.");
- // For each hostname, count the frequencies of packet sequences (i.e., count how many conversations exchange a
- // sequence of packets of some specific lengths).
+ // For each hostname, count the frequencies of packet sequences (i.e., count how many
+ // conversations exchange a sequence of packets of some specific lengths).
final Map<String, Map<String, Integer>> pktSeqFreqsByHostname = new HashMap<>();
- convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host, TcpConversationUtils.countPacketSequenceFrequencies(convs)));
+ convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host,
+ TcpConversationUtils.countPacketSequenceFrequencies(convs)));
System.out.println("Counted frequencies of packet sequences exchanged with each hostname.");
- // For each hostname, count frequencies of packet pairs exchanged with that hostname across all conversations
+ // For each hostname, count frequencies of packet pairs exchanged
+ // with that hostname across all conversations
final Map<String, Map<String, Integer>> pktPairFreqsByHostname =
TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap);
System.out.println("Counted frequencies of packet pairs per hostname");
// For each user action, reassemble the set of TCP connections occurring shortly after
- final Map<UserAction, List<Conversation>> userActionToConversations = trafficLabeler.getLabeledReassembledTcpTraffic();
- final Map<UserAction, Map<String, List<Conversation>>> userActionsToConvsByHostname = trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap);
+ final Map<UserAction, List<Conversation>> userActionToConversations =
+ trafficLabeler.getLabeledReassembledTcpTraffic();
+ final Map<UserAction, Map<String, List<Conversation>>> userActionsToConvsByHostname =
+ trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap);
System.out.println("Reassembled TCP conversations occurring shortly after each user event");
-
-
/*
* NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and
* sequence identifiers as we do not care about hostnames and sequences during clustering.
* We can simply use the UserAction->List<Conversation> map to generate ON/OFF groupings of conversations.
*/
-
// Contains all ON events: hostname -> sequence identifier -> list of conversations with that sequence
Map<String, Map<String, List<Conversation>>> ons = new HashMap<>();
// Contains all OFF events: hostname -> sequence identifier -> list of conversations with that sequence
});
});
-// System.out.println("==== ON ====");
-// // Print out all the pairs into a file for ON events
-// File fileOnEvents = new File(onPairsPath);
-// PrintWriter pwOn = null;
-// try {
-// pwOn = new PrintWriter(fileOnEvents);
-// } catch(Exception ex) {
-// ex.printStackTrace();
-// }
-// for(Map.Entry<String, Map<String, List<Conversation>>> entry : ons.entrySet()) {
-// Map<String, List<Conversation>> seqsToConvs = entry.getValue();
-// for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
-// List<Conversation> listConv = entryConv.getValue();
-// // Just get the first Conversation because all Conversations in this group
-// // should have the same pairs of Application Data.
-// for(Conversation conv : listConv) {
-// // Process only if it is a TLS packet
-// if (conv.isTls()) {
-// List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
-// for(PcapPacketPair pair: tlsAppDataList) {
-// System.out.println(PrintUtils.toCsv(pair, dnsMap));
-// pwOn.println(PrintUtils.toCsv(pair, dnsMap));
-// }
-// } else { // Non-TLS conversations
-// List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
-// for(PcapPacketPair pair: packetList) {
-// System.out.println(PrintUtils.toCsv(pair, dnsMap));
-// pwOn.println(PrintUtils.toCsv(pair, dnsMap));
-// }
-// }
-// }
-// }
-// }
-// pwOn.close();
-//
-// System.out.println("==== OFF ====");
-// // Print out all the pairs into a file for ON events
-// File fileOffEvents = new File(offPairsPath);
-// PrintWriter pwOff = null;
-// try {
-// pwOff = new PrintWriter(fileOffEvents);
-// } catch(Exception ex) {
-// ex.printStackTrace();
-// }
-// for(Map.Entry<String, Map<String, List<Conversation>>> entry : offs.entrySet()) {
-// Map<String, List<Conversation>> seqsToConvs = entry.getValue();
-// for(Map.Entry<String, List<Conversation>> entryConv : seqsToConvs.entrySet()) {
-// List<Conversation> listConv = entryConv.getValue();
-// // Just get the first Conversation because all Conversations in this group
-// // should have the same pairs of Application Data.
-// for(Conversation conv : listConv) {
-// // Process only if it is a TLS packet
-// if (conv.isTls()) {
-// List<PcapPacketPair> tlsAppDataList = TcpConversationUtils.extractTlsAppDataPacketPairs(conv);
-// for(PcapPacketPair pair: tlsAppDataList) {
-// System.out.println(PrintUtils.toCsv(pair, dnsMap));
-// pwOff.println(PrintUtils.toCsv(pair, dnsMap));
-// }
-// } else { // Non-TLS conversations
-// List<PcapPacketPair> packetList = TcpConversationUtils.extractPacketPairs(conv);
-// for (PcapPacketPair pair : packetList) {
-// System.out.println(PrintUtils.toCsv(pair, dnsMap));
-// pwOff.println(PrintUtils.toCsv(pair, dnsMap));
-// }
-// }
-// }
-// }
-// }
-// pwOff.close();
-
-
- // ================================================ CLUSTERING ================================================
- // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List<Conversation> map
- // when don't care about hostnames and sequences (see comment earlier).
+ // ============================================== PAIR CLUSTERING ============================================
+ // TODO: No need to use the more convoluted on/off maps; Can simply use the UserAction->List<Conversation> map
+ // TODO: when don't care about hostnames and sequences (see comment earlier).
+ // ===========================================================================================================
List<Conversation> onConversations = userActionToConversations.entrySet().stream().
filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
map(e -> e.getValue()). // no longer interested in the UserActions
// Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
// Perform clustering on conversation logged as part of all ON events.
- double eps = 10.0;
- int minPts = 45;
+ // Calculate number of events per type (only ON/only OFF), which means half of the number of all timestamps.
+ int numberOfEventsPerType = triggerTimes.size() / 2;
+ int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.1);
+ int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.1);
+ int minPts = lowerBound;
DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(eps, minPts);
List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
// Perform clustering on conversation logged as part of all OFF events.
for (Cluster<PcapPacketPair> c : onClusters) {
System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
System.out.print(PrintUtils.toSummaryString(c));
- if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+ if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) {
// Print to file
List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
// Check for overlaps and decide whether to do range-based or conservative checking
for (Cluster<PcapPacketPair> c : offClusters) {
System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
System.out.print(PrintUtils.toSummaryString(c));
- if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+ if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) {
// Print to file
List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
// Check for overlaps and decide whether to do range-based or conservative checking
ppListOfListListOff.add(ppListOfList);
}
}
- // TODO: Merging test
- ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
- // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
- // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
- // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
- // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+
+ // =========================================== SIGNATURE CREATION ===========================================
+ // Concatenate
+ ppListOfListListOn = PcapPacketUtils.concatSequences(ppListOfListListOn, sortedAllConversation);
// TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
// TODO: This sequence actually belongs to the local communication between the plug and the phone
// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
- ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+ // Remove sequences in the list that have overlap
+ StringTokenizer stringTokenizerOn = new StringTokenizer(deletedSequencesOn, ",");
+ while(stringTokenizerOn.hasMoreTokens()) {
+ int sequenceToDelete = Integer.parseInt(stringTokenizerOn.nextToken());
+ if (sequenceToDelete == -1) { // '-1' means there is no removal
+ break;
+ }
+ PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, sequenceToDelete);
+ }
+ ppListOfListListOn = PcapPacketUtils.sortSequences(ppListOfListListOn);
- // TODO: Merging test
- ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
- // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3);
- // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
- // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
- // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
+ // Concatenate
+ ppListOfListListOff = PcapPacketUtils.concatSequences(ppListOfListListOff, sortedAllConversation);
// TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
// TODO: This sequence actually belongs to the local communication between the plug and the phone
// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
- ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
+ // Remove sequences in the list that have overlap
+ StringTokenizer stringTokenizerOff = new StringTokenizer(deletedSequencesOff, ",");
+ while(stringTokenizerOff.hasMoreTokens()) {
+ int sequenceToDelete = Integer.parseInt(stringTokenizerOff.nextToken());
+ if (sequenceToDelete == -1) { // '-1' means there is no removal
+ break;
+ }
+ PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, sequenceToDelete);
+ }
+ ppListOfListListOff = PcapPacketUtils.sortSequences(ppListOfListListOff);
// Write the signatures into the screen
+ System.out.println("========================================");
+ System.out.println(" ON Signature ");
+ System.out.println("========================================");
PcapPacketUtils.printSignatures(ppListOfListListOn);
+ System.out.println("========================================");
+ System.out.println(" OFF Signature ");
+ System.out.println("========================================");
PcapPacketUtils.printSignatures(ppListOfListListOff);
-
// Printing signatures into files
- PrintUtils.serializeIntoFile("./onSignature.sig", ppListOfListListOn);
- PrintUtils.serializeIntoFile("./offSignature.sig", ppListOfListListOff);
- //ppListOfListReadOn = PrintUtils.deserializeFromFile("./onSignature.sig");
- //ppListOfListReadOff = PrintUtils.deserializeFromFile("./offSignature.sig");
-
+ PrintUtils.serializeIntoFile(onSignatureFile, ppListOfListListOn);
+ PrintUtils.serializeIntoFile(offSignatureFile, ppListOfListListOff);
// Printing cluster analyses into files
- PrintUtils.serializeIntoFile("./onClusters.cls", corePointRangeSignatureOn);
- PrintUtils.serializeIntoFile("./offClusters.cls", corePointRangeSignatureOff);
- System.out.println("========================================");
-
- // ============================================================================================================
-
-// // TODO: This part is just for DBSCAN sensitivity experiment
-// // TODO: This part is just for DBSCAN sensitivity experiment
-// // TODO: This part is just for DBSCAN sensitivity experiment
-// // TODO: This part is just for DBSCAN sensitivity experiment
-// // TODO: This part is just for DBSCAN sensitivity experiment
-// List<Conversation> onConversations = userActionToConversations.entrySet().stream().
-// filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
-// map(e -> e.getValue()). // no longer interested in the UserActions
-// flatMap(List::stream). // flatten List<List<T>> to a List<T>
-// collect(Collectors.toList());
-// List<Conversation> offConversations = userActionToConversations.entrySet().stream().
-// filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
-// map(e -> e.getValue()).
-// flatMap(List::stream).
-// collect(Collectors.toList());
-// //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
-//
-// List<PcapPacketPair> onPairs = onConversations.stream().
-// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
-// TcpConversationUtils.extractPacketPairs(c)).
-// flatMap(List::stream). // flatten List<List<>> to List<>
-// collect(Collectors.toList());
-// List<PcapPacketPair> offPairs = offConversations.stream().
-// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
-// TcpConversationUtils.extractPacketPairs(c)).
-// flatMap(List::stream). // flatten List<List<>> to List<>
-// collect(Collectors.toList());
-// // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
-// Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
-//
-// double eps = 10; // loop from eps 1-10
-// int minPts = 50; // loop from minPts 30-50
-// for(int epsCount = 7; epsCount <= eps; epsCount++) {
-// for(int minPtsCount = 30; minPtsCount <= minPts; minPtsCount++) {
-// System.out.println("Eps: " + epsCount + " --- minPts: " + minPtsCount);
-// DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
-// DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(epsCount, minPtsCount);
-// List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
-// List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
-// // Sort the conversations as reference
-// List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
-// // Output clusters
-// System.out.println("========================================");
-// System.out.println(" Clustering results for ON ");
-// System.out.println(" Number of clusters: " + onClusters.size());
-// int count = 0;
-// List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
-// for (Cluster<PcapPacketPair> c : onClusters) {
-// System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
-//// System.out.print(PrintUtils.toSummaryString(c));
-// if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-//// if(c.getPoints().size() > 25) {
-// // Print to file
-// List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-// ppListOfListListOn.add(ppListOfList);
-// }
-// }
-// PcapPacketUtils.printSignatures(ppListOfListListOn);
-//
-// System.out.println("========================================");
-// System.out.println(" Clustering results for OFF ");
-// System.out.println(" Number of clusters: " + offClusters.size());
-// count = 0;
-// List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
-// for (Cluster<PcapPacketPair> c : offClusters) {
-// System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
-//// System.out.print(PrintUtils.toSummaryString(c));
-// if (c.getPoints().size() > 45 && c.getPoints().size() < 55) {
-// //if(c.getPoints().size() > 25) {
-// // Print to file
-// List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
-// ppListOfListListOff.add(ppListOfList);
-// }
-// }
-// PcapPacketUtils.printSignatures(ppListOfListListOff);
-// System.out.println();
-// System.out.println();
-// System.out.println();
-// // ============================================================================================================
-// }
-// }
-
-
-// // ================================================================================================
-// // <<< Some work-in-progress/explorative code that extracts a "representative" sequence >>>
-// //
-// // Currently need to know relevant hostname in advance :(
-// String hostname = "events.tplinkra.com";
-//// String hostname = "rfe-us-west-1.dch.dlink.com";
-// // Conversations with 'hostname' for ON events.
-// List<Conversation> onsForHostname = new ArrayList<>();
-// // Conversations with 'hostname' for OFF events.
-// List<Conversation> offsForHostname = new ArrayList<>();
-// // "Unwrap" sequence groupings in ons/offs maps.
-// ons.get(hostname).forEach((k,v) -> onsForHostname.addAll(v));
-// offs.get(hostname).forEach((k,v) -> offsForHostname.addAll(v));
-//
-//
-// Map<String, List<Conversation>> onsForHostnameGroupedByTlsAppDataSequence = TcpConversationUtils.groupConversationsByTlsApplicationDataPacketSequence(onsForHostname);
-//
-//
-// // Extract representative sequence for ON and OFF by providing the list of conversations with
-// // 'hostname' observed for each event type (the training data).
-// SequenceExtraction seqExtraction = new SequenceExtraction();
-//// ExtractedSequence extractedSequenceForOn = seqExtraction.extract(onsForHostname);
-//// ExtractedSequence extractedSequenceForOff = seqExtraction.extract(offsForHostname);
-//
-// ExtractedSequence extractedSequenceForOn = seqExtraction.extractByTlsAppData(onsForHostname);
-// ExtractedSequence extractedSequenceForOff = seqExtraction.extractByTlsAppData(offsForHostname);
-//
-// // Let's check how many ONs align with OFFs and vice versa (that is, how many times an event is incorrectly
-// // labeled).
-// int onsLabeledAsOff = 0;
-// Integer[] representativeOnSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOn.getRepresentativeSequence());
-// Integer[] representativeOffSeq = TcpConversationUtils.getPacketLengthSequence(extractedSequenceForOff.getRepresentativeSequence());
-// SequenceAlignment<Integer> seqAlg = seqExtraction.getAlignmentAlgorithm();
-// for (Conversation c : onsForHostname) {
-// Integer[] onSeq = TcpConversationUtils.getPacketLengthSequence(c);
-// if (seqAlg.calculateAlignment(representativeOffSeq, onSeq) <= extractedSequenceForOff.getMaxAlignmentCost()) {
-// onsLabeledAsOff++;
-// }
-// }
-// int offsLabeledAsOn = 0;
-// for (Conversation c : offsForHostname) {
-// Integer[] offSeq = TcpConversationUtils.getPacketLengthSequence(c);
-// if (seqAlg.calculateAlignment(representativeOnSeq, offSeq) <= extractedSequenceForOn.getMaxAlignmentCost()) {
-// offsLabeledAsOn++;
-// }
-// }
-// System.out.println("");
-// // ================================================================================================
-//
-//
-// // -------------------------------------------------------------------------------------------------------------
-// // -------------------------------------------------------------------------------------------------------------
- }
-
- /**
- * Check if there is any overlap between the signature stored in this class and another signature.
- * Conditions:
- * 1) If the signatures do not have any range, then we need to do conservative checking (return true).
- * 2) If the signatures have the same number of packets/packet lengths, then we check the range; if the
- * numbers of packets/packet lengths are different then we assume that there is no overlap.
- * 3) If there is any range in the signatures, then we need to check for overlap.
- * 4) If there is overlap for every packet/packet length, then we return false (range-based checking); otherwise,
- * true (conservative checking).
- *
- * @param signatures Multiple {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects to be checked
- * for overlaps.
- * @return A boolean that is true if there is an overlap; false otherwise.
- */
- public boolean isConservativeChecking(List<List<List<PcapPacket>>>...signatures) {
-
- // If none of the pairs/sequences is range-based then we go conservative
- boolean isRange = false;
- for(List<List<List<PcapPacket>>> signature : signatures) {
- if (isRangeBased(signature)) {
- isRange = true;
+ PrintUtils.serializeIntoFile(onClusterAnalysisFile, corePointRangeSignatureOn);
+ PrintUtils.serializeIntoFile(offClusterAnalysisFile, corePointRangeSignatureOff);
+
+ // =========================================== SIGNATURE DURATION ===========================================
+ List<Instant> firstSignatureTimestamps = new ArrayList<>();
+ List<Instant> lastSignatureTimestamps = new ArrayList<>();
+ if (!ppListOfListListOn.isEmpty()) {
+ List<List<PcapPacket>> firstListOnSign = ppListOfListListOn.get(0);
+ List<List<PcapPacket>> lastListOnSign = ppListOfListListOn.get(ppListOfListListOn.size() - 1);
+ // Load ON signature first and last packet's timestamps
+ for (List<PcapPacket> list : firstListOnSign) {
+ // Get timestamp Instant from the last packet
+ firstSignatureTimestamps.add(list.get(0).getTimestamp());
+ }
+ for (List<PcapPacket> list : lastListOnSign) {
+ // Get timestamp Instant from the last packet
+ int lastPacketIndex = list.size() - 1;
+ lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
}
}
- return isRange;
- }
-
-
- /*
- * Check and see if there is any range in the signatures
- */
- private boolean isRangeBased(List<List<List<PcapPacket>>> signatureRanges) {
+ if (!ppListOfListListOn.isEmpty()) {
+ List<List<PcapPacket>> firstListOffSign = ppListOfListListOff.get(0);
+ List<List<PcapPacket>> lastListOffSign = ppListOfListListOff.get(ppListOfListListOff.size() - 1);
+ // Load OFF signature first and last packet's timestamps
+ for (List<PcapPacket> list : firstListOffSign) {
+ // Get timestamp Instant from the last packet
+ firstSignatureTimestamps.add(list.get(0).getTimestamp());
+ }
+ for (List<PcapPacket> list : lastListOffSign) {
+ // Get timestamp Instant from the last packet
+ int lastPacketIndex = list.size() - 1;
+ lastSignatureTimestamps.add(list.get(lastPacketIndex).getTimestamp());
+ }
+ }
+ // Sort the timestamps
+ firstSignatureTimestamps.sort((p1, p2) -> {
+ return p1.compareTo(p2);
+ });
+ // Sort the timestamps
+ lastSignatureTimestamps.sort((p1, p2) -> {
+ return p1.compareTo(p2);
+ });
- for(List<List<PcapPacket>> listListPcapPacket : signatureRanges) {
- // Lower bound of the range is in index 0
- // Upper bound of the range is in index 1
- List<PcapPacket> minSequence = listListPcapPacket.get(0);
- List<PcapPacket> maxSequence = listListPcapPacket.get(1);
- for(PcapPacket pcapPacket : minSequence) {
- int index = minSequence.indexOf(pcapPacket);
- if (pcapPacket.length() != maxSequence.get(index).length()) {
- // If there is any packet length that differs in the minSequence
- // and maxSequence, then it is range-based
- return true;
+ Iterator<Instant> iterFirst = firstSignatureTimestamps.iterator();
+ Iterator<Instant> iterLast = lastSignatureTimestamps.iterator();
+ long duration;
+ long maxDuration = Long.MIN_VALUE;
+ System.out.println("========================================");
+ System.out.println(" Signature Durations ");
+ System.out.println("========================================");
+ while (iterFirst.hasNext() && iterLast.hasNext()) {
+ Instant firstInst = (Instant) iterFirst.next();
+ Instant lastInst = (Instant) iterLast.next();
+ Duration dur = Duration.between(firstInst, lastInst);
+ duration = dur.toMillis();
+ // Check duration --- should be below 15 seconds
+ if (duration > TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) {
+ while (duration > TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS && iterFirst.hasNext()) {
+ // that means we have to move to the next trigger
+ firstInst = (Instant) iterFirst.next();
}
+ dur = Duration.between(firstInst, lastInst);
+ duration = dur.toMillis();
+ } else { // Below 0/Negative --- that means we have to move to the next signature
+ while (duration < 0 && iterLast.hasNext()) { // that means we have to move to the next trigger
+ lastInst = (Instant) iterLast.next();
+ }
+ dur = Duration.between(firstInst, lastInst);
+ duration = dur.toMillis();
}
+ System.out.println(duration);
+ // Update duration if this bigger than the max value and still less than the window inclusion time
+ maxDuration = maxDuration < duration && duration <= TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS ?
+ duration : maxDuration;
+ }
+ // Just assign the value 0 if there is no signature
+ if (maxDuration == Long.MIN_VALUE) {
+ maxDuration = 0;
}
+ System.out.println("========================================");
+ System.out.println("Max signature duration: " + maxDuration);
+ System.out.println("========================================");
- return false;
+ // ==========================================================================================================
}
-}
-// TP-Link MAC 50:c7:bf:33:1f:09 and usually IP 192.168.1.159 (remember to verify per file)
-// frame.len >= 556 && frame.len <= 558 && ip.addr == 192.168.1.159
\ No newline at end of file
+}
projects / pingpong.git / blobdiff