X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=blobdiff_plain;f=Code%2FProjects%2FPacketLevelSignatureExtractor%2Fsrc%2Fmain%2Fjava%2Fedu%2Fuci%2Fiotproject%2Fdetection%2Flayer2%2FLayer2SignatureDetector.java;h=57685f3c9d7bbf809236558c507a55aa1c8b2edc;hp=f5a314fb23861e828522f683e7568671f1ed0ca6;hb=30341a6e9f439f7abb2cf53f3b616a969eb2c3f5;hpb=49dd7a06d29cc71b962d5e0a322fc935a7565438 diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index f5a314f..57685f3 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -9,6 +9,7 @@ import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.jgrapht.GraphPath; import org.jgrapht.alg.shortestpath.DijkstraShortestPath; @@ -38,6 +39,17 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb */ private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + /** + * Router's MAC. + * This is only useful for the filter for direction when it is a WAN signature (Phone-Cloud or Device-Cloud). + * Phone-Device signatures do not have router MAC address in it. + */ + // TODO: We can remove the following constants if we do remove router's MAC filtering for directions + private static String TRAINING_ROUTER_WLAN_MAC = null; + private static String ROUTER_WLAN_MAC = null; + //private static String TRAINING_ROUTER_WLAN_MAC = "b0:b9:8a:73:69:8e"; + //private static String ROUTER_WLAN_MAC = "00:c1:b1:14:eb:31"; + private static List> parseSignatureMacFilters(String filtersString) { List> filters = new ArrayList<>(); String[] filterRegexes = filtersString.split(";"); @@ -50,39 +62,74 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb } public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { + String errMsg = String.format("SPECTO version 1.0\n" + + "Copyright (C) 2018-2019 Janus Varmarken and Rahmadi Trimananda.\n" + + "University of California, Irvine.\n" + + "All rights reserved.\n\n" + + "Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile " + + "resultsFile signatureDuration eps onMaxSkippedPackets offMaxSkippedPackets" + + "\n inputPcapFile: the target of the detection" + + "\n onAnalysisFile: the file that contains the ON clusters analysis" + + "\n offAnalysisFile: the file that contains the OFF clusters analysis" + + "\n onSignatureFile: the file that contains the ON signature to search for" + + "\n offSignatureFile: the file that contains the OFF signature to search for" + + "\n resultsFile: where to write the results of the detection" + + "\n signatureDuration: the maximum duration of signature detection" + + "\n epsilon: the epsilon value for the DBSCAN algorithm\n" + + "\n Additional options (add '-r' before the following two parameters):" + + "\n delta: delta for relaxed matching" + + "\n packetId: packet number in the sequence" + + "\n (could be more than one packet whose matching is relaxed, " + + "\n e.g., 0,1 for packets 0 and 1)", + Layer2SignatureDetector.class.getSimpleName()); + String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " + + "following options and associated positional arguments may be used:\n" + + " '-onmacfilters ;;...;' which specifies that sequence matching should ONLY" + + " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " + + "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " + + "aggressive filtering on parts of the signature (e.g., the communication that involves the " + + "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " + + "that matches the vendor's prefix).\n" + + " '-offmacfilters ;;...;' works exactly the same as onmacfilters, but " + + "applies to the OFF signature instead of the ON signature.\n" + + " '-sout ' true/false literal indicating if output should also be printed to std out; default is true.\n" + + " '-vpn ' router's MAC address; this is to simulate a VPN that combines all flows even when the traffic is not a VPN traffic.\n" + + " '-onskipped ' the maximum duration of ON signature detection.\n" + + " '-offskipped ' the maximum duration of OFF signature detection.\n"; // Parse required parameters. - if (args.length < 5) { - String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" + - "\n inputPcapFile: the target of the detection" + - "\n onSignatureFile: the file that contains the ON signature to search for" + - "\n offSignatureFile: the file that contains the OFF signature to search for" + - "\n resultsFile: where to write the results of the detection" + - "\n signatureDuration: the maximum duration of signature detection", - Layer2SignatureDetector.class.getSimpleName()); + if (args.length < 8) { System.out.println(errMsg); - String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " + - "following options and associated positional arguments may be used:\n" + - " '-onmacfilters ;;...;' which specifies that sequence matching should ONLY" + - " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " + - "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " + - "aggressive filtering on parts of the signature (e.g., the communication that involves the " + - "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " + - "that matches the vendor's prefix).\n" + - " '-offmacfilters ;;...;' works exactly the same as onmacfilters, but " + - "applies to the OFF signature instead of the ON signature.\n" + - " '-sout ' true/false literal indicating if output should also be printed to std out; default is true."; System.out.println(optParamsExplained); return; } final String pcapFile = args[0]; - final String onSignatureFile = args[1]; - final String offSignatureFile = args[2]; - final String resultsFile = args[3]; - final int signatureDuration = Integer.parseInt(args[4]); - + final String onClusterAnalysisFile = args[1]; + final String offClusterAnalysisFile = args[2]; + final String onSignatureFile = args[3]; + final String offSignatureFile = args[4]; + final String resultsFile = args[5]; + final int signatureDuration = Integer.parseInt(args[6]); + final double eps = Double.parseDouble(args[7]); + // Additional feature---relaxed matching + int delta = 0; + final Set packetSet = new HashSet<>(); + if (args.length > 8 && args[8].equals("-r")) { + delta = Integer.parseInt(args[9]); + StringTokenizer stringTokenizerOff = new StringTokenizer(args[10], ","); + // Add the list of packet IDs + while(stringTokenizerOff.hasMoreTokens()) { + int id = Integer.parseInt(stringTokenizerOff.nextToken()); + packetSet.add(id); + } + } // Parse optional parameters. List> onSignatureMacFilters = null, offSignatureMacFilters = null; - final int optParamsStartIdx = 5; + String vpnClientMacAddress = null; + // TODO: Currently the skipped packets implementation is not activated. + // TODO: We looked into limiting the number of skipped packets to declare a signature match at layer-2. + int onMaxSkippedPackets = -1; + int offMaxSkippedPackets = -1; + final int optParamsStartIdx = 8; if (args.length > optParamsStartIdx) { for (int i = optParamsStartIdx; i < args.length; i++) { if (args[i].equalsIgnoreCase("-onMacFilters")) { @@ -94,6 +141,16 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb } else if (args[i].equalsIgnoreCase("-sout")) { // Next argument is a boolean true/false literal. DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]); + } else if (args[i].equalsIgnoreCase("-vpn")) { + vpnClientMacAddress = args[i+1]; + } else if (args[i].equalsIgnoreCase("-onskipped")) { + if (i+2 > args.length - 1 || !args[i+2].equalsIgnoreCase("-offskipped")) { + throw new Error("Please make sure that the -onskipped and -offskipped options are both used at the same time..."); + } + if (args[i+2].equalsIgnoreCase("-offskipped")) { + onMaxSkippedPackets = Integer.parseInt(args[i+1]); + offMaxSkippedPackets = Integer.parseInt(args[i+3]); + } } } } @@ -105,29 +162,52 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Include metadata as comments at the top PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); resultsWriter.flush(); - // TODO: IMPLEMENT THE RANGE-BASED DETECTION HERE - boolean isRangeBased = true; - // Create signature detectors and add observers that output their detected events. List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); + // Load signature analyses + List>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile); + List>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile); + // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS + // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR + // TODO: STRICT MATCHING + // Check if we should use range-based matching + boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); + boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); + // Update the signature with ranges if it is range-based + if (isRangeBasedForOn) { + onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); + } + if (isRangeBasedForOff) { + offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); + } Layer2SignatureDetector onDetector = onSignatureMacFilters == null ? - new Layer2SignatureDetector(onSignature) : - new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration, + isRangeBasedForOn, eps, onMaxSkippedPackets, vpnClientMacAddress, delta, packetSet) : + new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, + onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps, onMaxSkippedPackets, + vpnClientMacAddress, delta, packetSet); Layer2SignatureDetector offDetector = offSignatureMacFilters == null ? - new Layer2SignatureDetector(offSignature) : - new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration, + isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress, delta, packetSet) : + new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, offSignatureMacFilters, + signatureDuration, isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress, delta, packetSet); + final List detectedEvents = new ArrayList<>(); onDetector.addObserver((signature, match) -> { UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp()); PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + detectedEvents.add(event); }); offDetector.addObserver((signature, match) -> { UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, match.get(0).get(0).getTimestamp()); PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + detectedEvents.add(event); }); // Load the PCAP file @@ -141,6 +221,27 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Parse the file reader.readFromHandle(); + String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count(); + String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count(); + String onMaximumSkippedPackets = "# Maximum number of skipped packets in ON signature " + + Integer.toString(onDetector.getMaxSkippedPackets()); + String offMaximumSkippedPackets = "# Maximum number of skipped packets in OFF signature " + + Integer.toString(offDetector.getMaxSkippedPackets()); + PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + // Perform the skipped packet analysis if needed + if (onMaxSkippedPackets != -1 && offMaxSkippedPackets != -1) { + PrintWriterUtils.println(onMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + for (Integer skippedPackets : onDetector.getSkippedPackets()) { + PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + } + PrintWriterUtils.println(offMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + for (Integer skippedPackets : offDetector.getSkippedPackets()) { + PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + } + } // Flush output to results file and close it. resultsWriter.flush(); resultsWriter.close(); @@ -173,27 +274,44 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb /** * In charge of reassembling layer 2 packet flows. */ - private final Layer2FlowReassembler mFlowReassembler = new Layer2FlowReassembler(); + private Layer2FlowReassembler mFlowReassembler; private final List mObservers = new ArrayList<>(); private int mInclusionTimeMillis; - public Layer2SignatureDetector(List>> searchedSignature) { - this(searchedSignature, null, 0, false); + /** + * Skipped-packet analysis. + */ + private int mMaxSkippedPackets; + private List mSkippedPackets; + + + + public Layer2SignatureDetector(List>> searchedSignature, String trainingRouterWlanMac, + String routerWlanMac, int signatureDuration, boolean isRangeBased, double eps, + int limitSkippedPackets, String vpnClientMacAddress, int delta, Set packetSet) { + this(searchedSignature, trainingRouterWlanMac, routerWlanMac, null, signatureDuration, isRangeBased, + eps, limitSkippedPackets, vpnClientMacAddress, delta, packetSet); } - public Layer2SignatureDetector(List>> searchedSignature, List> flowFilters, int inclusionTimeMillis, boolean isRangeBased) { + public Layer2SignatureDetector(List>> searchedSignature, String trainingRouterWlanMac, + String routerWlanMac, List> flowFilters, + int inclusionTimeMillis, boolean isRangeBased, double eps, int limitSkippedPackets, + String vpnClientMacAddress, int delta, Set packetSet) { if (flowFilters != null && flowFilters.size() != searchedSignature.size()) { - throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature."); + throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster " + + "of the signature."); } mSignature = Collections.unmodifiableList(searchedSignature); List clusterMatchers = new ArrayList<>(); for (int i = 0; i < mSignature.size(); i++) { List> cluster = mSignature.get(i); Layer2ClusterMatcher clusterMatcher = flowFilters == null ? - new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased); + new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, inclusionTimeMillis, + isRangeBased, eps, limitSkippedPackets, delta, packetSet) : + new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, flowFilters.get(i), + inclusionTimeMillis, isRangeBased, eps, limitSkippedPackets, delta, packetSet); clusterMatcher.addObserver(this); clusterMatchers.add(clusterMatcher); } @@ -208,9 +326,27 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb } mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); // Register all cluster matchers to receive a notification whenever a new flow is encountered. + if (vpnClientMacAddress != null) { + mFlowReassembler = new Layer2FlowReassembler(vpnClientMacAddress); + } else { + mFlowReassembler = new Layer2FlowReassembler(); + } mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm)); mInclusionTimeMillis = inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + mMaxSkippedPackets = 0; + mSkippedPackets = new ArrayList<>(); + } + + public int getMaxSkippedPackets() { + return mMaxSkippedPackets; + } + + public List getSkippedPackets() { + for (Layer2ClusterMatcher matcher : mClusterMatchers) { + mSkippedPackets.addAll(matcher.getSkippedPackets()); + } + return mSkippedPackets; } @Override @@ -226,6 +362,10 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Add the match at the corresponding index mPendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); checkSignatureMatch(); + // Update maximum number of skipped packets + if (mMaxSkippedPackets < ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets()) { + mMaxSkippedPackets = ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets(); + } } }