From 1035a19372c61da28a64ac4d27e907cbd89c4a67 Mon Sep 17 00:00:00 2001 From: Janus Varmarken Date: Tue, 22 Jan 2019 22:48:40 -0800 Subject: [PATCH] Add capability to filter flows when performing layer 2 detection --- .../.idea/modules/SmartPlugDetector_main.iml | 1 + .../.idea/modules/SmartPlugDetector_test.iml | 1 + .../layer2/Layer2ClusterMatcher.java | 35 +++++++- .../layer2/Layer2SignatureDetector.java | 79 ++++++++++++++++--- .../trafficreassembly/layer2/Layer2Flow.java | 20 +++++ 5 files changed, 121 insertions(+), 15 deletions(-) diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml index 15e9a30..9353a45 100644 --- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml +++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_main.iml @@ -5,6 +5,7 @@ + diff --git a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml index d8e4114..d10f1b0 100644 --- a/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml +++ b/Code/Projects/SmartPlugDetector/.idea/modules/SmartPlugDetector_test.iml @@ -5,6 +5,7 @@ + diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java index 79a945f..5021c31 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; /** * Attempts to detect members of a cluster (packet sequence mutations) in layer 2 flows. @@ -27,9 +28,30 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye */ private final Map mPerFlowSeqMatchers = new HashMap<>(); + private final Function mFlowFilter; + /** + * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. + * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. + */ public Layer2ClusterMatcher(List> cluster) { + // Consider all flows if no flow filter specified. + this(cluster, flow -> true); + } + + /** + * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. + * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. + * @param flowFilter A filter that defines what {@link Layer2Flow}s the new {@link Layer2ClusterMatcher} should + * search for {@code cluster}'s members in. If {@code flowFilter} returns {@code true}, the flow + * will be included (searched). Note that {@code flowFilter} is only queried once for each flow, + * namely when the {@link Layer2FlowReassembler} notifies the {@link Layer2ClusterMatcher} about + * the new flow. This functionality may for example come in handy when one only wants to search + * for matches in the subset of flows that involves a specific (range of) MAC(s). + */ + public Layer2ClusterMatcher(List> cluster, Function flowFilter) { super(cluster); + mFlowFilter = flowFilter; } @Override @@ -113,10 +135,19 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye return prunedCluster; } + private static final boolean DEBUG = false; @Override public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) { - // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. - newFlow.addFlowObserver(this); + // New flow detected. Check if we should consider it when searching for cluster member matches. + if (mFlowFilter.apply(newFlow)) { + if (DEBUG) { + System.out.println(">>> ACCEPTING FLOW: " + newFlow + " <<<"); + } + // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. + newFlow.addFlowObserver(this); + } else if (DEBUG) { + System.out.println(">>> IGNORING FLOW: " + newFlow + " <<<"); + } } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index 014e199..ba5ec7a 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -7,6 +7,7 @@ import edu.uci.iotproject.detection.ClusterMatcherObserver; import edu.uci.iotproject.detection.SignatureDetectorObserver; import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.io.PrintWriterUtils; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; import edu.uci.iotproject.util.PrintUtils; import org.jgrapht.GraphPath; @@ -21,6 +22,8 @@ import java.io.IOException; import java.io.PrintWriter; import java.time.Duration; import java.util.*; +import java.util.function.Function; +import java.util.regex.Pattern; /** * Performs layer 2 signature detection. @@ -35,24 +38,62 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb */ private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + private static List> parseSignatureMacFilters(String filtersString) { + List> filters = new ArrayList<>(); + String[] filterRegexes = filtersString.split(";"); + for (String filterRegex : filterRegexes) { + final Pattern regex = Pattern.compile(filterRegex); + // Create a filter that includes all flows where one of the two MAC addresses match the regex. + filters.add(flow -> regex.matcher(flow.getEndpoint1().toString()).matches() || regex.matcher(flow.getEndpoint2().toString()).matches()); + } + return filters; + } + public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { + // Parse required parameters. if (args.length < 4) { - String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile [stdOut]" + - "\n - inputPcapFile: the target of the detection" + - "\n - onSignatureFile: the file that contains the ON signature to search for" + - "\n - offSignatureFile: the file that contains the OFF signature to search for" + - "\n - resultsFile: where to write the results of the detection" + - "\n - stdOut: optional true/false literal indicating if output should also be printed to std out; default is true", + String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" + + "\n inputPcapFile: the target of the detection" + + "\n onSignatureFile: the file that contains the ON signature to search for" + + "\n offSignatureFile: the file that contains the OFF signature to search for" + + "\n resultsFile: where to write the results of the detection", Layer2SignatureDetector.class.getSimpleName()); System.out.println(errMsg); + String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " + + "following options and associated positional arguments may be used:\n" + + " '-onmacfilters ;;...;' which specifies that sequence matching should ONLY" + + " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " + + "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " + + "aggressive filtering on parts of the signature (e.g., the communication that involves the " + + "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " + + "that matches the vendor's prefix).\n" + + " '-offmacfilters ;;...;' works exactly the same as onmacfilters, but " + + "applies to the OFF signature instead of the ON signature.\n" + + " '-sout ' true/false literal indicating if output should also be printed to std out; default is true."; + System.out.println(optParamsExplained); return; } final String pcapFile = args[0]; final String onSignatureFile = args[1]; final String offSignatureFile = args[2]; final String resultsFile = args[3]; - if (args.length == 5) { - DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[4]); + + // Parse optional parameters. + List> onSignatureMacFilters = null, offSignatureMacFilters = null; + final int optParamsStartIdx = 4; + if (args.length > optParamsStartIdx) { + for (int i = optParamsStartIdx; i < args.length; i++) { + if (args[i].equalsIgnoreCase("-onMacFilters")) { + // Next argument is the cluster-wise MAC filters (separated by semicolons). + onSignatureMacFilters = parseSignatureMacFilters(args[i+1]); + } else if (args[i].equalsIgnoreCase("-offMacFilters")) { + // Next argument is the cluster-wise MAC filters (separated by semicolons). + offSignatureMacFilters = parseSignatureMacFilters(args[i+1]); + } else if (args[i].equalsIgnoreCase("-sout")) { + // Next argument is a boolean true/false literal. + DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]); + } + } } // Prepare file outputter. @@ -67,8 +108,12 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb resultsWriter.flush(); // Create signature detectors and add observers that output their detected events. - Layer2SignatureDetector onDetector = new Layer2SignatureDetector(PrintUtils.deserializeSignatureFromFile(onSignatureFile)); - Layer2SignatureDetector offDetector = new Layer2SignatureDetector(PrintUtils.deserializeSignatureFromFile(offSignatureFile)); + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); + Layer2SignatureDetector onDetector = onSignatureMacFilters == null ? + new Layer2SignatureDetector(onSignature) : new Layer2SignatureDetector(onSignature, onSignatureMacFilters); + Layer2SignatureDetector offDetector = offSignatureMacFilters == null ? + new Layer2SignatureDetector(offSignature) : new Layer2SignatureDetector(offSignature, offSignatureMacFilters); onDetector.addObserver((signature, match) -> { UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp()); PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); @@ -126,10 +171,19 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb private final List mObservers = new ArrayList<>(); public Layer2SignatureDetector(List>> searchedSignature) { + this(searchedSignature, null); + } + + public Layer2SignatureDetector(List>> searchedSignature, List> flowFilters) { + if (flowFilters != null && flowFilters.size() != searchedSignature.size()) { + throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature."); + } mSignature = Collections.unmodifiableList(searchedSignature); List clusterMatchers = new ArrayList<>(); - for (List> cluster : mSignature) { - Layer2ClusterMatcher clusterMatcher = new Layer2ClusterMatcher(cluster); + for (int i = 0; i < mSignature.size(); i++) { + List> cluster = mSignature.get(i); + Layer2ClusterMatcher clusterMatcher = flowFilters == null ? + new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i)); clusterMatcher.addObserver(this); clusterMatchers.add(clusterMatcher); } @@ -147,7 +201,6 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm)); } - @Override public void gotPacket(PcapPacket packet) { // Forward packet processing to the flow reassembler that in turn notifies the cluster matchers as appropriate diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java index 2d804bd..f8c3237 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java @@ -36,6 +36,22 @@ public class Layer2Flow { mEndpoint2 = endpoint2; } + /** + * Get the first endpoint of this flow. + * @return the first endpoint of this flow. + */ + public MacAddress getEndpoint1() { + return mEndpoint1; + } + + /** + * Get the second endpoint of this flow. + * @return the second endpoint of this flow. + */ + public MacAddress getEndpoint2() { + return mEndpoint2; + } + /** * Register as an observer of this flow. * @param observer The client that is to be notified whenever this flow changes (has new packets added). @@ -92,4 +108,8 @@ public class Layer2Flow { throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow"); } + @Override + public String toString() { + return getClass().getSimpleName() + String.format(" with mEndpoint1=%s and mEndpoint2=%s", mEndpoint1, mEndpoint2); + } } -- 2.34.1