From 92a31b0967b3acff9e473ce62c136e84298b3aab Mon Sep 17 00:00:00 2001 From: rtrimana Date: Mon, 11 Mar 2019 15:57:52 -0700 Subject: [PATCH] Adding range-based matching for Layer 2 and fusing it off for now (the results are terrible for signatures that only have 2 packets). --- .../layer2/Layer2AbstractMatcher.java | 102 +++++++++++++++ .../layer2/Layer2ClusterMatcher.java | 78 ++++++++++- .../detection/layer2/Layer2RangeMatcher.java | 123 ++++++++++++++++++ .../layer2/Layer2SequenceMatcher.java | 57 +------- .../layer2/Layer2SignatureDetector.java | 63 ++++++--- .../layer3/Layer3ClusterMatcher.java | 9 +- ...ctor.java => Layer3SignatureDetector.java} | 59 +++------ .../layer2/Layer2FlowObserver.java | 1 - 8 files changed, 364 insertions(+), 128 deletions(-) create mode 100644 Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java create mode 100644 Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java rename Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/{SignatureDetector.java => Layer3SignatureDetector.java} (93%) diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java new file mode 100644 index 0000000..1621c82 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java @@ -0,0 +1,102 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; + +import java.util.ArrayList; +import java.util.List; + +/** + * Base class for layer 2 matchers ({@code Layer2SequenceMatcher} and {@code Layer2RangeMatcher}). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +abstract public class Layer2AbstractMatcher { + + /** + * Buffer of actual packets seen so far that match the searched range (i.e., constitutes a subsequence). + */ + protected final List mMatchedPackets = new ArrayList<>(); + + /** + * Models the directions of packets. As the sequence matcher assumes that it is only presented + * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a + * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses + * of devices in advance during matching. + */ + protected final boolean[] mPacketDirections; + + /** + * Create a {@code Layer2AbstractMatcher}. + * @param sequence The sequence of the signature. + */ + public Layer2AbstractMatcher(List sequence) { + mPacketDirections = new boolean[sequence.size()]; + // Compute packet directions for sequence. + for (int i = 0; i < sequence.size(); i++) { + if (i == 0) { + // No previous packet; boolean parameter is ignored in this special case. + mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i)); + } else { + // Base direction marker on direction of previous packet. + PcapPacket prevPkt = sequence.get(i-1); + boolean prevPktDirection = mPacketDirections[i-1]; + mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i)); + } + } + } + + /** + * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction + * of {@code currPkt} is {@code true} by definition. + * @param prevPkt The previous packet, if any. + * @param prevPktDirection The computed direction of the previous packet + * @param currPkt The current packet for which the direction is to be determined. + * @return The direction of {@code currPkt}. + */ + protected boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { + if (prevPkt == null) { + // By definition, use true as direction marker for first packet + return true; + } + if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { + // Current packet goes in same direction as previous packet. + return prevPktDirection; + } else { + // Current packet goes in opposite direction of previous packet. + return !prevPktDirection; + } + } + + /** + * See the implementer class for the following method. + * + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public abstract boolean matchPacket(PcapPacket packet); + + /** + * See the implementer class for the following method. + */ + public abstract int getTargetSequencePacketCount(); + + public int getMatchedPacketsCount() { + return mMatchedPackets.size(); + } + + public List getMatchedPackets() { + return mMatchedPackets; + } + + /** + * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. + * @return The last matched packet, or {@code null} if no packets have been matched yet. + */ + public PcapPacket getLastPacket() { + //return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null; + return mMatchedPackets.size() > 0 ? mMatchedPackets.get(mMatchedPackets.size()-1) : null; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java index 88cb64e..e2a4aea 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -27,16 +27,27 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye * of {@link #mCluster} and has so far matched {@code j} packets of that particular sequence. */ private final Map mPerFlowSeqMatchers = new HashMap<>(); + private final Map mPerFlowRangeMatcher = new HashMap<>(); private final Function mFlowFilter; + /** + * Specifying range-based instead of conservative exact matching. + */ + private final boolean mRangeBased; + + /** + * Epsilon value used by the DBSCAN algorithm; it is used again for range-based matching here. + */ + private final double mEps; + /** * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. */ - public Layer2ClusterMatcher(List> cluster) { + public Layer2ClusterMatcher(List> cluster, boolean isRangeBased, double eps) { // Consider all flows if no flow filter specified. - this(cluster, flow -> true, false); + this(cluster, flow -> true, isRangeBased, eps); } /** @@ -49,15 +60,26 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye * the new flow. This functionality may for example come in handy when one only wants to search * for matches in the subset of flows that involves a specific (range of) MAC(s). * @param isRangeBased The boolean that decides if it is range-based vs. strict matching. + * @param eps The epsilon value used in the DBSCAN algorithm. */ public Layer2ClusterMatcher(List> cluster, Function flowFilter, - boolean isRangeBased) { + boolean isRangeBased, double eps) { super(cluster, isRangeBased); mFlowFilter = flowFilter; + mRangeBased = isRangeBased; + mEps = eps; } @Override public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { + if (mRangeBased) { + rangeBasedMatching(flow, newPacket); + } else { + conservativeMatching(flow, newPacket); + } + } + + private void conservativeMatching(Layer2Flow flow, PcapPacket newPacket) { if (mPerFlowSeqMatchers.get(flow) == null) { // If this is the first time we encounter this flow, we need to set up sequence matchers for it. // All sequences of the cluster have the same length, so we only need to compute the length of the nested @@ -114,6 +136,56 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye } } + private void rangeBasedMatching(Layer2Flow flow, PcapPacket newPacket) { + // TODO: For range-based matching, we only care about matching a range; therefore it is a matcher array. + if (mPerFlowRangeMatcher.get(flow) == null) { + // If this is the first time we encounter this flow, we need to set up a sequence matcher. + // All sequences of the cluster have the same length, so we only need to compute the length of the + // arrays once. We want to make room for a cluster matcher in each state, including the initial empty state + // but excluding the final "full match" state (as there is no point in keeping a terminated sequence matcher + // around), so the length of the array is simply the sequence length. + Layer2RangeMatcher[] matcher = new Layer2RangeMatcher[mCluster.get(0).size()]; + // Prepare a "state 0" sequence matcher. + matcher[0] = new Layer2RangeMatcher(mCluster.get(0), mCluster.get(1), mEps); + // Associate the new sequence matcher table with the new flow. + mPerFlowRangeMatcher.put(flow, matcher); + } + // Fetch table that contains sequence matchers for this flow. + Layer2RangeMatcher[] matcher = mPerFlowRangeMatcher.get(flow); + // Present packet to the sequence matcher. + for (int j = matcher.length - 1; j >= 0; j--) { + Layer2RangeMatcher sm = matcher[j]; + if (sm == null) { + // There is currently no sequence matcher that has managed to match j packets. + continue; + } + boolean matched = sm.matchPacket(newPacket); + if (matched) { + if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { + // Sequence matcher has a match. Report it to observers. + mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets())); + // Remove the now terminated sequence matcher. + matcher[j] = null; + } else { + // Sequence matcher advanced one step, so move it to its corresponding new position iff the + // packet that advanced it has a later timestamp than that of the last matched packet of the + // sequence matcher at the new index, if any. In most traces, a small amount of the packets + // appear out of order (with regards to their timestamp), which is why this check is required. + // Obviously it would not be needed if packets where guaranteed to be processed in timestamp + // order here. + if (matcher[j+1] == null || + newPacket.getTimestamp().isAfter(matcher[j+1].getLastPacket().getTimestamp())) { + matcher[j+1] = sm; + } + } + // We always want to have a sequence matcher in state 0, regardless of if the one that advanced + // from state zero completed its matching or if it replaced a different one in state 1 or not. + if (sm.getMatchedPacketsCount() == 1) { + matcher[j] = new Layer2RangeMatcher(sm.getTargetLowerBound(), sm.getTargetUpperBound(), mEps); + } + } + } + } @Override protected List> pruneCluster(List> cluster) { diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java new file mode 100644 index 0000000..db7e5b8 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java @@ -0,0 +1,123 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.List; + +/** + * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls + * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information. This class has the same flavor as the + * {@link Layer2SequenceMatcher} class. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2RangeMatcher extends Layer2AbstractMatcher { + /** + * The range this {@link Layer2RangeMatcher} is searching for. + */ + private final List mLowerBound; + private final List mUpperBound; + private final double mEps; + + /** + * Create a {@code Layer2RangeMatcher}. + * @param lowerBound The lower bound of the sequence to match against (search for). + * @param upperBound The upper bound of the sequence to match against (search for). + * @param eps The epsilon value used in the DBSCAN algorithm. + */ + public Layer2RangeMatcher(List lowerBound, List upperBound, double eps) { + // TODO: Just use the lower bound since both lower and upper bounds' packets essentially have the same direction + // TODO: for the same position in the array. Both arrays also have the same length. + super(lowerBound); + mLowerBound = lowerBound; + mUpperBound = upperBound; + mEps = eps; + } + + /** + * Attempt to advance this {@code Layer2RangeMatcher} by matching {@code packet} against the packet that this + * {@code Layer2RangeMatcher} expects as the next packet of the sequence it is searching for. + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public boolean matchPacket(PcapPacket packet) { + if (getMatchedPacketsCount() == getTargetSequencePacketCount()) { + // We already matched the entire sequence, so we can't match any more packets. + return false; + } + + // Verify that new packet pertains to same flow as previously matched packets, if any. + if (getMatchedPacketsCount() > 0) { + MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet); + MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet); + MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0)); + MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0)); + if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) || + pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) { + return false; + } + } + + // Get representative of the packet we expect to match next. + PcapPacket expectedLowerBound = mLowerBound.get(mMatchedPackets.size()); + PcapPacket expectedUpperBound = mUpperBound.get(mMatchedPackets.size()); + // First verify if the received packet has the length we're looking for (the length should be within the range). +// if (expectedLowerBound.getOriginalLength() - (int) mEps <= packet.getOriginalLength() && +// packet.getOriginalLength() <= expectedUpperBound.getOriginalLength() + (int) mEps){ + if (expectedLowerBound.getOriginalLength() - (int) mEps <= packet.getOriginalLength() && + packet.getOriginalLength() <= expectedUpperBound.getOriginalLength() + (int) mEps){ + // If this is the first packet, we only need to verify that its length is correct. Time constraints are + // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we + // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet + // must originate from this particular MAC". + if (getMatchedPacketsCount() == 0) { + // Store packet as matched and advance. + mMatchedPackets.add(packet); + return true; + } + // Check if direction of packet matches expected direction. + boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1), + mPacketDirections[getMatchedPacketsCount()-1], packet); + boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()]; + if (actualDirection != expectedDirection) { + return false; + } + // Next apply timing constraints: + // 1: to be a match, the packet must have a later timestamp than any other packet currently matched + // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded? + if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) { + return false; + } + if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). + plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { + return false; + } + // If we made it here, it means that this packet has the expected length, direction, and obeys the timing + // constraints, so we store it and advance. + mMatchedPackets.add(packet); + if (mMatchedPackets.size() == mLowerBound.size()) { + // TODO report (to observers?) that we are done? + } + return true; + } + return false; + } + + public int getTargetSequencePacketCount() { + return mLowerBound.size(); + } + + public List getTargetLowerBound() { + return mLowerBound; + } + + public List getTargetUpperBound() { + return mLowerBound; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java index 672fb72..2db2228 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -15,35 +15,21 @@ import java.util.List; * @author Janus Varmarken {@literal } * @author Rahmadi Trimananda {@literal } */ -public class Layer2SequenceMatcher { +public class Layer2SequenceMatcher extends Layer2AbstractMatcher { /** * The sequence this {@link Layer2SequenceMatcher} is searching for. */ private final List mSequence; - /** - * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the - * searched sequence). - */ - private final List mMatchedPackets = new ArrayList<>(); - - /** - * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented - * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a - * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses - * of devices in advance during matching. - */ - private final boolean[] mPacketDirections; - /** * Create a {@code Layer2SequenceMatcher}. * @param sequence The sequence to match against (search for). */ public Layer2SequenceMatcher(List sequence) { + super(sequence); mSequence = sequence; // Compute packet directions for sequence. - mPacketDirections = new boolean[sequence.size()]; for (int i = 0; i < sequence.size(); i++) { if (i == 0) { // No previous packet; boolean parameter is ignored in this special case. @@ -123,10 +109,6 @@ public class Layer2SequenceMatcher { return false; } - public int getMatchedPacketsCount() { - return mMatchedPackets.size(); - } - public int getTargetSequencePacketCount() { return mSequence.size(); } @@ -135,39 +117,4 @@ public class Layer2SequenceMatcher { return mSequence; } - public List getMatchedPackets() { - return mMatchedPackets; - } - - /** - * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. - * @return The last matched packet, or {@code null} if no packets have been matched yet. - */ - public PcapPacket getLastPacket() { - return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null; - } - - /** - * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction - * of {@code currPkt} is {@code true} by definition. - * @param prevPkt The previous packet, if any. - * @param prevPktDirection The computed direction of the previous packet - * @param currPkt The current packet for which the direction is to be determined. - * @return The direction of {@code currPkt}. - */ - private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { - if (prevPkt == null) { - // By definition, use true as direction marker for first packet - return true; - } - if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { - // Current packet goes in same direction as previous packet. - return prevPktDirection; - } else { - // Current packet goes in opposite direction of previous packet. - return !prevPktDirection; - } - } - - } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index f5a314f..505bfdc 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -9,6 +9,7 @@ import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.jgrapht.GraphPath; import org.jgrapht.alg.shortestpath.DijkstraShortestPath; @@ -51,9 +52,11 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { // Parse required parameters. - if (args.length < 5) { - String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" + + if (args.length < 7) { + String errMsg = String.format("Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile resultsFile" + "\n inputPcapFile: the target of the detection" + + "\n onAnalysisFile: the file that contains the ON clusters analysis" + + "\n offAnalysisFile: the file that contains the OFF clusters analysis" + "\n onSignatureFile: the file that contains the ON signature to search for" + "\n offSignatureFile: the file that contains the OFF signature to search for" + "\n resultsFile: where to write the results of the detection" + @@ -75,14 +78,16 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb return; } final String pcapFile = args[0]; - final String onSignatureFile = args[1]; - final String offSignatureFile = args[2]; - final String resultsFile = args[3]; - final int signatureDuration = Integer.parseInt(args[4]); + final String onClusterAnalysisFile = args[1]; + final String offClusterAnalysisFile = args[2]; + final String onSignatureFile = args[3]; + final String offSignatureFile = args[4]; + final String resultsFile = args[5]; + final int signatureDuration = Integer.parseInt(args[6]); // Parse optional parameters. List> onSignatureMacFilters = null, offSignatureMacFilters = null; - final int optParamsStartIdx = 5; + final int optParamsStartIdx = 7; if (args.length > optParamsStartIdx) { for (int i = optParamsStartIdx; i < args.length; i++) { if (args[i].equalsIgnoreCase("-onMacFilters")) { @@ -105,22 +110,40 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Include metadata as comments at the top PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); resultsWriter.flush(); - // TODO: IMPLEMENT THE RANGE-BASED DETECTION HERE - boolean isRangeBased = true; - + double eps = 10.0; // Create signature detectors and add observers that output their detected events. List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); + // Load signature analyses + List>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile); + List>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile); + // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS + // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR + // TODO: STRICT MATCHING + // Check if we should use range-based matching +// boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); +// boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); + // TODO: WE DON'T DO RANGE-BASED FOR NOW BECAUSE THE RESULTS ARE TERRIBLE FOR LAYER 2 MATCHING + // TODO: THIS WOULD ONLY WORK FOR SIGNATURES LONGER THAN 2 PACKETS + boolean isRangeBasedForOn = false; + boolean isRangeBasedForOff = false; + // Update the signature with ranges if it is range-based + if (isRangeBasedForOn && isRangeBasedForOff) { + onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); + offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); + } Layer2SignatureDetector onDetector = onSignatureMacFilters == null ? - new Layer2SignatureDetector(onSignature) : - new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(onSignature, isRangeBasedForOn, eps) : + new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps); Layer2SignatureDetector offDetector = offSignatureMacFilters == null ? - new Layer2SignatureDetector(offSignature) : - new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(offSignature, isRangeBasedForOff, eps) : + new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBasedForOff, eps); onDetector.addObserver((signature, match) -> { UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp()); PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); @@ -179,21 +202,23 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb private int mInclusionTimeMillis; - public Layer2SignatureDetector(List>> searchedSignature) { - this(searchedSignature, null, 0, false); + public Layer2SignatureDetector(List>> searchedSignature, boolean isRangeBased, double eps) { + this(searchedSignature, null, 0, isRangeBased, eps); } public Layer2SignatureDetector(List>> searchedSignature, List> flowFilters, int inclusionTimeMillis, boolean isRangeBased) { + Boolean>> flowFilters, int inclusionTimeMillis, boolean isRangeBased, double eps) { if (flowFilters != null && flowFilters.size() != searchedSignature.size()) { - throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature."); + throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster " + + "of the signature."); } mSignature = Collections.unmodifiableList(searchedSignature); List clusterMatchers = new ArrayList<>(); for (int i = 0; i < mSignature.size(); i++) { List> cluster = mSignature.get(i); Layer2ClusterMatcher clusterMatcher = flowFilters == null ? - new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased); + new Layer2ClusterMatcher(cluster, isRangeBased, eps) : + new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased, eps); clusterMatcher.addObserver(this); clusterMatchers.add(clusterMatcher); } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java index a4ad857..b070bd2 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java @@ -66,11 +66,6 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack */ private final String mRouterWanIp; - /** - * Range-based vs. strict matching. - */ - private final boolean mRangeBased; - /** * Epsilon value used by the DBSCAN algorithm; it is used again for range-based matching here. */ @@ -81,6 +76,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack * @param cluster The cluster that traffic is matched against. * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for * determining the direction of packets). + * @param eps The epsilon value used in the DBSCAN algorithm. * @param isRangeBased The boolean that decides if it is range-based vs. strict matching. * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that * (a subset of) the examined traffic is similar to the traffic that makes up @@ -103,8 +99,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted * in order to ensure correctness, especially during the development/debugging phase. */ - mRangeBased = isRangeBased; - if (!mRangeBased) { // Only when it is not range-based + if (!isRangeBased) { // Only when it is not range-based if (mCluster.stream(). anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) { throw new IllegalArgumentException( diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java similarity index 93% rename from Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java rename to Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java index 6e5b87c..859c056 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java @@ -28,7 +28,7 @@ import java.util.function.Consumer; * @author Janus Varmarken {@literal } * @author Rahmadi Trimananda {@literal } */ -public class SignatureDetector implements PacketListener, ClusterMatcherObserver { +public class Layer3SignatureDetector implements PacketListener, ClusterMatcherObserver { // Test client public static void main(String[] args) throws PcapNativeException, NotOpenException { @@ -242,15 +242,15 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver // final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; // final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; // final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; - final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; // final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; // Arlo Camera PHONE signatures // final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; // final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete"; - final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete"; - final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls"; - final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls"; +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls"; // TODO: NEST THERMOSTAT experiment // final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; @@ -282,15 +282,15 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver // final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; // final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; -//// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; -// // Blossom Sprinkler DEVICE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; -//// final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls"; -//// final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls"; -// // Blossom Sprinkler PHONE signatures +// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; + final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; + // Blossom Sprinkler DEVICE signatures + final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; + final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls"; + final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls"; + // Blossom Sprinkler PHONE signatures // final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; // final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; // final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-phone-side.cls"; @@ -357,30 +357,6 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver // final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-onClusters-phone-side.cls"; // final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-offClusters-phone-side.cls"; - - /* - // WeMo Plug experiment - final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; - // WeMo Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig"; - // WeMo Insight Plug experiment - final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; - // WeMo Insight Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig"; - */ - - // D-Link Siren experiment -// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; - // D-Link Siren DEVICE signatures - //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; - //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - - // Output file names used (to make it easy to catch if one forgets to change them) System.out.println("ON signature file in use is " + onSignatureFile); System.out.println("OFF signature file in use is " + offSignatureFile); @@ -408,9 +384,6 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); } - // LAN -// SignatureDetector onDetector = new SignatureDetector(onSignature, null); -// SignatureDetector offDetector = new SignatureDetector(offSignature, null); // WAN SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", 0, isRangeBasedForOn, eps); @@ -539,7 +512,7 @@ public class SignatureDetector implements PacketListener, ClusterMatcherObserver return listUserActionClean; } - public SignatureDetector(List>> searchedSignature, String routerWanIp, + public Layer3SignatureDetector(List>> searchedSignature, String routerWanIp, int inclusionTimeMillis, boolean isRangeBased, double eps) { // note: doesn't protect inner lists from changes :'( mSignature = Collections.unmodifiableList(searchedSignature); diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java index e1648ba..8f5874c 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java @@ -16,5 +16,4 @@ public interface Layer2FlowObserver { * @param newPacket The packet that was added to the flow. */ void onNewPacket(Layer2Flow flow, PcapPacket newPacket); - } -- 2.34.1