From: rtrimana Date: Mon, 11 Mar 2019 22:57:52 +0000 (-0700) Subject: Adding range-based matching for Layer 2 and fusing it off for now (the results are... X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=92a31b0967b3acff9e473ce62c136e84298b3aab Adding range-based matching for Layer 2 and fusing it off for now (the results are terrible for signatures that only have 2 packets). --- diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java new file mode 100644 index 0000000..1621c82 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java @@ -0,0 +1,102 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; + +import java.util.ArrayList; +import java.util.List; + +/** + * Base class for layer 2 matchers ({@code Layer2SequenceMatcher} and {@code Layer2RangeMatcher}). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +abstract public class Layer2AbstractMatcher { + + /** + * Buffer of actual packets seen so far that match the searched range (i.e., constitutes a subsequence). + */ + protected final List mMatchedPackets = new ArrayList<>(); + + /** + * Models the directions of packets. As the sequence matcher assumes that it is only presented + * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a + * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses + * of devices in advance during matching. + */ + protected final boolean[] mPacketDirections; + + /** + * Create a {@code Layer2AbstractMatcher}. + * @param sequence The sequence of the signature. + */ + public Layer2AbstractMatcher(List sequence) { + mPacketDirections = new boolean[sequence.size()]; + // Compute packet directions for sequence. + for (int i = 0; i < sequence.size(); i++) { + if (i == 0) { + // No previous packet; boolean parameter is ignored in this special case. + mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i)); + } else { + // Base direction marker on direction of previous packet. + PcapPacket prevPkt = sequence.get(i-1); + boolean prevPktDirection = mPacketDirections[i-1]; + mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i)); + } + } + } + + /** + * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction + * of {@code currPkt} is {@code true} by definition. + * @param prevPkt The previous packet, if any. + * @param prevPktDirection The computed direction of the previous packet + * @param currPkt The current packet for which the direction is to be determined. + * @return The direction of {@code currPkt}. + */ + protected boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { + if (prevPkt == null) { + // By definition, use true as direction marker for first packet + return true; + } + if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { + // Current packet goes in same direction as previous packet. + return prevPktDirection; + } else { + // Current packet goes in opposite direction of previous packet. + return !prevPktDirection; + } + } + + /** + * See the implementer class for the following method. + * + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public abstract boolean matchPacket(PcapPacket packet); + + /** + * See the implementer class for the following method. + */ + public abstract int getTargetSequencePacketCount(); + + public int getMatchedPacketsCount() { + return mMatchedPackets.size(); + } + + public List getMatchedPackets() { + return mMatchedPackets; + } + + /** + * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. + * @return The last matched packet, or {@code null} if no packets have been matched yet. + */ + public PcapPacket getLastPacket() { + //return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null; + return mMatchedPackets.size() > 0 ? mMatchedPackets.get(mMatchedPackets.size()-1) : null; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java index 88cb64e..e2a4aea 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -27,16 +27,27 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye * of {@link #mCluster} and has so far matched {@code j} packets of that particular sequence. */ private final Map mPerFlowSeqMatchers = new HashMap<>(); + private final Map mPerFlowRangeMatcher = new HashMap<>(); private final Function mFlowFilter; + /** + * Specifying range-based instead of conservative exact matching. + */ + private final boolean mRangeBased; + + /** + * Epsilon value used by the DBSCAN algorithm; it is used again for range-based matching here. + */ + private final double mEps; + /** * Create a new {@link Layer2ClusterMatcher} that attempts to find occurrences of {@code cluster}'s members. * @param cluster The sequence mutations that the new {@link Layer2ClusterMatcher} should search for. */ - public Layer2ClusterMatcher(List> cluster) { + public Layer2ClusterMatcher(List> cluster, boolean isRangeBased, double eps) { // Consider all flows if no flow filter specified. - this(cluster, flow -> true, false); + this(cluster, flow -> true, isRangeBased, eps); } /** @@ -49,15 +60,26 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye * the new flow. This functionality may for example come in handy when one only wants to search * for matches in the subset of flows that involves a specific (range of) MAC(s). * @param isRangeBased The boolean that decides if it is range-based vs. strict matching. + * @param eps The epsilon value used in the DBSCAN algorithm. */ public Layer2ClusterMatcher(List> cluster, Function flowFilter, - boolean isRangeBased) { + boolean isRangeBased, double eps) { super(cluster, isRangeBased); mFlowFilter = flowFilter; + mRangeBased = isRangeBased; + mEps = eps; } @Override public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { + if (mRangeBased) { + rangeBasedMatching(flow, newPacket); + } else { + conservativeMatching(flow, newPacket); + } + } + + private void conservativeMatching(Layer2Flow flow, PcapPacket newPacket) { if (mPerFlowSeqMatchers.get(flow) == null) { // If this is the first time we encounter this flow, we need to set up sequence matchers for it. // All sequences of the cluster have the same length, so we only need to compute the length of the nested @@ -114,6 +136,56 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye } } + private void rangeBasedMatching(Layer2Flow flow, PcapPacket newPacket) { + // TODO: For range-based matching, we only care about matching a range; therefore it is a matcher array. + if (mPerFlowRangeMatcher.get(flow) == null) { + // If this is the first time we encounter this flow, we need to set up a sequence matcher. + // All sequences of the cluster have the same length, so we only need to compute the length of the + // arrays once. We want to make room for a cluster matcher in each state, including the initial empty state + // but excluding the final "full match" state (as there is no point in keeping a terminated sequence matcher + // around), so the length of the array is simply the sequence length. + Layer2RangeMatcher[] matcher = new Layer2RangeMatcher[mCluster.get(0).size()]; + // Prepare a "state 0" sequence matcher. + matcher[0] = new Layer2RangeMatcher(mCluster.get(0), mCluster.get(1), mEps); + // Associate the new sequence matcher table with the new flow. + mPerFlowRangeMatcher.put(flow, matcher); + } + // Fetch table that contains sequence matchers for this flow. + Layer2RangeMatcher[] matcher = mPerFlowRangeMatcher.get(flow); + // Present packet to the sequence matcher. + for (int j = matcher.length - 1; j >= 0; j--) { + Layer2RangeMatcher sm = matcher[j]; + if (sm == null) { + // There is currently no sequence matcher that has managed to match j packets. + continue; + } + boolean matched = sm.matchPacket(newPacket); + if (matched) { + if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { + // Sequence matcher has a match. Report it to observers. + mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets())); + // Remove the now terminated sequence matcher. + matcher[j] = null; + } else { + // Sequence matcher advanced one step, so move it to its corresponding new position iff the + // packet that advanced it has a later timestamp than that of the last matched packet of the + // sequence matcher at the new index, if any. In most traces, a small amount of the packets + // appear out of order (with regards to their timestamp), which is why this check is required. + // Obviously it would not be needed if packets where guaranteed to be processed in timestamp + // order here. + if (matcher[j+1] == null || + newPacket.getTimestamp().isAfter(matcher[j+1].getLastPacket().getTimestamp())) { + matcher[j+1] = sm; + } + } + // We always want to have a sequence matcher in state 0, regardless of if the one that advanced + // from state zero completed its matching or if it replaced a different one in state 1 or not. + if (sm.getMatchedPacketsCount() == 1) { + matcher[j] = new Layer2RangeMatcher(sm.getTargetLowerBound(), sm.getTargetUpperBound(), mEps); + } + } + } + } @Override protected List> pruneCluster(List> cluster) { diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java new file mode 100644 index 0000000..db7e5b8 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java @@ -0,0 +1,123 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.List; + +/** + * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls + * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information. This class has the same flavor as the + * {@link Layer2SequenceMatcher} class. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2RangeMatcher extends Layer2AbstractMatcher { + /** + * The range this {@link Layer2RangeMatcher} is searching for. + */ + private final List mLowerBound; + private final List mUpperBound; + private final double mEps; + + /** + * Create a {@code Layer2RangeMatcher}. + * @param lowerBound The lower bound of the sequence to match against (search for). + * @param upperBound The upper bound of the sequence to match against (search for). + * @param eps The epsilon value used in the DBSCAN algorithm. + */ + public Layer2RangeMatcher(List lowerBound, List upperBound, double eps) { + // TODO: Just use the lower bound since both lower and upper bounds' packets essentially have the same direction + // TODO: for the same position in the array. Both arrays also have the same length. + super(lowerBound); + mLowerBound = lowerBound; + mUpperBound = upperBound; + mEps = eps; + } + + /** + * Attempt to advance this {@code Layer2RangeMatcher} by matching {@code packet} against the packet that this + * {@code Layer2RangeMatcher} expects as the next packet of the sequence it is searching for. + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public boolean matchPacket(PcapPacket packet) { + if (getMatchedPacketsCount() == getTargetSequencePacketCount()) { + // We already matched the entire sequence, so we can't match any more packets. + return false; + } + + // Verify that new packet pertains to same flow as previously matched packets, if any. + if (getMatchedPacketsCount() > 0) { + MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet); + MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet); + MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0)); + MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0)); + if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) || + pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) { + return false; + } + } + + // Get representative of the packet we expect to match next. + PcapPacket expectedLowerBound = mLowerBound.get(mMatchedPackets.size()); + PcapPacket expectedUpperBound = mUpperBound.get(mMatchedPackets.size()); + // First verify if the received packet has the length we're looking for (the length should be within the range). +// if (expectedLowerBound.getOriginalLength() - (int) mEps <= packet.getOriginalLength() && +// packet.getOriginalLength() <= expectedUpperBound.getOriginalLength() + (int) mEps){ + if (expectedLowerBound.getOriginalLength() - (int) mEps <= packet.getOriginalLength() && + packet.getOriginalLength() <= expectedUpperBound.getOriginalLength() + (int) mEps){ + // If this is the first packet, we only need to verify that its length is correct. Time constraints are + // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we + // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet + // must originate from this particular MAC". + if (getMatchedPacketsCount() == 0) { + // Store packet as matched and advance. + mMatchedPackets.add(packet); + return true; + } + // Check if direction of packet matches expected direction. + boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1), + mPacketDirections[getMatchedPacketsCount()-1], packet); + boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()]; + if (actualDirection != expectedDirection) { + return false; + } + // Next apply timing constraints: + // 1: to be a match, the packet must have a later timestamp than any other packet currently matched + // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded? + if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) { + return false; + } + if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). + plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { + return false; + } + // If we made it here, it means that this packet has the expected length, direction, and obeys the timing + // constraints, so we store it and advance. + mMatchedPackets.add(packet); + if (mMatchedPackets.size() == mLowerBound.size()) { + // TODO report (to observers?) that we are done? + } + return true; + } + return false; + } + + public int getTargetSequencePacketCount() { + return mLowerBound.size(); + } + + public List getTargetLowerBound() { + return mLowerBound; + } + + public List getTargetUpperBound() { + return mLowerBound; + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java index 672fb72..2db2228 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -15,35 +15,21 @@ import java.util.List; * @author Janus Varmarken {@literal } * @author Rahmadi Trimananda {@literal } */ -public class Layer2SequenceMatcher { +public class Layer2SequenceMatcher extends Layer2AbstractMatcher { /** * The sequence this {@link Layer2SequenceMatcher} is searching for. */ private final List mSequence; - /** - * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the - * searched sequence). - */ - private final List mMatchedPackets = new ArrayList<>(); - - /** - * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented - * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a - * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses - * of devices in advance during matching. - */ - private final boolean[] mPacketDirections; - /** * Create a {@code Layer2SequenceMatcher}. * @param sequence The sequence to match against (search for). */ public Layer2SequenceMatcher(List sequence) { + super(sequence); mSequence = sequence; // Compute packet directions for sequence. - mPacketDirections = new boolean[sequence.size()]; for (int i = 0; i < sequence.size(); i++) { if (i == 0) { // No previous packet; boolean parameter is ignored in this special case. @@ -123,10 +109,6 @@ public class Layer2SequenceMatcher { return false; } - public int getMatchedPacketsCount() { - return mMatchedPackets.size(); - } - public int getTargetSequencePacketCount() { return mSequence.size(); } @@ -135,39 +117,4 @@ public class Layer2SequenceMatcher { return mSequence; } - public List getMatchedPackets() { - return mMatchedPackets; - } - - /** - * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. - * @return The last matched packet, or {@code null} if no packets have been matched yet. - */ - public PcapPacket getLastPacket() { - return mSequence.size() > 0 ? mSequence.get(mSequence.size()-1) : null; - } - - /** - * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction - * of {@code currPkt} is {@code true} by definition. - * @param prevPkt The previous packet, if any. - * @param prevPktDirection The computed direction of the previous packet - * @param currPkt The current packet for which the direction is to be determined. - * @return The direction of {@code currPkt}. - */ - private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { - if (prevPkt == null) { - // By definition, use true as direction marker for first packet - return true; - } - if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { - // Current packet goes in same direction as previous packet. - return prevPktDirection; - } else { - // Current packet goes in opposite direction of previous packet. - return !prevPktDirection; - } - } - - } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index f5a314f..505bfdc 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -9,6 +9,7 @@ import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.jgrapht.GraphPath; import org.jgrapht.alg.shortestpath.DijkstraShortestPath; @@ -51,9 +52,11 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException { // Parse required parameters. - if (args.length < 5) { - String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile resultsFile" + + if (args.length < 7) { + String errMsg = String.format("Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile resultsFile" + "\n inputPcapFile: the target of the detection" + + "\n onAnalysisFile: the file that contains the ON clusters analysis" + + "\n offAnalysisFile: the file that contains the OFF clusters analysis" + "\n onSignatureFile: the file that contains the ON signature to search for" + "\n offSignatureFile: the file that contains the OFF signature to search for" + "\n resultsFile: where to write the results of the detection" + @@ -75,14 +78,16 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb return; } final String pcapFile = args[0]; - final String onSignatureFile = args[1]; - final String offSignatureFile = args[2]; - final String resultsFile = args[3]; - final int signatureDuration = Integer.parseInt(args[4]); + final String onClusterAnalysisFile = args[1]; + final String offClusterAnalysisFile = args[2]; + final String onSignatureFile = args[3]; + final String offSignatureFile = args[4]; + final String resultsFile = args[5]; + final int signatureDuration = Integer.parseInt(args[6]); // Parse optional parameters. List> onSignatureMacFilters = null, offSignatureMacFilters = null; - final int optParamsStartIdx = 5; + final int optParamsStartIdx = 7; if (args.length > optParamsStartIdx) { for (int i = optParamsStartIdx; i < args.length; i++) { if (args[i].equalsIgnoreCase("-onMacFilters")) { @@ -105,22 +110,40 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb // Include metadata as comments at the top PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); resultsWriter.flush(); - // TODO: IMPLEMENT THE RANGE-BASED DETECTION HERE - boolean isRangeBased = true; - + double eps = 10.0; // Create signature detectors and add observers that output their detected events. List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); + // Load signature analyses + List>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile); + List>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile); + // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS + // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR + // TODO: STRICT MATCHING + // Check if we should use range-based matching +// boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); +// boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); + // TODO: WE DON'T DO RANGE-BASED FOR NOW BECAUSE THE RESULTS ARE TERRIBLE FOR LAYER 2 MATCHING + // TODO: THIS WOULD ONLY WORK FOR SIGNATURES LONGER THAN 2 PACKETS + boolean isRangeBasedForOn = false; + boolean isRangeBasedForOff = false; + // Update the signature with ranges if it is range-based + if (isRangeBasedForOn && isRangeBasedForOff) { + onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); + offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); + } Layer2SignatureDetector onDetector = onSignatureMacFilters == null ? - new Layer2SignatureDetector(onSignature) : - new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(onSignature, isRangeBasedForOn, eps) : + new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps); Layer2SignatureDetector offDetector = offSignatureMacFilters == null ? - new Layer2SignatureDetector(offSignature) : - new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBased); + new Layer2SignatureDetector(offSignature, isRangeBasedForOff, eps) : + new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBasedForOff, eps); onDetector.addObserver((signature, match) -> { UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp()); PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); @@ -179,21 +202,23 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb private int mInclusionTimeMillis; - public Layer2SignatureDetector(List>> searchedSignature) { - this(searchedSignature, null, 0, false); + public Layer2SignatureDetector(List>> searchedSignature, boolean isRangeBased, double eps) { + this(searchedSignature, null, 0, isRangeBased, eps); } public Layer2SignatureDetector(List>> searchedSignature, List> flowFilters, int inclusionTimeMillis, boolean isRangeBased) { + Boolean>> flowFilters, int inclusionTimeMillis, boolean isRangeBased, double eps) { if (flowFilters != null && flowFilters.size() != searchedSignature.size()) { - throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster of the signature."); + throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster " + + "of the signature."); } mSignature = Collections.unmodifiableList(searchedSignature); List clusterMatchers = new ArrayList<>(); for (int i = 0; i < mSignature.size(); i++) { List> cluster = mSignature.get(i); Layer2ClusterMatcher clusterMatcher = flowFilters == null ? - new Layer2ClusterMatcher(cluster) : new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased); + new Layer2ClusterMatcher(cluster, isRangeBased, eps) : + new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased, eps); clusterMatcher.addObserver(this); clusterMatchers.add(clusterMatcher); } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java index a4ad857..b070bd2 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java @@ -66,11 +66,6 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack */ private final String mRouterWanIp; - /** - * Range-based vs. strict matching. - */ - private final boolean mRangeBased; - /** * Epsilon value used by the DBSCAN algorithm; it is used again for range-based matching here. */ @@ -81,6 +76,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack * @param cluster The cluster that traffic is matched against. * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for * determining the direction of packets). + * @param eps The epsilon value used in the DBSCAN algorithm. * @param isRangeBased The boolean that decides if it is range-based vs. strict matching. * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that * (a subset of) the examined traffic is similar to the traffic that makes up @@ -103,8 +99,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted * in order to ensure correctness, especially during the development/debugging phase. */ - mRangeBased = isRangeBased; - if (!mRangeBased) { // Only when it is not range-based + if (!isRangeBased) { // Only when it is not range-based if (mCluster.stream(). anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) { throw new IllegalArgumentException( diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java new file mode 100644 index 0000000..859c056 --- /dev/null +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java @@ -0,0 +1,692 @@ +package edu.uci.iotproject.detection.layer3; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.detection.ClusterMatcherObserver; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PcapPacketUtils; +import edu.uci.iotproject.util.PrintUtils; +import org.apache.commons.math3.distribution.AbstractRealDistribution; +import org.apache.commons.math3.distribution.NormalDistribution; +import org.jgrapht.GraphPath; +import org.jgrapht.alg.shortestpath.DijkstraShortestPath; +import org.jgrapht.graph.DefaultWeightedEdge; +import org.jgrapht.graph.SimpleDirectedWeightedGraph; +import org.pcap4j.core.*; + +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.*; +import java.util.function.Consumer; + +/** + * Detects an event signature that spans one or multiple TCP connections. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer3SignatureDetector implements PacketListener, ClusterMatcherObserver { + + // Test client + public static void main(String[] args) throws PcapNativeException, NotOpenException { +// if (args.length < 3) { +// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", +// SignatureDetector.class.getSimpleName()); +// System.out.println(errMsg); +// return; +// } +// final String inputPcapFile = args[0]; +// final String onSignatureFile = args[1]; +// final String offSignatureFile = args[2]; + + String path = "/scratch/July-2018"; // Rahmadi +// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus +// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) +// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) + + // No activity test + //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; + + // D-Link Siren experiment +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; + // D-Link Siren DEVICE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; + // TODO: EXPERIMENT - November 19, 2018 + // Hue Bulb experiment +// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + + /* + // Kwikset Doorlock Sep 12 experiment +// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; + final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; +// // Kwikset Doorlock PHONE signatures + final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; + final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; + */ + + // D-Link Plug experiment + //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; + + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; + + // TODO: The following are negative tests against the PCAP file from UNSW +// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! + // TODO: The following one is very long!!! - Split into smaller files! +// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; + // Negative test: dataset from UNB +// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap"; + + // TODO: The following are tests for signatures against training data + + // TODO: D-LINK PLUG experiment +// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - November 7, 2018 + // D-Link Plug experiment +// //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; +// // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-device-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-device-side.cls"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-phone-side.cls"; + + // TODO: EXPERIMENT - November 9, 2018 + // TODO: D-LINK SIREN experiment + //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; +// // D-Link Siren DEVICE signatures +// // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-offClusters-phone-side.cls"; + + // TODO: TP-LINK PLUG experiment +//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; +// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; + // TODO: EXPERIMENT - November 8, 2018 + // TP-Link Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; +//// // TP-Link Plug DEVICE signatures +//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; +// // TP-Link Plug PHONE signatures +//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; +//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; +// // TP-Link Plug cluster analyses +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-onClusters.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-offClusters.cls"; + + // Amazon Alexa experiment +// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; + + // TODO: KWIKSET DOORLOCK Sep 12 experiment +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; +// // Kwikset Doorlock PHONE signatures +// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; +// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; + // TODO: EXPERIMENT - November 10, 2018 + // Kwikset Door lock experiment +// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; +// // Kwikset Door lock PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; +//// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; +//// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-offClusters-phone-side.cls"; + + // TODO: SMARTTHINGS PLUG experiment +// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// // SmartThings Plug DEVICE signatures +// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; +// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; +// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 12, 2018 + // SmartThings Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; +//// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-offClusters-phone-side.cls"; + + // TODO: ARLO CAMERA experiment +// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +//// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 13, 2018 + // Arlo Camera experiment +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; +// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; + // Arlo Camera PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls"; + + // TODO: NEST THERMOSTAT experiment +// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +// // Nest Thermostat DEVICE signatures +//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; +//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; +// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; +// // TODO: EXPERIMENT - November 15, 2018 + // Nest Thermostat experiment +// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; +//// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-offClusters-phone-side.cls"; + + // TODO: EXPERIMENT - January 9, 2018 + // TODO: BLOSSOM SPRINKLER experiment + // Blossom Sprinkler experiment +// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; +// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; +// // Blossom Sprinkler DEVICE signatures +// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + +// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; + final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; + // Blossom Sprinkler DEVICE signatures + final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; + final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls"; + final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls"; + // Blossom Sprinkler PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls"; + + // LiFX Bulb experiment +// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; +// // LiFX Bulb DEVICE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; + // LiFX Bulb PHONE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; + + /* + // Hue Bulb experiment + final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures + final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; + final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + */ + + // TODO: TP-LINK BULB experiment +// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 16, 2018 + // TP-Link Bulb experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-offClusters-phone-side.cls"; + + // TODO: EXPERIMENT - November 20, 2018 + // TODO: WEMO PLUG experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; + // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE + // TODO: THE ACTUAL TRIGGERS +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; +// // WeMo Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-offClusters-phone-side.cls"; + + // TODO: EXPERIMENT - November 21, 2018 + // TODO: WEMO INSIGHT Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; + // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; + // WeMo Insight Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; +// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-onClusters-phone-side.cls"; +// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-offClusters-phone-side.cls"; + + // Output file names used (to make it easy to catch if one forgets to change them) + System.out.println("ON signature file in use is " + onSignatureFile); + System.out.println("OFF signature file in use is " + offSignatureFile); + System.out.println("PCAP file that is the target of detection is " + inputPcapFile); + + // Specify epsilon + // TODO: This would be specified through command line option + double eps = 10.0; + // Load signatures + List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); + // Load signature analyses + List>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile); + List>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile); + + // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS + // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR + // TODO: STRICT MATCHING + // Check if we should use range-based matching + boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); + boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); + // Update the signature with ranges if it is range-based + if (isRangeBasedForOn && isRangeBasedForOff) { + onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); + offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); + } + + // WAN + SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", + 0, isRangeBasedForOn, eps); + SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", + 0, isRangeBasedForOff, eps); + + final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). + withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); + + // Outputs information about a detected event to std.out + final Consumer outputter = ua -> { + String eventDescription; + switch (ua.getType()) { + case TOGGLE_ON: + eventDescription = "ON"; + break; + case TOGGLE_OFF: + eventDescription = "OFF"; + break; + default: + throw new AssertionError("unhandled event type"); + } + //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", + // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); + String output = String.format("%s", + dateTimeFormatter.format(ua.getTimestamp())); + System.out.println(output); + }; + + // Let's create observers that construct a UserAction representing the detected event. + final List detectedEvents = new ArrayList<>(); + onDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); + }); + offDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); + }); + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); + reader.readFromHandle(); + + // TODO: need a better way of triggering detection than this... + if (isRangeBasedForOn && isRangeBasedForOff) { + onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); + offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); + } else { + onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); + offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); + } + + // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger + // times file. + Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp)); + + // Output the detected events + detectedEvents.forEach(outputter); + + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); + + // TODO: Temporary clean up until we clean the pipeline +// List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); +// cleanedDetectedEvents.forEach(outputter); + } + + /** + * The signature that this {@link SignatureDetector} is searching for. + */ + private final List>> mSignature; + + /** + * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the + * the signature. + */ + private final List mClusterMatchers; + + /** + * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches + * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., + * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed + * of multiple packet sequences occurring shortly after one another on multiple connections). + */ + private final List>[] pendingMatches; + + /** + * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}. + */ + private final Map mClusterMatcherIds; + + private final List mObservers = new ArrayList<>(); + + private int mInclusionTimeMillis; + + /** + * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions + * that appear multiple times. + * TODO: This static method is probably just for temporary and we could get rid of this after we clean up + * TODO: the pipeline + * + * @param listUserAction A {@link List} of {@code UserAction}. + * + */ + public static List removeDuplicates(List listUserAction) { + + // Iterate and check for duplicates (check timestamps) + Set epochSecondSet = new HashSet<>(); + // Create a target list for cleaned up list + List listUserActionClean = new ArrayList<>(); + for(UserAction userAction : listUserAction) { + // Don't insert if any duplicate is found + if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) { + listUserActionClean.add(userAction); + epochSecondSet.add(userAction.getTimestamp().getEpochSecond()); + } + } + return listUserActionClean; + } + + public Layer3SignatureDetector(List>> searchedSignature, String routerWanIp, + int inclusionTimeMillis, boolean isRangeBased, double eps) { + // note: doesn't protect inner lists from changes :'( + mSignature = Collections.unmodifiableList(searchedSignature); + // Generate corresponding/appropriate ClusterMatchers based on the provided signature + List clusterMatchers = new ArrayList<>(); + for (List> cluster : mSignature) { + clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, isRangeBased, eps, this)); + } + mClusterMatchers = Collections.unmodifiableList(clusterMatchers); + + // < exploratory > + pendingMatches = new List[mClusterMatchers.size()]; + for (int i = 0; i < pendingMatches.length; i++) { + pendingMatches[i] = new ArrayList<>(); + } + Map clusterMatcherIds = new HashMap<>(); + for (int i = 0; i < mClusterMatchers.size(); i++) { + clusterMatcherIds.put(mClusterMatchers.get(i), i); + } + mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); + mInclusionTimeMillis = + inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + } + + public void addObserver(SignatureDetectionObserver observer) { + mObservers.add(observer); + } + + public boolean removeObserver(SignatureDetectionObserver observer) { + return mObservers.remove(observer); + } + + @Override + public void gotPacket(PcapPacket packet) { + // simply delegate packet reception to all ClusterMatchers. + mClusterMatchers.forEach(cm -> cm.gotPacket(packet)); + } + + @Override + public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { + // Add the match at the corresponding index + pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); + checkSignatureMatch(); + } + + private void checkSignatureMatch() { + // << Graph-based approach using Balint's idea. >> + // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. + + // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence. + if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) { + // Construct the DAG + final SimpleDirectedWeightedGraph graph = + new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); + // Add a vertex for each match found by all ClusterMatchers + // And maintain an array to keep track of what cluster matcher each vertex corresponds to + final List[] vertices = new List[pendingMatches.length]; + for (int i = 0; i < pendingMatches.length; i++) { + vertices[i] = new ArrayList<>(); + for (List sequence : pendingMatches[i]) { + Vertex v = new Vertex(sequence); + vertices[i].add(v); // retain reference for later when we are to add edges + graph.addVertex(v); // add to vertex to graph + } + } + // Add dummy source and sink vertices to facilitate search. + final Vertex source = new Vertex(null); + final Vertex sink = new Vertex(null); + graph.addVertex(source); + graph.addVertex(sink); + // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0. + // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. + for (Vertex v : vertices[0]) { + DefaultWeightedEdge edge = graph.addEdge(source, v); + graph.setEdgeWeight(edge, 0.0); + } + // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature + // are connected to the sink node. + for (Vertex v : vertices[vertices.length-1]) { + DefaultWeightedEdge edge = graph.addEdge(v, sink); + graph.setEdgeWeight(edge, 0.0); + } + // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index + // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former). + for (int i = 0; i < vertices.length; i++) { + int j = i + 1; + if (j < vertices.length) { + for (Vertex iv : vertices[i]) { + PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); + for (Vertex jv : vertices[j]) { + PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); + if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { + DefaultWeightedEdge edge = graph.addEdge(iv, jv); + // The weight is the duration of the i'th sequence plus the duration between the i'th + // and i+1'th sequence. + Duration d = Duration. + between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); + // Unfortunately weights are double values, so must convert from long to double. + // TODO: need nano second precision? If so, use d.toNanos(). + // TODO: risk of overflow when converting from long to double..? + graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); + } + // Alternative version if we cannot assume that sequences are ordered by timestamp: +// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() +// .getTimestamp().isBefore(jv.sequence.stream().min( +// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { +// +// } + } + } + } + } + // Graph construction complete, run shortest-path to find a (potential) signature match. + DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); + GraphPath shortestPath = dijkstra.getPath(source, sink); + if (shortestPath != null) { + // The total weight is the duration between the first packet of the first sequence and the last packet + // of the last sequence, so we simply have to compare the weight against the timeframe that we allow + // the signature to span. For now we just use the inclusion window we defined for training purposes. + // Note however, that we must convert back from double to long as the weight is stored as a double in + // JGraphT's API. + if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) { + // There's a signature match! + // Extract the match from the vertices + List> signatureMatch = new ArrayList<>(); + for(Vertex v : shortestPath.getVertexList()) { + if (v == source || v == sink) { + // Skip the dummy source and sink nodes. + continue; + } + signatureMatch.add(v.sequence); + // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that + // the sequence we've "consumed" for index i of the matched signature is also at index i in + // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct + // another signature match in a later call. + pendingMatches[signatureMatch.size()-1].remove(v.sequence); + } + // Declare success: notify observers + mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, + Collections.unmodifiableList(signatureMatch))); + } + } + } + } + + /** + * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. + */ + interface SignatureDetectionObserver { + + /** + * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's + * examining. + * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching + * for. + * @param matchingTraffic The actual traffic trace that matches the searched signature. + */ + void onSignatureDetected(List>> searchedSignature, + List> matchingTraffic); + } + + /** + * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding + * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. + * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} + * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not + * recognize two lists that contain the same items--from a value and not reference point of view--as the same + * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more + * appropriate. + */ + private static class Vertex { + private final List sequence; + private Vertex(List wrappedSequence) { + sequence = wrappedSequence; + } + } +} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java deleted file mode 100644 index 6e5b87c..0000000 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java +++ /dev/null @@ -1,719 +0,0 @@ -package edu.uci.iotproject.detection.layer3; - -import edu.uci.iotproject.analysis.TriggerTrafficExtractor; -import edu.uci.iotproject.analysis.UserAction; -import edu.uci.iotproject.detection.AbstractClusterMatcher; -import edu.uci.iotproject.detection.ClusterMatcherObserver; -import edu.uci.iotproject.io.PcapHandleReader; -import edu.uci.iotproject.util.PcapPacketUtils; -import edu.uci.iotproject.util.PrintUtils; -import org.apache.commons.math3.distribution.AbstractRealDistribution; -import org.apache.commons.math3.distribution.NormalDistribution; -import org.jgrapht.GraphPath; -import org.jgrapht.alg.shortestpath.DijkstraShortestPath; -import org.jgrapht.graph.DefaultWeightedEdge; -import org.jgrapht.graph.SimpleDirectedWeightedGraph; -import org.pcap4j.core.*; - -import java.time.Duration; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.*; -import java.util.function.Consumer; - -/** - * Detects an event signature that spans one or multiple TCP connections. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class SignatureDetector implements PacketListener, ClusterMatcherObserver { - - // Test client - public static void main(String[] args) throws PcapNativeException, NotOpenException { -// if (args.length < 3) { -// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", -// SignatureDetector.class.getSimpleName()); -// System.out.println(errMsg); -// return; -// } -// final String inputPcapFile = args[0]; -// final String onSignatureFile = args[1]; -// final String offSignatureFile = args[2]; - - String path = "/scratch/July-2018"; // Rahmadi -// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus -// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) -// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) - - // No activity test - //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; - - // D-Link Siren experiment -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; - // D-Link Siren DEVICE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - // TODO: EXPERIMENT - November 19, 2018 - // Hue Bulb experiment -// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - - /* - // Kwikset Doorlock Sep 12 experiment -// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; - final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; -// // Kwikset Doorlock PHONE signatures - final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; - final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; - */ - - // D-Link Plug experiment - //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; - - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; - - // TODO: The following are negative tests against the PCAP file from UNSW -// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! - // TODO: The following one is very long!!! - Split into smaller files! -// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; - // Negative test: dataset from UNB -// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint-001.pcap"; - - // TODO: The following are tests for signatures against training data - - // TODO: D-LINK PLUG experiment -// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; - - // TODO: EXPERIMENT - November 7, 2018 - // D-Link Plug experiment -// //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; -// // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-device-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-device-side.cls"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-plug/analysis/dlink-plug-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 9, 2018 - // TODO: D-LINK SIREN experiment - //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; -// // D-Link Siren DEVICE signatures -// // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/dlink-siren/analysis/dlink-siren-offClusters-phone-side.cls"; - - // TODO: TP-LINK PLUG experiment -//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; -// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; - // TODO: EXPERIMENT - November 8, 2018 - // TP-Link Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; -//// // TP-Link Plug DEVICE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; -// // TP-Link Plug PHONE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; -// // TP-Link Plug cluster analyses -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-onClusters.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-plug/analysis/tplink-plug-offClusters.cls"; - - // Amazon Alexa experiment -// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; - - // TODO: KWIKSET DOORLOCK Sep 12 experiment -// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; -// // Kwikset Doorlock PHONE signatures -// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; -// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; - // TODO: EXPERIMENT - November 10, 2018 - // Kwikset Door lock experiment -// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; -// // Kwikset Door lock PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; -//// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; -//// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/kwikset-doorlock/analysis/kwikset-doorlock-offClusters-phone-side.cls"; - - // TODO: SMARTTHINGS PLUG experiment -// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// // SmartThings Plug DEVICE signatures -// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; -// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; -// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 12, 2018 - // SmartThings Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; -//// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/st-plug/analysis/st-plug-offClusters-phone-side.cls"; - - // TODO: ARLO CAMERA experiment -// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -//// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 13, 2018 - // Arlo Camera experiment -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; - final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; -// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; - // Arlo Camera PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig.complete"; - final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig.complete"; - final String onClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-onClusters-phone-side.cls"; - final String offClusterAnalysisFile = path + "/experimental_result/standalone/arlo-camera/analysis/arlo-camera-offClusters-phone-side.cls"; - - // TODO: NEST THERMOSTAT experiment -// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -// // Nest Thermostat DEVICE signatures -//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; -// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; -// // TODO: EXPERIMENT - November 15, 2018 - // Nest Thermostat experiment -// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; -//// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/nest-thermostat/analysis/nest-thermostat-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - January 9, 2018 - // TODO: BLOSSOM SPRINKLER experiment - // Blossom Sprinkler experiment -// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; -// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; -// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; -// // Blossom Sprinkler DEVICE signatures -// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; - -//// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; -// // Blossom Sprinkler DEVICE signatures -//// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; -//// final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-device-side.cls"; -//// final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-device-side.cls"; -// // Blossom Sprinkler PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/blossom-sprinkler/analysis/blossom-sprinkler-offClusters-phone-side.cls"; - - // LiFX Bulb experiment -// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; -// // LiFX Bulb DEVICE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; - // LiFX Bulb PHONE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; - - /* - // Hue Bulb experiment - final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures - final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; - final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - */ - - // TODO: TP-LINK BULB experiment -// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 16, 2018 - // TP-Link Bulb experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/tplink-bulb/analysis/tplink-bulb-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 20, 2018 - // TODO: WEMO PLUG experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; - // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE - // TODO: THE ACTUAL TRIGGERS -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; -// // WeMo Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-plug/analysis/wemo-plug-offClusters-phone-side.cls"; - - // TODO: EXPERIMENT - November 21, 2018 - // TODO: WEMO INSIGHT Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; - // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; - // WeMo Insight Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; -// final String onClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-onClusters-phone-side.cls"; -// final String offClusterAnalysisFile = path + "/experimental_result/standalone/wemo-insight-plug/analysis/wemo-insight-plug-offClusters-phone-side.cls"; - - - /* - // WeMo Plug experiment - final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; - // WeMo Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig"; - // WeMo Insight Plug experiment - final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; - // WeMo Insight Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig"; - */ - - // D-Link Siren experiment -// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; - // D-Link Siren DEVICE signatures - //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; - //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - - - // Output file names used (to make it easy to catch if one forgets to change them) - System.out.println("ON signature file in use is " + onSignatureFile); - System.out.println("OFF signature file in use is " + offSignatureFile); - System.out.println("PCAP file that is the target of detection is " + inputPcapFile); - - // Specify epsilon - // TODO: This would be specified through command line option - double eps = 10.0; - // Load signatures - List>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile); - List>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile); - // Load signature analyses - List>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile); - List>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile); - - // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS - // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR - // TODO: STRICT MATCHING - // Check if we should use range-based matching - boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature); - boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature); - // Update the signature with ranges if it is range-based - if (isRangeBasedForOn && isRangeBasedForOff) { - onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis); - offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis); - } - - // LAN -// SignatureDetector onDetector = new SignatureDetector(onSignature, null); -// SignatureDetector offDetector = new SignatureDetector(offSignature, null); - // WAN - SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105", - 0, isRangeBasedForOn, eps); - SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105", - 0, isRangeBasedForOff, eps); - - final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). - withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); - - // Outputs information about a detected event to std.out - final Consumer outputter = ua -> { - String eventDescription; - switch (ua.getType()) { - case TOGGLE_ON: - eventDescription = "ON"; - break; - case TOGGLE_OFF: - eventDescription = "OFF"; - break; - default: - throw new AssertionError("unhandled event type"); - } - //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", - // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); - String output = String.format("%s", - dateTimeFormatter.format(ua.getTimestamp())); - System.out.println(output); - }; - - // Let's create observers that construct a UserAction representing the detected event. - final List detectedEvents = new ArrayList<>(); - onDetector.addObserver((searched, match) -> { - PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); - }); - offDetector.addObserver((searched, match) -> { - PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); - }); - - PcapHandle handle; - try { - handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); - } catch (PcapNativeException pne) { - handle = Pcaps.openOffline(inputPcapFile); - } - PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); - reader.readFromHandle(); - - // TODO: need a better way of triggering detection than this... - if (isRangeBasedForOn && isRangeBasedForOff) { - onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); - offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionRangeBased()); - } else { - onDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); - offDetector.mClusterMatchers.forEach(cm -> cm.performDetectionConservative()); - } - - // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger - // times file. - Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp)); - - // Output the detected events - detectedEvents.forEach(outputter); - - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); - - // TODO: Temporary clean up until we clean the pipeline -// List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); -// cleanedDetectedEvents.forEach(outputter); - } - - /** - * The signature that this {@link SignatureDetector} is searching for. - */ - private final List>> mSignature; - - /** - * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the - * the signature. - */ - private final List mClusterMatchers; - - /** - * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches - * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., - * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed - * of multiple packet sequences occurring shortly after one another on multiple connections). - */ - private final List>[] pendingMatches; - - /** - * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}. - */ - private final Map mClusterMatcherIds; - - private final List mObservers = new ArrayList<>(); - - private int mInclusionTimeMillis; - - /** - * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions - * that appear multiple times. - * TODO: This static method is probably just for temporary and we could get rid of this after we clean up - * TODO: the pipeline - * - * @param listUserAction A {@link List} of {@code UserAction}. - * - */ - public static List removeDuplicates(List listUserAction) { - - // Iterate and check for duplicates (check timestamps) - Set epochSecondSet = new HashSet<>(); - // Create a target list for cleaned up list - List listUserActionClean = new ArrayList<>(); - for(UserAction userAction : listUserAction) { - // Don't insert if any duplicate is found - if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) { - listUserActionClean.add(userAction); - epochSecondSet.add(userAction.getTimestamp().getEpochSecond()); - } - } - return listUserActionClean; - } - - public SignatureDetector(List>> searchedSignature, String routerWanIp, - int inclusionTimeMillis, boolean isRangeBased, double eps) { - // note: doesn't protect inner lists from changes :'( - mSignature = Collections.unmodifiableList(searchedSignature); - // Generate corresponding/appropriate ClusterMatchers based on the provided signature - List clusterMatchers = new ArrayList<>(); - for (List> cluster : mSignature) { - clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, isRangeBased, eps, this)); - } - mClusterMatchers = Collections.unmodifiableList(clusterMatchers); - - // < exploratory > - pendingMatches = new List[mClusterMatchers.size()]; - for (int i = 0; i < pendingMatches.length; i++) { - pendingMatches[i] = new ArrayList<>(); - } - Map clusterMatcherIds = new HashMap<>(); - for (int i = 0; i < mClusterMatchers.size(); i++) { - clusterMatcherIds.put(mClusterMatchers.get(i), i); - } - mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); - mInclusionTimeMillis = - inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; - } - - public void addObserver(SignatureDetectionObserver observer) { - mObservers.add(observer); - } - - public boolean removeObserver(SignatureDetectionObserver observer) { - return mObservers.remove(observer); - } - - @Override - public void gotPacket(PcapPacket packet) { - // simply delegate packet reception to all ClusterMatchers. - mClusterMatchers.forEach(cm -> cm.gotPacket(packet)); - } - - @Override - public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { - // Add the match at the corresponding index - pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); - checkSignatureMatch(); - } - - private void checkSignatureMatch() { - // << Graph-based approach using Balint's idea. >> - // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. - - // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence. - if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) { - // Construct the DAG - final SimpleDirectedWeightedGraph graph = - new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); - // Add a vertex for each match found by all ClusterMatchers - // And maintain an array to keep track of what cluster matcher each vertex corresponds to - final List[] vertices = new List[pendingMatches.length]; - for (int i = 0; i < pendingMatches.length; i++) { - vertices[i] = new ArrayList<>(); - for (List sequence : pendingMatches[i]) { - Vertex v = new Vertex(sequence); - vertices[i].add(v); // retain reference for later when we are to add edges - graph.addVertex(v); // add to vertex to graph - } - } - // Add dummy source and sink vertices to facilitate search. - final Vertex source = new Vertex(null); - final Vertex sink = new Vertex(null); - graph.addVertex(source); - graph.addVertex(sink); - // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0. - // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. - for (Vertex v : vertices[0]) { - DefaultWeightedEdge edge = graph.addEdge(source, v); - graph.setEdgeWeight(edge, 0.0); - } - // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature - // are connected to the sink node. - for (Vertex v : vertices[vertices.length-1]) { - DefaultWeightedEdge edge = graph.addEdge(v, sink); - graph.setEdgeWeight(edge, 0.0); - } - // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index - // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former). - for (int i = 0; i < vertices.length; i++) { - int j = i + 1; - if (j < vertices.length) { - for (Vertex iv : vertices[i]) { - PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); - for (Vertex jv : vertices[j]) { - PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); - if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { - DefaultWeightedEdge edge = graph.addEdge(iv, jv); - // The weight is the duration of the i'th sequence plus the duration between the i'th - // and i+1'th sequence. - Duration d = Duration. - between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); - // Unfortunately weights are double values, so must convert from long to double. - // TODO: need nano second precision? If so, use d.toNanos(). - // TODO: risk of overflow when converting from long to double..? - graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); - } - // Alternative version if we cannot assume that sequences are ordered by timestamp: -// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() -// .getTimestamp().isBefore(jv.sequence.stream().min( -// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { -// -// } - } - } - } - } - // Graph construction complete, run shortest-path to find a (potential) signature match. - DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); - GraphPath shortestPath = dijkstra.getPath(source, sink); - if (shortestPath != null) { - // The total weight is the duration between the first packet of the first sequence and the last packet - // of the last sequence, so we simply have to compare the weight against the timeframe that we allow - // the signature to span. For now we just use the inclusion window we defined for training purposes. - // Note however, that we must convert back from double to long as the weight is stored as a double in - // JGraphT's API. - if (((long)shortestPath.getWeight()) < mInclusionTimeMillis) { - // There's a signature match! - // Extract the match from the vertices - List> signatureMatch = new ArrayList<>(); - for(Vertex v : shortestPath.getVertexList()) { - if (v == source || v == sink) { - // Skip the dummy source and sink nodes. - continue; - } - signatureMatch.add(v.sequence); - // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that - // the sequence we've "consumed" for index i of the matched signature is also at index i in - // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct - // another signature match in a later call. - pendingMatches[signatureMatch.size()-1].remove(v.sequence); - } - // Declare success: notify observers - mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, - Collections.unmodifiableList(signatureMatch))); - } - } - } - } - - /** - * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. - */ - interface SignatureDetectionObserver { - - /** - * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's - * examining. - * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching - * for. - * @param matchingTraffic The actual traffic trace that matches the searched signature. - */ - void onSignatureDetected(List>> searchedSignature, - List> matchingTraffic); - } - - /** - * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding - * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. - * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} - * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not - * recognize two lists that contain the same items--from a value and not reference point of view--as the same - * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more - * appropriate. - */ - private static class Vertex { - private final List sequence; - private Vertex(List wrappedSequence) { - sequence = wrappedSequence; - } - } -} diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java index e1648ba..8f5874c 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java @@ -16,5 +16,4 @@ public interface Layer2FlowObserver { * @param newPacket The packet that was added to the flow. */ void onNewPacket(Layer2Flow flow, PcapPacket newPacket); - }