From 39172356d48f5cd574ef15ec276a33de9146155a Mon Sep 17 00:00:00 2001 From: rtrimana Date: Mon, 8 Apr 2019 16:55:56 -0700 Subject: [PATCH] Adding skipped packets analysis. --- .../uci/iotproject/SignatureGenerator.java | 9 +++++++-- .../detection/ClusterMatcherObserver.java | 3 ++- .../layer2/Layer2AbstractMatcher.java | 12 ++++++++++++ .../layer2/Layer2ClusterMatcher.java | 6 ++++-- .../detection/layer2/Layer2RangeMatcher.java | 4 +++- .../layer2/Layer2SequenceMatcher.java | 8 ++++++++ .../layer2/Layer2SignatureDetector.java | 19 ++++++++++++++++++- .../layer3/Layer3ClusterMatcher.java | 16 ++++------------ .../layer3/Layer3SignatureDetector.java | 2 +- .../uci/iotproject/util/PcapPacketUtils.java | 13 +++++++++---- 10 files changed, 68 insertions(+), 24 deletions(-) diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java index 3104e08..0195f1d 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java @@ -283,6 +283,8 @@ public class SignatureGenerator { PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, sequenceToDelete); } ppListOfListListOn = PcapPacketUtils.sortSequences(ppListOfListListOn); + PrintWriterUtils.println("Concatenated and sorted ON signature sequences...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); // Concatenate ppListOfListListOff = PcapPacketUtils.concatSequences(ppListOfListListOff, sortedAllConversation); @@ -296,6 +298,9 @@ public class SignatureGenerator { PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, sequenceToDelete); } ppListOfListListOff = PcapPacketUtils.sortSequences(ppListOfListListOff); + PrintWriterUtils.println("Concatenated and sorted OFF signature sequences...", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + // Write the signatures into the screen PrintWriterUtils.println("========================================", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); @@ -318,7 +323,7 @@ public class SignatureGenerator { PrintUtils.serializeIntoFile(onClusterAnalysisFile, corePointRangeSignatureOn); PrintUtils.serializeIntoFile(offClusterAnalysisFile, corePointRangeSignatureOff); - // =========================================== SIGNATURE DURATION =========================================== + // =========================================== SIGNATURE DURATIONS ============================================= List firstSignatureTimestamps = new ArrayList<>(); List lastSignatureTimestamps = new ArrayList<>(); if (!ppListOfListListOn.isEmpty()) { @@ -336,7 +341,7 @@ public class SignatureGenerator { } } - if (!ppListOfListListOn.isEmpty()) { + if (!ppListOfListListOff.isEmpty()) { List> firstListOffSign = ppListOfListListOff.get(0); List> lastListOffSign = ppListOfListListOff.get(ppListOfListListOff.size() - 1); // Load OFF signature first and last packet's timestamps diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java index d67c520..9108858 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/ClusterMatcherObserver.java @@ -20,7 +20,8 @@ public interface ClusterMatcherObserver { * @param clusterMatcher The {@link AbstractClusterMatcher} that detected a match (i.e., classified traffic as * pertaining to its associated cluster). * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}. + * @param maxSkippedPackets Maximum number of skipped packets. */ - void onMatch(AbstractClusterMatcher clusterMatcher, List match); + void onMatch(AbstractClusterMatcher clusterMatcher, List match, int maxSkippedPackets); } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java index 1621c82..6a7de57 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2AbstractMatcher.java @@ -27,6 +27,12 @@ abstract public class Layer2AbstractMatcher { */ protected final boolean[] mPacketDirections; + /** + * Keep track of the numbers of skipped packets + */ + protected int mSkippedPackets; + protected int mMaxSkippedPackets; + /** * Create a {@code Layer2AbstractMatcher}. * @param sequence The sequence of the signature. @@ -45,6 +51,8 @@ abstract public class Layer2AbstractMatcher { mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i)); } } + mSkippedPackets = 0; + mMaxSkippedPackets = 0; } /** @@ -91,6 +99,10 @@ abstract public class Layer2AbstractMatcher { return mMatchedPackets; } + public int getMaxSkippedPackets() { + return mMaxSkippedPackets; + } + /** * Utility for {@code getMatchedPackets().get(getMatchedPackets().size()-1)}. * @return The last matched packet, or {@code null} if no packets have been matched yet. diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java index bec7840..159146e 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -120,7 +120,8 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye if (matched) { if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { // Sequence matcher has a match. Report it to observers. - mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets())); + mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets(), + sm.getMaxSkippedPackets())); // Remove the now terminated sequence matcher. matchers[i][j] = null; } else { @@ -192,7 +193,8 @@ public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Laye if (matched) { if (sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { // Sequence matcher has a match. Report it to observers. - mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets())); + mObservers.forEach(o -> o.onMatch(this, sm.getMatchedPackets(), + sm.getMaxSkippedPackets())); // Terminate sequence matcher since matching is complete. listMatchers.remove(matcher); } diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java index 5f92df7..97fa072 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2RangeMatcher.java @@ -24,6 +24,7 @@ public class Layer2RangeMatcher extends Layer2AbstractMatcher { private final List mUpperBound; private final double mEps; private int mInclusionTimeMillis; + private int mSkippedPackets; /** * Create a {@code Layer2RangeMatcher}. @@ -41,6 +42,7 @@ public class Layer2RangeMatcher extends Layer2AbstractMatcher { mEps = eps; mInclusionTimeMillis = inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + mSkippedPackets = 0; } /** @@ -108,7 +110,7 @@ public class Layer2RangeMatcher extends Layer2AbstractMatcher { return false; } // If we made it here, it means that this packet has the expected length, direction, and obeys the timing - // constraints, so we store it and advance. + // constraints, so we store it and advance.zzzz mMatchedPackets.add(packet); if (mMatchedPackets.size() == mLowerBound.size()) { // TODO report (to observers?) that we are done? diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java index a9d6241..49951f0 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -24,6 +24,7 @@ public class Layer2SequenceMatcher extends Layer2AbstractMatcher { private int mInclusionTimeMillis; + /** * Create a {@code Layer2SequenceMatcher}. * @param sequence The sequence to match against (search for). @@ -90,22 +91,29 @@ public class Layer2SequenceMatcher extends Layer2AbstractMatcher { mPacketDirections[getMatchedPacketsCount()-1], packet); boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()]; if (actualDirection != expectedDirection) { + mSkippedPackets++; return false; } // Next apply timing constraints: // 1: to be a match, the packet must have a later timestamp than any other packet currently matched // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded? if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) { + mSkippedPackets++; return false; } // if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). // plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). plusMillis(mInclusionTimeMillis))) { + mSkippedPackets++; return false; } // If we made it here, it means that this packet has the expected length, direction, and obeys the timing // constraints, so we store it and advance. + if (mMaxSkippedPackets < mSkippedPackets) { + mMaxSkippedPackets = mSkippedPackets; + mSkippedPackets = 0; + } mMatchedPackets.add(packet); if (mMatchedPackets.size() == mSequence.size()) { // TODO report (to observers?) that we are done? diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java index 1980348..51883c0 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SignatureDetector.java @@ -178,8 +178,14 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count(); String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count(); + String onMaxSkippedPackets = "# Number of skipped packets in ON signature " + + Integer.toString(onDetector.getMaxSkippedPackets()); + String offMaxSkippedPackets = "# Number of skipped packets in OFF signature " + + Integer.toString(offDetector.getMaxSkippedPackets()); PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(onMaxSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(offMaxSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // Flush output to results file and close it. resultsWriter.flush(); @@ -219,6 +225,8 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb private int mInclusionTimeMillis; + private int mMaxSkippedPackets; + public Layer2SignatureDetector(List>> searchedSignature, int signatureDuration, boolean isRangeBased, double eps) { this(searchedSignature, null, signatureDuration, isRangeBased, eps); } @@ -253,6 +261,11 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm)); mInclusionTimeMillis = inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis; + mMaxSkippedPackets = 0; + } + + public int getMaxSkippedPackets() { + return mMaxSkippedPackets; } @Override @@ -262,7 +275,11 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb } @Override - public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { + public void onMatch(AbstractClusterMatcher clusterMatcher, List match, int maxSkippedPackets) { + // Update the number of skipped packets + if (mMaxSkippedPackets < maxSkippedPackets) { + mMaxSkippedPackets = maxSkippedPackets; + } // TODO: a cluster matcher found a match if (clusterMatcher instanceof Layer2ClusterMatcher) { // Add the match at the corresponding index diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java index 398ac1e..56b4b0a 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java @@ -132,12 +132,8 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack isPresent()) { List matchSeq = match.get(); // Notify observers about the match. - mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); -// if (!matchSeq.get(matchSeq.size()-1).getTimestamp().isAfter(matchSeq.get(0).getTimestamp(). -// plusMillis(mInclusionTimeMillis))) { -// // Notify observers about the match. -// mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); -// } + // Max number of skipped packets in layer 3 is 0 (no skipped packets) + mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq, 0)); /* * Get the index in cPkts of the last packet in the sequence of packets that matches the searched * signature sequence. @@ -179,12 +175,8 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack isPresent()) { List matchSeq = match.get(); // Notify observers about the match. - mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); -// if (!matchSeq.get(matchSeq.size()-1).getTimestamp().isAfter(matchSeq.get(0).getTimestamp(). -// plusMillis(mInclusionTimeMillis))) { -// // Notify observers about the match. -// mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); -// } + // Max number of skipped packets in layer 3 is 0 (no skipped packets) + mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq, 0)); /* * Get the index in cPkts of the last packet in the sequence of packets that matches the searched * signature sequence. diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java index 03e4bd1..dbd9046 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3SignatureDetector.java @@ -290,7 +290,7 @@ public class Layer3SignatureDetector implements PacketListener, ClusterMatcherOb } @Override - public void onMatch(AbstractClusterMatcher clusterMatcher, List match) { + public void onMatch(AbstractClusterMatcher clusterMatcher, List match, int maxSkippedPackets) { // Add the match at the corresponding index pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); checkSignatureMatch(); diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java index b453566..c1a1a25 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -348,7 +348,8 @@ public final class PcapPacketUtils { if (Math.abs(timestamp1 - timestamp2) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { // If these two are within INCLUSION_WINDOW_MILLIS window then compare! compare = p1.get(count1).get(0).getTimestamp().compareTo(p2.get(count2).get(0).getTimestamp()); - overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2)); + overlapChecking(compare, comparePrev, p1.get(count1), p2.get(count2), + signatures.indexOf(p1), signatures.indexOf(p2)); comparePrev = compare; count1++; count2++; @@ -372,8 +373,12 @@ public final class PcapPacketUtils { * @param comparePrev Previous comparison value between packet sequences p1 and p2 * @param sequence1 The packet sequence ({@link List} of {@link PcapPacket} objects). * @param sequence2 The packet sequence ({@link List} of {@link PcapPacket} objects). + * @param indexSequence1 The index of packet sequence ({@link List} of {@link PcapPacket} objects). + * @param indexSequence2 The index of packet sequence ({@link List} of {@link PcapPacket} objects). */ - private static void overlapChecking(int compare, int comparePrev, List sequence1, List sequence2) { + private static void overlapChecking(int compare, int comparePrev, + List sequence1, List sequence2, + int indexSequence1, int indexSequence2) { // Check if p1 occurs before p2 but both have same overlap if (comparePrev != 0) { // First time since it is 0 @@ -382,8 +387,8 @@ public final class PcapPacketUtils { // E.g., 111, 222, 333 in one occassion and 222, 333, 111 in the other. throw new Error("OVERLAP WARNING: " + "" + "Two sequences have some overlap. Please remove one of the sequences: " + - sequence1.get(0).length() + "... OR " + - sequence2.get(0).length() + "..."); + sequence1.get(0).length() + " with index " + indexSequence1 + " OR " + + sequence2.get(0).length() + " with index " + indexSequence2); } } // Check if p1 is longer than p2 and p2 occurs during the occurrence of p1 -- 2.34.1