/**
* For reassembling the observed traffic into TCP connections.
*/
- private final TcpReassembler mTcpReassembler = new TcpReassembler();
+ private final TcpReassembler mTcpReassembler;
/**
* IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
*/
private int mInclusionTimeMillis;
+ /**
+ * Relaxed matching
+ */
+ private int mDelta;
+ private Set<Integer> mPacketSet;
+
/**
* Create a {@link Layer3ClusterMatcher}.
* @param cluster The cluster that traffic is matched against.
* {@code cluster}.
*/
public Layer3ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp, int inclusionTimeMillis,
- boolean isRangeBased, double eps,
+ boolean isRangeBased, double eps, int delta, Set<Integer> packetSet,
ClusterMatcherObserver... detectionObservers) {
super(cluster, isRangeBased);
Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null");
}
mEps = eps;
mRouterWanIp = routerWanIp;
+ mTcpReassembler = new TcpReassembler(mRouterWanIp);
mInclusionTimeMillis =
inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
+ mDelta = delta;
+ mPacketSet = packetSet;
}
@Override
isPresent()) {
List<PcapPacket> matchSeq = match.get();
// Notify observers about the match.
-// mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
- if (!matchSeq.get(matchSeq.size()-1).getTimestamp().isAfter(matchSeq.get(0).getTimestamp().
- plusMillis(mInclusionTimeMillis))) {
- // Notify observers about the match.
- mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
- }
+ // Max number of skipped packets in layer 3 is 0 (no skipped packets)
+ mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
/*
* Get the index in cPkts of the last packet in the sequence of packets that matches the searched
* signature sequence.
}
}
+ // TODO: Relaxed matching with delta is only applied to conservative matching for now
public void performDetectionConservative() {
/*
* Let's start out simple by building a version that only works for signatures that do not span across multiple
* won't have to be recomputed internally in each call to findSubsequenceInSequence().
*/
Optional<List<PcapPacket>> match;
- while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)).
+ while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null, mDelta, mPacketSet)).
isPresent()) {
List<PcapPacket> matchSeq = match.get();
// Notify observers about the match.
-// mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
- if (!matchSeq.get(matchSeq.size()-1).getTimestamp().isAfter(matchSeq.get(0).getTimestamp().
- plusMillis(mInclusionTimeMillis))) {
- // Notify observers about the match.
- mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
- }
+ // Max number of skipped packets in layer 3 is 0 (no skipped packets)
+ mObservers.forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
/*
* Get the index in cPkts of the last packet in the sequence of packets that matches the searched
* signature sequence.
return Optional.empty();
}
+ /**
+ * Overload the same method with relaxed matching.
+ *
+ * @param subsequence The sequence to search for.
+ * @param sequence The sequence to search.
+ * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i},
+ * {@code subsequenceDirections[i]} is the direction of the packet returned by
+ * {@code subsequence.get(i)}. May be set to {@code null}, in which this call will
+ * internally compute the packet directions.
+ * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i},
+ * {@code sequenceDirections[i]} is the direction of the packet returned by
+ * {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally
+ * compute the packet directions.
+ * @param delta The delta for relaxed matching
+ * @param packetSet The set of unique packet lengths, whose matching is to be relaxed
+ *
+ * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty
+ * {@link Optional} if no part of {@code sequence} matches {@code subsequence}.
+ */
+ private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+ List<PcapPacket> sequence,
+ Conversation.Direction[] subsequenceDirections,
+ Conversation.Direction[] sequenceDirections,
+ int delta,
+ Set<Integer> packetSet) {
+ if (sequence.size() < subsequence.size()) {
+ // If subsequence is longer, it cannot be contained in sequence.
+ return Optional.empty();
+ }
+ if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
+ // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+ return Optional.empty();
+ }
+ // If packet directions have not been precomputed by calling code, we need to construct them.
+ if (subsequenceDirections == null) {
+ subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
+ }
+ if (sequenceDirections == null) {
+ sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+ }
+ int subseqIdx = 0;
+ int seqIdx = 0;
+ while (seqIdx < sequence.size()) {
+ PcapPacket subseqPkt = subsequence.get(subseqIdx);
+ PcapPacket seqPkt = sequence.get(seqIdx);
+ // We only have a match if packet lengths and directions match.
+ // Do relaxed matching here if applicable
+ if ((delta > 0 && packetSet.contains(subseqPkt.getOriginalLength()) &&
+ subseqPkt.getOriginalLength() - delta <= seqPkt.getOriginalLength() &&
+ seqPkt.getOriginalLength() <= subseqPkt.getOriginalLength() + delta &&
+ subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) ||
+ // Or just exact matching
+ (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() &&
+ subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx])) {
+ // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+ subseqIdx++;
+ seqIdx++;
+ if (subseqIdx == subsequence.size()) {
+ // We managed to match the entire subsequence in sequence.
+ // Return the sublist of sequence that matches subsequence.
+ /*
+ * TODO:
+ * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+ * for live traces!
+ */
+ return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
+ }
+ } else {
+ // Mismatch.
+ if (subseqIdx > 0) {
+ /*
+ * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+ * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+ * leave seqIdx untouched.
+ */
+ subseqIdx = 0;
+ } else {
+ /*
+ * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+ * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+ * sequence.
+ */
+ seqIdx++;
+ }
+ }
+ }
+ return Optional.empty();
+ }
+
/**
* Overloading the method {@code findSubsequenceInSequence} for range-based matching. Instead of a sequence,
* we have sequences of lower and upper bounds.
PcapPacket seqPkt = sequence.get(seqIdx);
// We only have a match if packet lengths and directions match.
// The packet lengths have to be in the range of [lowerBound - eps, upperBound+eps]
- // TODO: Maybe we could do better here for the double to integer conversion?
- int epsLowerBound = lowBndPkt.length() - (int) mEps;
- int epsUpperBound = upBndPkt.length() + (int) mEps;
- // TODO: TEMPORARILY REMOVE EPS BOUNDS
-// int epsLowerBound = lowBndPkt.length();
-// int epsUpperBound = upBndPkt.length();
+ // We initialize the lower and upper bounds first
+ int epsLowerBound = lowBndPkt.length();
+ int epsUpperBound = upBndPkt.length();
+ // Do strict matching if the lower and upper bounds are the same length
+ // Do range matching with eps otherwise
+ if (epsLowerBound != epsUpperBound) {
+ // TODO: Maybe we could do better here for the double to integer conversion?
+ epsLowerBound = epsLowerBound - (int) mEps;
+ epsUpperBound = epsUpperBound + (int) mEps;
+ }
if (epsLowerBound <= seqPkt.getOriginalLength() &&
seqPkt.getOriginalLength() <= epsUpperBound &&
subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
return Optional.empty();
}
+ // TODO: EXPERIMENT WITH ONLY PACKET DIRECTION AND TIMING
+// private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+// List<PcapPacket> sequence,
+// Conversation.Direction[] subsequenceDirections,
+// Conversation.Direction[] sequenceDirections) {
+// if (sequence.size() < subsequence.size()) {
+// // If subsequence is longer, it cannot be contained in sequence.
+// return Optional.empty();
+// }
+// if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
+// // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+// return Optional.empty();
+// }
+// // If packet directions have not been precomputed by calling code, we need to construct them.
+// if (subsequenceDirections == null) {
+// subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
+// }
+// if (sequenceDirections == null) {
+// sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+// }
+// int subseqIdx = 0;
+// int seqIdx = 0;
+// while (subseqIdx < subsequence.size() && seqIdx < sequence.size()) {
+// // We only have a match if packet lengths and directions match.
+// if (subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+// // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+// subseqIdx++;
+// seqIdx++;
+// if (subseqIdx == subsequence.size()) {
+// // We managed to match the entire subsequence in sequence.
+// // Return the sublist of sequence that matches subsequence.
+// /*
+// * TODO:
+// * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+// * for live traces!
+// */
+// // TODO: ALSO CHECK TIMING CONSTRAINT
+// PcapPacket firstPacket = sequence.get(seqIdx - subsequence.size());
+// PcapPacket lastPacket = sequence.get(seqIdx-1);
+// if (!lastPacket.getTimestamp().isAfter(firstPacket.getTimestamp().plusMillis(mInclusionTimeMillis))) {
+// return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
+// }
+// }
+// } else {
+// // Mismatch.
+// if (subseqIdx > 0) {
+// /*
+// * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+// * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+// * leave seqIdx untouched.
+// */
+// subseqIdx = 0;
+// } else {
+// /*
+// * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+// * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+// * sequence.
+// */
+// seqIdx++;
+// }
+// }
+// }
+// return Optional.empty();
+// }
+//
+// private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> lowerBound,
+// List<PcapPacket> upperBound,
+// List<PcapPacket> sequence,
+// Conversation.Direction[] subsequenceDirections,
+// Conversation.Direction[] sequenceDirections) {
+// // Just do the checks for either lower or upper bound!
+// // TODO: For now we use just the lower bound
+// if (sequence.size() < lowerBound.size()) {
+// // If subsequence is longer, it cannot be contained in sequence.
+// return Optional.empty();
+// }
+// if (isTlsSequence(lowerBound) != isTlsSequence(sequence)) {
+// // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+// return Optional.empty();
+// }
+// // If packet directions have not been precomputed by calling code, we need to construct them.
+// if (subsequenceDirections == null) {
+// subsequenceDirections = getPacketDirections(lowerBound, mRouterWanIp);
+// }
+// if (sequenceDirections == null) {
+// sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+// }
+// int subseqIdx = 0;
+// int seqIdx = 0;
+// while (subseqIdx < lowerBound.size() && seqIdx < sequence.size()) {
+// // TODO: ONLY MATCH PACKET DIRECTIONS
+// if (subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+// // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+// subseqIdx++;
+// seqIdx++;
+// if (subseqIdx == lowerBound.size()) {
+// // We managed to match the entire subsequence in sequence.
+// // Return the sublist of sequence that matches subsequence.
+// /*
+// * TODO:
+// * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+// * for live traces!
+// */
+// // TODO: ALSO CHECK TIMING CONSTRAINT
+// PcapPacket firstPacket = sequence.get(seqIdx - lowerBound.size());
+// PcapPacket lastPacket = sequence.get(seqIdx);
+// if (!lastPacket.getTimestamp().isAfter(firstPacket.getTimestamp().plusMillis(mInclusionTimeMillis))) {
+// return Optional.of(sequence.subList(seqIdx - lowerBound.size(), seqIdx));
+// }
+// }
+// } else {
+// // Mismatch.
+// if (subseqIdx > 0) {
+// /*
+// * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+// * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+// * leave seqIdx untouched.
+// */
+// subseqIdx = 0;
+// } else {
+// /*
+// * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+// * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+// * sequence.
+// */
+// seqIdx++;
+// }
+// }
+// }
+// return Optional.empty();
+// }
+
/**
* Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
* members. Two cluster members are considered identical if their packets lengths and packet directions are